diff --git a/configure.ac b/configure.ac index d4143bce0..de2165ab2 100644 --- a/configure.ac +++ b/configure.ac @@ -108,6 +108,27 @@ AC_ARG_WITH([swift-toolchain], AM_CONDITIONAL(HAVE_SWIFT, $have_swift) AC_SUBST([SWIFTC]) +# +# Enable __thread based TSD on platforms where it is efficient +# Allow override based on command line argument to configure +# +AC_CANONICAL_TARGET +AC_ARG_ENABLE([thread-local-storage], + [AS_HELP_STRING([--enable-thread-local-storage], + [Enable usage of thread local storage via __thread])],, + [case $target_os in + linux*) + enable_thread_local_storage=yes + ;; + *) + enable_thread_local_storage=no + esac] +) +AS_IF([test "x$enable_thread_local_storage" = "xyes"], + [AC_DEFINE(DISPATCH_USE_THREAD_LOCAL_STORAGE, 1, + [Enable usage of thread local storage via __thread])] +) + AC_USE_SYSTEM_EXTENSIONS AM_INIT_AUTOMAKE([foreign no-dependencies subdir-objects]) LT_INIT([disable-static]) diff --git a/src/init.c b/src/init.c index e18dbe732..8b1a98c52 100644 --- a/src/init.c +++ b/src/init.c @@ -67,7 +67,10 @@ void *(*_dispatch_begin_NSAutoReleasePool)(void); void (*_dispatch_end_NSAutoReleasePool)(void *); #endif -#if !DISPATCH_USE_DIRECT_TSD +#if DISPATCH_USE_THREAD_LOCAL_STORAGE +__thread struct dispatch_tsd __dispatch_tsd; +pthread_key_t __dispatch_tsd_key; +#elif !DISPATCH_USE_DIRECT_TSD pthread_key_t dispatch_queue_key; pthread_key_t dispatch_frame_key; pthread_key_t dispatch_cache_key; @@ -82,7 +85,7 @@ pthread_key_t dispatch_bcounter_key; pthread_key_t dispatch_sema4_key; pthread_key_t dispatch_voucher_key; pthread_key_t dispatch_deferred_items_key; -#endif // !DISPATCH_USE_DIRECT_TSD +#endif // !DISPATCH_USE_DIRECT_TSD && !DISPATCH_USE_THREAD_LOCAL_STORAGE #if VOUCHER_USE_MACH_VOUCHER dispatch_once_t _voucher_task_mach_voucher_pred; @@ -184,10 +187,6 @@ const struct dispatch_tsd_indexes_s dispatch_tsd_indexes = { .dti_voucher_index = dispatch_voucher_key, .dti_qos_class_index = dispatch_priority_key, }; -#else // DISPATCH_USE_DIRECT_TSD -#ifndef __LINUX_PORT_HDD__ -#error Not implemented on this platform -#endif #endif // DISPATCH_USE_DIRECT_TSD // 6618342 Contact the team that owns the Instrument DTrace probe before diff --git a/src/queue.c b/src/queue.c index 9fb419f90..dec60d3cb 100644 --- a/src/queue.c +++ b/src/queue.c @@ -894,6 +894,9 @@ libdispatch_init(void) #endif #endif +#if DISPATCH_USE_THREAD_LOCAL_STORAGE + _dispatch_thread_key_create(&__dispatch_tsd_key, _libdispatch_tsd_cleanup); +#else _dispatch_thread_key_create(&dispatch_queue_key, _dispatch_queue_cleanup); _dispatch_thread_key_create(&dispatch_deferred_items_key, _dispatch_deferred_items_cleanup); @@ -913,6 +916,7 @@ libdispatch_init(void) _dispatch_thread_semaphore_dispose); } #endif +#endif #if DISPATCH_USE_RESOLVERS // rdar://problem/8541707 _dispatch_main_q.do_targetq = &_dispatch_root_queues[ @@ -967,6 +971,59 @@ _dispatch_get_mach_host_port(void) } #endif +#if DISPATCH_USE_THREAD_LOCAL_STORAGE +#include +#include + +#ifdef SYS_gettid +DISPATCH_ALWAYS_INLINE +static inline pid_t +gettid(void) +{ + return (pid_t) syscall(SYS_gettid); +} +#else +#error "SYS_gettid unavailable on this system" +#endif + +#define _tsd_call_cleanup(k, f) do { \ + if ((f) && tsd->k) ((void(*)(void*))(f))(tsd->k); \ + } while (0) + +void +_libdispatch_tsd_cleanup(void *ctx) +{ + struct dispatch_tsd *tsd = (struct dispatch_tsd*) ctx; + + _tsd_call_cleanup(dispatch_queue_key, _dispatch_queue_cleanup); + _tsd_call_cleanup(dispatch_frame_key, _dispatch_frame_cleanup); + _tsd_call_cleanup(dispatch_cache_key, _dispatch_cache_cleanup); + _tsd_call_cleanup(dispatch_context_key, _dispatch_context_cleanup); + _tsd_call_cleanup(dispatch_pthread_root_queue_observer_hooks_key, + NULL); + _tsd_call_cleanup(dispatch_defaultpriority_key, NULL); +#if DISPATCH_PERF_MON && !DISPATCH_INTROSPECTION + _tsd_call_cleanup(dispatch_bcounter_key, NULL); +#endif +#if DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK + _tsd_call_cleanup(dispatch_sema4_key, _dispatch_thread_semaphore_dispose); +#endif + _tsd_call_cleanup(dispatch_priority_key, NULL); + _tsd_call_cleanup(dispatch_voucher_key, _voucher_thread_cleanup); + _tsd_call_cleanup(dispatch_deferred_items_key, + _dispatch_deferred_items_cleanup); + tsd->tid = 0; +} + +DISPATCH_NOINLINE +void +libdispatch_tsd_init(void) +{ + pthread_setspecific(__dispatch_tsd_key, &__dispatch_tsd); + __dispatch_tsd.tid = gettid(); +} +#endif + DISPATCH_EXPORT DISPATCH_NOTHROW void dispatch_atfork_child(void) diff --git a/src/shims/linux_stubs.c b/src/shims/linux_stubs.c index d5618d572..33a4578ac 100644 --- a/src/shims/linux_stubs.c +++ b/src/shims/linux_stubs.c @@ -57,3 +57,4 @@ unsigned short dispatch_callout__return_semaphore; unsigned short dispatch_queue__push_semaphore; void (*_dispatch_block_special_invoke)(void*); struct dispatch_queue_attr_s _dispatch_queue_attr_concurrent; + diff --git a/src/shims/lock.h b/src/shims/lock.h index 0786d9947..5c2dfc502 100644 --- a/src/shims/lock.h +++ b/src/shims/lock.h @@ -98,7 +98,8 @@ typedef pid_t dispatch_lock_owner; #define DLOCK_OWNER_MASK ((dispatch_lock)FUTEX_TID_MASK) #define DLOCK_WAITERS_BIT ((dispatch_lock)FUTEX_WAITERS) #define DLOCK_FAILED_TRYLOCK_BIT ((dispatch_lock)FUTEX_OWNER_DIED) -#define _dispatch_tid_self() syscall(SYS_gettid) /* FIXME: should be cached in TSD instead of doing syscall each time */ +#define _dispatch_tid_self() \ + ((dispatch_lock_owner)(_dispatch_get_tsd_base()->tid)) DISPATCH_ALWAYS_INLINE static inline bool diff --git a/src/shims/tsd.h b/src/shims/tsd.h index ef67a90d3..2edba3959 100644 --- a/src/shims/tsd.h +++ b/src/shims/tsd.h @@ -92,10 +92,76 @@ _dispatch_thread_key_create(const unsigned long *k, void (*d)(void *)) if (!*k || !d) return; dispatch_assert_zero(pthread_key_init_np((int)*k, d)); } +#elif DISPATCH_USE_THREAD_LOCAL_STORAGE + +DISPATCH_TSD_INLINE +static inline void +_dispatch_thread_key_create(pthread_key_t *k, void (*d)(void *)) +{ + dispatch_assert_zero(pthread_key_create(k, d)); +} + +struct dispatch_tsd { + pid_t tid; + void *dispatch_queue_key; + void *dispatch_frame_key; + void *dispatch_cache_key; + void *dispatch_context_key; + void *dispatch_pthread_root_queue_observer_hooks_key; + void *dispatch_defaultpriority_key; +#if DISPATCH_INTROSPECTION + void *dispatch_introspection_key; +#elif DISPATCH_PERF_MON + void *dispatch_bcounter_key; +#endif +#if DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK + void *dispatch_sema4_key; +#endif + void *dispatch_priority_key; + void *dispatch_voucher_key; + void *dispatch_deferred_items_key; +}; + +extern __thread struct dispatch_tsd __dispatch_tsd; +extern pthread_key_t __dispatch_tsd_key; +extern void libdispatch_tsd_init(void); +extern void _libdispatch_tsd_cleanup(void *ctx); + +DISPATCH_ALWAYS_INLINE +static inline struct dispatch_tsd * +_dispatch_get_tsd_base(void) +{ + if (unlikely(__dispatch_tsd.tid == 0)) { + libdispatch_tsd_init(); + } + OS_COMPILER_CAN_ASSUME(__dispatch_tsd.tid != 0); + return &__dispatch_tsd; +} + +#define _dispatch_thread_getspecific(key) \ + (_dispatch_get_tsd_base()->key) +#define _dispatch_thread_setspecific(key, value) \ + (void)(_dispatch_get_tsd_base()->key = (value)) + +#define _dispatch_thread_getspecific_pair(k1, p1, k2, p2) \ + ( *(p1) = _dispatch_thread_getspecific(k1), \ + *(p2) = _dispatch_thread_getspecific(k2) ) + +#define _dispatch_thread_getspecific_packed_pair(k1, k2, p) \ + ( (p)[0] = _dispatch_thread_getspecific(k1), \ + (p)[1] = _dispatch_thread_getspecific(k2) ) + +#define _dispatch_thread_setspecific_pair(k1, p1, k2, p2) \ + ( _dispatch_thread_setspecific(k1,p1), \ + _dispatch_thread_setspecific(k2,p2) ) + +#define _dispatch_thread_setspecific_packed_pair(k1, k2, p) \ + ( _dispatch_thread_setspecific(k1,(p)[0]), \ + _dispatch_thread_setspecific(k2,(p)[1]) ) + #else extern pthread_key_t dispatch_queue_key; extern pthread_key_t dispatch_frame_key; -extern pthread_key_t dispatch_sema4_key; extern pthread_key_t dispatch_cache_key; extern pthread_key_t dispatch_context_key; extern pthread_key_t dispatch_pthread_root_queue_observer_hooks_key; @@ -105,7 +171,8 @@ extern pthread_key_t dispatch_introspection_key; #elif DISPATCH_PERF_MON extern pthread_key_t dispatch_bcounter_key; #endif -extern pthread_key_t dispatch_cache_key; +extern pthread_key_t dispatch_sema4_key; +extern pthread_key_t dispatch_priority_key; extern pthread_key_t dispatch_voucher_key; extern pthread_key_t dispatch_deferred_items_key; @@ -117,6 +184,7 @@ _dispatch_thread_key_create(pthread_key_t *k, void (*d)(void *)) } #endif +#ifndef DISPATCH_USE_THREAD_LOCAL_STORAGE DISPATCH_TSD_INLINE static inline void _dispatch_thread_setspecific(pthread_key_t k, void *v) @@ -210,6 +278,7 @@ _dispatch_thread_setspecific_packed_pair(pthread_key_t k1, pthread_key_t k2, _dispatch_thread_setspecific(k1, p[0]); _dispatch_thread_setspecific(k2, p[1]); } +#endif #if TARGET_OS_WIN32 #define _dispatch_thread_self() ((uintptr_t)GetCurrentThreadId()) @@ -228,6 +297,8 @@ _dispatch_thread_setspecific_packed_pair(pthread_key_t k1, pthread_key_t k2, #if DISPATCH_USE_DIRECT_TSD #define _dispatch_thread_port() ((mach_port_t)(uintptr_t)\ _dispatch_thread_getspecific(_PTHREAD_TSD_SLOT_MACH_THREAD_SELF)) +#elif DISPATCH_USE_THREAD_LOCAL_STORAGE +#define _dispatch_thread_port() ((mach_port_t)(_dispatch_get_tsd_base()->tid)) #else #define _dispatch_thread_port() pthread_mach_thread_np(_dispatch_thread_self()) #endif