diff --git a/configure.ac b/configure.ac index 670863f98..09941599e 100644 --- a/configure.ac +++ b/configure.ac @@ -101,6 +101,27 @@ AS_IF([test "x$enable_apple_tsd_optimizations" = "xyes"], [Define to use non-portable pthread TSD optimizations for Mac OS X)])] ) +# +# Enable __thread based TSD on platforms where it is efficient +# Allow override based on command line argument to configure +# +AC_CANONICAL_TARGET +AC_ARG_ENABLE([thread-local-storage], + [AS_HELP_STRING([--enable-thread-local-storage], + [Enable usage of thread local storage via __thread])],, + [case $target_os in + linux*) + enable_thread_local_storage=yes + ;; + *) + enable_thread_local_storage=no + esac] +) +AS_IF([test "x$enable_thread_local_storage" = "xyes"], + [AC_DEFINE(DISPATCH_USE_THREAD_LOCAL_STORAGE, 1, + [Enable usage of thread local storage via __thread])] +) + AC_USE_SYSTEM_EXTENSIONS AM_INIT_AUTOMAKE([foreign no-dependencies subdir-objects]) LT_INIT([disable-static]) diff --git a/src/init.c b/src/init.c index 8e538b913..1cbaa7ac7 100644 --- a/src/init.c +++ b/src/init.c @@ -65,7 +65,7 @@ void *(*_dispatch_begin_NSAutoReleasePool)(void); void (*_dispatch_end_NSAutoReleasePool)(void *); #endif -#if !DISPATCH_USE_DIRECT_TSD +#if !DISPATCH_USE_DIRECT_TSD && !DISPATCH_USE_THREAD_LOCAL_STORAGE pthread_key_t dispatch_queue_key; pthread_key_t dispatch_sema4_key; pthread_key_t dispatch_cache_key; @@ -77,7 +77,7 @@ pthread_key_t dispatch_introspection_key; #elif DISPATCH_PERF_MON pthread_key_t dispatch_bcounter_key; #endif -#endif // !DISPATCH_USE_DIRECT_TSD +#endif // !DISPATCH_USE_DIRECT_TSD && !DISPATCH_USE_THREAD_LOCAL_STORAGE #if VOUCHER_USE_MACH_VOUCHER dispatch_once_t _voucher_task_mach_voucher_pred; @@ -161,10 +161,6 @@ const struct dispatch_tsd_indexes_s dispatch_tsd_indexes = { .dti_voucher_index = dispatch_voucher_key, .dti_qos_class_index = dispatch_priority_key, }; -#else // DISPATCH_USE_DIRECT_TSD -#ifndef __LINUX_PORT_HDD__ -#error Not implemented on this platform -#endif #endif // DISPATCH_USE_DIRECT_TSD // 6618342 Contact the team that owns the Instrument DTrace probe before diff --git a/src/queue.c b/src/queue.c index db414f69d..d7c4640f9 100644 --- a/src/queue.c +++ b/src/queue.c @@ -913,6 +913,60 @@ _dispatch_root_queues_init(void *context DISPATCH_UNUSED) #define countof(x) (sizeof(x) / sizeof(x[0])) +#if DISPATCH_USE_THREAD_LOCAL_STORAGE +#include +#include + +#ifdef SYS_gettid +DISPATCH_ALWAYS_INLINE +static inline pid_t +gettid(void) +{ + return (pid_t) syscall(SYS_gettid); +} +#else +#error "SYS_gettid unavailable on this system" +#endif + +#define _tsd_call_cleanup(k, f) do { \ + if (f && tsd->k) ((void(*)(void*))(f))(tsd->k); \ + } while (0) + +static void +_libdispatch_tsd_cleanup(void *ctx) +{ + struct dispatch_tsd *tsd = (struct dispatch_tsd*) ctx; + + _tsd_call_cleanup(dispatch_queue_key, _dispatch_queue_cleanup); + _tsd_call_cleanup(dispatch_voucher_key, _voucher_thread_cleanup); + _tsd_call_cleanup(dispatch_cache_key, _dispatch_cache_cleanup); + _tsd_call_cleanup(dispatch_io_key, NULL); + _tsd_call_cleanup(dispatch_apply_key, NULL); + _tsd_call_cleanup(dispatch_defaultpriority_key, NULL); + _tsd_call_cleanup(dispatch_pthread_root_queue_observer_hooks_key, + NULL); +#if DISPATCH_PERF_MON && !DISPATCH_INTROSPECTION + _tsd_call_cleanup(dispatch_bcounter_key, NULL); +#endif +#if !DISPATCH_USE_OS_SEMAPHORE_CACHE + _tsd_call_cleanup(dispatch_sema4_key, + (void (*)(void *))_dispatch_thread_semaphore_dispose); +#endif + tsd->tid = -1; + tsd->initialized = false; +} + +DISPATCH_EXPORT DISPATCH_NOINLINE +void +libdispatch_tsd_init(void) +{ + __dispatch_tsd.tid = gettid(); + pthread_setspecific(__dispatch_tsd_key, &__dispatch_tsd); + __dispatch_tsd.initialized = true; +} + +#endif + DISPATCH_EXPORT DISPATCH_NOTHROW void libdispatch_init(void) @@ -951,6 +1005,9 @@ libdispatch_init(void) dispatch_assert(sizeof(struct dispatch_root_queue_context_s) % DISPATCH_CACHELINE_SIZE == 0); +#if DISPATCH_USE_THREAD_LOCAL_STORAGE + _dispatch_thread_key_create(&__dispatch_tsd_key, _libdispatch_tsd_cleanup); +#else _dispatch_thread_key_create(&dispatch_queue_key, _dispatch_queue_cleanup); _dispatch_thread_key_create(&dispatch_voucher_key, _voucher_thread_cleanup); _dispatch_thread_key_create(&dispatch_cache_key, _dispatch_cache_cleanup); @@ -966,6 +1023,7 @@ libdispatch_init(void) _dispatch_thread_key_create(&dispatch_sema4_key, (void (*)(void *))_dispatch_thread_semaphore_dispose); #endif +#endif #if DISPATCH_USE_RESOLVERS // rdar://problem/8541707 _dispatch_main_q.do_targetq = &_dispatch_root_queues[ diff --git a/src/shims/linux_stubs.c b/src/shims/linux_stubs.c index e62ee8fbd..bf9af6254 100644 --- a/src/shims/linux_stubs.c +++ b/src/shims/linux_stubs.c @@ -53,8 +53,9 @@ char* mach_error_string(mach_msg_return_t x) { void mach_vm_deallocate() { LINUX_PORT_ERROR(); } mach_port_t pthread_mach_thread_np(void) { - return (pid_t)syscall(SYS_gettid); + return (mach_port_t)pthread_self(); } + mach_port_t mach_task_self(void) { return (mach_port_t)pthread_self(); } @@ -72,3 +73,8 @@ unsigned short dispatch_timer__fire_semaphore; unsigned short dispatch_timer__configure_semaphore; void (*_dispatch_block_special_invoke)(void*); struct dispatch_queue_attr_s _dispatch_queue_attr_concurrent; + +#if DISPATCH_USE_THREAD_LOCAL_STORAGE +__thread struct dispatch_tsd __dispatch_tsd; +pthread_key_t __dispatch_tsd_key; +#endif diff --git a/src/shims/tsd.h b/src/shims/tsd.h index 40c4a583e..6da78ca83 100644 --- a/src/shims/tsd.h +++ b/src/shims/tsd.h @@ -73,6 +73,56 @@ _dispatch_thread_key_create(const unsigned long *k, void (*d)(void *)) if (!*k || !d) return; dispatch_assert_zero(pthread_key_init_np((int)*k, d)); } +#elif DISPATCH_USE_THREAD_LOCAL_STORAGE + +DISPATCH_TSD_INLINE +static inline void +_dispatch_thread_key_create(pthread_key_t *k, void (*d)(void *)) +{ + dispatch_assert_zero(pthread_key_create(k, d)); +} + +struct dispatch_tsd { + bool initialized; + pid_t tid; + void *dispatch_queue_key; + void *dispatch_voucher_key; +#if DISPATCH_USE_OS_SEMAPHORE_CACHE +#error "Invalid DISPATCH_USE_OS_SEMAPHORE_CACHE configuration" +#else + void *dispatch_sema4_key; +#endif + void *dispatch_cache_key; + void *dispatch_io_key; + void *dispatch_apply_key; + void *dispatch_defaultpriority_key; +#if DISPATCH_INTROSPECTION + void *dispatch_introspection_key; +#elif DISPATCH_PERF_MON + void *dispatch_bcounter_key; +#endif + void *dispatch_pthread_root_queue_observer_hooks_key; +}; + +extern __thread struct dispatch_tsd __dispatch_tsd; +extern pthread_key_t __dispatch_tsd_key; +extern void libdispatch_tsd_init(void); + +DISPATCH_ALWAYS_INLINE +static inline struct dispatch_tsd * +_dispatch_get_tsd_base(void) +{ + if (slowpath(!__dispatch_tsd.initialized)) { + libdispatch_tsd_init(); + } + return &__dispatch_tsd; +} + +#define _dispatch_thread_getspecific(key) \ + (_dispatch_get_tsd_base()->key) +#define _dispatch_thread_setspecific(key, value) \ + (void)(_dispatch_get_tsd_base()->key = (value)) + #else extern pthread_key_t dispatch_queue_key; extern pthread_key_t dispatch_voucher_key; @@ -100,7 +150,7 @@ _dispatch_thread_key_create(pthread_key_t *k, void (*d)(void *)) } #endif -#if DISPATCH_USE_TSD_BASE && !DISPATCH_DEBUG +#if (DISPATCH_USE_TSD_BASE && !DISPATCH_DEBUG) || DISPATCH_USE_THREAD_LOCAL_STORAGE #else // DISPATCH_USE_TSD_BASE DISPATCH_TSD_INLINE static inline void @@ -134,6 +184,8 @@ _dispatch_thread_getspecific(pthread_key_t k) #if DISPATCH_USE_DIRECT_TSD #define _dispatch_thread_self() ((uintptr_t)_dispatch_thread_getspecific( \ _PTHREAD_TSD_SLOT_PTHREAD_SELF)) +#elif DISPATCH_USE_THREAD_LOCAL_STORAGE +#define _dispatch_thread_self() ((uintptr_t)pthread_self()) #else #define _dispatch_thread_self() ((uintptr_t)pthread_self()) #endif @@ -145,6 +197,8 @@ _dispatch_thread_getspecific(pthread_key_t k) #if DISPATCH_USE_DIRECT_TSD #define _dispatch_thread_port() ((mach_port_t)_dispatch_thread_getspecific(\ _PTHREAD_TSD_SLOT_MACH_THREAD_SELF)) +#elif DISPATCH_USE_THREAD_LOCAL_STORAGE +#define _dispatch_thread_port() ((mach_port_t)(_dispatch_get_tsd_base()->tid)) #else #define _dispatch_thread_port() (pthread_mach_thread_np(_dispatch_thread_self())) #endif