From ba884f0d9b539b7349ac77230270e3c167662189 Mon Sep 17 00:00:00 2001 From: Hubertus Franke Date: Thu, 4 Feb 2016 11:57:33 -0500 Subject: [PATCH 1/3] introduce usage of thread local storage structure --- configure.ac | 21 +++++++++++++++++ src/queue.c | 50 +++++++++++++++++++++++++++++++++++++++ src/shims/linux_stubs.c | 11 +++++++++ src/shims/tsd.h | 52 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 133 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 670863f98..09941599e 100644 --- a/configure.ac +++ b/configure.ac @@ -101,6 +101,27 @@ AS_IF([test "x$enable_apple_tsd_optimizations" = "xyes"], [Define to use non-portable pthread TSD optimizations for Mac OS X)])] ) +# +# Enable __thread based TSD on platforms where it is efficient +# Allow override based on command line argument to configure +# +AC_CANONICAL_TARGET +AC_ARG_ENABLE([thread-local-storage], + [AS_HELP_STRING([--enable-thread-local-storage], + [Enable usage of thread local storage via __thread])],, + [case $target_os in + linux*) + enable_thread_local_storage=yes + ;; + *) + enable_thread_local_storage=no + esac] +) +AS_IF([test "x$enable_thread_local_storage" = "xyes"], + [AC_DEFINE(DISPATCH_USE_THREAD_LOCAL_STORAGE, 1, + [Enable usage of thread local storage via __thread])] +) + AC_USE_SYSTEM_EXTENSIONS AM_INIT_AUTOMAKE([foreign no-dependencies subdir-objects]) LT_INIT([disable-static]) diff --git a/src/queue.c b/src/queue.c index db414f69d..63182b36e 100644 --- a/src/queue.c +++ b/src/queue.c @@ -913,6 +913,52 @@ _dispatch_root_queues_init(void *context DISPATCH_UNUSED) #define countof(x) (sizeof(x) / sizeof(x[0])) +#if DISPATCH_USE_THREAD_LOCAL_STORAGE +#include +#include + +#ifdef SYS_gettid +static inline pid_t gettid() { return (pid_t) syscall(SYS_gettid); } +#else +#error "SYS_gettid unavailable on this system" +#endif + +#define _tsd_call_cleanup(k,f) \ + do { if (f && __dispatch_tsd.k) ((void(*)(void*))(f))(__dispatch_tsd.k); \ + } while (0) + +static +void +_libdispatch_thread_cleanup(void) +{ + _tsd_call_cleanup(dispatch_queue_key, _dispatch_queue_cleanup); + _tsd_call_cleanup(dispatch_voucher_key, _voucher_thread_cleanup); + _tsd_call_cleanup(dispatch_cache_key, _dispatch_cache_cleanup); + _tsd_call_cleanup(dispatch_io_key, NULL); + _tsd_call_cleanup(dispatch_apply_key, NULL); + _tsd_call_cleanup(dispatch_defaultpriority_key, NULL); + _tsd_call_cleanup(dispatch_pthread_root_queue_observer_hooks_key, + NULL); +#if DISPATCH_PERF_MON && !DISPATCH_INTROSPECTION + _tsd_call_cleanup(dispatch_bcounter_key, NULL); +#endif +#if !DISPATCH_USE_OS_SEMAPHORE_CACHE + _tsd_call_cleanup(dispatch_sema4_key, + (void (*)(void *))_dispatch_thread_semaphore_dispose); +#endif +} + +DISPATCH_EXPORT +void +libdispatch_thread_init(void) +{ + __dispatch_tsd.tid = gettid(); + pthread_setspecific(__dispatch_tsd_key, &__dispatch_tsd); + __dispatch_tsd.initialized = true; +} + +#endif + DISPATCH_EXPORT DISPATCH_NOTHROW void libdispatch_init(void) @@ -951,6 +997,9 @@ libdispatch_init(void) dispatch_assert(sizeof(struct dispatch_root_queue_context_s) % DISPATCH_CACHELINE_SIZE == 0); +#if DISPATCH_USE_THREAD_LOCAL_STORAGE + _dispatch_thread_key_create(&__dispatch_tsd_key, (void(*)(void*))_libdispatch_thread_cleanup); +#else _dispatch_thread_key_create(&dispatch_queue_key, _dispatch_queue_cleanup); _dispatch_thread_key_create(&dispatch_voucher_key, _voucher_thread_cleanup); _dispatch_thread_key_create(&dispatch_cache_key, _dispatch_cache_cleanup); @@ -966,6 +1015,7 @@ libdispatch_init(void) _dispatch_thread_key_create(&dispatch_sema4_key, (void (*)(void *))_dispatch_thread_semaphore_dispose); #endif +#endif #if DISPATCH_USE_RESOLVERS // rdar://problem/8541707 _dispatch_main_q.do_targetq = &_dispatch_root_queues[ diff --git a/src/shims/linux_stubs.c b/src/shims/linux_stubs.c index e62ee8fbd..2eee1470c 100644 --- a/src/shims/linux_stubs.c +++ b/src/shims/linux_stubs.c @@ -52,9 +52,15 @@ char* mach_error_string(mach_msg_return_t x) { } void mach_vm_deallocate() { LINUX_PORT_ERROR(); } +#if DISPATCH_USE_THREAD_LOCAL_STORAGE +mach_port_t pthread_mach_thread_np(void) { + return __dispatch_tsd.tid; +} +#else mach_port_t pthread_mach_thread_np(void) { return (pid_t)syscall(SYS_gettid); } +#endif mach_port_t mach_task_self(void) { return (mach_port_t)pthread_self(); } @@ -72,3 +78,8 @@ unsigned short dispatch_timer__fire_semaphore; unsigned short dispatch_timer__configure_semaphore; void (*_dispatch_block_special_invoke)(void*); struct dispatch_queue_attr_s _dispatch_queue_attr_concurrent; + +#if DISPATCH_USE_THREAD_LOCAL_STORAGE +__thread struct dispatch_tsd __dispatch_tsd; +pthread_key_t __dispatch_tsd_key; +#endif diff --git a/src/shims/tsd.h b/src/shims/tsd.h index 40c4a583e..5c484150a 100644 --- a/src/shims/tsd.h +++ b/src/shims/tsd.h @@ -73,6 +73,56 @@ _dispatch_thread_key_create(const unsigned long *k, void (*d)(void *)) if (!*k || !d) return; dispatch_assert_zero(pthread_key_init_np((int)*k, d)); } +#elif DISPATCH_USE_THREAD_LOCAL_STORAGE + +DISPATCH_TSD_INLINE +static inline void +_dispatch_thread_key_create(pthread_key_t *k, void (*d)(void *)) +{ + dispatch_assert_zero(pthread_key_create(k, d)); +} + +struct dispatch_tsd { + bool initialized; + pid_t tid; + void* dispatch_queue_key; + void* dispatch_voucher_key; +#if DISPATCH_USE_OS_SEMAPHORE_CACHE +#error "Invalid DISPATCH_USE_OS_SEMAPHORE_CACHE configuration" +#else + void* dispatch_sema4_key; +#endif + void* dispatch_cache_key; + void* dispatch_io_key; + void* dispatch_apply_key; + void* dispatch_defaultpriority_key; +#if DISPATCH_INTROSPECTION + void* dispatch_introspection_key; +#elif DISPATCH_PERF_MON + void* dispatch_bcounter_key; +#endif + void* dispatch_pthread_root_queue_observer_hooks_key; +}; + +extern __thread struct dispatch_tsd __dispatch_tsd; +extern pthread_key_t __dispatch_tsd_key; +extern void libdispatch_thread_init(void); + +DISPATCH_ALWAYS_INLINE +static inline +struct dispatch_tsd * +_dispatch_get_tsd_base(void) +{ + if (slowpath(!__dispatch_tsd.initialized)) + libdispatch_thread_init(); + return &__dispatch_tsd; +} + +#define _dispatch_thread_getspecific(key) \ + (_dispatch_get_tsd_base()->key) +#define _dispatch_thread_setspecific(key, value) \ + (void)(_dispatch_get_tsd_base()->key = (value)) + #else extern pthread_key_t dispatch_queue_key; extern pthread_key_t dispatch_voucher_key; @@ -100,7 +150,7 @@ _dispatch_thread_key_create(pthread_key_t *k, void (*d)(void *)) } #endif -#if DISPATCH_USE_TSD_BASE && !DISPATCH_DEBUG +#if (DISPATCH_USE_TSD_BASE && !DISPATCH_DEBUG) || DISPATCH_USE_THREAD_LOCAL_STORAGE #else // DISPATCH_USE_TSD_BASE DISPATCH_TSD_INLINE static inline void From b79a638b78646576095264fa4dafd6cd48f1dd16 Mon Sep 17 00:00:00 2001 From: Hubertus Franke Date: Thu, 4 Feb 2016 19:01:18 -0500 Subject: [PATCH 2/3] fix format and content issues --- src/init.c | 8 ++------ src/queue.c | 28 ++++++++++++++++++---------- src/shims/linux_stubs.c | 9 ++------- src/shims/tsd.h | 40 ++++++++++++++++++++++------------------ 4 files changed, 44 insertions(+), 41 deletions(-) diff --git a/src/init.c b/src/init.c index 8e538b913..1cbaa7ac7 100644 --- a/src/init.c +++ b/src/init.c @@ -65,7 +65,7 @@ void *(*_dispatch_begin_NSAutoReleasePool)(void); void (*_dispatch_end_NSAutoReleasePool)(void *); #endif -#if !DISPATCH_USE_DIRECT_TSD +#if !DISPATCH_USE_DIRECT_TSD && !DISPATCH_USE_THREAD_LOCAL_STORAGE pthread_key_t dispatch_queue_key; pthread_key_t dispatch_sema4_key; pthread_key_t dispatch_cache_key; @@ -77,7 +77,7 @@ pthread_key_t dispatch_introspection_key; #elif DISPATCH_PERF_MON pthread_key_t dispatch_bcounter_key; #endif -#endif // !DISPATCH_USE_DIRECT_TSD +#endif // !DISPATCH_USE_DIRECT_TSD && !DISPATCH_USE_THREAD_LOCAL_STORAGE #if VOUCHER_USE_MACH_VOUCHER dispatch_once_t _voucher_task_mach_voucher_pred; @@ -161,10 +161,6 @@ const struct dispatch_tsd_indexes_s dispatch_tsd_indexes = { .dti_voucher_index = dispatch_voucher_key, .dti_qos_class_index = dispatch_priority_key, }; -#else // DISPATCH_USE_DIRECT_TSD -#ifndef __LINUX_PORT_HDD__ -#error Not implemented on this platform -#endif #endif // DISPATCH_USE_DIRECT_TSD // 6618342 Contact the team that owns the Instrument DTrace probe before diff --git a/src/queue.c b/src/queue.c index 63182b36e..3ad1b7c0f 100644 --- a/src/queue.c +++ b/src/queue.c @@ -918,19 +918,25 @@ _dispatch_root_queues_init(void *context DISPATCH_UNUSED) #include #ifdef SYS_gettid -static inline pid_t gettid() { return (pid_t) syscall(SYS_gettid); } +DISPATCH_ALWAYS_INLINE +static inline pid_t +gettid(void) +{ + return (pid_t) syscall(SYS_gettid); +} #else #error "SYS_gettid unavailable on this system" #endif -#define _tsd_call_cleanup(k,f) \ - do { if (f && __dispatch_tsd.k) ((void(*)(void*))(f))(__dispatch_tsd.k); \ +#define _tsd_call_cleanup(k, f) do { \ + if (f && tsd->k) ((void(*)(void*))(f))(tsd->k); \ } while (0) -static -void -_libdispatch_thread_cleanup(void) +static void +_libdispatch_tsd_cleanup(void *ctx) { + struct dispatch_tsd *tsd = (struct dispatch_tsd*) ctx; + _tsd_call_cleanup(dispatch_queue_key, _dispatch_queue_cleanup); _tsd_call_cleanup(dispatch_voucher_key, _voucher_thread_cleanup); _tsd_call_cleanup(dispatch_cache_key, _dispatch_cache_cleanup); @@ -946,15 +952,17 @@ _libdispatch_thread_cleanup(void) _tsd_call_cleanup(dispatch_sema4_key, (void (*)(void *))_dispatch_thread_semaphore_dispose); #endif + tsd->tid = -1; + tsd->initialized = false; } DISPATCH_EXPORT void -libdispatch_thread_init(void) +libdispatch_tsd_init(void) { __dispatch_tsd.tid = gettid(); - pthread_setspecific(__dispatch_tsd_key, &__dispatch_tsd); - __dispatch_tsd.initialized = true; + pthread_setspecific(__dispatch_tsd_key, &__dispatch_tsd); + __dispatch_tsd.initialized = true; } #endif @@ -998,7 +1006,7 @@ libdispatch_init(void) DISPATCH_CACHELINE_SIZE == 0); #if DISPATCH_USE_THREAD_LOCAL_STORAGE - _dispatch_thread_key_create(&__dispatch_tsd_key, (void(*)(void*))_libdispatch_thread_cleanup); + _dispatch_thread_key_create(&__dispatch_tsd_key, _libdispatch_tsd_cleanup); #else _dispatch_thread_key_create(&dispatch_queue_key, _dispatch_queue_cleanup); _dispatch_thread_key_create(&dispatch_voucher_key, _voucher_thread_cleanup); diff --git a/src/shims/linux_stubs.c b/src/shims/linux_stubs.c index 2eee1470c..bf9af6254 100644 --- a/src/shims/linux_stubs.c +++ b/src/shims/linux_stubs.c @@ -52,15 +52,10 @@ char* mach_error_string(mach_msg_return_t x) { } void mach_vm_deallocate() { LINUX_PORT_ERROR(); } -#if DISPATCH_USE_THREAD_LOCAL_STORAGE -mach_port_t pthread_mach_thread_np(void) { - return __dispatch_tsd.tid; -} -#else mach_port_t pthread_mach_thread_np(void) { - return (pid_t)syscall(SYS_gettid); + return (mach_port_t)pthread_self(); } -#endif + mach_port_t mach_task_self(void) { return (mach_port_t)pthread_self(); } diff --git a/src/shims/tsd.h b/src/shims/tsd.h index 5c484150a..6da78ca83 100644 --- a/src/shims/tsd.h +++ b/src/shims/tsd.h @@ -83,39 +83,39 @@ _dispatch_thread_key_create(pthread_key_t *k, void (*d)(void *)) } struct dispatch_tsd { - bool initialized; - pid_t tid; - void* dispatch_queue_key; - void* dispatch_voucher_key; + bool initialized; + pid_t tid; + void *dispatch_queue_key; + void *dispatch_voucher_key; #if DISPATCH_USE_OS_SEMAPHORE_CACHE #error "Invalid DISPATCH_USE_OS_SEMAPHORE_CACHE configuration" #else - void* dispatch_sema4_key; + void *dispatch_sema4_key; #endif - void* dispatch_cache_key; - void* dispatch_io_key; - void* dispatch_apply_key; - void* dispatch_defaultpriority_key; + void *dispatch_cache_key; + void *dispatch_io_key; + void *dispatch_apply_key; + void *dispatch_defaultpriority_key; #if DISPATCH_INTROSPECTION - void* dispatch_introspection_key; + void *dispatch_introspection_key; #elif DISPATCH_PERF_MON - void* dispatch_bcounter_key; + void *dispatch_bcounter_key; #endif - void* dispatch_pthread_root_queue_observer_hooks_key; + void *dispatch_pthread_root_queue_observer_hooks_key; }; extern __thread struct dispatch_tsd __dispatch_tsd; extern pthread_key_t __dispatch_tsd_key; -extern void libdispatch_thread_init(void); +extern void libdispatch_tsd_init(void); DISPATCH_ALWAYS_INLINE -static inline -struct dispatch_tsd * +static inline struct dispatch_tsd * _dispatch_get_tsd_base(void) { - if (slowpath(!__dispatch_tsd.initialized)) - libdispatch_thread_init(); - return &__dispatch_tsd; + if (slowpath(!__dispatch_tsd.initialized)) { + libdispatch_tsd_init(); + } + return &__dispatch_tsd; } #define _dispatch_thread_getspecific(key) \ @@ -184,6 +184,8 @@ _dispatch_thread_getspecific(pthread_key_t k) #if DISPATCH_USE_DIRECT_TSD #define _dispatch_thread_self() ((uintptr_t)_dispatch_thread_getspecific( \ _PTHREAD_TSD_SLOT_PTHREAD_SELF)) +#elif DISPATCH_USE_THREAD_LOCAL_STORAGE +#define _dispatch_thread_self() ((uintptr_t)pthread_self()) #else #define _dispatch_thread_self() ((uintptr_t)pthread_self()) #endif @@ -195,6 +197,8 @@ _dispatch_thread_getspecific(pthread_key_t k) #if DISPATCH_USE_DIRECT_TSD #define _dispatch_thread_port() ((mach_port_t)_dispatch_thread_getspecific(\ _PTHREAD_TSD_SLOT_MACH_THREAD_SELF)) +#elif DISPATCH_USE_THREAD_LOCAL_STORAGE +#define _dispatch_thread_port() ((mach_port_t)(_dispatch_get_tsd_base()->tid)) #else #define _dispatch_thread_port() (pthread_mach_thread_np(_dispatch_thread_self())) #endif From e3c1862d16c9b5d7e02fabd8e625a03b7f2850b8 Mon Sep 17 00:00:00 2001 From: Hubertus Franke Date: Tue, 9 Feb 2016 14:38:34 -0500 Subject: [PATCH 3/3] disable inlining libdispatch_tsd_init for code efficiency --- src/queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queue.c b/src/queue.c index 3ad1b7c0f..d7c4640f9 100644 --- a/src/queue.c +++ b/src/queue.c @@ -956,7 +956,7 @@ _libdispatch_tsd_cleanup(void *ctx) tsd->initialized = false; } -DISPATCH_EXPORT +DISPATCH_EXPORT DISPATCH_NOINLINE void libdispatch_tsd_init(void) {