diff --git a/Zend/zend.c b/Zend/zend.c index 9f6e709efbecf..2d3aa4f8e3300 100644 --- a/Zend/zend.c +++ b/Zend/zend.c @@ -735,14 +735,16 @@ static void compiler_globals_ctor(zend_compiler_globals *compiler_globals) /* {{ compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(NULL); compiler_globals->map_ptr_size = 0; compiler_globals->map_ptr_last = global_map_ptr_last; - if (compiler_globals->map_ptr_last) { + compiler_globals->internal_run_time_cache = NULL; + if (compiler_globals->map_ptr_last || zend_map_ptr_static_size) { /* Allocate map_ptr table */ compiler_globals->map_ptr_size = ZEND_MM_ALIGNED_SIZE_EX(compiler_globals->map_ptr_last, 4096); - void *base = pemalloc(compiler_globals->map_ptr_size * sizeof(void*), 1); + void *base = pemalloc((zend_map_ptr_static_size + compiler_globals->map_ptr_size) * sizeof(void*), 1); compiler_globals->map_ptr_real_base = base; compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(base); - memset(base, 0, compiler_globals->map_ptr_last * sizeof(void*)); + memset(base, 0, (zend_map_ptr_static_size + compiler_globals->map_ptr_last) * sizeof(void*)); } + zend_init_internal_run_time_cache(); } /* }}} */ @@ -785,6 +787,10 @@ static void compiler_globals_dtor(zend_compiler_globals *compiler_globals) /* {{ compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(NULL); compiler_globals->map_ptr_size = 0; } + if (compiler_globals->internal_run_time_cache) { + pefree(compiler_globals->internal_run_time_cache, 1); + compiler_globals->internal_run_time_cache = NULL; + } } /* }}} */ @@ -1115,6 +1121,10 @@ zend_result zend_post_startup(void) /* {{{ */ } compiler_globals->map_ptr_real_base = NULL; compiler_globals->map_ptr_base = ZEND_MAP_PTR_BIASED_BASE(NULL); + if (compiler_globals->internal_run_time_cache) { + pefree(compiler_globals->internal_run_time_cache, 1); + } + compiler_globals->internal_run_time_cache = NULL; if ((script_encoding_list = (zend_encoding **)compiler_globals->script_encoding_list)) { compiler_globals_ctor(compiler_globals); compiler_globals->script_encoding_list = (const zend_encoding **)script_encoding_list; @@ -1198,6 +1208,9 @@ void zend_shutdown(void) /* {{{ */ CG(script_encoding_list_size) = 0; } #endif + zend_map_ptr_static_last = 0; + zend_map_ptr_static_size = 0; + zend_destroy_rsrc_list_dtors(); zend_unload_modules(); @@ -1297,9 +1310,9 @@ ZEND_API void zend_activate(void) /* {{{ */ init_executor(); startup_scanner(); if (CG(map_ptr_last)) { - memset(CG(map_ptr_real_base), 0, CG(map_ptr_last) * sizeof(void*)); + memset((void **)CG(map_ptr_real_base) + zend_map_ptr_static_size, 0, CG(map_ptr_last) * sizeof(void*)); } - zend_init_internal_run_time_cache(); + zend_reset_internal_run_time_cache(); zend_observer_activate(); } /* }}} */ @@ -1984,6 +1997,9 @@ void free_estring(char **str_p) /* {{{ */ } /* }}} */ +ZEND_API size_t zend_map_ptr_static_size; +ZEND_API size_t zend_map_ptr_static_last; + ZEND_API void zend_map_ptr_reset(void) { CG(map_ptr_last) = global_map_ptr_last; @@ -1996,15 +2012,36 @@ ZEND_API void *zend_map_ptr_new(void) if (CG(map_ptr_last) >= CG(map_ptr_size)) { /* Grow map_ptr table */ CG(map_ptr_size) = ZEND_MM_ALIGNED_SIZE_EX(CG(map_ptr_last) + 1, 4096); - CG(map_ptr_real_base) = perealloc(CG(map_ptr_real_base), CG(map_ptr_size) * sizeof(void*), 1); + CG(map_ptr_real_base) = perealloc(CG(map_ptr_real_base), (zend_map_ptr_static_size + CG(map_ptr_size)) * sizeof(void*), 1); CG(map_ptr_base) = ZEND_MAP_PTR_BIASED_BASE(CG(map_ptr_real_base)); } - ptr = (void**)CG(map_ptr_real_base) + CG(map_ptr_last); + ptr = (void**)CG(map_ptr_real_base) + zend_map_ptr_static_size + CG(map_ptr_last); *ptr = NULL; CG(map_ptr_last)++; return ZEND_MAP_PTR_PTR2OFFSET(ptr); } +ZEND_API void *zend_map_ptr_new_static(void) +{ + void **ptr; + + if (zend_map_ptr_static_last >= zend_map_ptr_static_size) { + zend_map_ptr_static_size += 4096; + /* Grow map_ptr table */ + void *new_base = pemalloc((zend_map_ptr_static_size + CG(map_ptr_size)) * sizeof(void*), 1); + if (CG(map_ptr_real_base)) { + memcpy((void **)new_base + 4096, CG(map_ptr_real_base), (CG(map_ptr_last) + zend_map_ptr_static_size - 4096) * sizeof(void *)); + pefree(CG(map_ptr_real_base), 1); + } + CG(map_ptr_real_base) = new_base; + CG(map_ptr_base) = ZEND_MAP_PTR_BIASED_BASE(new_base); + } + ptr = (void**)CG(map_ptr_real_base) + (zend_map_ptr_static_last & 4095); + *ptr = NULL; + zend_map_ptr_static_last++; + return ZEND_MAP_PTR_PTR2OFFSET(ptr); +} + ZEND_API void zend_map_ptr_extend(size_t last) { if (last > CG(map_ptr_last)) { @@ -2013,10 +2050,10 @@ ZEND_API void zend_map_ptr_extend(size_t last) if (last >= CG(map_ptr_size)) { /* Grow map_ptr table */ CG(map_ptr_size) = ZEND_MM_ALIGNED_SIZE_EX(last, 4096); - CG(map_ptr_real_base) = perealloc(CG(map_ptr_real_base), CG(map_ptr_size) * sizeof(void*), 1); + CG(map_ptr_real_base) = perealloc(CG(map_ptr_real_base), (zend_map_ptr_static_size + CG(map_ptr_size)) * sizeof(void*), 1); CG(map_ptr_base) = ZEND_MAP_PTR_BIASED_BASE(CG(map_ptr_real_base)); } - ptr = (void**)CG(map_ptr_real_base) + CG(map_ptr_last); + ptr = (void**)CG(map_ptr_real_base) + zend_map_ptr_static_size + CG(map_ptr_last); memset(ptr, 0, (last - CG(map_ptr_last)) * sizeof(void*)); CG(map_ptr_last) = last; } diff --git a/Zend/zend_API.c b/Zend/zend_API.c index b4f13bedecc56..2319862488ca6 100644 --- a/Zend/zend_API.c +++ b/Zend/zend_API.c @@ -2958,7 +2958,11 @@ ZEND_API zend_result zend_register_functions(zend_class_entry *scope, const zend if (EG(active)) { // at run-time: this ought to only happen if registered with dl() or somehow temporarily at runtime ZEND_MAP_PTR_INIT(internal_function->run_time_cache, zend_arena_calloc(&CG(arena), 1, zend_internal_run_time_cache_reserved_size())); } else { - ZEND_MAP_PTR_NEW(internal_function->run_time_cache); +#if ZTS + ZEND_MAP_PTR_NEW_STATIC(internal_function->run_time_cache); +#else + ZEND_MAP_PTR_INIT(internal_function->run_time_cache, NULL); +#endif } if (ptr->flags) { if (!(ptr->flags & ZEND_ACC_PPP_MASK)) { diff --git a/Zend/zend_enum.c b/Zend/zend_enum.c index 309d7da1cf7aa..e6731aed40d2b 100644 --- a/Zend/zend_enum.c +++ b/Zend/zend_enum.c @@ -421,7 +421,11 @@ static void zend_enum_register_func(zend_class_entry *ce, zend_known_string_id n if (EG(active)) { // at run-time ZEND_MAP_PTR_INIT(zif->run_time_cache, zend_arena_calloc(&CG(arena), 1, zend_internal_run_time_cache_reserved_size())); } else { - ZEND_MAP_PTR_NEW(zif->run_time_cache); +#if ZTS + ZEND_MAP_PTR_NEW_STATIC(zif->run_time_cache); +#else + ZEND_MAP_PTR_INIT(zif->run_time_cache, NULL); +#endif } if (!zend_hash_add_ptr(&ce->function_table, name, zif)) { diff --git a/Zend/zend_extensions.c b/Zend/zend_extensions.c index c3719219655c2..fecea4fa0b5ea 100644 --- a/Zend/zend_extensions.c +++ b/Zend/zend_extensions.c @@ -331,19 +331,22 @@ ZEND_API void zend_init_internal_run_time_cache(void) { functions += zend_hash_num_elements(&ce->function_table); } ZEND_HASH_FOREACH_END(); - char *ptr = zend_arena_calloc(&CG(arena), functions, rt_size); + size_t alloc_size = functions * rt_size; + char *ptr = pemalloc(alloc_size, 1); + + CG(internal_run_time_cache) = ptr; + CG(internal_run_time_cache_size) = alloc_size; + zend_internal_function *zif; ZEND_HASH_MAP_FOREACH_PTR(CG(function_table), zif) { - if (!ZEND_USER_CODE(zif->type) && ZEND_MAP_PTR_GET(zif->run_time_cache) == NULL) - { + if (!ZEND_USER_CODE(zif->type) && ZEND_MAP_PTR_GET(zif->run_time_cache) == NULL) { ZEND_MAP_PTR_SET(zif->run_time_cache, (void *)ptr); ptr += rt_size; } } ZEND_HASH_FOREACH_END(); ZEND_HASH_MAP_FOREACH_PTR(CG(class_table), ce) { ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, zif) { - if (!ZEND_USER_CODE(zif->type) && ZEND_MAP_PTR_GET(zif->run_time_cache) == NULL) - { + if (!ZEND_USER_CODE(zif->type) && ZEND_MAP_PTR_GET(zif->run_time_cache) == NULL) { ZEND_MAP_PTR_SET(zif->run_time_cache, (void *)ptr); ptr += rt_size; } @@ -352,6 +355,12 @@ ZEND_API void zend_init_internal_run_time_cache(void) { } } +ZEND_API void zend_reset_internal_run_time_cache(void) { + if (CG(internal_run_time_cache)) { + memset(CG(internal_run_time_cache), 0, CG(internal_run_time_cache_size)); + } +} + ZEND_API zend_extension *zend_get_extension(const char *extension_name) { zend_llist_element *element; diff --git a/Zend/zend_extensions.h b/Zend/zend_extensions.h index 908c90e7d0485..15aaa14f91513 100644 --- a/Zend/zend_extensions.h +++ b/Zend/zend_extensions.h @@ -149,6 +149,7 @@ void zend_shutdown_extensions(void); ZEND_API size_t zend_internal_run_time_cache_reserved_size(void); ZEND_API void zend_init_internal_run_time_cache(void); +ZEND_API void zend_reset_internal_run_time_cache(void); BEGIN_EXTERN_C() ZEND_API zend_result zend_load_extension(const char *path); diff --git a/Zend/zend_globals.h b/Zend/zend_globals.h index 539793c8325a6..62a97d753634a 100644 --- a/Zend/zend_globals.h +++ b/Zend/zend_globals.h @@ -154,6 +154,9 @@ struct _zend_compiler_globals { uint32_t rtd_key_counter; + void *internal_run_time_cache; + uint32_t internal_run_time_cache_size; + zend_stack short_circuiting_opnums; #ifdef ZTS uint32_t copied_functions_count; diff --git a/Zend/zend_map_ptr.h b/Zend/zend_map_ptr.h index 7a0d853dbd166..ebcda89411d0e 100644 --- a/Zend/zend_map_ptr.h +++ b/Zend/zend_map_ptr.h @@ -42,6 +42,9 @@ typedef struct _zend_string zend_string; #define ZEND_MAP_PTR_NEW(ptr) do { \ ZEND_MAP_PTR(ptr) = zend_map_ptr_new(); \ } while (0) +#define ZEND_MAP_PTR_NEW_STATIC(ptr) do { \ + ZEND_MAP_PTR(ptr) = zend_map_ptr_new_static(); \ + } while (0) #if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET # define ZEND_MAP_PTR_NEW_OFFSET() \ @@ -53,7 +56,7 @@ typedef struct _zend_string zend_string; ZEND_MAP_PTR_GET_IMM(ptr) : \ ((void*)(ZEND_MAP_PTR(ptr))))) # define ZEND_MAP_PTR_GET_IMM(ptr) \ - (*ZEND_MAP_PTR_OFFSET2PTR((uintptr_t)ZEND_MAP_PTR(ptr))) + (*ZEND_MAP_PTR_OFFSET2PTR((intptr_t)ZEND_MAP_PTR(ptr))) # define ZEND_MAP_PTR_SET(ptr, val) do { \ if (ZEND_MAP_PTR_IS_OFFSET(ptr)) { \ ZEND_MAP_PTR_SET_IMM(ptr, val); \ @@ -62,11 +65,11 @@ typedef struct _zend_string zend_string; } \ } while (0) # define ZEND_MAP_PTR_SET_IMM(ptr, val) do { \ - void **__p = ZEND_MAP_PTR_OFFSET2PTR((uintptr_t)ZEND_MAP_PTR(ptr)); \ + void **__p = ZEND_MAP_PTR_OFFSET2PTR((intptr_t)ZEND_MAP_PTR(ptr)); \ *__p = (val); \ } while (0) # define ZEND_MAP_PTR_BIASED_BASE(real_base) \ - ((void*)(((uintptr_t)(real_base)) - 1)) + ((void*)(((uintptr_t)(real_base)) + zend_map_ptr_static_size * sizeof(void *) - 1)) #else # error "Unknown ZEND_MAP_PTR_KIND" #endif @@ -75,9 +78,13 @@ BEGIN_EXTERN_C() ZEND_API void zend_map_ptr_reset(void); ZEND_API void *zend_map_ptr_new(void); +ZEND_API void *zend_map_ptr_new_static(void); ZEND_API void zend_map_ptr_extend(size_t last); ZEND_API void zend_alloc_ce_cache(zend_string *type_name); +ZEND_API extern size_t zend_map_ptr_static_last; +ZEND_API extern size_t zend_map_ptr_static_size; + END_EXTERN_C() #endif /* ZEND_MAP_PTR_H */ diff --git a/ext/ffi/ffi.c b/ext/ffi/ffi.c index 6d9216b6f7f3f..4e5dfc632c9ba 100644 --- a/ext/ffi/ffi.c +++ b/ext/ffi/ffi.c @@ -5393,6 +5393,11 @@ static zend_result ffi_fixup_temporaries(void) { ++zend_ffi_cast_fn.T; ++zend_ffi_type_fn.T; } +#if !ZTS + ZEND_MAP_PTR(zend_ffi_new_fn.run_time_cache) = ZEND_MAP_PTR(((zend_internal_function *)zend_hash_str_find_ptr(&zend_ffi_ce->function_table, "new", sizeof("new")-1))->run_time_cache); + ZEND_MAP_PTR(zend_ffi_cast_fn.run_time_cache) = ZEND_MAP_PTR(((zend_internal_function *)zend_hash_str_find_ptr(&zend_ffi_ce->function_table, "cast", sizeof("cast")-1))->run_time_cache); + ZEND_MAP_PTR(zend_ffi_type_fn.run_time_cache) = ZEND_MAP_PTR(((zend_internal_function *)zend_hash_str_find_ptr(&zend_ffi_ce->function_table, "type", sizeof("type")-1))->run_time_cache); +#endif if (prev_zend_post_startup_cb) { return prev_zend_post_startup_cb(); } diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index 0f79b83832e6a..848c3308b3a51 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -4534,8 +4534,12 @@ static struct jit_observer_fcall_is_unobserved_data jit_observer_fcall_is_unobse if (func && (func->common.fn_flags & ZEND_ACC_CLOSURE) == 0 && ZEND_MAP_PTR_IS_OFFSET(func->common.run_time_cache)) { // JIT: ZEND_MAP_PTR_GET_IMM(func->common.runtime_cache) run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CG(map_ptr_base)), (uintptr_t)ZEND_MAP_PTR(func->common.run_time_cache))); +#if !ZTS + } else if (func && rx == IS_UNUSED) { // happens for internal functions only + ZEND_ASSERT(!ZEND_USER_CODE(func->type)); + run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_CONST_ADDR(func), offsetof(zend_op_array, run_time_cache__ptr))); +#endif } else { - ZEND_ASSERT(rx != IR_UNUSED); // Closures may be duplicated and have a different runtime cache. Use the regular run_time_cache access pattern for these if (func && ZEND_USER_CODE(func->type)) { // not a closure and definitely not an internal function run_time_cache = ir_LOAD_A(jit_CALL(rx, run_time_cache)); diff --git a/ext/xmlreader/php_xmlreader.c b/ext/xmlreader/php_xmlreader.c index 528fd8b351c61..e8f63bade5bef 100644 --- a/ext/xmlreader/php_xmlreader.c +++ b/ext/xmlreader/php_xmlreader.c @@ -1276,6 +1276,10 @@ static zend_result xmlreader_fixup_temporaries(void) { ++xmlreader_open_fn.T; ++xmlreader_xml_fn.T; } +#if !ZTS + ZEND_MAP_PTR(xmlreader_open_fn.run_time_cache) = ZEND_MAP_PTR(((zend_internal_function *)zend_hash_str_find_ptr(&xmlreader_class_entry->function_table, "open", sizeof("open")-1))->run_time_cache); + ZEND_MAP_PTR(xmlreader_xml_fn.run_time_cache) = ZEND_MAP_PTR(((zend_internal_function *)zend_hash_str_find_ptr(&xmlreader_class_entry->function_table, "xml", sizeof("xml")-1))->run_time_cache); +#endif if (prev_zend_post_startup_cb) { return prev_zend_post_startup_cb(); } diff --git a/main/main.c b/main/main.c index 26d38b83e1644..d31de566f48b8 100644 --- a/main/main.c +++ b/main/main.c @@ -2316,6 +2316,9 @@ zend_result php_module_startup(sapi_module_struct *sf, zend_module_entry *additi /* freeze the list of observer fcall_init handlers */ zend_observer_post_startup(); + /* freeze the list of persistent internal functions */ + zend_init_internal_run_time_cache(); + /* Extensions that add engine hooks after this point do so at their own peril */ zend_finalize_system_id();