From 2d9784b2e6a8141c33b5a482a2a78edbdb9cbdc6 Mon Sep 17 00:00:00 2001 From: wxue1 Date: Mon, 20 Mar 2023 23:16:17 -0700 Subject: [PATCH] JIT inlined function separately if trace is too long Duplicated JITTed code brings overhead for the instruction cache. This patch reduces duplication by JITting inlined function separately if trace is too long because the same function is JITTed in different root trace or side trace sometimes. It increases 2%~3% performance of our workload in tracing mode. Signed-off-by: Wang, Xue Signed-off-by: Yang, Lin A Signed-off-by: Su, Tao --- ext/opcache/jit/zend_jit.h | 1 + ext/opcache/jit/zend_jit_internal.h | 4 +++- ext/opcache/jit/zend_jit_trace.c | 4 ++-- ext/opcache/jit/zend_jit_vm_helpers.c | 7 +++++++ ext/opcache/zend_accelerator_module.c | 2 ++ 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index d22422181af9c..f6ecbab72104d 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -116,6 +116,7 @@ typedef struct _zend_jit_globals { zend_long max_recursive_calls; /* max number of recursive inlined call unrolls */ zend_long max_recursive_returns; /* max number of recursive inlined return unrolls */ zend_long max_polymorphic_calls; /* max number of inlined polymorphic calls */ + zend_long max_inline_func_length; /* max length of inlined functions in one trace */ zend_sym_node *symbols; /* symbols for disassembler */ diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 606efebe0ea9d..0ad9e0822d33e 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -359,6 +359,7 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key); _(RECURSION_EXIT, "return from recursive function") \ _(BLACK_LIST, "trace blacklisted") \ _(INNER_LOOP, "inner loop") /* trace it */ \ + _(JIT_INLINE_FUNC, "JIT inlined function and skip current trace") /* trace inlined function */ \ _(COMPILED_LOOP, "compiled loop") \ _(TRAMPOLINE, "trampoline call") \ _(BAD_FUNC, "bad function call") \ @@ -383,8 +384,9 @@ typedef enum _zend_jit_trace_stop { #define ZEND_JIT_TRACE_STOP_DONE(ret) \ (ret < ZEND_JIT_TRACE_STOP_ERROR) +/* restart to trace an inner loop or inlined function */ #define ZEND_JIT_TRACE_STOP_REPEAT(ret) \ - (ret == ZEND_JIT_TRACE_STOP_INNER_LOOP) + (ret == ZEND_JIT_TRACE_STOP_INNER_LOOP || ret == ZEND_JIT_TRACE_STOP_JIT_INLINE_FUNC) #define ZEND_JIT_TRACE_STOP_MAY_RECOVER(ret) \ (ret <= ZEND_JIT_TRACE_STOP_COMPILER_ERROR) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 862ee5eda9d08..eb013d454956d 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -7696,7 +7696,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_jit_trace_stop_description[stop]); } if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop) - || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) { + || (stop != ZEND_JIT_TRACE_STOP_JIT_INLINE_FUNC && zend_jit_trace_is_bad_root(orig_opline, stop, offset))) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { fprintf(stderr, "---- TRACE %d blacklisted\n", trace_num); @@ -8039,7 +8039,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 zend_jit_trace_stop_description[stop]); } if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop) - || zend_jit_trace_exit_is_bad(parent_num, exit_num)) { + || (stop != ZEND_JIT_TRACE_STOP_JIT_INLINE_FUNC && zend_jit_trace_exit_is_bad(parent_num, exit_num))) { zend_jit_blacklist_trace_exit(parent_num, exit_num); if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { fprintf(stderr, "---- EXIT %d/%d blacklisted\n", diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 8e02fbbbfeac2..cf54236de9a66 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -1062,6 +1062,13 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags; if (trace_flags) { + /* if inlined functions are too long, stop current tracing and restart a new one */ + if (trace_buffer[idx-1].op == ZEND_JIT_TRACE_ENTER && idx > JIT_G(max_inline_func_length)) { + if (!(trace_flags & ZEND_JIT_TRACE_JITED)) { + stop = ZEND_JIT_TRACE_STOP_JIT_INLINE_FUNC; + break; + } + } if (trace_flags & ZEND_JIT_TRACE_JITED) { if (trace_flags & ZEND_JIT_TRACE_START_LOOP) { if ((start & ZEND_JIT_TRACE_START_LOOP) != 0 diff --git a/ext/opcache/zend_accelerator_module.c b/ext/opcache/zend_accelerator_module.c index a50110b17572f..35fa927e7064b 100644 --- a/ext/opcache/zend_accelerator_module.c +++ b/ext/opcache/zend_accelerator_module.c @@ -326,6 +326,7 @@ ZEND_INI_BEGIN() STD_PHP_INI_ENTRY("opcache.jit_max_root_traces" , "1024", PHP_INI_SYSTEM, OnUpdateLong, max_root_traces, zend_jit_globals, jit_globals) STD_PHP_INI_ENTRY("opcache.jit_max_side_traces" , "128", PHP_INI_SYSTEM, OnUpdateLong, max_side_traces, zend_jit_globals, jit_globals) STD_PHP_INI_ENTRY("opcache.jit_max_exit_counters" , "8192", PHP_INI_SYSTEM, OnUpdateLong, max_exit_counters, zend_jit_globals, jit_globals) + STD_PHP_INI_ENTRY("opcache.jit_max_inline_func_length" , "16", PHP_INI_SYSTEM, OnUpdateLong, max_inline_func_length,zend_jit_globals, jit_globals) STD_PHP_INI_ENTRY("opcache.jit_hot_loop" , "64", PHP_INI_SYSTEM, OnUpdateCounter, hot_loop, zend_jit_globals, jit_globals) STD_PHP_INI_ENTRY("opcache.jit_hot_func" , "127", PHP_INI_SYSTEM, OnUpdateCounter, hot_func, zend_jit_globals, jit_globals) STD_PHP_INI_ENTRY("opcache.jit_hot_return" , "8", PHP_INI_SYSTEM, OnUpdateCounter, hot_return, zend_jit_globals, jit_globals) @@ -849,6 +850,7 @@ ZEND_FUNCTION(opcache_get_configuration) add_assoc_long(&directives, "opcache.jit_max_root_traces", JIT_G(max_root_traces)); add_assoc_long(&directives, "opcache.jit_max_side_traces", JIT_G(max_side_traces)); add_assoc_long(&directives, "opcache.jit_prof_threshold", JIT_G(prof_threshold)); + add_assoc_long(&directives, "opcache.jit_max_inline_func_length", JIT_G(max_inline_func_length)); #endif add_assoc_zval(return_value, "directives", &directives);