Skip to content

Allow JIT for passing arguments to trampolines and "bad" functions #16365

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Oct 15, 2024
Merged

Allow JIT for passing arguments to trampolines and "bad" functions #16365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
0fed37a
Better trace coverage (JIT trampoline calls)
dstogov Oct 10, 2024
8243338
clenup trampoline by zend_jit_free_trampoline()
dstogov Oct 11, 2024
995cf5d
Fix ZEND_JIT_TRACE_INIT_CALL/ZEND_JIT_TRACE_DO_ICALL num_args mismatch
dstogov Oct 11, 2024
c6dd5ae
cleanup
dstogov Oct 11, 2024
cab8472
cleanup
dstogov Oct 11, 2024
ceffc85
Don't record function that may be temporary
dstogov Oct 11, 2024
8112c60
cleanup
dstogov Oct 11, 2024
6f0b4a8
Prevent invalid run_time_cache allocation for "bad" internal functions
dstogov Oct 14, 2024
b2cf43a
Update zend_jit_trace_record_fake_init_call_ex() accordingly
dstogov Oct 15, 2024
271f553
Better handling of "bad" functions and fake closures
dstogov Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ext/opcache/jit/zend_jit_helpers.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ static zend_never_inline zend_op_array* ZEND_FASTCALL zend_jit_init_func_run_tim
{
void **run_time_cache;

if (!RUN_TIME_CACHE(op_array)) {
if (op_array->type == ZEND_USER_FUNCTION && !RUN_TIME_CACHE(op_array)) {
run_time_cache = zend_arena_alloc(&CG(arena), op_array->cache_size);
memset(run_time_cache, 0, op_array->cache_size);
ZEND_MAP_PTR_SET(op_array->run_time_cache, run_time_cache);
Expand Down
14 changes: 10 additions & 4 deletions ext/opcache/jit/zend_jit_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,8 +248,11 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key);
_(RECURSIVE_CALL, "recursive call") \
_(RECURSIVE_RET, "recursive return") \
_(RETURN, "return") \
_(INTERPRETER, "exit to VM interpreter") \
_(LINK, "link to another trace") \
_(INTERPRETER, "exit to VM interpreter") \
_(TRAMPOLINE, "trampoline call") \
_(PROP_HOOK_CALL, "property hook call") \
_(BAD_FUNC, "bad function call") \
/* compilation and linking successful */ \
_(COMPILED, "compiled") \
_(ALREADY_DONE, "already prcessed") \
Expand All @@ -267,9 +270,6 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key);
_(BLACK_LIST, "trace blacklisted") \
_(INNER_LOOP, "inner loop") /* trace it */ \
_(COMPILED_LOOP, "compiled loop") \
_(TRAMPOLINE, "trampoline call") \
_(PROP_HOOK_CALL, "property hook call") \
_(BAD_FUNC, "bad function call") \
_(COMPILER_ERROR, "JIT compilation error") \
/* no recoverable error (blacklist immediately) */ \
_(NO_SHM, "insufficient shared memory") \
Expand Down Expand Up @@ -380,6 +380,12 @@ typedef enum _zend_jit_trace_op {
#define ZEND_JIT_TRACE_FAKE_INFO(level) \
(((level) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT) | ZEND_JIT_TRACE_FAKE_INIT_CALL)

#define ZEND_JIT_TRACE_NUM_ARGS_INFO(count) \
((count) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT)

#define ZEND_JIT_TRACE_NUM_ARGS(info) \
(((info) & ZEND_JIT_TRACE_FAKE_LEVEL_MASK) >> ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT)

#define ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(_info, var) do { \
_info |= (var << ZEND_JIT_TRACE_SSA_VAR_SHIFT); \
} while (0)
Expand Down
34 changes: 25 additions & 9 deletions ext/opcache/jit/zend_jit_ir.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8431,13 +8431,21 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co
used_stack_ref);

if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
bool may_be_trampoline = !func && (opline->opcode == ZEND_INIT_METHOD_CALL);
int32_t exit_point = zend_jit_trace_get_exit_point(opline,
may_be_trampoline ?
(ZEND_JIT_EXIT_TO_VM | ZEND_JIT_EXIT_METHOD_CALL) : ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

if (!exit_addr) {
return 0;
}

if (may_be_trampoline) {
jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
}

ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
if_enough_stack = ir_IF(ref);
Expand Down Expand Up @@ -9064,6 +9072,14 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit,
jit->delayed_call_level = call_level;
}

if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) {
if (!zend_jit_set_ip(jit, opline + 1)) {
return 0;
}
}

return 1;
}

Expand Down Expand Up @@ -9324,7 +9340,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit,

if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) {
if (!zend_jit_set_ip(jit, opline + 1)) {
return 0;
}
Expand Down Expand Up @@ -9933,7 +9949,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen

if (trace && !func) {
if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION);
ZEND_ASSERT(!trace->func || trace->func->type == ZEND_INTERNAL_FUNCTION);
#ifndef ZEND_WIN32
// TODO: ASLR may cause different addresses in different workers ???
func = trace->func;
Expand Down Expand Up @@ -10115,7 +10131,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen

if (call_num_args <= func->op_array.num_args) {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) {
uint32_t num_args;

if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
Expand Down Expand Up @@ -10149,7 +10165,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
}
} else {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) {
ir_ref ip;

if (zend_accel_in_shm(func->op_array.opcodes)) {
Expand Down Expand Up @@ -10275,7 +10291,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
ir_ref observer_handler;
ir_ref rx = jit_FP(jit);
struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref);
if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) {
ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END);
jit_SET_EX_OPLINE(jit, trace[1].opline);
} else if (GCC_GLOBAL_REGS) {
Expand Down Expand Up @@ -10568,7 +10584,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
jit_LOAD_IP_ADDR(jit, opline + 1);
} else if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) {
jit_LOAD_IP_ADDR(jit, opline + 1);
}
}
Expand Down Expand Up @@ -16908,7 +16924,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
if (trace->op != ZEND_JIT_TRACE_END ||
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) {
/* this check may be handled by the following OPLINE guard or jmp [IP] */
ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)),
jit_STUB_ADDR(jit, jit_stub_trace_halt));
Expand All @@ -16926,7 +16942,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr
}
if (trace->op != ZEND_JIT_TRACE_END ||
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) {

const zend_op *next_opline = trace->opline;
const zend_op *exit_opline = NULL;
Expand Down
Loading