From a08efa7f5bc89b9d623efbc72573072b0acd1dab Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 13:06:47 +0100 Subject: [PATCH 01/34] ext/opcache/jit/zend_jit: use true/false for bool variables --- ext/opcache/jit/zend_jit.c | 334 ++++++++++++++++++------------------- 1 file changed, 167 insertions(+), 167 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 429d8dbd49f5..5b1ed7500ed3 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -111,7 +111,7 @@ int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; const zend_op *zend_jit_halt_op = NULL; static int zend_jit_vm_kind = 0; #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP -static int zend_write_protect = 1; +static bool zend_write_protect = true; #endif static void *dasm_buf = NULL; @@ -171,7 +171,7 @@ static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa * zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; do { if (!ssa->vars[phi->ssa_var].no_val) { - return 0; + return false; } phi = zend_ssa_next_use_phi(ssa, var, phi); } while (phi); @@ -186,7 +186,7 @@ static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa * if (b != ssa->cfg.map[prev_use] && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use]) && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) { - return 0; + return false; } prev_use = zend_ssa_next_use(ssa->ops, var, prev_use); } @@ -194,11 +194,11 @@ static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa * next_use = zend_ssa_next_use(ssa->ops, var, use); if (next_use < 0) { - return 1; + return true; } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) { - return 1; + return true; } - return 0; + return false; } static bool zend_ival_is_last_use(const zend_lifetime_interval *ival, int use) @@ -211,7 +211,7 @@ static bool zend_ival_is_last_use(const zend_lifetime_interval *ival, int use) } return range->end == use; } - return 0; + return false; } static bool zend_is_commutative(zend_uchar opcode) @@ -224,12 +224,12 @@ static bool zend_is_commutative(zend_uchar opcode) opcode == ZEND_BW_XOR; } -static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, - zend_ssa_range *op1_range, - zend_jit_addr op1_addr, - zend_ssa_range *op2_range, - zend_jit_addr op2_addr, - bool *result) +static bool zend_jit_is_constant_cmp_long_long(const zend_op *opline, + zend_ssa_range *op1_range, + zend_jit_addr op1_addr, + zend_ssa_range *op2_range, + zend_jit_addr op2_addr, + bool *result) { zend_long op1_min; zend_long op1_max; @@ -243,7 +243,7 @@ static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG); op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr)); } else { - return 0; + return false; } if (op2_range) { @@ -253,7 +253,7 @@ static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG); op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr)); } else { - return 0; + return false; } switch (opline->opcode) { @@ -262,56 +262,56 @@ static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, case ZEND_CASE: case ZEND_CASE_STRICT: if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { - *result = 1; - return 1; + *result = true; + return true; } else if (op1_max < op2_min || op1_min > op2_max) { - *result = 0; - return 1; + *result = false; + return true; } - return 0; + return false; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { - *result = 0; - return 1; + *result = false; + return true; } else if (op1_max < op2_min || op1_min > op2_max) { - *result = 1; - return 1; + *result = true; + return true; } - return 0; + return false; case ZEND_IS_SMALLER: if (op1_max < op2_min) { - *result = 1; - return 1; + *result = true; + return true; } else if (op1_min >= op2_max) { - *result = 0; - return 1; + *result = false; + return true; } - return 0; + return false; case ZEND_IS_SMALLER_OR_EQUAL: if (op1_max <= op2_min) { - *result = 1; - return 1; + *result = true; + return true; } else if (op1_min > op2_max) { - *result = 0; - return 1; + *result = false; + return true; } - return 0; + return false; default: ZEND_UNREACHABLE(); } - return 0; + return false; } -static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace) +static bool zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace) { - int skip; + bool skip; if (trace) { zend_jit_trace_rec *p = trace; ssa_op++; - while (1) { + while (true) { if (p->op == ZEND_JIT_TRACE_VM) { switch (p->opline->opcode) { case ZEND_SEND_ARRAY: @@ -341,13 +341,13 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons case ZEND_DECLARE_ANON_CLASS: case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: - return 1; + return true; case ZEND_DO_ICALL: case ZEND_DO_UCALL: case ZEND_DO_FCALL_BY_NAME: case ZEND_DO_FCALL: case ZEND_CALLABLE_CONVERT: - return 0; + return false; case ZEND_SEND_VAL: case ZEND_SEND_VAR: case ZEND_SEND_VAL_EX: @@ -360,14 +360,14 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons break; default: if (zend_may_throw(opline, ssa_op, op_array, ssa)) { - return 1; + return true; } } ssa_op += zend_jit_trace_op_len(opline); } else if (p->op == ZEND_JIT_TRACE_ENTER || p->op == ZEND_JIT_TRACE_BACK || p->op == ZEND_JIT_TRACE_END) { - return 1; + return true; } p++; } @@ -382,7 +382,7 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons while (opline != end) { if (!skip) { if (zend_may_throw(opline, ssa_op, op_array, ssa)) { - return 1; + return true; } } switch (opline->opcode) { @@ -394,7 +394,7 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons case ZEND_SEND_REF: case ZEND_SEND_VAR_NO_REF: case ZEND_SEND_VAR_NO_REF_EX: - skip = 0; + skip = false; break; case ZEND_SEND_ARRAY: case ZEND_SEND_USER: @@ -423,7 +423,7 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons case ZEND_DECLARE_ANON_CLASS: case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: - return 1; + return true; case ZEND_DO_ICALL: case ZEND_DO_UCALL: case ZEND_DO_FCALL_BY_NAME: @@ -432,22 +432,22 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons end = opline; if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { /* INIT_FCALL and DO_FCALL in different BasicBlocks */ - return 1; + return true; } - return 0; + return false; } opline++; ssa_op++; } - return 1; + return true; } else { const zend_op *end = call_info->caller_call_opline; /* end may be null if an opcode like EXIT is part of the argument list. */ if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { /* INIT_FCALL and DO_FCALL in different BasicBlocks */ - return 1; + return true; } opline++; @@ -464,23 +464,23 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons case ZEND_SEND_REF: case ZEND_SEND_VAR_NO_REF: case ZEND_SEND_VAR_NO_REF_EX: - skip = 0; + skip = false; break; case ZEND_SEND_ARRAY: case ZEND_SEND_USER: case ZEND_SEND_UNPACK: - return 1; + return true; } } else { if (zend_may_throw(opline, ssa_op, op_array, ssa)) { - return 1; + return true; } } opline++; ssa_op++; } - return 0; + return false; } } @@ -556,7 +556,7 @@ static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_i if (!JIT_G(current_frame) || !JIT_G(current_frame)->call->func || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { - return 0; + return false; } /* break missing intentionally */ case ZEND_FETCH_OBJ_R: @@ -572,7 +572,7 @@ static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_i if (!JIT_G(current_frame) || !JIT_G(current_frame)->call->func || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { - return 0; + return false; } /* break missing intentionally */ case ZEND_FETCH_DIM_R: @@ -584,7 +584,7 @@ static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_i } break; } - return 0; + return false; } static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags) @@ -692,7 +692,7 @@ static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *memb return 1; } - return 0; + return false; } #define OP_RANGE(ssa_op, opN) \ @@ -1105,13 +1105,13 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, return entry; } -static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa) +static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa) { int res; zend_long op1_min, op1_max, op2_min, op2_max; if (!ssa->ops || !ssa->var_info) { - return 1; + return true; } switch (opline->opcode) { case ZEND_PRE_INC: @@ -1121,14 +1121,14 @@ static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, c || !ssa->var_info[res].has_range || ssa->var_info[res].range.overflow) { if (!OP1_HAS_RANGE()) { - return 1; + return true; } op1_max = OP1_MAX_RANGE(); if (op1_max == ZEND_LONG_MAX) { - return 1; + return true; } } - return 0; + return false; case ZEND_PRE_DEC: case ZEND_POST_DEC: res = ssa_op->op1_def; @@ -1136,68 +1136,68 @@ static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, c || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow) { if (!OP1_HAS_RANGE()) { - return 1; + return true; } op1_min = OP1_MIN_RANGE(); if (op1_min == ZEND_LONG_MIN) { - return 1; + return true; } } - return 0; + return false; case ZEND_ADD: res = ssa_op->result_def; if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } op1_min = OP1_MIN_RANGE(); op2_min = OP2_MIN_RANGE(); if (zend_add_will_overflow(op1_min, op2_min)) { - return 1; + return true; } } if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.overflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } op1_max = OP1_MAX_RANGE(); op2_max = OP2_MAX_RANGE(); if (zend_add_will_overflow(op1_max, op2_max)) { - return 1; + return true; } } - return 0; + return false; case ZEND_SUB: res = ssa_op->result_def; if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } op1_min = OP1_MIN_RANGE(); op2_max = OP2_MAX_RANGE(); if (zend_sub_will_overflow(op1_min, op2_max)) { - return 1; + return true; } } if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.overflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } op1_max = OP1_MAX_RANGE(); op2_min = OP2_MIN_RANGE(); if (zend_sub_will_overflow(op1_max, op2_min)) { - return 1; + return true; } } - return 0; + return false; case ZEND_MUL: res = ssa_op->result_def; return (res < 0 || @@ -1211,54 +1211,54 @@ static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, c || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } op1_min = OP1_MIN_RANGE(); op2_min = OP2_MIN_RANGE(); if (zend_add_will_overflow(op1_min, op2_min)) { - return 1; + return true; } } if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.overflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } op1_max = OP1_MAX_RANGE(); op2_max = OP2_MAX_RANGE(); if (zend_add_will_overflow(op1_max, op2_max)) { - return 1; + return true; } } - return 0; + return false; } else if (opline->extended_value == ZEND_SUB) { res = ssa_op->op1_def; if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } op1_min = OP1_MIN_RANGE(); op2_max = OP2_MAX_RANGE(); if (zend_sub_will_overflow(op1_min, op2_max)) { - return 1; + return true; } } if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.overflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } op1_max = OP1_MAX_RANGE(); op2_min = OP2_MIN_RANGE(); if (zend_sub_will_overflow(op1_max, op2_min)) { - return 1; + return true; } } - return 0; + return false; } else if (opline->extended_value == ZEND_MUL) { res = ssa_op->op1_def; return (res < 0 || @@ -1268,7 +1268,7 @@ static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, c } ZEND_FALLTHROUGH; default: - return 1; + return true; } } @@ -1580,11 +1580,11 @@ static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval ** static ZEND_ATTRIBUTE_UNUSED void zend_jit_print_regset(zend_regset regset) { zend_reg reg; - int first = 1; + bool first = true; ZEND_REGSET_FOREACH(regset, reg) { if (first) { - first = 0; + first = false; fprintf(stderr, "%s", zend_reg_name[reg]); } else { fprintf(stderr, ", %s", zend_reg_name[reg]); @@ -1628,7 +1628,7 @@ static bool zend_jit_in_loop(zend_ssa *ssa, int header, zend_basic_block *b) } b = ssa->cfg.blocks + b->loop_header; } - return 0; + return false; } static void zend_jit_compute_loop_body(zend_ssa *ssa, int header, int n, zend_bitset loop_body) @@ -2017,12 +2017,12 @@ static bool zend_interval_covers(zend_lifetime_interval *ival, uint32_t position do { if (position >= range->start && position <= range->end) { - return 1; + return true; } range = range->next; } while (range); - return 0; + return false; } static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_lifetime_interval *ival2) @@ -2047,7 +2047,7 @@ static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_l /* See "Optimized Interval Splitting in a Linear Scan Register Allocator", Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling */ -static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_regset *hints, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free) +static bool zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_regset *hints, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free) { zend_lifetime_interval *it; uint32_t freeUntilPos[ZREG_NUM]; @@ -2108,7 +2108,7 @@ static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const z } if (hint == ZREG_NONE && ZEND_REGSET_IS_EMPTY(available)) { - return 0; + return false; } /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and @@ -2204,11 +2204,11 @@ static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const z if (current->used_as_hint) { ZEND_REGSET_INCL(*hints, hint); } - return 1; + return true; } if (ZEND_REGSET_IS_EMPTY(available)) { - return 0; + return false; } pos = 0; reg = ZREG_NONE; @@ -2241,14 +2241,14 @@ static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const z if (reg == ZREG_NONE) { /* no register available without spilling */ - return 0; + return false; } else if (zend_interval_end(current) < pos) { /* register available for the whole interval */ current->reg = reg; if (current->used_as_hint) { ZEND_REGSET_INCL(*hints, reg); } - return 1; + return true; #if 0 // TODO: allow low priority register usage } else if (reg2 != ZREG_NONE && zend_interval_end(current) < pos2) { @@ -2257,19 +2257,19 @@ static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const z if (current->used_as_hint) { ZEND_REGSET_INCL(*hints, reg2); } - return 1; + return true; #endif } else { /* TODO: enable interval splitting ??? */ /* register available for the first part of the interval */ if (1 || zend_jit_split_interval(current, pos, list, free) != SUCCESS) { - return 0; + return false; } current->reg = reg; if (current->used_as_hint) { ZEND_REGSET_INCL(*hints, reg); } - return 1; + return true; } } @@ -2520,7 +2520,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array } } } else { - int need_move = 0; + bool need_move = false; for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) { src = phi->sources[k]; @@ -2533,12 +2533,12 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array } if (intervals[i]) { if (!intervals[src]) { - need_move = 1; + need_move = true; } else if (intervals[i]->reg != intervals[src]->reg) { - need_move = 1; + need_move = true; } } else if (intervals[src]) { - need_move = 1; + need_move = true; } } } @@ -2570,13 +2570,13 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array ((intervals[i]->flags & ZREG_LOAD) || ((intervals[i]->flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) && ssa->vars[i].use_chain < 0) { - bool may_remove = 1; + bool may_remove = true; zend_ssa_phi *phi = ssa->vars[i].phi_use_chain; while (phi) { if (intervals[phi->ssa_var] && !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) { - may_remove = 0; + may_remove = false; break; } phi = zend_ssa_next_use_phi(ssa, i, phi); @@ -2593,13 +2593,13 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array (intervals[i]->flags & ZREG_STORE) && (ssa->vars[i].use_chain < 0 || zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) { - bool may_remove = 1; + bool may_remove = true; zend_ssa_phi *phi = ssa->vars[i].phi_use_chain; while (phi) { if (intervals[phi->ssa_var] && !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) { - may_remove = 0; + may_remove = false; break; } phi = zend_ssa_next_use_phi(ssa, i, phi); @@ -2641,9 +2641,9 @@ static bool zend_jit_next_is_send_result(const zend_op *opline) && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op2_type != IS_CONST && (opline+1)->op1.var == opline->result.var) { - return 1; + return true; } - return 0; + return false; } static bool zend_jit_supported_binary_op(zend_uchar op, uint32_t op1_info, uint32_t op2_info) @@ -2683,8 +2683,8 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op int call_level = 0; void *checkpoint = NULL; zend_lifetime_interval **ra = NULL; - bool is_terminated = 1; /* previous basic block is terminated by jump */ - bool recv_emitted = 0; /* emitted at least one RECV opcode */ + bool is_terminated = true; /* previous basic block is terminated by jump */ + bool recv_emitted = false; /* emitted at least one RECV opcode */ zend_uchar smart_branch_opcode; uint32_t target_label, target_label2; uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info; @@ -2771,7 +2771,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } zend_jit_prologue(&dasm_state); } - recv_emitted = 1; + recv_emitted = true; } else if (opline->opcode == ZEND_RECV) { if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { /* skip */ @@ -2797,7 +2797,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); zend_jit_prologue(&dasm_state); - recv_emitted = 1; + recv_emitted = true; } } else { if (recv_emitted) { @@ -2815,7 +2815,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); zend_jit_prologue(&dasm_state); - recv_emitted = 1; + recv_emitted = true; } } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE && ssa->cfg.blocks[b].len == 1 && @@ -2833,7 +2833,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } } - is_terminated = 0; + is_terminated = false; zend_jit_label(&dasm_state, b); if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) { @@ -3160,14 +3160,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op break; } ce = NULL; - ce_is_instanceof = 0; - on_this = 0; + ce_is_instanceof = false; + on_this = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_OBJECT)) { @@ -3208,14 +3208,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op break; } ce = NULL; - ce_is_instanceof = 0; - on_this = 0; + ce_is_instanceof = false; + on_this = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_OBJECT)) { @@ -3249,14 +3249,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op break; } ce = NULL; - ce_is_instanceof = 0; - on_this = 0; + ce_is_instanceof = false; + on_this = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_OBJECT)) { @@ -3418,7 +3418,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } goto done; case ZEND_DO_UCALL: - is_terminated = 1; + is_terminated = true; ZEND_FALLTHROUGH; case ZEND_DO_ICALL: case ZEND_DO_FCALL_BY_NAME: @@ -3547,7 +3547,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } } else { int j; - bool left_frame = 0; + bool left_frame = false; if (!zend_jit_return(&dasm_state, opline, op_array, op1_info, OP1_REG_ADDR())) { @@ -3568,7 +3568,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_free_cvs(&dasm_state)) { goto jit_failure; } - left_frame = 1; + left_frame = true; } if (!left_frame) { for (j = 0 ; j < op_array->last_var; j++) { @@ -3576,7 +3576,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (!left_frame) { - left_frame = 1; + left_frame = true; if (!zend_jit_leave_frame(&dasm_state)) { goto jit_failure; } @@ -3751,14 +3751,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op break; } ce = NULL; - ce_is_instanceof = 0; - on_this = 0; + ce_is_instanceof = false; + on_this = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; op1_addr = 0; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_OBJECT)) { @@ -3902,14 +3902,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op break; } ce = NULL; - ce_is_instanceof = 0; - on_this = 0; + ce_is_instanceof = false; + on_this = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; op1_addr = 0; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_OBJECT)) { @@ -3979,7 +3979,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) { goto jit_failure; } - is_terminated = 1; + is_terminated = true; break; case ZEND_CATCH: case ZEND_FAST_CALL: @@ -3996,7 +3996,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_tail_handler(&dasm_state, opline)) { goto jit_failure; } - is_terminated = 1; + is_terminated = true; break; /* stackless execution */ case ZEND_INCLUDE_OR_EVAL: @@ -4006,7 +4006,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_call(&dasm_state, opline, b + 1)) { goto jit_failure; } - is_terminated = 1; + is_terminated = true; break; case ZEND_JMPZ: case ZEND_JMPNZ: @@ -4067,7 +4067,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op zend_jit_cond_jmp(&dasm_state, next_opline, ssa->cfg.blocks[b].successors[0]); if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) { zend_jit_call(&dasm_state, next_opline, b + 1); - is_terminated = 1; + is_terminated = true; } else { zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1, NULL); } @@ -4091,7 +4091,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b+1].successors[1])) { goto jit_failure; } - is_terminated = 1; + is_terminated = true; } } done: @@ -4237,7 +4237,7 @@ static int ZEND_FASTCALL zend_runtime_jit(void) zend_op_array *op_array = &EX(func)->op_array; zend_op *opline = op_array->opcodes; zend_jit_op_array_extension *jit_extension; - bool do_bailout = 0; + bool do_bailout = false; zend_shared_alloc_lock(); @@ -4314,7 +4314,7 @@ void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend zend_op_array *op_array = &EX(func)->op_array; zend_jit_op_array_hot_extension *jit_extension; uint32_t i; - bool do_bailout = 0; + bool do_bailout = false; zend_shared_alloc_lock(); jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array); @@ -4331,7 +4331,7 @@ void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend /* perform real JIT for this function */ zend_real_jit_func(op_array, NULL, opline); } zend_catch { - do_bailout = 1; + do_bailout = true; } zend_end_try(); zend_jit_protect(); @@ -4703,7 +4703,7 @@ static void zend_jit_init_handlers(void) } } -static int zend_jit_make_stubs(void) +static bool zend_jit_make_stubs(void) { dasm_State* dasm_state = NULL; uint32_t i; @@ -4714,18 +4714,18 @@ static int zend_jit_make_stubs(void) for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) { dasm_setup(&dasm_state, dasm_actions); if (!zend_jit_stubs[i].stub(&dasm_state)) { - return 0; + return false; } if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name, 0, zend_jit_stubs[i].offset, zend_jit_stubs[i].adjustment)) { - return 0; + return false; } } zend_jit_init_handlers(); dasm_free(&dasm_state); - return 1; + return true; } static void zend_jit_globals_ctor(zend_jit_globals *jit_globals) @@ -4737,7 +4737,7 @@ static void zend_jit_globals_ctor(zend_jit_globals *jit_globals) static int zend_jit_parse_config_num(zend_long jit) { if (jit == 0) { - JIT_G(on) = 0; + JIT_G(on) = false; return SUCCESS; } @@ -4760,7 +4760,7 @@ static int zend_jit_parse_config_num(zend_long jit) if (jit / 10 != 0) return FAILURE; - JIT_G(on) = 1; + JIT_G(on) = true; return SUCCESS; } @@ -4776,30 +4776,30 @@ ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage) if (ZSTR_LEN(jit) == 0 || zend_string_equals_literal_ci(jit, "disable")) { - JIT_G(enabled) = 0; - JIT_G(on) = 0; + JIT_G(enabled) = false; + JIT_G(on) = false; return SUCCESS; } else if (zend_string_equals_literal_ci(jit, "0") || zend_string_equals_literal_ci(jit, "off") || zend_string_equals_literal_ci(jit, "no") || zend_string_equals_literal_ci(jit, "false")) { - JIT_G(enabled) = 1; - JIT_G(on) = 0; + JIT_G(enabled) = true; + JIT_G(on) = false; return SUCCESS; } else if (zend_string_equals_literal_ci(jit, "1") || zend_string_equals_literal_ci(jit, "on") || zend_string_equals_literal_ci(jit, "yes") || zend_string_equals_literal_ci(jit, "true") || zend_string_equals_literal_ci(jit, "tracing")) { - JIT_G(enabled) = 1; - JIT_G(on) = 1; + JIT_G(enabled) = true; + JIT_G(on) = true; JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS; JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE; JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX; return SUCCESS; } else if (zend_string_equals_literal_ci(jit, "function")) { - JIT_G(enabled) = 1; - JIT_G(on) = 1; + JIT_G(enabled) = true; + JIT_G(on) = true; JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT; JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD; JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX; @@ -4810,14 +4810,14 @@ ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage) if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) { goto failure; } - JIT_G(enabled) = 1; + JIT_G(enabled) = true; return SUCCESS; } failure: zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number"); - JIT_G(enabled) = 0; - JIT_G(on) = 0; + JIT_G(enabled) = false; + JIT_G(on) = false; return FAILURE; } @@ -4860,8 +4860,8 @@ ZEND_EXT_API int zend_jit_check_support(void) if (zend_jit_vm_kind != ZEND_VM_KIND_CALL && zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled."); - JIT_G(enabled) = 0; - JIT_G(on) = 0; + JIT_G(enabled) = false; + JIT_G(on) = false; return FAILURE; } @@ -4869,8 +4869,8 @@ ZEND_EXT_API int zend_jit_check_support(void) if (strcmp(sapi_module.name, "phpdbg") != 0) { zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled."); } - JIT_G(enabled) = 0; - JIT_G(on) = 0; + JIT_G(enabled) = false; + JIT_G(on) = false; return FAILURE; } @@ -4884,8 +4884,8 @@ ZEND_EXT_API int zend_jit_check_support(void) default: if (zend_get_user_opcode_handler(i) != NULL) { zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled."); - JIT_G(enabled) = 0; - JIT_G(on) = 0; + JIT_G(enabled) = false; + JIT_G(on) = false; return FAILURE; } } From 5e0affc799438e96c047d1e550b20f53a5411085 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 11:56:12 +0100 Subject: [PATCH 02/34] ext/opcache/jit/zend_jit: make zend_jit_check_funcs() static --- ext/opcache/jit/zend_jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 5b1ed7500ed3..00eea0be2d6a 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -4276,7 +4276,7 @@ static int ZEND_FASTCALL zend_runtime_jit(void) return 0; } -void zend_jit_check_funcs(HashTable *function_table, bool is_method) { +static void zend_jit_check_funcs(HashTable *function_table, bool is_method) { zend_op *opline; zend_function *func; zend_op_array *op_array; From 71db1016bc7e29b39986e3824bad3f543c33b27c Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 12:29:24 +0100 Subject: [PATCH 03/34] ext/opcache/jit/zend_jit_trace: make zend_jit_trace_hot_side() static Dropping ZEND_FASTCALL because this is an internal function only called from C code. --- ext/opcache/jit/zend_jit_trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 55db634ec882..c4d4c0493276 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -7902,7 +7902,7 @@ exit:; return ret; } -int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num) +static int zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num) { zend_jit_trace_stop stop; int ret = 0; From e7eb6bbe026c759ef2669656ca21bdd7f1288aec Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 12:32:44 +0100 Subject: [PATCH 04/34] ext/opcache/jit/zend_jit_trace: make zend_jit_trace_exit() static Only used in zend_jit_*.dasc, which is the same compilation unit. For these, this commit adds a (static) forward declaration. --- ext/opcache/jit/zend_jit.c | 2 ++ ext/opcache/jit/zend_jit_internal.h | 1 - ext/opcache/jit/zend_jit_trace.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 00eea0be2d6a..2575286e45a7 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -756,6 +756,8 @@ typedef enum _sp_adj_kind { static int sp_adj[SP_ADJ_LAST]; +static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs); + /* The generated code may contain tautological comparisons, ignore them. */ #if defined(__clang__) # pragma clang diagnostic push diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index fb640ff9b97c..8d8ac7d9c18d 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -707,7 +707,6 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HAND ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS); int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline); -int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs); zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start, uint32_t is_megamorphc); static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit_trace_rec *trace, const zend_op *opline, bool *exit_if_true) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index c4d4c0493276..5d43b134f935 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -8048,7 +8048,7 @@ static int zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t par return ret; } -int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs) +static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs) { uint32_t trace_num = EG(jit_trace_num); zend_execute_data *execute_data = EG(current_execute_data); From a9e57ce8f722bc3cff1315b0bb795818a1bae674 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 12:46:10 +0100 Subject: [PATCH 05/34] ext/opcache/jit/zend_jit: make zend_jit_hot_counters static --- ext/opcache/jit/zend_jit.c | 3 ++- ext/opcache/jit/zend_jit_internal.h | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 2575286e45a7..d6b0bb6fd87f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -106,7 +106,8 @@ typedef struct _zend_jit_stub { zend_ulong zend_jit_profile_counter = 0; int zend_jit_profile_counter_rid = -1; -int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; +#define ZEND_HOT_COUNTERS_COUNT 128 +static int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; const zend_op *zend_jit_halt_op = NULL; static int zend_jit_vm_kind = 0; diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 8d8ac7d9c18d..17b1d3727e8d 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -233,10 +233,6 @@ extern int zend_jit_profile_counter_rid; /* Hot Counters */ -#define ZEND_HOT_COUNTERS_COUNT 128 - -extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; - static zend_always_inline zend_long zend_jit_hash(const void *ptr) { uintptr_t x; From d8bcac79ca82ebe917728fbe888bda74faaf4307 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 13:06:06 +0100 Subject: [PATCH 06/34] ext/opcache/jit/zend_jit: make the "ssa" parameter of zend_may_overflow() const --- ext/opcache/jit/zend_jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index d6b0bb6fd87f..c733609ae7c8 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -1108,7 +1108,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, return entry; } -static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa) +static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, const zend_ssa *ssa) { int res; zend_long op1_min, op1_max, op2_min, op2_max; From e1812fcdb565f725f9e1b6ec920cc45e00595de1 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 13:12:00 +0100 Subject: [PATCH 07/34] ext/opcache/jit/zend_jit: move redundant code to zend_jit_new_op_array_extension() --- ext/opcache/jit/zend_jit.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index c733609ae7c8..70ccf18c2e33 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -4430,6 +4430,23 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array) #include "jit/zend_jit_trace.c" +/** + * Allocate and initialize a new #zend_jit_op_array_extension for the + * given #zend_op_array. + */ +static zend_jit_op_array_extension *zend_jit_new_op_array_extension(zend_op_array *op_array) +{ + zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension)); + if (!jit_extension) { + return NULL; + } + + memset(&jit_extension->func_info, 0, sizeof(zend_func_info)); + ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); + zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension); + return jit_extension; +} + ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) { if (dasm_ptr == NULL) { @@ -4453,16 +4470,13 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) opline++; } } - jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension)); + jit_extension = zend_jit_new_op_array_extension(op_array); if (!jit_extension) { return FAILURE; } - memset(&jit_extension->func_info, 0, sizeof(zend_func_info)); jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC; jit_extension->orig_handler = (void*)opline->handler; - ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); opline->handler = (const void*)zend_jit_runtime_jit_handler; - zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension); return SUCCESS; } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) { @@ -4482,16 +4496,13 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) opline++; } } - jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension)); + jit_extension = zend_jit_new_op_array_extension(op_array); if (!jit_extension) { return FAILURE; } - memset(&jit_extension->func_info, 0, sizeof(zend_func_info)); jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST; jit_extension->orig_handler = (void*)opline->handler; - ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); opline->handler = (const void*)zend_jit_profile_jit_handler; - zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension); } return SUCCESS; From d7bb24ee0c024a8a2d2f7520fff5279c8edf1ec2 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 11:58:02 +0100 Subject: [PATCH 08/34] ext/opcache/jit/zend_jit_trace: use true/false instead of 1/0 --- ext/opcache/jit/zend_jit.c | 2 +- ext/opcache/jit/zend_jit_trace.c | 420 +++++++++++++++---------------- 2 files changed, 211 insertions(+), 211 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 70ccf18c2e33..93dea44d703b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -134,7 +134,7 @@ static const void *zend_jit_loop_trace_counter_handler = NULL; static int ZEND_FASTCALL zend_runtime_jit(void); static int zend_jit_trace_op_len(const zend_op *opline); -static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline); +static bool zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline); static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags); static const void *zend_jit_trace_get_exit_addr(uint32_t n); static void zend_jit_trace_add_code(const void *start, uint32_t size); diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 5d43b134f935..e394c56fa421 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -290,7 +290,7 @@ static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t return buf.s; } -static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline) +static bool zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline) { switch (opline->opcode) { case ZEND_IS_IDENTICAL: @@ -313,7 +313,7 @@ static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op case ZEND_ARRAY_KEY_EXISTS: if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) { /* smart branch */ - return 1; + return true; } break; case ZEND_JMPZ: @@ -332,11 +332,11 @@ static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op case ZEND_SWITCH_STRING: case ZEND_MATCH: /* branch opcodes */ - return 1; + return true; case ZEND_NEW: if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) { /* NEW may skip constructor without arguments */ - return 1; + return true; } break; case ZEND_CATCH: @@ -350,16 +350,16 @@ static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op case ZEND_INCLUDE_OR_EVAL: case ZEND_MATCH_ERROR: /* unsupported */ - return 1; + return true; case ZEND_DO_FCALL: /* potentially polymorphic call */ - return 1; + return true; #if 0 case ZEND_DO_UCALL: case ZEND_DO_FCALL_BY_NAME: /* monomorphic call */ // TODO: recompilation may change target ??? - return 0; + return false; #endif case ZEND_RETURN_BY_REF: case ZEND_RETURN: @@ -368,7 +368,7 @@ static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op default: break; } - return 0; + return false; } static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(zend_uchar type, uint32_t info) @@ -523,7 +523,7 @@ static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, if (type != IS_UNKNOWN && type < IS_STRING && ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type)) { - return 0; + return false; } } if (call_info && arg_num < call_info->num_args && call_info->arg_info[arg_num].opline) { @@ -538,7 +538,7 @@ static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, // TODO: few functions (e.g. pcntl_exec) modify arrays in-place ??? if (type != IS_ARRAY && (ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) { - return 0; + return false; } } } @@ -546,7 +546,7 @@ static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, } } - return 1; + return true; } static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script) @@ -816,7 +816,7 @@ static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_ return ssa_vars_count; } -static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +static bool zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) { int var, use; zend_ssa_op *op; @@ -848,14 +848,14 @@ static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const info = ssa->var_info + op->result_use; } else { assert(0); - return 0; + return false; } tssa->vars[ssa_var].no_val = no_val; tssa->vars[ssa_var].alias = alias; memcpy(&tssa->var_info[ssa_var], info, sizeof(zend_ssa_var_info)); - return 1; + return true; } - return 0; + return false; } static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) @@ -869,7 +869,7 @@ static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const tssa->var_info[ssa_var].range.max = tmp.max; tssa->var_info[ssa_var].range.underflow = tmp.underflow; tssa->var_info[ssa_var].range.overflow = tmp.overflow; - tssa->var_info[ssa_var].has_range = 1; + tssa->var_info[ssa_var].has_range = true; } } @@ -916,14 +916,14 @@ static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, con tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow; tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow; } else { - tssa->var_info[ssa_var].has_range = 1; + tssa->var_info[ssa_var].has_range = true; tssa->var_info[ssa_var].range = info->range; } } } } -static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +static bool zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) { int def; zend_ssa_op *op; @@ -941,7 +941,7 @@ static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, c info = ssa->var_info + op->result_def; } else { assert(0); - return 0; + return false; } tssa->var_info[ssa_var].type &= info->type; if (info->ce) { @@ -970,13 +970,13 @@ static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, c tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow; tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow; } else { - tssa->var_info[ssa_var].has_range = 1; + tssa->var_info[ssa_var].has_range = true; tssa->var_info[ssa_var].range = info->range; } } - return 1; + return true; } - return 0; + return false; } static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op) @@ -1071,7 +1071,7 @@ static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, c return NULL; } -static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var) +static bool is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var) { if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG && !(tssa->var_info[var].type & MAY_BE_REF)) { @@ -1086,12 +1086,12 @@ static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, u || opline->opcode == ZEND_POST_INC) { if (tssa->ops[idx].op1_use >= 0 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) { - return 0; + return false; } if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - return 0; + return false; } - return 1; + return true; } else if (opline->opcode == ZEND_ASSIGN_OP && (opline->extended_value == ZEND_ADD || opline->extended_value == ZEND_SUB @@ -1099,20 +1099,20 @@ static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, u if ((opline->op2_type & (IS_VAR|IS_CV)) && tssa->ops[idx].op2_use >= 0 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) { - return 0; + return false; } if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - return 0; + return false; } if (opline->op2_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op2); if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) { - return 0; + return false; } } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - return 0; + return false; } - return 1; + return true; } } if (tssa->ops[idx].result_def == var) { @@ -1123,28 +1123,28 @@ static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, u if ((opline->op1_type & (IS_VAR|IS_CV)) && tssa->ops[idx].op1_use >= 0 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) { - return 0; + return false; } if ((opline->op2_type & (IS_VAR|IS_CV)) && tssa->ops[idx].op2_use >= 0 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) { - return 0; + return false; } if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) { - return 0; + return false; } } else if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - return 0; + return false; } if (opline->op2_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op2); if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) { - return 0; + return false; } } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - return 0; + return false; } return 1; } else if (opline->opcode == ZEND_PRE_DEC @@ -1154,17 +1154,17 @@ static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, u if ((opline->op1_type & (IS_VAR|IS_CV)) && tssa->ops[idx].op1_use >= 0 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) { - return 0; + return false; } if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - return 0; + return false; } - return 1; + return true; } } } } - return 0; + return false; } typedef struct _zend_tssa { @@ -1545,7 +1545,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } ssa_var_info[i].type = tmp; ssa_var_info[i].ce = ce; - ssa_var_info[i].is_instanceof = 1; + ssa_var_info[i].is_instanceof = true; } else { ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } @@ -2353,7 +2353,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } ssa_var_info[v].type = tmp; ssa_var_info[v].ce = ce; - ssa_var_info[v].is_instanceof = 1; + ssa_var_info[v].is_instanceof = true; } else { ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } @@ -2822,7 +2822,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace && ssa_op->op1_def >= 0 && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) { /* avoid register allocation in case of possibility of indirect modification*/ - support_opline = 0; + support_opline = false; } if (ssa_op->op1_use >= 0 @@ -3502,7 +3502,7 @@ static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ss || ssa->vars[var].phi_use_chain || ssa->ops[use].op1_use != var || ssa->ops[use].op1_use_chain != -1) { - return 0; + return false; } opline = ssa_opcodes[use]; @@ -3514,7 +3514,7 @@ static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ss || !JIT_G(current_frame)->call || !JIT_G(current_frame)->call->func || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { - return 0; + return false; } } else if (opline->opcode != ZEND_FETCH_OBJ_R && opline->opcode != ZEND_FETCH_OBJ_IS @@ -3525,13 +3525,13 @@ static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ss && opline->opcode != ZEND_PRE_DEC_OBJ && opline->opcode != ZEND_POST_INC_OBJ && opline->opcode != ZEND_POST_DEC_OBJ) { - return 0; + return false; } if (opline->op2_type != IS_CONST || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { - return 0; + return false; } if (opline->opcode == ZEND_ASSIGN_OBJ_OP) { @@ -3539,11 +3539,11 @@ static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ss && (opline+1)->op1_type == IS_CV && (opline+1)->op1.var == opline->op1.var) { /* skip $a->prop += $a; */ - return 0; + return false; } if (!zend_jit_supported_binary_op( opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) { - return 0; + return false; } } @@ -3552,27 +3552,27 @@ static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ss || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME || ssa_opcodes[i]->opcode == ZEND_DO_FCALL || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) { - return 0; + return false; } } - return 1; + return true; } -static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size) +static bool zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size) { uint32_t i; for (i = 0; i < stack_size; i++) { if (STACK_REG(stack, i) != ZREG_NONE && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) { - return 1; + return true; } } - return 0; + return false; } -static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num) +static bool zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num) { const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags; @@ -3580,7 +3580,7 @@ static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t zend_jit_trace_stack *stack; if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) { - return 1; + return true; } stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size; @@ -3588,19 +3588,19 @@ static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t return zend_jit_trace_stack_needs_deoptimization(stack, stack_size); } -static int zend_jit_trace_deoptimization(dasm_State **Dst, - uint32_t flags, - const zend_op *opline, - zend_jit_trace_stack *parent_stack, - int parent_vars_count, - zend_ssa *ssa, - zend_jit_trace_stack *stack, - zend_lifetime_interval **ra, - bool polymorphic_side_trace) +static bool zend_jit_trace_deoptimization(dasm_State **Dst, + uint32_t flags, + const zend_op *opline, + zend_jit_trace_stack *parent_stack, + int parent_vars_count, + zend_ssa *ssa, + zend_jit_trace_stack *stack, + zend_lifetime_interval **ra, + bool polymorphic_side_trace) { int i; - bool has_constants = 0; - bool has_unsaved_vars = 0; + bool has_constants = false; + bool has_unsaved_vars = false; // TODO: Merge this loop with the following register LOAD loop to implement parallel move ??? for (i = 0; i < parent_vars_count; i++) { @@ -3615,14 +3615,14 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, if (stack) { SET_STACK_REG_EX(stack, i, reg, STACK_FLAGS(parent_stack, i)); } - has_unsaved_vars = 1; + has_unsaved_vars = true; } else { uint8_t type = STACK_TYPE(parent_stack, i); if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) && !zend_jit_store_var(Dst, 1 << type, i, reg, STACK_MEM_TYPE(parent_stack, i) != type)) { - return 0; + return false; } if (stack) { SET_STACK_TYPE(stack, i, type, 1); @@ -3630,7 +3630,7 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, } } else { /* delay custom deoptimization instructions to prevent register clobbering */ - has_constants = 1; + has_constants = true; } } } @@ -3654,7 +3654,7 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) && !zend_jit_store_var(Dst, 1 << type, i, reg, STACK_MEM_TYPE(parent_stack, i) != type)) { - return 0; + return false; } } } @@ -3671,20 +3671,20 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, /* pass */ } else if (reg == ZREG_THIS) { if (polymorphic_side_trace) { - ssa->var_info[i].delayed_fetch_this = 1; + ssa->var_info[i].delayed_fetch_this = true; if (stack) { SET_STACK_REG(stack, i, ZREG_THIS); } } else if (!zend_jit_load_this(Dst, EX_NUM_TO_VAR(i))) { - return 0; + return false; } } else { if (reg == ZREG_ZVAL_COPY_GPR0 &&!zend_jit_escape_if_undef_r0(Dst, i, flags, opline)) { - return 0; + return false; } if (!zend_jit_store_const(Dst, i, reg)) { - return 0; + return false; } } } @@ -3693,7 +3693,7 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { if (!zend_jit_save_call_chain(Dst, -1)) { - return 0; + return false; } } @@ -3701,7 +3701,7 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, const zend_op *op = opline - 1; if (!zend_jit_free_op(Dst, op, -1, op->op2.var)) { - return 0; + return false; } } @@ -3709,32 +3709,32 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, const zend_op *op = opline - 1; if (!zend_jit_free_op(Dst, op, -1, op->op1.var)) { - return 0; + return false; } } if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) { if (!zend_jit_check_exception(Dst)) { - return 0; + return false; } } if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) { if (!zend_jit_free_trampoline(Dst)) { - return 0; + return false; } } - return 1; + return true; } static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max) { - info->has_range = 1; + info->has_range = true; info->range.min = min; info->range.max = max; - info->range.underflow = 0; - info->range.overflow = 0; + info->range.underflow = false; + info->range.overflow = false; } static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true) @@ -3933,19 +3933,19 @@ static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_o || prev_opcode == ZEND_CASE_STRICT) { if (ssa_op->op1_use < 0) { if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) { - return 0; + return false; } } if (ssa_op->op2_use < 0) { if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) { - return 0; + return false; } } - return 1; + return true; } } } - return 0; + return false; } static bool zend_jit_trace_next_is_send_result(const zend_op *opline, @@ -3972,9 +3972,9 @@ static bool zend_jit_trace_next_is_send_result(const zend_op *oplin zend_jit_trace_send_type(opline+1, frame->call, res_type); } } - return 1; + return true; } - return 0; + return false; } static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num) @@ -3996,14 +3996,14 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par zend_uchar smart_branch_opcode; const void *exit_addr; uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info; - bool send_result = 0; + bool send_result = false; bool skip_comparison; zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; zend_class_entry *ce; bool ce_is_instanceof; - bool on_this = 0; - bool delayed_fetch_this = 0; - bool avoid_refcounting = 0; + bool on_this = false; + bool delayed_fetch_this = false; + bool avoid_refcounting = false; bool polymorphic_side_trace = parent_trace && (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL); @@ -4340,7 +4340,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { - gen_handler = 0; + gen_handler = false; switch (opline->opcode) { case ZEND_PRE_INC: case ZEND_PRE_DEC: @@ -4412,7 +4412,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par res_addr = RES_REG_ADDR(); if (Z_MODE(res_addr) != IS_REG && zend_jit_trace_next_is_send_result(opline, p, frame)) { - send_result = 1; + send_result = true; res_use_info = -1; res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); if (!zend_jit_reuse_ip(&dasm_state)) { @@ -4450,7 +4450,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par goto jit_failure; } if (ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -4465,7 +4465,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par goto jit_failure; } if (ssa->vars[ssa_op->op2_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op2_use].guarded_reference = 1; + ssa->var_info[ssa_op->op2_use].guarded_reference = true; } } else { CHECK_OP2_TRACE_TYPE(); @@ -4484,7 +4484,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par res_addr = RES_REG_ADDR(); if (Z_MODE(res_addr) != IS_REG && zend_jit_trace_next_is_send_result(opline, p, frame)) { - send_result = 1; + send_result = true; res_use_info = -1; res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); if (!zend_jit_reuse_ip(&dasm_state)) { @@ -4534,7 +4534,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } res_addr = RES_REG_ADDR(); if (zend_jit_trace_next_is_send_result(opline, p, frame)) { - send_result = 1; + send_result = true; res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); if (!zend_jit_reuse_ip(&dasm_state)) { goto jit_failure; @@ -4614,7 +4614,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -4641,15 +4641,15 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par break; } ce = NULL; - ce_is_instanceof = 0; - on_this = delayed_fetch_this = 0; - op1_indirect = 0; + ce_is_instanceof = false; + on_this = delayed_fetch_this = false; + op1_indirect = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { if (ssa_op->op1_use >= 0) { delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; @@ -4662,7 +4662,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (opline->op1_type == IS_VAR) { if (orig_op1_type != IS_UNKNOWN && (orig_op1_type & IS_TRACE_INDIRECT)) { - op1_indirect = 1; + op1_indirect = true; if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { goto jit_failure; @@ -4677,7 +4677,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -4695,7 +4695,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } if (delayed_fetch_this) { - on_this = 1; + on_this = true; } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; } else if (op_array_ssa->ops @@ -4732,15 +4732,15 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par break; } ce = NULL; - ce_is_instanceof = 0; - on_this = delayed_fetch_this = 0; - op1_indirect = 0; + ce_is_instanceof = false; + on_this = delayed_fetch_this = false; + op1_indirect = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { if (ssa_op->op1_use >= 0) { delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; @@ -4753,7 +4753,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (opline->op1_type == IS_VAR) { if (orig_op1_type != IS_UNKNOWN && (orig_op1_type & IS_TRACE_INDIRECT)) { - op1_indirect = 1; + op1_indirect = true; if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { goto jit_failure; @@ -4768,7 +4768,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -4786,7 +4786,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } if (delayed_fetch_this) { - on_this = 1; + on_this = true; } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; } else if (op_array_ssa->ops @@ -4812,15 +4812,15 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par break; } ce = NULL; - ce_is_instanceof = 0; - on_this = delayed_fetch_this = 0; - op1_indirect = 0; + ce_is_instanceof = false; + on_this = delayed_fetch_this = false; + op1_indirect = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { if (ssa_op->op1_use >= 0) { delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; @@ -4833,7 +4833,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (opline->op1_type == IS_VAR) { if (orig_op1_type != IS_UNKNOWN && (orig_op1_type & IS_TRACE_INDIRECT)) { - op1_indirect = 1; + op1_indirect = true; if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { goto jit_failure; @@ -4848,7 +4848,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -4866,7 +4866,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } if (delayed_fetch_this) { - on_this = 1; + on_this = true; } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; } else if (op_array_ssa->ops @@ -4920,7 +4920,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -4985,7 +4985,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } if (opline->result_type == IS_UNUSED) { res_addr = 0; @@ -5013,7 +5013,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par res_info = RES_INFO(); if (Z_MODE(res_addr) != IS_REG && zend_jit_trace_next_is_send_result(opline, p, frame)) { - send_result = 1; + send_result = true; res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); if (!zend_jit_reuse_ip(&dasm_state)) { goto jit_failure; @@ -5119,7 +5119,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } goto done; case ZEND_SEND_VAR: @@ -5217,7 +5217,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par CHECK_OP1_TRACE_TYPE(); CHECK_OP2_TRACE_TYPE(); if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point; @@ -5265,7 +5265,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par CHECK_OP1_TRACE_TYPE(); CHECK_OP2_TRACE_TYPE(); if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point; @@ -5305,7 +5305,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par goto done; case ZEND_DEFINED: if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); @@ -5330,7 +5330,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op1_info = OP1_INFO(); CHECK_OP1_TRACE_TYPE(); if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point; @@ -5369,8 +5369,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } else { int j; - int may_throw = 0; - bool left_frame = 0; + bool may_throw = false; + bool left_frame = false; if (!zend_jit_return(&dasm_state, opline, op_array, op1_info, OP1_REG_ADDR())) { @@ -5381,7 +5381,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (!zend_jit_free_cvs(&dasm_state)) { goto jit_failure; } - left_frame = 1; + left_frame = true; } if (!left_frame) { for (j = 0 ; j < op_array->last_var; j++) { @@ -5403,7 +5403,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (!left_frame) { - left_frame = 1; + left_frame = true; if (!zend_jit_leave_frame(&dasm_state)) { goto jit_failure; } @@ -5413,7 +5413,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) { if (info & MAY_BE_RC1) { - may_throw = 1; + may_throw = true; } } } @@ -5524,13 +5524,13 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); } if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); @@ -5560,7 +5560,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par break; } if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); @@ -5600,9 +5600,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; if (ssa_op->op1_def >= 0) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } } else { @@ -5673,7 +5673,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -5689,7 +5689,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W) && !(op1_info & (MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { - ssa->var_info[ssa_op->result_def].indirect_reference = 1; + ssa->var_info[ssa_op->result_def].indirect_reference = true; } goto done; case ZEND_ISSET_ISEMPTY_DIM_OBJ: @@ -5707,7 +5707,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -5715,7 +5715,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op2_info = OP2_INFO(); CHECK_OP2_TRACE_TYPE(); if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point; @@ -5780,22 +5780,22 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par case ZEND_FETCH_OBJ_R: case ZEND_FETCH_OBJ_IS: case ZEND_FETCH_OBJ_W: - on_this = delayed_fetch_this = 0; - avoid_refcounting = 0; + on_this = delayed_fetch_this = false; + avoid_refcounting = false; if (opline->op2_type != IS_CONST || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { break; } ce = NULL; - ce_is_instanceof = 0; - op1_indirect = 0; + ce_is_instanceof = false; + op1_indirect = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_OBJECT)) { @@ -5806,7 +5806,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && opline->opcode == ZEND_FETCH_OBJ_W) { if (orig_op1_type != IS_UNKNOWN && (orig_op1_type & IS_TRACE_INDIRECT)) { - op1_indirect = 1; + op1_indirect = true; if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { goto jit_failure; @@ -5821,7 +5821,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -5843,7 +5843,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting; } if (delayed_fetch_this) { - on_this = 1; + on_this = true; } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; } else if (op_array_ssa->ops @@ -5870,7 +5870,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op1_info = OP1_INFO(); } if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) { goto jit_failure; @@ -5937,7 +5937,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -5959,7 +5959,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -5972,11 +5972,11 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } goto done; case ZEND_FETCH_THIS: - delayed_fetch_this = 0; + delayed_fetch_this = false; if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) { if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) { - ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1; - delayed_fetch_this = 1; + ssa->var_info[ssa_op->result_def].delayed_fetch_this = true; + delayed_fetch_this = true; } } if (!zend_jit_fetch_this(&dasm_state, opline, op_array, delayed_fetch_this)) { @@ -6066,15 +6066,15 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) { break; } - on_this = delayed_fetch_this = 0; + on_this = delayed_fetch_this = false; ce = NULL; - ce_is_instanceof = 0; + ce_is_instanceof = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); op1_addr = OP1_REG_ADDR(); @@ -6089,7 +6089,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -6107,7 +6107,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; } if (delayed_fetch_this) { - on_this = 1; + on_this = true; } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; } else if (op_array_ssa->ops @@ -6162,7 +6162,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) { - gen_handler = 1; + gen_handler = true; op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (op1_info & MAY_BE_GUARD) { @@ -6284,9 +6284,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->result_def, &tmp)) { ssa->var_info[ssa_op->result_def].range.min = tmp.min; ssa->var_info[ssa_op->result_def].range.max = tmp.max; - ssa->var_info[ssa_op->result_def].range.underflow = 0; - ssa->var_info[ssa_op->result_def].range.overflow = 0; - ssa->var_info[ssa_op->result_def].has_range = 1; + ssa->var_info[ssa_op->result_def].range.underflow = false; + ssa->var_info[ssa_op->result_def].range.overflow = false; + ssa->var_info[ssa_op->result_def].has_range = true; } } if (ssa_op->op1_def >= 0 @@ -6345,9 +6345,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) { ssa->var_info[ssa_op->op1_def].range.min = tmp.min; ssa->var_info[ssa_op->op1_def].range.max = tmp.max; - ssa->var_info[ssa_op->op1_def].range.underflow = 0; - ssa->var_info[ssa_op->op1_def].range.overflow = 0; - ssa->var_info[ssa_op->op1_def].has_range = 1; + ssa->var_info[ssa_op->op1_def].range.underflow = false; + ssa->var_info[ssa_op->op1_def].range.overflow = false; + ssa->var_info[ssa_op->op1_def].has_range = true; } } if (ssa_op->op2_def >= 0 @@ -6386,9 +6386,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op2_def, &tmp)) { ssa->var_info[ssa_op->op2_def].range.min = tmp.min; ssa->var_info[ssa_op->op2_def].range.max = tmp.max; - ssa->var_info[ssa_op->op2_def].range.underflow = 0; - ssa->var_info[ssa_op->op2_def].range.overflow = 0; - ssa->var_info[ssa_op->op2_def].has_range = 1; + ssa->var_info[ssa_op->op2_def].range.underflow = false; + ssa->var_info[ssa_op->op2_def].range.overflow = false; + ssa->var_info[ssa_op->op2_def].has_range = true; } } @@ -6429,9 +6429,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) { ssa->var_info[ssa_op->op1_def].range.min = tmp.min; ssa->var_info[ssa_op->op1_def].range.max = tmp.max; - ssa->var_info[ssa_op->op1_def].range.underflow = 0; - ssa->var_info[ssa_op->op1_def].range.overflow = 0; - ssa->var_info[ssa_op->op1_def].has_range = 1; + ssa->var_info[ssa_op->op1_def].range.underflow = false; + ssa->var_info[ssa_op->op1_def].range.overflow = false; + ssa->var_info[ssa_op->op1_def].has_range = true; } } ssa_op++; @@ -6491,7 +6491,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) { p++; } - send_result = 0; + send_result = false; } } } else if (p->op == ZEND_JIT_TRACE_ENTER) { @@ -6700,7 +6700,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) { - int skip_guard = 0; + bool skip_guard = false; if (init_opline) { zend_call_info *call_info = jit_extension->func_info.callee_info; @@ -6716,7 +6716,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par break; } } - skip_guard = 1; + skip_guard = true; break; } call_info = call_info->next_callee; @@ -6724,7 +6724,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (!skip_guard && !zend_jit_may_be_polymorphic_call(init_opline) && !zend_jit_may_be_modified(p->func, op_array)) { - skip_guard = 1; + skip_guard = true; } } @@ -6988,7 +6988,7 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n const zend_op *opline; uint32_t stack_size; zend_jit_trace_stack *stack; - bool original_handler = 0; + bool original_handler = false; if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) { return dasm_labels[zend_lbtrace_escape]; @@ -7019,7 +7019,7 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n if (opline) { if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) { /* prevent endless loop */ - original_handler = 1; + original_handler = true; } zend_jit_set_ip_ex(&dasm_state, opline, original_handler); } @@ -7041,7 +7041,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace uint8_t orig_trigger; zend_jit_trace_info *t = NULL; zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; - bool do_bailout = 0; + bool do_bailout = false; zend_shared_alloc_lock(); @@ -7138,7 +7138,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace exit:; } zend_catch { - do_bailout = 1; + do_bailout = true; } zend_end_try(); zend_jit_protect(); @@ -7263,7 +7263,7 @@ static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_sto if (cache_opline[i] == opline) { if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) { cache_opline[i] = NULL; - return 1; + return true; } else { #if 0 if (ZEND_OP_TRACE_INFO(opline, offset)->counter) { @@ -7273,7 +7273,7 @@ static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_sto #endif cache_count[i]++; cache_stop[i] = stop; - return 0; + return false; } } } @@ -7283,7 +7283,7 @@ static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_sto cache_stop[i] = stop; cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS; JIT_G(bad_root_slot) = cache_slot; - return 0; + return false; } static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa) @@ -7615,10 +7615,10 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const goto abort; } - JIT_G(tracing) = 1; + JIT_G(tracing) = true; stop = zend_jit_trace_execute(execute_data, opline, trace_buffer, ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0); - JIT_G(tracing) = 0; + JIT_G(tracing) = false; if (stop & ZEND_JIT_TRACE_HALT) { ret = -1; @@ -7709,7 +7709,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num) { const void *handler; - bool do_bailout = 0; + bool do_bailout = false; zend_shared_alloc_lock(); @@ -7730,7 +7730,7 @@ static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num) } zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED; } zend_catch { - do_bailout = 1; + do_bailout = true; } zend_end_try(); zend_jit_protect(); @@ -7750,10 +7750,10 @@ static bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num) zend_jit_traces[trace_num].exit_counters + exit_num; if (*counter + 1 >= JIT_G(hot_side_exit) + JIT_G(blacklist_side_trace)) { - return 1; + return true; } (*counter)++; - return 0; + return false; } static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num) @@ -7762,10 +7762,10 @@ static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num) zend_jit_traces[trace_num].exit_counters + exit_num; if (*counter + 1 >= JIT_G(hot_side_exit)) { - return 1; + return true; } (*counter)++; - return 0; + return false; } static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism) @@ -7775,7 +7775,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace uint8_t orig_trigger; zend_jit_trace_info *t; zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; - bool do_bailout = 0; + bool do_bailout = false; zend_shared_alloc_lock(); @@ -7880,7 +7880,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace exit:; } zend_catch { - do_bailout = 1; + do_bailout = true; } zend_end_try(); zend_jit_protect(); @@ -7954,9 +7954,9 @@ static int zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t par } } - JIT_G(tracing) = 1; + JIT_G(tracing) = true; stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic); - JIT_G(tracing) = 0; + JIT_G(tracing) = false; if (stop & ZEND_JIT_TRACE_HALT) { ret = -1; @@ -8055,7 +8055,7 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe const zend_op *orig_opline = EX(opline); const zend_op *opline; zend_jit_trace_info *t = &zend_jit_traces[trace_num]; - int repeat_last_opline = 0; + bool repeat_last_opline = false; /* Deoptimization of VM stack state */ uint32_t i; @@ -8110,7 +8110,7 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) { /* Undefined array index or property */ - repeat_last_opline = 1; + repeat_last_opline = true; } else { ZVAL_COPY(EX_VAR_NUM(i), val); } @@ -8127,7 +8127,7 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe && EX(opline)->opcode != ZEND_FETCH_LIST_R) { Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var)); } - return 1; + return true; } opline = t->exit_info[exit_num].opline; @@ -8153,7 +8153,7 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe } if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) { if (EG(exception)) { - return 1; + return true; } } if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { @@ -8163,7 +8163,7 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe zend_string_release_ex(func->common.function_name, 0); zend_free_trampoline(func); EX(opline) = opline; - return 1; + return true; } } @@ -8172,10 +8172,10 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe } if (zend_atomic_bool_load_ex(&EG(vm_interrupt)) || JIT_G(tracing)) { - return 1; + return true; /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */ } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) { - return 0; + return false; } ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION); @@ -8232,7 +8232,7 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe zend_shared_alloc_unlock(); - return 0; + return false; } if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) { @@ -8242,7 +8242,7 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe fprintf(stderr, "---- EXIT %d/%d blacklisted\n", trace_num, exit_num); } - return 0; + return false; } } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) { return zend_jit_trace_hot_side(execute_data, trace_num, exit_num); @@ -8377,7 +8377,7 @@ static void zend_jit_trace_init_caches(void) static void zend_jit_trace_reset_caches(void) { - JIT_G(tracing) = 0; + JIT_G(tracing) = false; #ifdef ZTS if (!JIT_G(exit_counters)) { JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1); From fc291bc768ba3b9a6e43c098a7862ddf20ec6c1a Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 12:42:56 +0100 Subject: [PATCH 09/34] ext/opcache/jit: add some API documentation --- ext/opcache/jit/zend_jit.c | 47 +++++++++++++++++++ ext/opcache/jit/zend_jit.h | 30 ++++++++++++ ext/opcache/jit/zend_jit_internal.h | 71 +++++++++++++++++++++++++++++ ext/opcache/jit/zend_jit_trace.c | 4 ++ 4 files changed, 152 insertions(+) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 93dea44d703b..253f4d9c6b86 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -115,12 +115,48 @@ static int zend_jit_vm_kind = 0; static bool zend_write_protect = true; #endif +/** + * Start of the shared memory area to contain JIT machine code. Call + * zend_jit_unprotect() before writing to it, and call + * zend_jit_protect() when done. + * + * #dasmn_end is the end of the area and #dasm_ptr points to the end + * of the portion that is really used currently. + * + * This area is allocated during startup and is never changed. + * Allocations are linear by incrementing #dasm_ptr. + */ static void *dasm_buf = NULL; + +/** + * End of the shared memory area to contain JIT machine code. + */ static void *dasm_end = NULL; + +/** + * Pointer to pointer to end of the used portion of the shared memory + * area to contain JIT machine code. + * + * This "pointer to pointer" points to inside the shared memory area, + * because it needs to be shared across all processes/threads. All + * accesses to the pointed-to pointer need to be protected using + * zend_shared_alloc_lock(). + * + * Additionally, dasm_ptr[1] contains a pointer to the end of the + * stubs and veneers. This is used by zend_jit_restart() to free all + * JIT-generated functions, but keep those stubs and veneers. + */ static void **dasm_ptr = NULL; +/** + * The total size of the #dasm_buf, including the trailer which + * #dasm_ptr points to (after #dasm_end). + */ static size_t dasm_size = 0; +/** + * Counter for checking the opcache.jit_bisect_limit setting. + */ static zend_long jit_bisect_pos = 0; static const void *zend_jit_runtime_jit_handler = NULL; @@ -805,6 +841,10 @@ ZEND_EXT_API void zend_jit_status(zval *ret) add_assoc_zval(ret, "jit", &stats); } +/** + * Generate a name for a JIT-generated native function (for the + * disassembler and for external debuggers/profilers/tracers). + */ static zend_string *zend_jit_func_name(const zend_op_array *op_array) { smart_str buf = {0}; @@ -2677,6 +2717,10 @@ static bool zend_jit_supported_binary_op(zend_uchar op, uint32_t op1_info, uint3 } } +/** + * Caller must have called zend_shared_alloc_lock() and + * zend_jit_unprotect(). + */ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline) { int b, i, end; @@ -4717,6 +4761,9 @@ static void zend_jit_init_handlers(void) } } +/** + * Generate all JIT stub functions. + */ static bool zend_jit_make_stubs(void) { dasm_State* dasm_state = NULL; diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index d22422181af9..73ebfd9fbcdd 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -119,6 +119,10 @@ typedef struct _zend_jit_globals { zend_sym_node *symbols; /* symbols for disassembler */ + /** + * True while zend_jit_trace_execute() runs, to avoid + * recursively calling it twice. + */ bool tracing; zend_jit_trace_rec *current_trace; @@ -140,10 +144,36 @@ extern int jit_globals_id; extern zend_jit_globals jit_globals; #endif +/** + * Activate the JIT on the given #zend_op_array. + * + * @return SUCCESS or FAILURE + */ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); + +/** + * Activate the JIT on the given #zend_script. + * + * @return SUCCESS or FAILURE + */ ZEND_EXT_API int zend_jit_script(zend_script *script); + +/** + * Make the JIT machine code area (i.e. #dasm_buf) writable (but not + * executable). Call this before generating new machine code. Call + * zend_jit_protect() when done. + * + * (These write/execute permissions affect only the current process.) + */ ZEND_EXT_API void zend_jit_unprotect(void); + +/** + * Make the JIT machine code area (i.e. #dasm_buf) executable (but not + * writable). Call this after generating new machine code to undo the + * effect of zend_jit_unprotect(). + */ ZEND_EXT_API void zend_jit_protect(void); + ZEND_EXT_API void zend_jit_init(void); ZEND_EXT_API int zend_jit_config(zend_string *jit_options, int stage); ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage); diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 17b1d3727e8d..e16b32418a9d 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -219,8 +219,18 @@ static zend_always_inline bool zend_jit_same_addr(zend_jit_addr addr1, zend_jit_ return 0; } +/** + * #ZEND_FUNC_INFO for functions handled by the JIT with trigger + * #ZEND_JIT_ON_FIRST_EXEC or #ZEND_JIT_ON_PROF_REQUEST. The handler + * #override is zend_jit_runtime_jit_handler(). + */ typedef struct _zend_jit_op_array_extension { zend_func_info func_info; + + /** + * The original opcode handler, just in case we need to + * restore it. + */ const void *orig_handler; } zend_jit_op_array_extension; @@ -252,9 +262,24 @@ static zend_always_inline zend_long zend_jit_hash(const void *ptr) void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline); +/** + * #ZEND_FUNC_INFO for functions being handled by the JIT with trigger + * #ZEND_JIT_ON_HOT_COUNTERS. The handler override is + * #zend_jit_func_hot_counter_handler(). + */ typedef struct _zend_jit_op_array_hot_extension { zend_func_info func_info; + + /** + * Pointer to the counter. Points to inside global variable + * #zend_jit_hot_counters. + */ int16_t *counter; + + /** + * The original handler for each opline, just in case we need + * to restore it. + */ const void *orig_handlers[1]; } zend_jit_op_array_hot_extension; @@ -392,10 +417,29 @@ typedef enum _zend_jit_trace_stop { #define ZEND_JIT_TRACE_START_RETURN (1<<2) #define ZEND_JIT_TRACE_START_SIDE (1<<3) /* used for side traces */ +/** + * Was this opline already processed by the JIT? This is used to skip + * further JIT calls. + */ #define ZEND_JIT_TRACE_JITED (1<<4) + +/** + * Was this opline blacklisted by the JIT? Blacklisted oplines will + * never be inspected again by the JIT. + */ #define ZEND_JIT_TRACE_BLACKLISTED (1<<5) + +/** + * Is this opline supported by the JIT? This is determined by + * zend_jit_trace_supported(). + */ #define ZEND_JIT_TRACE_UNSUPPORTED (1<<6) +/** + * Not an actual trace_flag; just a descriptive macro indicating the + * absence of #ZEND_JIT_TRACE_UNSUPPORTED for the + * zend_jit_trace_supported() return value. + */ #define ZEND_JIT_TRACE_SUPPORTED 0 #define ZEND_JIT_EXIT_JITED (1<<0) @@ -410,20 +454,47 @@ typedef enum _zend_jit_trace_stop { #define ZEND_JIT_EXIT_METHOD_CALL (1<<9) /* exit because of polymorphic INIT_METHOD_CALL call */ #define ZEND_JIT_EXIT_INVALIDATE (1<<10) /* invalidate current trace */ +/** + * Per-opline information for #zend_jit_op_array_trace_extension. + */ typedef union _zend_op_trace_info { zend_op dummy; /* the size of this structure must be the same as zend_op */ struct { + /** + * The original opcode handler, just in case we need + * to restore it. + */ const void *orig_handler; const void *call_handler; + + /** + * Pointer to the counter. Points to inside global variable + * #zend_jit_hot_counters. + */ int16_t *counter; + + /** + * Bit mask of the ZEND_JIT_TRACE_* flags. + */ uint8_t trace_flags; }; } zend_op_trace_info; +/** + * #ZEND_FUNC_INFO for functions handled by the JIT with trigger + * #ZEND_JIT_ON_HOT_TRACE. The handler override is + * #zend_jit_loop_trace_counter_handler() or + * #zend_jit_func_trace_counter_handler(), configured by + * #zend_jit_setup_hot_trace_counters(). + */ typedef struct _zend_jit_op_array_trace_extension { zend_func_info func_info; const zend_op_array *op_array; size_t offset; /* offset from "zend_op" to corresponding "op_info" */ + + /** + * Information for each opline. + */ zend_op_trace_info trace_info[1]; } zend_jit_op_array_trace_extension; diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index e394c56fa421..71180c189728 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -3977,6 +3977,10 @@ static bool zend_jit_trace_next_is_send_result(const zend_op *oplin return false; } +/** + * Caller must have called zend_shared_alloc_lock() and + * zend_jit_unprotect(). + */ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num) { const void *handler = NULL; From 404667899d736f1250f7207306ba7b164c6c644d Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 17:10:52 +0100 Subject: [PATCH 10/34] ext/opcache/jit/zend_jit_internal: remove unused macro zend_jit_opline_hash() --- ext/opcache/jit/zend_jit_internal.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index e16b32418a9d..7b5027032c47 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -354,8 +354,6 @@ zend_constant* ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t fla zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key); /* Tracer */ -#define zend_jit_opline_hash(opline) \ - zend_jit_hash(opline) #define ZEND_JIT_TRACE_STOP(_) \ _(LOOP, "loop") \ From 8fea73f5bf71320fa7d4b8bcaf5f0bd0882a0f38 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 17:11:14 +0100 Subject: [PATCH 11/34] ext/opcache/jit/zend_jit_internal: zend_jit_hash() returns unsigned A negative hash value is not useful for anything, but can cause harm if not used correctly. --- ext/opcache/jit/zend_jit_internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 7b5027032c47..31a5b014e6d8 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -243,7 +243,7 @@ extern int zend_jit_profile_counter_rid; /* Hot Counters */ -static zend_always_inline zend_long zend_jit_hash(const void *ptr) +static zend_always_inline zend_ulong zend_jit_hash(const void *ptr) { uintptr_t x; From 4ac322eb84842aef2d01c3b595f146e5cafbdf5b Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 17:12:13 +0100 Subject: [PATCH 12/34] ext/opcache/jit/zend_jit_internal: convert zend_jit_op_array_hash() to function --- ext/opcache/jit/zend_jit_internal.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 31a5b014e6d8..fdb59ea28190 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -283,8 +283,10 @@ typedef struct _zend_jit_op_array_hot_extension { const void *orig_handlers[1]; } zend_jit_op_array_hot_extension; -#define zend_jit_op_array_hash(op_array) \ - zend_jit_hash((op_array)->opcodes) +static zend_always_inline zend_ulong zend_jit_op_array_hash(const zend_op_array *op_array) +{ + return zend_jit_hash(op_array->opcodes); +} extern const zend_op *zend_jit_halt_op; From 7f2ca1e6980e537880c292c8f59b501bf4bb468e Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 17:08:15 +0100 Subject: [PATCH 13/34] ext/opcache/jit/zend_jit: use modulo instead of bitwise and This is clearer and more robust (just in case ZEND_HOT_COUNTERS_COUNT some days gets changed to not be a power of two), and results in the exact same machine code, thanks to compiler optimizations. --- ext/opcache/jit/zend_jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 253f4d9c6b86..aa83c4193c43 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -4459,7 +4459,7 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array) } memset(&jit_extension->func_info, 0, sizeof(zend_func_info)); jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS; - jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)]; + jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) % ZEND_HOT_COUNTERS_COUNT]; for (i = 0; i < op_array->last; i++) { jit_extension->orig_handlers[i] = op_array->opcodes[i].handler; } From 74ca778fb43b91b37cab3908dfc17c9d339daffd Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 16:51:36 +0100 Subject: [PATCH 14/34] ext/opcache/jit/zend_jit_trace: cache ZEND_OP_TRACE_INFO() in variable Eliminates a few redundant memory reads. --- ext/opcache/jit/zend_jit_trace.c | 65 ++++++++++++++++----------- ext/opcache/jit/zend_jit_vm_helpers.c | 11 ++--- 2 files changed, 44 insertions(+), 32 deletions(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 71180c189728..53a6e6df01c7 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -3478,15 +3478,16 @@ static void zend_jit_trace_clenup_stack(zend_jit_trace_stack *stack, const zend_ static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset) { zend_op *next_opline = (zend_op*)(opline + 1); + zend_op_trace_info *const next_trace_info = ZEND_OP_TRACE_INFO(next_opline, offset); - if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) { + if (JIT_G(hot_return) && !next_trace_info->trace_flags) { ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL); - if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) { - ZEND_OP_TRACE_INFO(next_opline, offset)->counter = + if (!next_trace_info->counter) { + next_trace_info->counter = &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; } - ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN; + next_trace_info->trace_flags = ZEND_JIT_TRACE_START_RETURN; next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler; } } @@ -7049,8 +7050,10 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace zend_shared_alloc_lock(); + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, offset); + /* Checks under lock */ - if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) { + if ((trace_info->trace_flags & ZEND_JIT_TRACE_JITED)) { ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE; } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) { ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; @@ -7122,7 +7125,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace ((zend_op*)opline)->handler = handler; ZEND_JIT_TRACE_NUM++; - ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED; + trace_info->trace_flags |= ZEND_JIT_TRACE_JITED; ret = ZEND_JIT_TRACE_STOP_COMPILED; } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || @@ -7239,14 +7242,16 @@ static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset) { zend_shared_alloc_lock(); - if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) { + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, offset); + + if (!(trace_info->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) { SHM_UNPROTECT(); zend_jit_unprotect(); ((zend_op*)opline)->handler = - ZEND_OP_TRACE_INFO(opline, offset)->orig_handler; + trace_info->orig_handler; - ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED; + trace_info->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED; zend_jit_protect(); SHM_PROTECT(); @@ -7270,8 +7275,9 @@ static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_sto return true; } else { #if 0 - if (ZEND_OP_TRACE_INFO(opline, offset)->counter) { - *ZEND_OP_TRACE_INFO(opline, offset)->counter = + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, offset); + if (trace_info->counter) { + *trace_info->counter = random() % ZEND_JIT_TRACE_COUNTER_MAX; } #endif @@ -7591,20 +7597,22 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const EX(opline) = opline; + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, offset); + /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */ - if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) { + if (trace_info->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) { return 0; } if (JIT_G(tracing)) { - ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter); + ++(*trace_info->counter); return 0; } if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n", trace_num, - zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags), + zend_jit_trace_star_desc(trace_info->trace_flags), EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "", EX(func)->op_array.scope ? "::" : "", EX(func)->op_array.function_name ? @@ -7621,7 +7629,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const JIT_G(tracing) = true; stop = zend_jit_trace_execute(execute_data, opline, trace_buffer, - ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0); + trace_info->trace_flags & ZEND_JIT_TRACE_START_MASK, 0); JIT_G(tracing) = false; if (stop & ZEND_JIT_TRACE_HALT) { @@ -8210,9 +8218,10 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe zend_shared_alloc_lock(); jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array); + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset); /* Checks under lock, just in case something has changed while we were waiting for the lock */ - if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) { + if (!(trace_info->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) { /* skip: not JIT-ed nor blacklisted */ } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) { /* skip: too many root traces */ @@ -8220,14 +8229,14 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe SHM_UNPROTECT(); zend_jit_unprotect(); - if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) { + if (trace_info->trace_flags & ZEND_JIT_TRACE_START_LOOP) { ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler; - } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) { + } else if (trace_info->trace_flags & ZEND_JIT_TRACE_START_ENTER) { ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler; - } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) { + } else if (trace_info->trace_flags & ZEND_JIT_TRACE_START_RETURN) { ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler; } - ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &= + trace_info->trace_flags &= ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN; zend_jit_protect(); @@ -8327,14 +8336,15 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) { /* loop header */ opline = op_array->opcodes + cfg.blocks[i].start; - if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, jit_extension->offset); + if (!(trace_info->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { opline->handler = (const void*)zend_jit_loop_trace_counter_handler; - if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) { - ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = + if (!trace_info->counter) { + trace_info->counter = &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; } - ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |= + trace_info->trace_flags |= ZEND_JIT_TRACE_START_LOOP; } } @@ -8351,13 +8361,14 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) } } - if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) { + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, jit_extension->offset); + if (!trace_info->trace_flags) { /* function entry */ opline->handler = (const void*)zend_jit_func_trace_counter_handler; - ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = + trace_info->counter = &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; - ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |= + trace_info->trace_flags |= ZEND_JIT_TRACE_START_ENTER; } } diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 8e02fbbbfeac..8c17b0096b15 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -308,15 +308,16 @@ static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_c { zend_jit_op_array_trace_extension *jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array); - size_t offset = jit_extension->offset; #ifndef HAVE_GCC_GLOBAL_REGS const zend_op *opline = EX(opline); #endif - *(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost; + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, jit_extension->offset); - if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) { - *(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_COUNTER_INIT; + *(trace_info->counter) -= cost; + + if (UNEXPECTED(*(trace_info->counter) <= 0)) { + *(trace_info->counter) = ZEND_JIT_COUNTER_INIT; if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) { #ifdef HAVE_GCC_GLOBAL_REGS opline = NULL; @@ -333,7 +334,7 @@ static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_c return 1; #endif } else { - zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->orig_handler; + zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)trace_info->orig_handler; ZEND_OPCODE_TAIL_CALL(handler); } } From 5ff06c750a2e6d94fa9acbd2e96365ba9deb1b37 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 17:15:56 +0100 Subject: [PATCH 15/34] ext/opcache/jit/zend_jit_trace: move redundant code to zend_jit_hot_counter_allocate() --- ext/opcache/jit/zend_jit_trace.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 53a6e6df01c7..0a9fb1b35f4c 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -8297,6 +8297,17 @@ static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array) return SUCCESS; } +/** + * Allocate a new "hot counter" from #zend_jit_hot_counters. + */ +static int16_t *zend_jit_hot_counter_allocate(void) +{ + int16_t *counter = + &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; + ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; + return counter; +} + static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) { zend_op *opline; @@ -8341,8 +8352,7 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) opline->handler = (const void*)zend_jit_loop_trace_counter_handler; if (!trace_info->counter) { trace_info->counter = - &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; - ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; + zend_jit_hot_counter_allocate(); } trace_info->trace_flags |= ZEND_JIT_TRACE_START_LOOP; @@ -8366,8 +8376,7 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) /* function entry */ opline->handler = (const void*)zend_jit_func_trace_counter_handler; trace_info->counter = - &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; - ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; + zend_jit_hot_counter_allocate(); trace_info->trace_flags |= ZEND_JIT_TRACE_START_ENTER; } From ac8047da75b187c7852bec6aa662a41842e610b9 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Thu, 22 Dec 2022 14:44:56 +0100 Subject: [PATCH 16/34] ext/opcache/jit: move redundant code to zend_find_function_entry() --- ext/opcache/jit/zend_jit.c | 45 ++++------------------------- ext/opcache/jit/zend_jit_internal.h | 19 ++++++++++++ ext/opcache/jit/zend_jit_trace.c | 8 +---- 3 files changed, 26 insertions(+), 46 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index aa83c4193c43..1a7a13cab922 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -4282,7 +4282,6 @@ static int ZEND_FASTCALL zend_runtime_jit(void) { zend_execute_data *execute_data = EG(current_execute_data); zend_op_array *op_array = &EX(func)->op_array; - zend_op *opline = op_array->opcodes; zend_jit_op_array_extension *jit_extension; bool do_bailout = false; @@ -4295,11 +4294,7 @@ static int ZEND_FASTCALL zend_runtime_jit(void) zend_try { /* restore original opcode handlers */ - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } + zend_op *const opline = zend_find_function_entry(op_array); jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); opline->handler = jit_extension->orig_handler; @@ -4324,7 +4319,6 @@ static int ZEND_FASTCALL zend_runtime_jit(void) } static void zend_jit_check_funcs(HashTable *function_table, bool is_method) { - zend_op *opline; zend_function *func; zend_op_array *op_array; uintptr_t counter; @@ -4335,12 +4329,7 @@ static void zend_jit_check_funcs(HashTable *function_table, bool is_method) { break; } op_array = &func->op_array; - opline = op_array->opcodes; - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } + zend_op *const opline = zend_find_function_entry(op_array); if (opline->handler == zend_jit_profile_jit_handler) { if (!RUN_TIME_CACHE(op_array)) { continue; @@ -4396,14 +4385,7 @@ void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg) { if (JIT_G(hot_func)) { - zend_op *opline = op_array->opcodes; - - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } - + zend_op *const opline = zend_find_function_entry(op_array); opline->handler = (const void*)zend_jit_func_hot_counter_handler; } @@ -4499,7 +4481,6 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) { zend_jit_op_array_extension *jit_extension; - zend_op *opline = op_array->opcodes; if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) { ZEND_SET_FUNC_INFO(op_array, NULL); @@ -4509,11 +4490,7 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) /* Set run-time JIT handler */ ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL); - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } + zend_op *const opline = zend_find_function_entry(op_array); jit_extension = zend_jit_new_op_array_extension(op_array); if (!jit_extension) { return FAILURE; @@ -4525,7 +4502,6 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) return SUCCESS; } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) { zend_jit_op_array_extension *jit_extension; - zend_op *opline = op_array->opcodes; if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) { ZEND_SET_FUNC_INFO(op_array, NULL); @@ -4535,11 +4511,7 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) ZEND_ASSERT(zend_jit_profile_jit_handler != NULL); if (op_array->function_name) { - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } + zend_op *const opline = zend_find_function_entry(op_array); jit_extension = zend_jit_new_op_array_extension(op_array); if (!jit_extension) { return FAILURE; @@ -5182,15 +5154,10 @@ static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array) #if 0 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ??? } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) { - zend_op *opline = op_array->opcodes; zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)func_info; - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } + zend_op *const opline = zend_find_function_entry(op_array); if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) { opline->handler = (const void*)zend_jit_runtime_jit_handler; } else { diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index fdb59ea28190..7c68ad13afe0 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -838,6 +838,25 @@ static zend_always_inline bool zend_jit_may_be_polymorphic_call(const zend_op *o } } +/** + * Look up the function entry, i.e. the first #zend_op that is not + * #ZEND_RECV or #ZEND_RECV_INIT. + */ +static inline zend_op *zend_find_function_entry(zend_op_array *op_array) +{ + zend_op *opline = op_array->opcodes; + + if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + opline++; + + ZEND_ASSERT(opline < op_array->opcodes + op_array->last); + } + } + + return opline; +} + /* Instruction cache flush */ #ifndef JIT_CACHE_FLUSH # if ZEND_JIT_TARGET_ARM64 diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 0a9fb1b35f4c..5904c33ae0e3 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -8364,13 +8364,7 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) if (JIT_G(hot_func)) { ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL); - opline = op_array->opcodes; - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } - + zend_op *const opline = zend_find_function_entry(op_array); zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, jit_extension->offset); if (!trace_info->trace_flags) { /* function entry */ From e8a9f7c08d37ace775b568438c8bc003bc3ab076 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Thu, 22 Dec 2022 14:56:20 +0100 Subject: [PATCH 17/34] ext/opcache/jit: eliminate duplicate ZEND_FUNC_INFO() call --- ext/opcache/jit/zend_jit.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1a7a13cab922..dbea12c751ac 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -4282,12 +4282,14 @@ static int ZEND_FASTCALL zend_runtime_jit(void) { zend_execute_data *execute_data = EG(current_execute_data); zend_op_array *op_array = &EX(func)->op_array; - zend_jit_op_array_extension *jit_extension; bool do_bailout = false; zend_shared_alloc_lock(); - if (ZEND_FUNC_INFO(op_array)) { + zend_jit_op_array_extension *const jit_extension = + (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); + + if (jit_extension) { SHM_UNPROTECT(); zend_jit_unprotect(); @@ -4295,7 +4297,6 @@ static int ZEND_FASTCALL zend_runtime_jit(void) zend_try { /* restore original opcode handlers */ zend_op *const opline = zend_find_function_entry(op_array); - jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); opline->handler = jit_extension->orig_handler; /* perform real JIT for this function */ From 865ff4965dca5ef48b944ff8983e1b853644b3bb Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Thu, 22 Dec 2022 15:12:27 +0100 Subject: [PATCH 18/34] ext/opcache/jit/zend_jit_internal: convert ZEND_OP_TRACE_INFO() to function --- ext/opcache/jit/zend_jit_internal.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 7c68ad13afe0..e0d424f8793d 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -498,8 +498,10 @@ typedef struct _zend_jit_op_array_trace_extension { zend_op_trace_info trace_info[1]; } zend_jit_op_array_trace_extension; -#define ZEND_OP_TRACE_INFO(opline, offset) \ - ((zend_op_trace_info*)(((char*)opline) + offset)) +static zend_always_inline zend_op_trace_info *ZEND_OP_TRACE_INFO(const zend_op *opline, size_t offset) +{ + return (zend_op_trace_info*)((char*)opline + offset); +} /* Recorder */ typedef enum _zend_jit_trace_op { From 2211034d708911d351a72b0d48abd096abafa6a9 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Thu, 22 Dec 2022 15:44:25 +0100 Subject: [PATCH 19/34] ext/opcache/jit/zend_jit_trace: remove pointless variable assignments Those variables are never used again. --- ext/opcache/jit/zend_jit_trace.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 5904c33ae0e3..a2fe1854a034 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -7639,9 +7639,6 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) { orig_opline = trace_buffer[1].opline; - op_array = (zend_op_array*)trace_buffer[0].op_array; - jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - offset = jit_extension->offset; if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { const zend_op_array *op_array = trace_buffer[0].op_array; const zend_op *opline = trace_buffer[1].opline; From e96f55fa7b27d1eec22c1cf069c0a87a83e8fae2 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Thu, 22 Dec 2022 15:38:35 +0100 Subject: [PATCH 20/34] ext/opcache/jit/zend_jit_internal: add ZEND_OP_TRACE_INFO_OFFSET() Shortcut which allows eliminating several zend_jit_op_array_trace_extension* variables. --- ext/opcache/jit/zend_jit_internal.h | 8 ++++++++ ext/opcache/jit/zend_jit_trace.c | 9 ++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index e0d424f8793d..60286b926f7a 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -498,6 +498,14 @@ typedef struct _zend_jit_op_array_trace_extension { zend_op_trace_info trace_info[1]; } zend_jit_op_array_trace_extension; +static zend_always_inline size_t ZEND_OP_TRACE_INFO_OFFSET(const zend_op_array *op_array) +{ + const zend_jit_op_array_trace_extension *jit_extension = + (const zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + ZEND_ASSERT(jit_extension != NULL); + return jit_extension->offset; +} + static zend_always_inline zend_op_trace_info *ZEND_OP_TRACE_INFO(const zend_op *opline, size_t offset) { return (zend_op_trace_info*)((char*)opline + offset); diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index a2fe1854a034..a4f8b93db96c 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -7579,8 +7579,6 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_jit_trace_stop stop; int ret = 0; zend_op_array *op_array; - zend_jit_op_array_trace_extension *jit_extension; - size_t offset; uint32_t trace_num; zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH]; @@ -7592,8 +7590,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const trace_num = ZEND_JIT_TRACE_NUM; orig_opline = opline; op_array = &EX(func)->op_array; - jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - offset = jit_extension->offset; + const size_t offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); EX(opline) = opline; @@ -7642,9 +7639,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { const zend_op_array *op_array = trace_buffer[0].op_array; const zend_op *opline = trace_buffer[1].opline; - zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - size_t offset = jit_extension->offset; + const size_t offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n", trace_num, From 0a59bdc2f1624a64f91f0a3e0f6d8e1440f7738e Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Thu, 22 Dec 2022 15:40:15 +0100 Subject: [PATCH 21/34] ext/opcache/jit/zend_jit_trace: copy offset to local variable Don't remember the zend_jit_op_array_trace_extension pointer unless we need more than the offset. This eliminates a few memory reads. --- ext/opcache/jit/zend_jit_trace.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index a4f8b93db96c..7a7d45507852 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -6906,23 +6906,20 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par const zend_op_array *rec_op_array; rec_op_array = op_array = trace_buffer->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + size_t offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; for (;;p++) { if (p->op == ZEND_JIT_TRACE_VM) { opline = p->opline; } else if (p->op == ZEND_JIT_TRACE_ENTER) { if (p->op_array == rec_op_array) { - zend_jit_trace_setup_ret_counter(opline, jit_extension->offset); + zend_jit_trace_setup_ret_counter(opline, offset); } op_array = p->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); } else if (p->op == ZEND_JIT_TRACE_BACK) { op_array = p->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); } else if (p->op == ZEND_JIT_TRACE_END) { break; } @@ -6944,9 +6941,10 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par do { if (frame->call_opline) { op_array = &frame->func->op_array; + const size_t offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset); + zend_jit_trace_setup_ret_counter(frame->call_opline, offset); } frame = frame->prev; } while (frame); From 39b092b41c57695eea6141c380b513fb46547ef4 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Thu, 22 Dec 2022 15:15:33 +0100 Subject: [PATCH 22/34] ext/opcache/jit/zend_jit_internal: add ZEND_OP_TRACE_INFO2() Allows eliminating several local variables. --- ext/opcache/jit/zend_jit_arm64.dasc | 5 +---- ext/opcache/jit/zend_jit_internal.h | 16 ++++++++++++++++ ext/opcache/jit/zend_jit_trace.c | 10 +++------- ext/opcache/jit/zend_jit_vm_helpers.c | 4 +--- ext/opcache/jit/zend_jit_x86.dasc | 5 +---- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 712e3ba11360..7c2839587b93 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3243,11 +3243,8 @@ static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32 static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) { - zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - size_t offset = jit_extension->offset; const void *handler = - (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; + (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO2(op_array, opline)->call_handler; if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 60286b926f7a..081ea615326f 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -511,6 +511,22 @@ static zend_always_inline zend_op_trace_info *ZEND_OP_TRACE_INFO(const zend_op * return (zend_op_trace_info*)((char*)opline + offset); } +/** + * Access the #zend_op_trace_info of an #zend_op by looking up the + * offset from the specified #zend_op_array. + */ +static zend_always_inline zend_op_trace_info *ZEND_OP_TRACE_INFO2(const zend_op_array *op_array, const zend_op *opline) +{ + /* the opline must be from the specified function */ + ZEND_ASSERT(opline >= op_array->opcodes); + ZEND_ASSERT(opline < op_array->opcodes + op_array->last); + + const zend_jit_op_array_trace_extension *jit_extension = + (const zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + ZEND_ASSERT(jit_extension != NULL); + return ZEND_OP_TRACE_INFO(opline, jit_extension->offset); +} + /* Recorder */ typedef enum _zend_jit_trace_op { ZEND_JIT_TRACE_VM, diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 7a7d45507852..c744ba08da33 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -7969,13 +7969,11 @@ static int zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t par if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { const zend_op_array *op_array = trace_buffer[0].op_array; const zend_op *opline = trace_buffer[1].opline; - zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - size_t offset = jit_extension->offset; + const zend_op_trace_info *trace_info = ZEND_OP_TRACE_INFO2(op_array, opline); fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n", trace_num, - zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags), + zend_jit_trace_star_desc(trace_info->trace_flags), op_array->scope ? ZSTR_VAL(op_array->scope->name) : "", op_array->scope ? "::" : "", op_array->function_name ? @@ -8197,7 +8195,6 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe } if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_INVALIDATE) { - zend_jit_op_array_trace_extension *jit_extension; uint32_t num = trace_num; while (t->root != num) { @@ -8207,8 +8204,7 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe zend_shared_alloc_lock(); - jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array); - zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset); + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO2(t->op_array, t->opline); /* Checks under lock, just in case something has changed while we were waiting for the lock */ if (!(trace_info->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) { diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 8c17b0096b15..959ccb05609c 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -306,13 +306,11 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key) static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC) { - zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array); #ifndef HAVE_GCC_GLOBAL_REGS const zend_op *opline = EX(opline); #endif - zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, jit_extension->offset); + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO2(&EX(func)->op_array, opline); *(trace_info->counter) -= cost; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index de2fa51526d0..489da1c99fdf 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3594,11 +3594,8 @@ static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32 static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) { - zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - size_t offset = jit_extension->offset; const void *handler = - (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; + (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO2(op_array, opline)->call_handler; if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; From ff74acce54a8fd93d0fd4f147e51cb73896e5dd0 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Thu, 22 Dec 2022 16:12:34 +0100 Subject: [PATCH 23/34] ext/opcache/jit/zend_jit_trace: do not reset stack_map of unallocated traces Clearing stack_map is pointless if ZEND_JIT_TRACE_NUM was not incremented. --- ext/opcache/jit/zend_jit_trace.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index c744ba08da33..ad9d40794478 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -7096,7 +7096,6 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace if (!shared_exit_info) { if (t->stack_map) { efree(t->stack_map); - t->stack_map = NULL; } ret = ZEND_JIT_TRACE_STOP_NO_SHM; goto exit; @@ -7130,13 +7129,11 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { if (t->stack_map) { efree(t->stack_map); - t->stack_map = NULL; } ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; } else { if (t->stack_map) { efree(t->stack_map); - t->stack_map = NULL; } ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR; } @@ -7828,7 +7825,6 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace if (!shared_exit_info) { if (t->stack_map) { efree(t->stack_map); - t->stack_map = NULL; } ret = ZEND_JIT_TRACE_STOP_NO_SHM; goto exit; @@ -7869,13 +7865,11 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { if (t->stack_map) { efree(t->stack_map); - t->stack_map = NULL; } ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; } else { if (t->stack_map) { efree(t->stack_map); - t->stack_map = NULL; } ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR; } From 6eb0745a8cb87a4a149ef283ebad8372e66a2719 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 17:52:31 +0100 Subject: [PATCH 24/34] ext/opcache/jit/zend_jit_trace: fix memory leak in _compile_root_trace() A copy of this piece of code exists in zend_jit_compile_side_trace(), but there, the leak bug does not exist. This bug exists since both copies of this piece of code were added in commit 4bf2d09edeb14 --- ext/opcache/jit/zend_jit_trace.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index ad9d40794478..891047e7d7bd 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -7108,6 +7108,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace if (t->stack_map_size) { zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack)); if (!shared_stack_map) { + efree(t->stack_map); ret = ZEND_JIT_TRACE_STOP_NO_SHM; goto exit; } From b83ab47ef2cb85d44d717d18a0fdb10235050002 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Wed, 21 Dec 2022 17:44:14 +0100 Subject: [PATCH 25/34] ext/opcache/zend_shared_alloc: add zend_shared_alloc_copy() Allows simplifying some code in zend_jit_trace.c. --- ext/opcache/jit/zend_jit_trace.c | 38 ++++++++++++-------------------- ext/opcache/zend_shared_alloc.c | 8 +++++++ ext/opcache/zend_shared_alloc.h | 8 +++++++ 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 891047e7d7bd..1ad3aeac859e 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -7085,36 +7085,31 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace JIT_G(trigger) = orig_trigger; if (handler) { - zend_jit_trace_exit_info *shared_exit_info = NULL; - t->exit_info = NULL; if (t->exit_count) { /* reallocate exit_info into shared memory */ - shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc( + t->exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc_copy(exit_info, sizeof(zend_jit_trace_exit_info) * t->exit_count); - if (!shared_exit_info) { + if (!t->exit_info) { if (t->stack_map) { efree(t->stack_map); } ret = ZEND_JIT_TRACE_STOP_NO_SHM; goto exit; } - memcpy(shared_exit_info, exit_info, - sizeof(zend_jit_trace_exit_info) * t->exit_count); - t->exit_info = shared_exit_info; } if (t->stack_map_size) { - zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack)); + zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc_copy( + t->stack_map, + t->stack_map_size * sizeof(zend_jit_trace_stack)); + efree(t->stack_map); + t->stack_map = shared_stack_map; if (!shared_stack_map) { - efree(t->stack_map); ret = ZEND_JIT_TRACE_STOP_NO_SHM; goto exit; } - memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); - efree(t->stack_map); - t->stack_map = shared_stack_map; } t->exit_counters = ZEND_JIT_EXIT_COUNTERS; @@ -7815,36 +7810,31 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace JIT_G(trigger) = orig_trigger; if (handler) { - zend_jit_trace_exit_info *shared_exit_info = NULL; - t->exit_info = NULL; if (t->exit_count) { /* reallocate exit_info into shared memory */ - shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc( + t->exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc_copy(exit_info, sizeof(zend_jit_trace_exit_info) * t->exit_count); - if (!shared_exit_info) { + if (!t->exit_info) { if (t->stack_map) { efree(t->stack_map); } ret = ZEND_JIT_TRACE_STOP_NO_SHM; goto exit; } - memcpy(shared_exit_info, exit_info, - sizeof(zend_jit_trace_exit_info) * t->exit_count); - t->exit_info = shared_exit_info; } if (t->stack_map_size) { - zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack)); + zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc_copy( + t->stack_map, + t->stack_map_size * sizeof(zend_jit_trace_stack)); + efree(t->stack_map); + t->stack_map = shared_stack_map; if (!shared_stack_map) { - efree(t->stack_map); ret = ZEND_JIT_TRACE_STOP_NO_SHM; goto exit; } - memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); - efree(t->stack_map); - t->stack_map = shared_stack_map; } zend_jit_link_side_trace( diff --git a/ext/opcache/zend_shared_alloc.c b/ext/opcache/zend_shared_alloc.c index 703796d51d85..2c5fe381b7be 100644 --- a/ext/opcache/zend_shared_alloc.c +++ b/ext/opcache/zend_shared_alloc.c @@ -363,6 +363,14 @@ void *zend_shared_alloc(size_t size) return NULL; } +void *zend_shared_alloc_copy(const void *src, size_t size) +{ + void *dest = zend_shared_alloc(size); + if (dest != NULL) + memcpy(dest, src, size); + return dest; +} + static zend_always_inline zend_ulong zend_rotr3(zend_ulong key) { return (key >> 3) | (key << ((sizeof(key) * 8) - 3)); diff --git a/ext/opcache/zend_shared_alloc.h b/ext/opcache/zend_shared_alloc.h index 1caf678e36fe..09a0928f20d5 100644 --- a/ext/opcache/zend_shared_alloc.h +++ b/ext/opcache/zend_shared_alloc.h @@ -133,6 +133,14 @@ void zend_shared_alloc_shutdown(void); /* allocate shared memory block */ void *zend_shared_alloc(size_t size); +/** + * Allocate a shared memory block and copy raw data from the given + * source pointer into it. + * + * @return the new allocation or NULL if out of memory + */ +void *zend_shared_alloc_copy(const void *src, size_t size); + /** * Wrapper for zend_shared_alloc() which aligns at 64-byte boundary if * AVX or SSE2 are used. From 4dd445ce419df3da23e3f773aca575f0dca1619b Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Thu, 22 Dec 2022 16:23:33 +0100 Subject: [PATCH 26/34] ext/opcache/jit/zend_jit_trace: move code to _zend_jit_compile_root_trace() This eliminates `goto` and simplifies error handling. --- ext/opcache/jit/zend_jit_trace.c | 170 +++++++++++++++---------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 1ad3aeac859e..3d9df3029072 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -7037,13 +7037,93 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n return handler; } +/** + * Helper function for zend_jit_compile_root_trace(). + */ +static zend_jit_trace_stop _zend_jit_compile_root_trace(zend_jit_trace_rec *const trace_buffer, const zend_op *opline, zend_op_trace_info *const trace_info) +{ + zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; + zend_jit_trace_info *const t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; + + t->id = ZEND_JIT_TRACE_NUM; + t->root = ZEND_JIT_TRACE_NUM; + t->parent = 0; + t->link = 0; + t->exit_count = 0; + t->child_count = 0; + t->stack_map_size = 0; + t->flags = 0; + t->polymorphism = 0; + t->jmp_table_size = 0; + t->op_array = trace_buffer[0].op_array; + t->opline = trace_buffer[1].opline; + t->exit_info = exit_info; + t->stack_map = NULL; + + const uint8_t orig_trigger = JIT_G(trigger); + JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE; + + const void *handler = zend_jit_trace(trace_buffer, 0, 0); + + JIT_G(trigger) = orig_trigger; + + if (handler) { + t->exit_info = NULL; + if (t->exit_count) { + /* reallocate exit_info into shared memory */ + t->exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc_copy(exit_info, + sizeof(zend_jit_trace_exit_info) * t->exit_count); + + if (!t->exit_info) { + if (t->stack_map) { + efree(t->stack_map); + } + return ZEND_JIT_TRACE_STOP_NO_SHM; + } + } + + if (t->stack_map_size) { + zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc_copy( + t->stack_map, + t->stack_map_size * sizeof(zend_jit_trace_stack)); + efree(t->stack_map); + t->stack_map = shared_stack_map; + if (!shared_stack_map) { + return ZEND_JIT_TRACE_STOP_NO_SHM; + } + } + + t->exit_counters = ZEND_JIT_EXIT_COUNTERS; + ZEND_JIT_EXIT_COUNTERS += t->exit_count; + + ((zend_op*)opline)->handler = handler; + + ZEND_JIT_TRACE_NUM++; + trace_info->trace_flags |= ZEND_JIT_TRACE_JITED; + + if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 + && t->exit_count > 0) { + zend_jit_dump_exit_info(t); + } + + return ZEND_JIT_TRACE_STOP_COMPILED; + } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || + ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { + if (t->stack_map) { + efree(t->stack_map); + } + return ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; + } else { + if (t->stack_map) { + efree(t->stack_map); + } + return ZEND_JIT_TRACE_STOP_COMPILER_ERROR; + } +} + static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset) { zend_jit_trace_stop ret; - const void *handler; - uint8_t orig_trigger; - zend_jit_trace_info *t = NULL; - zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; bool do_bailout = false; zend_shared_alloc_lock(); @@ -7060,81 +7140,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace SHM_UNPROTECT(); zend_jit_unprotect(); - t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; - - t->id = ZEND_JIT_TRACE_NUM; - t->root = ZEND_JIT_TRACE_NUM; - t->parent = 0; - t->link = 0; - t->exit_count = 0; - t->child_count = 0; - t->stack_map_size = 0; - t->flags = 0; - t->polymorphism = 0; - t->jmp_table_size = 0; - t->op_array = trace_buffer[0].op_array; - t->opline = trace_buffer[1].opline; - t->exit_info = exit_info; - t->stack_map = NULL; - - orig_trigger = JIT_G(trigger); - JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE; - - handler = zend_jit_trace(trace_buffer, 0, 0); - - JIT_G(trigger) = orig_trigger; - - if (handler) { - t->exit_info = NULL; - if (t->exit_count) { - /* reallocate exit_info into shared memory */ - t->exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc_copy(exit_info, - sizeof(zend_jit_trace_exit_info) * t->exit_count); - - if (!t->exit_info) { - if (t->stack_map) { - efree(t->stack_map); - } - ret = ZEND_JIT_TRACE_STOP_NO_SHM; - goto exit; - } - } - - if (t->stack_map_size) { - zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc_copy( - t->stack_map, - t->stack_map_size * sizeof(zend_jit_trace_stack)); - efree(t->stack_map); - t->stack_map = shared_stack_map; - if (!shared_stack_map) { - ret = ZEND_JIT_TRACE_STOP_NO_SHM; - goto exit; - } - } - - t->exit_counters = ZEND_JIT_EXIT_COUNTERS; - ZEND_JIT_EXIT_COUNTERS += t->exit_count; - - ((zend_op*)opline)->handler = handler; - - ZEND_JIT_TRACE_NUM++; - trace_info->trace_flags |= ZEND_JIT_TRACE_JITED; - - ret = ZEND_JIT_TRACE_STOP_COMPILED; - } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || - ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { - if (t->stack_map) { - efree(t->stack_map); - } - ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; - } else { - if (t->stack_map) { - efree(t->stack_map); - } - ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR; - } - -exit:; + ret = _zend_jit_compile_root_trace(trace_buffer, opline, trace_info); } zend_catch { do_bailout = true; } zend_end_try(); @@ -7149,12 +7155,6 @@ exit:; zend_bailout(); } - if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 - && ret == ZEND_JIT_TRACE_STOP_COMPILED - && t->exit_count > 0) { - zend_jit_dump_exit_info(t); - } - return ret; } From 9b6b07ecc4359cfcd0f28370c8edc037e5830e87 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Thu, 22 Dec 2022 16:35:55 +0100 Subject: [PATCH 27/34] ext/opcache/jit/zend_jit_trace: move code to _zend_jit_compile_side_trace() --- ext/opcache/jit/zend_jit_trace.c | 180 +++++++++++++++---------------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 3d9df3029072..62788a943683 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -7763,13 +7763,98 @@ static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num) return false; } +/** + * Helper function for zend_jit_compile_side_trace(). + */ +static zend_jit_trace_stop _zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism) +{ + zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; + zend_jit_trace_info *const t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; + + t->id = ZEND_JIT_TRACE_NUM; + t->root = zend_jit_traces[parent_num].root; + t->parent = parent_num; + t->link = 0; + t->exit_count = 0; + t->child_count = 0; + t->stack_map_size = 0; + t->flags = 0; + t->polymorphism = polymorphism; + t->jmp_table_size = 0; + t->opline = NULL; + t->exit_info = exit_info; + t->stack_map = NULL; + + const uint8_t orig_trigger = JIT_G(trigger); + JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE; + + const void *const handler = zend_jit_trace(trace_buffer, parent_num, exit_num); + + JIT_G(trigger) = orig_trigger; + + if (handler) { + t->exit_info = NULL; + if (t->exit_count) { + /* reallocate exit_info into shared memory */ + t->exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc_copy(exit_info, + sizeof(zend_jit_trace_exit_info) * t->exit_count); + + if (!t->exit_info) { + if (t->stack_map) { + efree(t->stack_map); + } + return ZEND_JIT_TRACE_STOP_NO_SHM; + } + } + + if (t->stack_map_size) { + zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc_copy( + t->stack_map, + t->stack_map_size * sizeof(zend_jit_trace_stack)); + efree(t->stack_map); + t->stack_map = shared_stack_map; + if (!shared_stack_map) { + return ZEND_JIT_TRACE_STOP_NO_SHM; + } + } + + zend_jit_link_side_trace( + zend_jit_traces[parent_num].code_start, + zend_jit_traces[parent_num].code_size, + zend_jit_traces[parent_num].jmp_table_size, + exit_num, + handler); + + t->exit_counters = ZEND_JIT_EXIT_COUNTERS; + ZEND_JIT_EXIT_COUNTERS += t->exit_count; + + zend_jit_traces[zend_jit_traces[parent_num].root].child_count++; + ZEND_JIT_TRACE_NUM++; + zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED; + + if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 + && t->exit_count > 0) { + zend_jit_dump_exit_info(t); + } + + return ZEND_JIT_TRACE_STOP_COMPILED; + } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || + ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { + if (t->stack_map) { + efree(t->stack_map); + } + return ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; + } else { + if (t->stack_map) { + efree(t->stack_map); + } + return ZEND_JIT_TRACE_STOP_COMPILER_ERROR; + } +} + static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism) { zend_jit_trace_stop ret; - const void *handler; - uint8_t orig_trigger; - zend_jit_trace_info *t; - zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; bool do_bailout = false; zend_shared_alloc_lock(); @@ -7786,86 +7871,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace zend_jit_unprotect(); zend_try { - t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; - - t->id = ZEND_JIT_TRACE_NUM; - t->root = zend_jit_traces[parent_num].root; - t->parent = parent_num; - t->link = 0; - t->exit_count = 0; - t->child_count = 0; - t->stack_map_size = 0; - t->flags = 0; - t->polymorphism = polymorphism; - t->jmp_table_size = 0; - t->opline = NULL; - t->exit_info = exit_info; - t->stack_map = NULL; - - orig_trigger = JIT_G(trigger); - JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE; - - handler = zend_jit_trace(trace_buffer, parent_num, exit_num); - - JIT_G(trigger) = orig_trigger; - - if (handler) { - t->exit_info = NULL; - if (t->exit_count) { - /* reallocate exit_info into shared memory */ - t->exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc_copy(exit_info, - sizeof(zend_jit_trace_exit_info) * t->exit_count); - - if (!t->exit_info) { - if (t->stack_map) { - efree(t->stack_map); - } - ret = ZEND_JIT_TRACE_STOP_NO_SHM; - goto exit; - } - } - - if (t->stack_map_size) { - zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc_copy( - t->stack_map, - t->stack_map_size * sizeof(zend_jit_trace_stack)); - efree(t->stack_map); - t->stack_map = shared_stack_map; - if (!shared_stack_map) { - ret = ZEND_JIT_TRACE_STOP_NO_SHM; - goto exit; - } - } - - zend_jit_link_side_trace( - zend_jit_traces[parent_num].code_start, - zend_jit_traces[parent_num].code_size, - zend_jit_traces[parent_num].jmp_table_size, - exit_num, - handler); - - t->exit_counters = ZEND_JIT_EXIT_COUNTERS; - ZEND_JIT_EXIT_COUNTERS += t->exit_count; - - zend_jit_traces[zend_jit_traces[parent_num].root].child_count++; - ZEND_JIT_TRACE_NUM++; - zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED; - - ret = ZEND_JIT_TRACE_STOP_COMPILED; - } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || - ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { - if (t->stack_map) { - efree(t->stack_map); - } - ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; - } else { - if (t->stack_map) { - efree(t->stack_map); - } - ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR; - } - -exit:; + ret = _zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism); } zend_catch { do_bailout = true; } zend_end_try(); @@ -7880,12 +7886,6 @@ exit:; zend_bailout(); } - if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 - && ret == ZEND_JIT_TRACE_STOP_COMPILED - && t->exit_count > 0) { - zend_jit_dump_exit_info(t); - } - return ret; } From 751cd30427567872965717a20007ab213555f6fc Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 23 Dec 2022 08:18:41 +0100 Subject: [PATCH 28/34] ext/opcache/jit/zend_jit: declare variables where they are used (C99) --- ext/opcache/jit/zend_jit.c | 370 +++++++++++++++---------------------- 1 file changed, 145 insertions(+), 225 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index dbea12c751ac..f30f4c1e8fe4 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -202,8 +202,6 @@ static bool dominates(const zend_basic_block *blocks, int a, int b) { static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use) { - int next_use; - if (ssa->vars[var].phi_use_chain) { zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; do { @@ -229,7 +227,7 @@ static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa * } } - next_use = zend_ssa_next_use(ssa->ops, var, use); + const int next_use = zend_ssa_next_use(ssa->ops, var, use); if (next_use < 0) { return true; } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) { @@ -342,8 +340,6 @@ static bool zend_jit_is_constant_cmp_long_long(const zend_op *opline, static bool zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace) { - bool skip; - if (trace) { zend_jit_trace_rec *p = trace; @@ -415,7 +411,7 @@ static bool zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, con opline++; ssa_op++; - skip = (call_level == 1); + bool skip = (call_level == 1); while (opline != end) { if (!skip) { if (zend_may_throw(opline, ssa_op, op_array, ssa)) { @@ -489,7 +485,7 @@ static bool zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, con opline++; ssa_op++; - skip = (call_level == 1); + bool skip = (call_level == 1); while (opline != end) { if (skip) { switch (opline->opcode) { @@ -550,11 +546,11 @@ static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ss static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var) { - uint32_t j, info; + uint32_t info; if (ssa->vars && ssa->var_info) { info = ssa->var_info[var].type; - for (j = op_array->last_var; j < ssa->vars_count; j++) { + for (uint32_t j = op_array->last_var; j < ssa->vars_count; j++) { if (ssa->vars[j].var == var) { info |= ssa->var_info[j].type; } @@ -567,7 +563,7 @@ static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, u #ifdef ZEND_JIT_USE_RC_INFERENCE /* Refcount may be increased by RETURN opcode */ if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) { - for (j = 0; j < ssa->cfg.blocks_count; j++) { + for (uint32_t j = 0; j < ssa->cfg.blocks_count; j++) { if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) && ssa->cfg.blocks[j].len > 0) { const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1; @@ -626,11 +622,10 @@ static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_i static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags) { - zval *zv; zend_constant *c = NULL; /* null/true/false are resolved during compilation, so don't check for them here. */ - zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key)); + zval *zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key)); if (zv) { c = (zend_constant*)Z_PTR_P(zv); } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) { @@ -645,8 +640,6 @@ static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags) static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename) { - zend_property_info *info = NULL; - if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) || !ce || !(ce->ce_flags & ZEND_ACC_LINKED) || @@ -681,7 +674,7 @@ static zend_property_info* zend_get_known_property_info(const zend_op_array *op_ } } - info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); + zend_property_info *const info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); if (info == NULL || !IS_VALID_PROPERTY_OFFSET(info->offset) || (info->flags & ZEND_ACC_STATIC)) { @@ -704,8 +697,6 @@ static zend_property_info* zend_get_known_property_info(const zend_op_array *op_ static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename) { - zend_property_info *info; - if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) { return 1; } @@ -717,7 +708,7 @@ static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *memb } } - info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); + zend_property_info *const info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); if (info == NULL || !IS_VALID_PROPERTY_OFFSET(info->offset) || (info->flags & ZEND_ACC_STATIC)) { @@ -938,12 +929,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, uint32_t sp_offset, uint32_t sp_adjustment) { - size_t size; int ret; - void *entry; -#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE) - zend_string *str = NULL; -#endif if (rt_opline && ssa && ssa->cfg.map) { /* Create additional entry point, to switch from interpreter to JIT-ed @@ -984,6 +970,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, } } + size_t size; ret = dasm_link(dasm_state, &size); if (ret != DASM_S_OK) { #if ZEND_DEBUG @@ -1014,7 +1001,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, size += dasm_venners_size; #endif - entry = *dasm_ptr; + void *const entry = *dasm_ptr; *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT)); /* flush the hardware I-cache */ @@ -1053,6 +1040,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, } #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE) + zend_string *str = NULL; if (!name) { if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) { str = zend_jit_func_name(op_array); @@ -1151,7 +1139,6 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, const zend_ssa *ssa) { int res; - zend_long op1_min, op1_max, op2_min, op2_max; if (!ssa->ops || !ssa->var_info) { return true; @@ -1166,7 +1153,7 @@ static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, if (!OP1_HAS_RANGE()) { return true; } - op1_max = OP1_MAX_RANGE(); + const zend_long op1_max = OP1_MAX_RANGE(); if (op1_max == ZEND_LONG_MAX) { return true; } @@ -1181,7 +1168,7 @@ static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, if (!OP1_HAS_RANGE()) { return true; } - op1_min = OP1_MIN_RANGE(); + const zend_long op1_min = OP1_MIN_RANGE(); if (op1_min == ZEND_LONG_MIN) { return true; } @@ -1195,8 +1182,8 @@ static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { return true; } - op1_min = OP1_MIN_RANGE(); - op2_min = OP2_MIN_RANGE(); + const zend_long op1_min = OP1_MIN_RANGE(); + const zend_long op2_min = OP2_MIN_RANGE(); if (zend_add_will_overflow(op1_min, op2_min)) { return true; } @@ -1207,8 +1194,8 @@ static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { return true; } - op1_max = OP1_MAX_RANGE(); - op2_max = OP2_MAX_RANGE(); + const zend_long op1_max = OP1_MAX_RANGE(); + const zend_long op2_max = OP2_MAX_RANGE(); if (zend_add_will_overflow(op1_max, op2_max)) { return true; } @@ -1222,8 +1209,8 @@ static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { return true; } - op1_min = OP1_MIN_RANGE(); - op2_max = OP2_MAX_RANGE(); + const zend_long op1_min = OP1_MIN_RANGE(); + const zend_long op2_max = OP2_MAX_RANGE(); if (zend_sub_will_overflow(op1_min, op2_max)) { return true; } @@ -1234,8 +1221,8 @@ static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { return true; } - op1_max = OP1_MAX_RANGE(); - op2_min = OP2_MIN_RANGE(); + const zend_long op1_max = OP1_MAX_RANGE(); + const zend_long op2_min = OP2_MIN_RANGE(); if (zend_sub_will_overflow(op1_max, op2_min)) { return true; } @@ -1256,8 +1243,8 @@ static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { return true; } - op1_min = OP1_MIN_RANGE(); - op2_min = OP2_MIN_RANGE(); + const zend_long op1_min = OP1_MIN_RANGE(); + const zend_long op2_min = OP2_MIN_RANGE(); if (zend_add_will_overflow(op1_min, op2_min)) { return true; } @@ -1268,8 +1255,8 @@ static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { return true; } - op1_max = OP1_MAX_RANGE(); - op2_max = OP2_MAX_RANGE(); + const zend_long op1_max = OP1_MAX_RANGE(); + const zend_long op2_max = OP2_MAX_RANGE(); if (zend_add_will_overflow(op1_max, op2_max)) { return true; } @@ -1283,8 +1270,8 @@ static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { return true; } - op1_min = OP1_MIN_RANGE(); - op2_max = OP2_MAX_RANGE(); + const zend_long op1_min = OP1_MIN_RANGE(); + const zend_long op2_max = OP2_MAX_RANGE(); if (zend_sub_will_overflow(op1_min, op2_max)) { return true; } @@ -1295,8 +1282,8 @@ static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { return true; } - op1_max = OP1_MAX_RANGE(); - op2_min = OP2_MIN_RANGE(); + const zend_long op1_max = OP1_MAX_RANGE(); + const zend_long op2_min = OP2_MIN_RANGE(); if (zend_sub_will_overflow(op1_max, op2_min)) { return true; } @@ -1317,9 +1304,7 @@ static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg) { - uint32_t flags; - - flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY; + const uint32_t flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY; zend_build_cfg(&CG(arena), op_array, flags, cfg); @@ -1569,10 +1554,9 @@ static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval **intervals, int count) { zend_lifetime_interval *list, *last; - int i; list = NULL; - i = 0; + int i = 0; while (i < count) { list = intervals[i]; i++; @@ -1677,11 +1661,10 @@ static bool zend_jit_in_loop(zend_ssa *ssa, int header, zend_basic_block *b) static void zend_jit_compute_loop_body(zend_ssa *ssa, int header, int n, zend_bitset loop_body) { zend_basic_block *b = ssa->cfg.blocks + n; - uint32_t i; tail_call: if (b->len) { - for (i = b->start; i < b->start + b->len; i++) { + for (uint32_t i = b->start; i < b->start + b->len; i++) { zend_bitset_incl(loop_body, i); } } @@ -1724,34 +1707,27 @@ static void zend_jit_add_hint(zend_lifetime_interval **intervals, int dst, int s Michael Franz, CGO'10 (2010), Figure 4. */ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list) { - int set_size, i, j, k, l; - uint32_t n; - zend_bitset live, live_in, pi_vars, loop_body; - int *block_order; - zend_ssa_phi *phi; - zend_lifetime_interval **intervals; - size_t mem_size; ALLOCA_FLAG(use_heap); - set_size = zend_bitset_len(ssa->vars_count); - mem_size = + const int set_size = zend_bitset_len(ssa->vars_count); + const size_t mem_size = ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE) + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE) + ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int)); - intervals = do_alloca(mem_size, use_heap); + zend_lifetime_interval **const intervals = do_alloca(mem_size, use_heap); if (!intervals) { *list = NULL; return FAILURE; } - live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*))); - live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE)); - pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); - loop_body = (zend_bitset)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); - block_order = (int*)((char*)loop_body + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE)); + const zend_bitset live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*))); + zend_bitset live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE)); + const zend_bitset pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); + const zend_bitset loop_body = (zend_bitset)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); + int *const block_order = (int*)((char*)loop_body + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE)); memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*)); zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count); @@ -1760,21 +1736,19 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss * are before this block, and where all blocks belonging to the same loop * are contiguous ??? */ - for (l = zend_jit_compute_block_order(ssa, block_order) - 1; l >= 0; l--) { - zend_basic_block *b; - - i = block_order[l]; - b = ssa->cfg.blocks + i; + for (int l = zend_jit_compute_block_order(ssa, block_order) - 1; l >= 0; l--) { + int i = block_order[l]; + zend_basic_block *const b = ssa->cfg.blocks + i; /* live = UNION of successor.liveIn for each successor of b */ /* live.add(phi.inputOf(b)) for each phi of successors of b */ zend_bitset_clear(live, set_size); - for (j = 0; j < b->successors_count; j++) { + for (int j = 0; j < b->successors_count; j++) { int succ = b->successors[j]; zend_bitset_union(live, live_in + set_size * succ, set_size); zend_bitset_clear(pi_vars, set_size); - for (phi = ssa->blocks[succ].phis; phi; phi = phi->next) { + for (zend_ssa_phi *phi = ssa->blocks[succ].phis; phi; phi = phi->next) { if (ssa->vars[phi->ssa_var].no_val) { /* skip */ } else if (phi->pi >= 0) { @@ -1785,7 +1759,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss zend_bitset_incl(pi_vars, phi->var); } } else if (!zend_bitset_in(pi_vars, phi->var)) { - for (k = 0; k < ssa->cfg.blocks[succ].predecessors_count; k++) { + for (int k = 0; k < ssa->cfg.blocks[succ].predecessors_count; k++) { if (ssa->cfg.predecessors[ssa->cfg.blocks[succ].predecessor_offset + k] == i) { if (phi->sources[k] >= 0 && zend_bitset_in(candidates, phi->sources[k])) { zend_bitset_incl(live, phi->sources[k]); @@ -1798,6 +1772,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss } /* addRange(var, b.from, b.to) for each var in live */ + int j; ZEND_BITSET_FOREACH(live, set_size, j) { if (zend_bitset_in(candidates, j)) { if (zend_jit_add_range(intervals, j, b->start, b->start + b->len - 1) != SUCCESS) { @@ -1807,15 +1782,12 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss } ZEND_BITSET_FOREACH_END(); /* for each operation op of b in reverse order */ - for (n = b->start + b->len; n > b->start;) { - zend_ssa_op *op; - const zend_op *opline; - uint32_t num; - + for (uint32_t n = b->start + b->len; n > b->start;) { n--; - op = ssa->ops + n; - opline = op_array->opcodes + n; + zend_ssa_op *const op = ssa->ops + n; + const zend_op *const opline = op_array->opcodes + n; + uint32_t num; if (UNEXPECTED(opline->opcode == ZEND_OP_DATA)) { num = n - 1; } else { @@ -1874,7 +1846,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss } /* live.remove(phi.output) for each phi of b */ - for (phi = ssa->blocks[i].phis; phi; phi = phi->next) { + for (zend_ssa_phi *phi = ssa->blocks[i].phis; phi; phi = phi->next) { zend_bitset_excl(live, phi->ssa_var); } @@ -1882,7 +1854,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss zend_bitset_copy(live_in + set_size * i, live, set_size); } - for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) { + for (int i = ssa->cfg.blocks_count - 1; i >= 0; i--) { zend_basic_block *b = ssa->cfg.blocks + i; /* if b is loop header */ @@ -1904,6 +1876,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss } while (zend_bitset_in(loop_body, to)); to--; + int j; ZEND_BITSET_FOREACH(live, set_size, j) { if (zend_jit_add_range(intervals, j, from, to) != SUCCESS) { goto failure; @@ -1917,21 +1890,19 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) { /* Register hinting (a cheap way for register coalescing) */ - for (i = 0; i < ssa->vars_count; i++) { + for (int i = 0; i < ssa->vars_count; i++) { if (intervals[i]) { - int src; - if (ssa->vars[i].definition_phi) { zend_ssa_phi *phi = ssa->vars[i].definition_phi; if (phi->pi >= 0) { - src = phi->sources[0]; + const int src = phi->sources[0]; if (intervals[src]) { zend_jit_add_hint(intervals, i, src); } } else { - for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) { - src = phi->sources[k]; + for (int k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) { + int src = phi->sources[k]; if (src >= 0) { if (ssa->vars[src].definition_phi && ssa->vars[src].definition_phi->pi >= 0 @@ -1948,7 +1919,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss } } } - for (i = 0; i < ssa->vars_count; i++) { + for (int i = 0; i < ssa->vars_count; i++) { if (intervals[i] && !intervals[i]->hint) { if (ssa->vars[i].definition >= 0) { @@ -2094,11 +2065,8 @@ static bool zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const { zend_lifetime_interval *it; uint32_t freeUntilPos[ZREG_NUM]; - uint32_t pos, pos2; - zend_reg i, reg, reg2; + zend_reg i; zend_reg hint = ZREG_NONE; - zend_regset low_priority_regs; - zend_life_range *range; if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_FP); @@ -2173,7 +2141,7 @@ static bool zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const /* Handle Scratch Registers */ /* TODO: Optimize ??? */ - range = ¤t->range; + zend_life_range *range = ¤t->range; do { uint32_t line = range->start; uint32_t last_use_line = (uint32_t)-1; @@ -2254,9 +2222,9 @@ static bool zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const return false; } - pos = 0; reg = ZREG_NONE; - pos2 = 0; reg2 = ZREG_NONE; - low_priority_regs = *hints; + uint32_t pos = 0; zend_reg reg = ZREG_NONE; + uint32_t pos2 = 0; zend_reg reg2 = ZREG_NONE; + zend_regset low_priority_regs = *hints; if (current->used_as_hint) { /* TODO: Avoid most often scratch registers. Find a better way ??? */ low_priority_regs = ZEND_REGSET_UNION(low_priority_regs, ZEND_REGSET_LOW_PRIORITY); @@ -2330,25 +2298,22 @@ static int zend_jit_allocate_blocked_reg(void) Christian Wimmer VEE'10 (2005), Figure 2. */ static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *list) { - zend_lifetime_interval *unhandled, *active, *inactive, *handled, *free; - zend_lifetime_interval *current, **p, *q; - uint32_t position; zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); zend_regset hints = ZEND_REGSET_EMPTY; - unhandled = list; + zend_lifetime_interval *unhandled = list; /* active = inactive = handled = free = {} */ - active = inactive = handled = free = NULL; + zend_lifetime_interval *active = NULL, *inactive = NULL, *handled = NULL, *free = NULL; while (unhandled != NULL) { - current = unhandled; + zend_lifetime_interval *current = unhandled; unhandled = unhandled->list_next; - position = current->range.start; + const uint32_t position = current->range.start; - p = &active; + zend_lifetime_interval **p = &active; while (*p) { uint32_t end = zend_interval_end(*p); - q = *p; + zend_lifetime_interval *const q = *p; if (end < position) { /* move ival from active to handled */ ZEND_REGSET_INCL(available, q->reg); @@ -2370,7 +2335,7 @@ static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_arra while (*p) { uint32_t end = zend_interval_end(*p); - q = *p; + zend_lifetime_interval *const q = *p; if (end < position) { /* move ival from inactive to handled */ *p = q->list_next; @@ -2400,7 +2365,7 @@ static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_arra /* move active to handled */ while (active) { - current = active; + zend_lifetime_interval *current = active; active = active->list_next; current->list_next = handled; handled = current; @@ -2408,7 +2373,7 @@ static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_arra /* move inactive to handled */ while (inactive) { - current = inactive; + zend_lifetime_interval *current = inactive; inactive = inactive->list_next; current->list_next = handled; handled = current; @@ -2458,11 +2423,6 @@ static void zend_jit_dump_lifetime_interval(const zend_op_array *op_array, const static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array *op_array, zend_ssa *ssa) { - void *checkpoint; - int set_size, candidates_count, i; - zend_bitset candidates = NULL; - zend_lifetime_interval *list, *ival; - zend_lifetime_interval **intervals; ALLOCA_FLAG(use_heap); if (!ssa->var_info) { @@ -2470,14 +2430,14 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array } /* Identify SSA variables suitable for register allocation */ - set_size = zend_bitset_len(ssa->vars_count); - candidates = ZEND_BITSET_ALLOCA(set_size, use_heap); + const int set_size = zend_bitset_len(ssa->vars_count); + const zend_bitset candidates = ZEND_BITSET_ALLOCA(set_size, use_heap); if (!candidates) { return NULL; } - candidates_count = 0; + int candidates_count = 0; zend_bitset_clear(candidates, set_size); - for (i = 0; i < ssa->vars_count; i++) { + for (int i = 0; i < ssa->vars_count; i++) { if (zend_jit_may_be_in_reg(op_array, ssa, i)) { zend_bitset_incl(candidates, i); candidates_count++; @@ -2488,17 +2448,17 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array return NULL; } - checkpoint = zend_arena_checkpoint(CG(arena)); + void *const checkpoint = zend_arena_checkpoint(CG(arena)); /* Find life-time intervals */ + zend_lifetime_interval *list; if (zend_jit_compute_liveness(op_array, ssa, candidates, &list) != SUCCESS) { goto failure; } if (list) { /* Set ZREG_LAST_USE flags */ - ival = list; - while (ival) { + for (zend_lifetime_interval *ival = list; ival; ival = ival->list_next) { zend_life_range *range = &ival->range; while (range->next) { @@ -2507,17 +2467,14 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array if (zend_ssa_is_last_use(op_array, ssa, ival->ssa_var, range->end)) { ival->flags |= ZREG_LAST_USE; } - ival = ival->list_next; } } if (list) { if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); - ival = list; - while (ival) { + for (zend_lifetime_interval *ival = list; ival; ival = ival->list_next) { zend_jit_dump_lifetime_interval(op_array, ssa, ival); - ival = ival->list_next; } fprintf(stderr, "\n"); } @@ -2526,13 +2483,12 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array list = zend_jit_linear_scan(op_array, NULL, ssa, list); if (list) { - intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*)); + zend_lifetime_interval **const intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*)); if (!intervals) { goto failure; } - ival = list; - while (ival != NULL) { + for (zend_lifetime_interval *ival = list; ival;) { zend_lifetime_interval *next = ival->list_next; ival->list_next = intervals[ival->ssa_var]; @@ -2542,7 +2498,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) { /* Naive SSA resolution */ - for (i = 0; i < ssa->vars_count; i++) { + for (int i = 0; i < ssa->vars_count; i++) { if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) { zend_ssa_phi *phi = ssa->vars[i].definition_phi; int k, src; @@ -2608,7 +2564,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array } } /* Remove useless register allocation */ - for (i = 0; i < ssa->vars_count; i++) { + for (int i = 0; i < ssa->vars_count; i++) { if (intervals[i] && ((intervals[i]->flags & ZREG_LOAD) || ((intervals[i]->flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) && @@ -2630,7 +2586,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array } } /* Remove intervals used once */ - for (i = 0; i < ssa->vars_count; i++) { + for (int i = 0; i < ssa->vars_count; i++) { if (intervals[i] && (intervals[i]->flags & ZREG_LOAD) && (intervals[i]->flags & ZREG_STORE) && @@ -2656,11 +2612,9 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); - for (i = 0; i < ssa->vars_count; i++) { - ival = intervals[i]; - while (ival) { + for (int i = 0; i < ssa->vars_count; i++) { + for (zend_lifetime_interval *ival = list; ival; ival = ival->list_next) { zend_jit_dump_lifetime_interval(op_array, ssa, ival); - ival = ival->list_next; } } fprintf(stderr, "\n"); @@ -2723,22 +2677,9 @@ static bool zend_jit_supported_binary_op(zend_uchar op, uint32_t op1_info, uint3 */ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline) { - int b, i, end; - zend_op *opline; - dasm_State* dasm_state = NULL; - void *handler; int call_level = 0; - void *checkpoint = NULL; - zend_lifetime_interval **ra = NULL; bool is_terminated = true; /* previous basic block is terminated by jump */ bool recv_emitted = false; /* emitted at least one RECV opcode */ - zend_uchar smart_branch_opcode; - uint32_t target_label, target_label2; - uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info; - zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; - zend_class_entry *ce; - bool ce_is_instanceof; - bool on_this; if (JIT_G(bisect_limit)) { jit_bisect_pos++; @@ -2754,17 +2695,19 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } } + void *checkpoint = NULL; + zend_lifetime_interval **ra = NULL; if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) { checkpoint = zend_arena_checkpoint(CG(arena)); ra = zend_jit_allocate_registers(op_array, ssa); } /* mark hidden branch targets */ - for (b = 0; b < ssa->cfg.blocks_count; b++) { + for (int b = 0; b < ssa->cfg.blocks_count; b++) { if (ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE && ssa->cfg.blocks[b].len > 1) { - opline = op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1; + zend_op *const opline = op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1; if (opline->opcode == ZEND_DO_FCALL && (opline-1)->opcode == ZEND_NEW) { ssa->cfg.blocks[ssa->cfg.blocks[b].successors[0]].flags |= ZEND_BB_TARGET; @@ -2772,6 +2715,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } } + dasm_State* dasm_state = NULL; dasm_init(&dasm_state, DASM_MAXSECTION); dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); dasm_setup(&dasm_state, dasm_actions); @@ -2779,7 +2723,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op dasm_growpc(&dasm_state, ssa->cfg.blocks_count * 2 + 1); zend_jit_align_func(&dasm_state); - for (b = 0; b < ssa->cfg.blocks_count; b++) { + for (int b = 0; b < ssa->cfg.blocks_count; b++) { if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) { continue; } @@ -2804,7 +2748,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } else //#endif if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { - opline = op_array->opcodes + ssa->cfg.blocks[b].start; + zend_op *const opline = op_array->opcodes + ssa->cfg.blocks[b].start; if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) { if (opline->opcode == ZEND_RECV_INIT) { if (opline == op_array->opcodes || @@ -2813,7 +2757,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op zend_jit_jmp(&dasm_state, b); } zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); - for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) { + for (int i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) { zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b + i); } zend_jit_prologue(&dasm_state); @@ -2934,10 +2878,10 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op phi = phi->next; } } - end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1; - for (i = ssa->cfg.blocks[b].start; i <= end; i++) { + const int end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1; + for (int i = ssa->cfg.blocks[b].start; i <= end; i++) { zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL; - opline = op_array->opcodes + i; + zend_op *const opline = op_array->opcodes + i; switch (opline->opcode) { case ZEND_INIT_FCALL: case ZEND_INIT_FCALL_BY_NAME: @@ -2951,6 +2895,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { + zend_uchar smart_branch_opcode; + uint32_t target_label, target_label2; + uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info; + zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; + zend_class_entry *ce; + bool ce_is_instanceof; + bool on_this; + switch (opline->opcode) { case ZEND_PRE_INC: case ZEND_PRE_DEC: @@ -4153,7 +4105,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } } - handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0, + void *const handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0, (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET, SP_ADJ_JIT); if (!handler) { goto jit_failure; @@ -4193,12 +4145,8 @@ static void zend_jit_collect_calls(zend_op_array *op_array, zend_script *script) static void zend_jit_cleanup_func_info(zend_op_array *op_array) { zend_func_info *func_info = ZEND_FUNC_INFO(op_array); - zend_call_info *caller_info, *callee_info; if (func_info) { - caller_info = func_info->caller_info; - callee_info = func_info->callee_info; - if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) { @@ -4212,34 +4160,30 @@ static void zend_jit_cleanup_func_info(zend_op_array *op_array) ZEND_SET_FUNC_INFO(op_array, NULL); } - while (caller_info) { + for (zend_call_info *caller_info = func_info->caller_info; caller_info; caller_info = caller_info->next_caller) { if (caller_info->caller_op_array) { zend_jit_cleanup_func_info(caller_info->caller_op_array); } - caller_info = caller_info->next_caller; } - while (callee_info) { + + for (zend_call_info *callee_info = func_info->callee_info; callee_info; callee_info = callee_info->next_callee) { if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) { zend_jit_cleanup_func_info(&callee_info->callee_func->op_array); } - callee_info = callee_info->next_callee; } } } static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline) { - zend_ssa ssa; - void *checkpoint; - zend_func_info *func_info; - if (*dasm_ptr == dasm_end) { return FAILURE; } - checkpoint = zend_arena_checkpoint(CG(arena)); + void *const checkpoint = zend_arena_checkpoint(CG(arena)); /* Build SSA */ + zend_ssa ssa; memset(&ssa, 0, sizeof(zend_ssa)); if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) { @@ -4248,7 +4192,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) { zend_jit_collect_calls(op_array, script); - func_info = ZEND_FUNC_INFO(op_array); + zend_func_info *const func_info = ZEND_FUNC_INFO(op_array); func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array); if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { zend_init_func_return_info(op_array, script, &func_info->return_info); @@ -4321,23 +4265,20 @@ static int ZEND_FASTCALL zend_runtime_jit(void) static void zend_jit_check_funcs(HashTable *function_table, bool is_method) { zend_function *func; - zend_op_array *op_array; - uintptr_t counter; - zend_jit_op_array_extension *jit_extension; ZEND_HASH_MAP_REVERSE_FOREACH_PTR(function_table, func) { if (func->type == ZEND_INTERNAL_FUNCTION) { break; } - op_array = &func->op_array; + zend_op_array *const op_array = &func->op_array; zend_op *const opline = zend_find_function_entry(op_array); if (opline->handler == zend_jit_profile_jit_handler) { if (!RUN_TIME_CACHE(op_array)) { continue; } - counter = (uintptr_t)ZEND_COUNTER_INFO(op_array); + const uintptr_t counter = (uintptr_t)ZEND_COUNTER_INFO(op_array); ZEND_COUNTER_INFO(op_array) = 0; - jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); + zend_jit_op_array_extension *const jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); opline->handler = jit_extension->orig_handler; if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) { zend_real_jit_func(op_array, NULL, NULL); @@ -4349,19 +4290,17 @@ static void zend_jit_check_funcs(HashTable *function_table, bool is_method) { void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline) { zend_op_array *op_array = &EX(func)->op_array; - zend_jit_op_array_hot_extension *jit_extension; - uint32_t i; bool do_bailout = false; zend_shared_alloc_lock(); - jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array); + zend_jit_op_array_hot_extension *const jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array); if (jit_extension) { SHM_UNPROTECT(); zend_jit_unprotect(); zend_try { - for (i = 0; i < op_array->last; i++) { + for (uint32_t i = 0; i < op_array->last; i++) { op_array->opcodes[i].handler = jit_extension->orig_handlers[i]; } @@ -4405,15 +4344,12 @@ static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cf static int zend_jit_restart_hot_counters(zend_op_array *op_array) { - zend_jit_op_array_hot_extension *jit_extension; - zend_cfg cfg; - uint32_t i; - - jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array); - for (i = 0; i < op_array->last; i++) { + zend_jit_op_array_hot_extension *const jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array); + for (uint32_t i = 0; i < op_array->last; i++) { op_array->opcodes[i].handler = jit_extension->orig_handlers[i]; } + zend_cfg cfg; if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) { return FAILURE; } @@ -4425,25 +4361,22 @@ static int zend_jit_restart_hot_counters(zend_op_array *op_array) static int zend_jit_setup_hot_counters(zend_op_array *op_array) { - zend_jit_op_array_hot_extension *jit_extension; - zend_cfg cfg; - uint32_t i; - ZEND_ASSERT(zend_jit_func_hot_counter_handler != NULL); ZEND_ASSERT(zend_jit_loop_hot_counter_handler != NULL); + zend_cfg cfg; if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) { return FAILURE; } - jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*)); + zend_jit_op_array_hot_extension *const jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*)); if (!jit_extension) { return FAILURE; } memset(&jit_extension->func_info, 0, sizeof(zend_func_info)); jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS; jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) % ZEND_HOT_COUNTERS_COUNT]; - for (i = 0; i < op_array->last; i++) { + for (uint32_t i = 0; i < op_array->last; i++) { jit_extension->orig_handlers[i] = op_array->opcodes[i].handler; } ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); @@ -4502,8 +4435,6 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) return SUCCESS; } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) { - zend_jit_op_array_extension *jit_extension; - if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) { ZEND_SET_FUNC_INFO(op_array, NULL); zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT"); @@ -4513,7 +4444,7 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) ZEND_ASSERT(zend_jit_profile_jit_handler != NULL); if (op_array->function_name) { zend_op *const opline = zend_find_function_entry(op_array); - jit_extension = zend_jit_new_op_array_extension(op_array); + zend_jit_op_array_extension *const jit_extension = zend_jit_new_op_array_extension(op_array); if (!jit_extension) { return FAILURE; } @@ -4536,17 +4467,13 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) ZEND_EXT_API int zend_jit_script(zend_script *script) { - void *checkpoint; - zend_call_graph call_graph; - zend_func_info *info; - int i; - if (dasm_ptr == NULL || *dasm_ptr == dasm_end) { return FAILURE; } - checkpoint = zend_arena_checkpoint(CG(arena)); + void *const checkpoint = zend_arena_checkpoint(CG(arena)); + zend_call_graph call_graph; call_graph.op_arrays_count = 0; zend_build_call_graph(&CG(arena), script, &call_graph); @@ -4556,14 +4483,14 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - for (i = 0; i < call_graph.op_arrays_count; i++) { + for (int i = 0; i < call_graph.op_arrays_count; i++) { if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) { goto jit_failure; } } } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) { - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + for (int i = 0; i < call_graph.op_arrays_count; i++) { + zend_func_info *const info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) { goto jit_failure; @@ -4572,8 +4499,8 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) } } - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + for (int i = 0; i < call_graph.op_arrays_count; i++) { + zend_func_info *const info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]); if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { @@ -4582,8 +4509,8 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) } } - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + for (int i = 0; i < call_graph.op_arrays_count; i++) { + zend_func_info *const info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) { goto jit_failure; @@ -4593,16 +4520,16 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) } if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) { - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + for (int i = 0; i < call_graph.op_arrays_count; i++) { + zend_func_info *const info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa); } } } - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + for (int i = 0; i < call_graph.op_arrays_count; i++) { + zend_func_info *const info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) { goto jit_failure; @@ -4610,7 +4537,7 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) } } - for (i = 0; i < call_graph.op_arrays_count; i++) { + for (int i = 0; i < call_graph.op_arrays_count; i++) { ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); } } else { @@ -4624,9 +4551,9 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { zend_class_entry *ce; - zend_op_array *op_array; ZEND_HASH_MAP_FOREACH_PTR(&script->class_table, ce) { + zend_op_array *op_array; ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) { if (!ZEND_FUNC_INFO(op_array)) { void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes); @@ -4643,7 +4570,7 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) jit_failure: if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) { - for (i = 0; i < call_graph.op_arrays_count; i++) { + for (int i = 0; i < call_graph.op_arrays_count; i++) { ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); } } @@ -4740,12 +4667,10 @@ static void zend_jit_init_handlers(void) static bool zend_jit_make_stubs(void) { dasm_State* dasm_state = NULL; - uint32_t i; - dasm_init(&dasm_state, DASM_MAXSECTION); dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); - for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) { + for (uint32_t i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) { dasm_setup(&dasm_state, dasm_actions); if (!zend_jit_stubs[i].stub(&dasm_state)) { return false; @@ -5095,9 +5020,7 @@ ZEND_EXT_API void zend_jit_shutdown(void) static void zend_jit_reset_counters(void) { - int i; - - for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) { + for (int i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) { zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT; } } @@ -5155,9 +5078,6 @@ static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array) #if 0 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ??? } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) { - zend_jit_op_array_extension *jit_extension = - (zend_jit_op_array_extension*)func_info; - zend_op *const opline = zend_find_function_entry(op_array); if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) { opline->handler = (const void*)zend_jit_runtime_jit_handler; From 3b265fb1e6bec1017ed844b69c20092251b7b705 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Mon, 2 Jan 2023 20:29:51 +0100 Subject: [PATCH 29/34] Zend/Optimizer: make pointers const --- ext/opcache/jit/zend_jit.c | 10 +++++----- ext/opcache/jit/zend_jit_trace.c | 22 +++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index f30f4c1e8fe4..f4ae712ce571 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -338,7 +338,7 @@ static bool zend_jit_is_constant_cmp_long_long(const zend_op *opline, return false; } -static bool zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace) +static bool zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, const zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace) { if (trace) { zend_jit_trace_rec *p = trace; @@ -517,7 +517,7 @@ static bool zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, con } } -static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info) +static uint32_t skip_valid_arguments(const zend_op_array *op_array, const zend_ssa *ssa, const zend_call_info *call_info) { uint32_t num_args = 0; zend_function *func = call_info->callee_func; @@ -529,8 +529,8 @@ static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ss if (ZEND_TYPE_IS_SET(arg_info->type)) { if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) { - zend_op *opline = call_info->arg_info[num_args].opline; - zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; + const zend_op *opline = call_info->arg_info[num_args].opline; + const zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type); if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) { break; @@ -544,7 +544,7 @@ static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ss return num_args; } -static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var) +static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var) { uint32_t info; diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 62788a943683..9cbb408680ba 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -489,7 +489,7 @@ static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack return; } if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { - zend_arg_info *arg_info; + const zend_arg_info *arg_info; ZEND_ASSERT(arg_num <= op_array->num_args); arg_info = &op_array->arg_info[arg_num-1]; @@ -816,7 +816,7 @@ static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_ return ssa_vars_count; } -static bool zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +static bool zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *const *tssa_opcodes, zend_ssa *tssa, int ssa_var) { int var, use; zend_ssa_op *op; @@ -858,7 +858,7 @@ static bool zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, cons return false; } -static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op *const *tssa_opcodes, zend_ssa *tssa, int ssa_var) { zend_ssa_range tmp; int def = tssa->vars[ssa_var].definition; @@ -873,7 +873,7 @@ static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const } } -static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *const*tssa_opcodes, zend_ssa *tssa, int ssa_var) { int def; zend_ssa_op *op; @@ -923,7 +923,7 @@ static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, con } } -static bool zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +static bool zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *const*tssa_opcodes, zend_ssa *tssa, int ssa_var) { int def; zend_ssa_op *op; @@ -1071,7 +1071,7 @@ static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, c return NULL; } -static bool is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var) +static bool is_checked_guard(const zend_ssa *tssa, const zend_op *const*ssa_opcodes, uint32_t var, uint32_t phi_var) { if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG && !(tssa->var_info[var].type & MAY_BE_REF)) { @@ -1536,7 +1536,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i); } if (op_array->arg_info) { - zend_arg_info *arg_info = &op_array->arg_info[i]; + const zend_arg_info *arg_info = &op_array->arg_info[i]; zend_class_entry *ce; uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce); @@ -2001,7 +2001,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD; } if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { - zend_arg_info *arg_info; + const zend_arg_info *arg_info; ZEND_ASSERT(frame->call->func->op_array.arg_info); arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1]; @@ -2344,7 +2344,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } else { ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i); if (op_array->arg_info) { - zend_arg_info *arg_info = &op_array->arg_info[i]; + const zend_arg_info *arg_info = &op_array->arg_info[i]; zend_class_entry *ce; uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce); @@ -2513,7 +2513,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin if (opline->result_type != IS_UNDEF) { zend_class_entry *ce; const zend_function *func = p->func; - zend_arg_info *ret_info = func->common.arg_info - 1; + const zend_arg_info *ret_info = func->common.arg_info - 1; uint32_t ret_type = zend_fetch_arg_info_type(NULL, ret_info, &ce); ssa_var_info[ssa_ops[idx-1].result_def].type &= ret_type; @@ -3856,7 +3856,7 @@ static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const } } -static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array) +static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op *const*ssa_opcodes, const zend_op_array *op_array) { zend_uchar prev_opcode; From 94af4daf9b67c79720e6dfe11d831de9d588aa7f Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Fri, 23 Dec 2022 09:11:02 +0100 Subject: [PATCH 30/34] ext/opcache/jit/zend_jit_trace: declare variables where they are used (C99) --- ext/opcache/jit/zend_jit_trace.c | 633 ++++++++++++------------------- 1 file changed, 251 insertions(+), 382 deletions(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 9cbb408680ba..ff287a12088f 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -88,7 +88,6 @@ static int zend_jit_trace_startup(zend_bool reattached) static const void *zend_jit_trace_allocate_exit_group(uint32_t n) { dasm_State* dasm_state = NULL; - const void *entry; char name[32]; dasm_init(&dasm_state, DASM_MAXSECTION); @@ -97,14 +96,12 @@ static const void *zend_jit_trace_allocate_exit_group(uint32_t n) zend_jit_trace_exit_group_stub(&dasm_state, n); sprintf(name, "jit$$trace_exit_%d", n); - entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0, SP_ADJ_JIT, SP_ADJ_NONE); + const void *const entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0, SP_ADJ_JIT, SP_ADJ_NONE); dasm_free(&dasm_state); #ifdef HAVE_DISASM if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { - uint32_t i; - - for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { + for (uint32_t i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { sprintf(name, "jit$$trace_exit_%d", n + i); zend_jit_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING); } @@ -154,9 +151,8 @@ static zend_jit_trace_info *zend_jit_get_current_trace_info(void) static uint32_t zend_jit_trace_find_exit_point(const void* addr) { uint32_t n = ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP; - uint32_t i; - for (i = 0; i < n; i++) { + for (uint32_t i = 0; i < n; i++) { if ((const char*)addr >= (const char*)zend_jit_exit_groups[i] && (const char*)addr < (const char*)zend_jit_exit_groups[i] + (ZEND_JIT_EXIT_POINTS_PER_GROUP * ZEND_JIT_EXIT_POINTS_SPACING)) { @@ -171,7 +167,6 @@ static uint32_t zend_jit_trace_find_exit_point(const void* addr) static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags) { zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; - uint32_t exit_point; const zend_op_array *op_array; uint32_t stack_offset = (uint32_t)-1; uint32_t stack_size; @@ -219,7 +214,7 @@ static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t } while (i > 0); } - exit_point = t->exit_count; + const uint32_t exit_point = t->exit_count; if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) { if (stack_size != 0 && stack_offset == (uint32_t)-1) { stack_offset = t->stack_map_size; @@ -256,9 +251,7 @@ static void zend_jit_trace_add_code(const void *start, uint32_t size) */ static uint32_t zend_jit_find_trace(const void *addr) { - uint32_t i; - - for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) { + for (uint32_t i = 1; i < ZEND_JIT_TRACE_NUM; i++) { if (zend_jit_traces[i].code_start == addr) { return i; } @@ -551,10 +544,7 @@ static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script) { - zend_jit_op_array_trace_extension *jit_extension; - zend_ssa *ssa; - - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); jit_extension->func_info.num = 0; jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC @@ -562,7 +552,7 @@ static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_sc | ZEND_FUNC_JIT_ON_HOT_COUNTERS | ZEND_FUNC_JIT_ON_HOT_TRACE; memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa)); - ssa = &jit_extension->func_info.ssa; + zend_ssa *const ssa = &jit_extension->func_info.ssa; if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) { do { @@ -673,29 +663,21 @@ static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline) static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack) { - const zend_op_array *op_array; - zend_jit_trace_rec *p; - int k, vars_count; - zend_bitset use, def; uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS; - uint32_t set_size; - zend_ssa_phi *prev = NULL; int level = 0; ALLOCA_FLAG(use_heap); - op_array = trace_buffer->op_array; - set_size = zend_bitset_len(op_array->last_var + op_array->T); - use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap); + const zend_op_array *const op_array = trace_buffer->op_array; + const uint32_t set_size = zend_bitset_len(op_array->last_var + op_array->T); + const zend_bitset use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap); memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE); - def = use + set_size; - p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; - for (;;p++) { + const zend_bitset def = use + set_size; + for (zend_jit_trace_rec *p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;;p++) { if (p->op == ZEND_JIT_TRACE_VM && level == 0) { const zend_op *opline = p->opline; - int len; zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def); - len = zend_jit_trace_op_len(opline); + int len = zend_jit_trace_op_len(opline); while (len > 1) { opline++; if (opline->opcode != ZEND_OP_DATA) { @@ -721,12 +703,14 @@ static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ss zend_bitset_intersection(use, def, set_size); + int vars_count; if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { vars_count = op_array->last_var; } else { vars_count = op_array->last_var + op_array->T; } - for (k = 0; k < vars_count; k++) { + zend_ssa_phi *prev = NULL; + for (int k = 0; k < vars_count; k++) { if (zend_bitset_in(use, k)) { zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1, ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) + @@ -762,9 +746,8 @@ static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32 const zend_op_array *op_array = trace_buffer->op_array; const zend_op *opline = trace_buffer[1].opline; int count = opline - op_array->opcodes; - int i; - for(i = 0; i < count; i++) { + for(int i = 0; i < count; i++) { zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1, ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) + @@ -792,7 +775,6 @@ static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32 static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack) { const zend_op *opline = trace_buffer[1].opline - 1; - int i; if (RETURN_VALUE_USED(opline)) { zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1, @@ -800,7 +782,7 @@ static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_ ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) + sizeof(void*) * 2); - i = EX_VAR_TO_NUM(opline->result.var); + const int i = EX_VAR_TO_NUM(opline->result.var); phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi))); phi->sources[0] = STACK_VAR(stack, i); phi->sources[1] = -1; @@ -818,11 +800,7 @@ static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_ static bool zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *const *tssa_opcodes, zend_ssa *tssa, int ssa_var) { - int var, use; - zend_ssa_op *op; - zend_ssa_var_info *info; - unsigned int no_val; - zend_ssa_alias_kind alias; + int var; if (tssa->vars[ssa_var].phi_use_chain) { // TODO: this may be incorrect ??? @@ -830,10 +808,13 @@ static bool zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, cons } else { var = ssa_var; } - use = tssa->vars[var].use_chain; + const int use = tssa->vars[var].use_chain; if (use >= 0) { ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last); - op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes); + zend_ssa_op *const op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes); + unsigned int no_val; + zend_ssa_alias_kind alias; + zend_ssa_var_info *info; if (tssa->ops[use].op1_use == var) { no_val = ssa->vars[op->op1_use].no_val; alias = ssa->vars[op->op1_use].alias; @@ -875,16 +856,13 @@ static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *const*tssa_opcodes, zend_ssa *tssa, int ssa_var) { - int def; - zend_ssa_op *op; - zend_ssa_var_info *info; - unsigned int no_val; - zend_ssa_alias_kind alias; - - def = tssa->vars[ssa_var].definition; + const int def = tssa->vars[ssa_var].definition; if (def >= 0) { ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last); - op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes); + zend_ssa_op *const op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes); + zend_ssa_var_info *info; + unsigned int no_val; + zend_ssa_alias_kind alias; if (tssa->ops[def].op1_def == ssa_var) { no_val = ssa->vars[op->op1_def].no_val; alias = ssa->vars[op->op1_def].alias; @@ -925,14 +903,11 @@ static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, con static bool zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *const*tssa_opcodes, zend_ssa *tssa, int ssa_var) { - int def; - zend_ssa_op *op; - zend_ssa_var_info *info; - - def = tssa->vars[ssa_var].definition; + const int def = tssa->vars[ssa_var].definition; if (def >= 0) { ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last); - op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes); + zend_ssa_op *const op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes); + zend_ssa_var_info *info; if (tssa->ops[def].op1_def == ssa_var) { info = ssa->var_info + op->op1_def; } else if (tssa->ops[def].op2_def == ssa_var) { @@ -1177,42 +1152,26 @@ static const zend_op _nop_opcode = {0}; static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr) { - zend_ssa *tssa; - zend_ssa_op *ssa_ops, *op; - zend_ssa_var *ssa_vars; - zend_ssa_var_info *ssa_var_info; - const zend_op_array *op_array; - const zend_op *opline; - const zend_op **ssa_opcodes; - zend_jit_trace_rec *p; - int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count; - zend_jit_trace_stack *stack; - uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS; - uint32_t optimization_level = 0; - int call_level, level, num_op_arrays, used_stack, max_used_stack; - size_t frame_size, stack_top, stack_size, stack_bottom; - zend_jit_op_array_trace_extension *jit_extension; - zend_ssa *ssa; - zend_jit_trace_stack_frame *frame, *top, *call; - zend_ssa_var_info return_value_info; + const uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS; + const uint32_t optimization_level = 0; /* 1. Count number of TSSA opcodes; * Count number of activation frames; * Calculate size of abstract stack; * Construct regular SSA for involved op_array */ - op_array = trace_buffer->op_array; - stack_top = stack_size = zend_jit_trace_frame_size(op_array); - stack_bottom = 0; - p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; - ssa_ops_count = 0; - call_level = 0; - level = 0; - num_op_arrays = 0; + const zend_op_array *op_array = trace_buffer->op_array; + size_t stack_size = zend_jit_trace_frame_size(op_array); + size_t stack_top = stack_size; + size_t stack_bottom = 0; + int ssa_ops_count = 0; + int call_level = 0; + int level = 0; + int num_op_arrays = 0; /* Remember op_array to cleanup */ op_arrays[num_op_arrays++] = op_array; /* Build SSA */ - ssa = zend_jit_trace_build_ssa(op_array, script); - for (;;p++) { + zend_ssa *ssa = zend_jit_trace_build_ssa(op_array, script); + for (zend_jit_trace_rec *p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;; p++) { if (p->op == ZEND_JIT_TRACE_VM) { if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) { const zend_op *opline = p->opline; @@ -1254,7 +1213,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS; } } - frame_size = zend_jit_trace_frame_size(p->op_array); + const size_t frame_size = zend_jit_trace_frame_size(p->op_array); if (call_level == 0) { if (stack_top + frame_size > stack_size) { stack_size = stack_top + frame_size; @@ -1274,7 +1233,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin call_level--; } level++; - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); ssa = &jit_extension->func_info.ssa; if (ssa->cfg.blocks_count) { @@ -1292,7 +1251,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } else if (p->op == ZEND_JIT_TRACE_BACK) { if (level == 0) { stack_bottom += zend_jit_trace_frame_size(p->op_array); - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); ssa = &jit_extension->func_info.ssa; if (ssa->cfg.blocks_count) { @@ -1319,10 +1278,10 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin *num_op_arrays_ptr = num_op_arrays; /* Allocate space for abstract stack */ - JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom); + zend_jit_trace_stack_frame *frame = JIT_G(current_frame) = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom); /* 2. Construct TSSA */ - tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa)); + zend_ssa *const tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa)); tssa->cfg.flags = ZEND_SSA_TSSA; tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block)); tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block)); @@ -1362,20 +1321,21 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin return tssa; } - tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op)); + zend_ssa_op *const ssa_ops = tssa->ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op)); memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op)); - ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*)); + const zend_op **const ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*)); ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes; ssa_opcodes[ssa_ops_count] = &_nop_opcode; op_array = trace_buffer->op_array; + int ssa_vars_count; if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { ssa_vars_count = op_array->last_var; } else { ssa_vars_count = op_array->last_var + op_array->T; } - stack = frame->stack; - for (i = 0; i < ssa_vars_count; i++) { + zend_jit_trace_stack *stack = frame->stack; + for (int i = 0; i < ssa_vars_count; i++) { SET_STACK_VAR(stack, i, i); } @@ -1389,24 +1349,21 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack); } - p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; - idx = 0; + int idx = 0; level = 0; - for (;;p++) { + for (zend_jit_trace_rec *p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;; p++) { if (p->op == ZEND_JIT_TRACE_VM) { - opline = p->opline; + const zend_op *opline = p->opline; ssa_opcodes[idx] = opline; ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack); idx++; - len = zend_jit_trace_op_len(p->opline); - while (len > 1) { + for (int len = zend_jit_trace_op_len(p->opline); len > 1; --len) { opline++; ssa_opcodes[idx] = opline; if (opline->opcode != ZEND_OP_DATA) { ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack); } idx++; - len--; } } else if (p->op == ZEND_JIT_TRACE_ENTER) { frame = zend_jit_trace_call_frame(frame, op_array); @@ -1417,7 +1374,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin return NULL; } ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count); - for (i = 0; i < op_array->last_var; i++) { + for (int i = 0; i < op_array->last_var; i++) { SET_STACK_VAR(stack, i, ssa_vars_count++); } } else if (p->op == ZEND_JIT_TRACE_BACK) { @@ -1429,7 +1386,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin return NULL; } ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count); - for (i = 0; i < op_array->last_var + op_array->T; i++) { + for (int i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_VAR(stack, i, ssa_vars_count++); } } else { @@ -1442,13 +1399,14 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin op_array = trace_buffer->op_array; tssa->vars_count = ssa_vars_count; - tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var)); + zend_ssa_var *const ssa_vars = tssa->vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var)); + int vars_count; if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { vars_count = op_array->last_var; } else { vars_count = op_array->last_var + op_array->T; } - i = 0; + int i = 0; while (i < vars_count) { ssa_vars[i].var = i; ssa_vars[i].scc = -1; @@ -1482,9 +1440,9 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin /* 3. Compute use-def chains */ idx = (ssa_ops_count - 1); - op = ssa_ops + idx; + zend_ssa_op *op = ssa_ops + idx; while (idx >= 0) { - opline = ssa_opcodes[idx]; + const zend_op *const opline = ssa_opcodes[idx]; if (op->op1_use >= 0) { op->op1_use_chain = ssa_vars[op->op1_use].use_chain; ssa_vars[op->op1_use].use_chain = idx; @@ -1515,11 +1473,11 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin /* 4. Type inference */ op_array = trace_buffer->op_array; - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); ssa = &jit_extension->func_info.ssa; - tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info)); + zend_ssa_var_info *const ssa_var_info = tssa->var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info)); if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { i = 0; @@ -1569,7 +1527,6 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin int parent_vars_count = 0; zend_jit_trace_stack *parent_stack = NULL; - i = 0; if (parent_trace) { parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size, op_array->last_var + op_array->T); @@ -1579,7 +1536,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset; } } - while (i < op_array->last_var + op_array->T) { + for (int i = 0; i < op_array->last_var + op_array->T; ++i) { if (!ssa->var_info || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) { if (ssa->vars && i < ssa->vars_count) { @@ -1607,7 +1564,6 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } } - i++; } } @@ -1615,50 +1571,44 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { /* Propagate initial value through Phi functions */ - zend_ssa_phi *phi = tssa->blocks[1].phis; - while (phi) { + for (zend_ssa_phi *phi = tssa->blocks[1].phis; phi; phi = phi->next) { if (!ssa->var_info || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var)) { ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias; ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type; } - phi = phi->next; } } frame = JIT_G(current_frame); - top = zend_jit_trace_call_frame(frame, op_array); + zend_jit_trace_stack_frame *top = zend_jit_trace_call_frame(frame, op_array); TRACE_FRAME_INIT(frame, op_array, 0, 0); TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); frame->used_stack = 0; for (i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1); } + zend_ssa_var_info return_value_info; memset(&return_value_info, 0, sizeof(return_value_info)); + int used_stack, max_used_stack; if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { max_used_stack = used_stack = 0; } else { max_used_stack = used_stack = -1; } - p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; idx = 0; level = 0; - opline = NULL; - for (;;p++) { + const zend_op *opline = NULL; + for (zend_jit_trace_rec *p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;; p++) { if (p->op == ZEND_JIT_TRACE_VM) { - uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type; - uint8_t val_type = IS_UNKNOWN; -// zend_class_entry *op1_ce = NULL; - zend_class_entry *op2_ce = NULL; + const uint8_t orig_op1_type = p->op1_type, orig_op2_type = p->op2_type; opline = p->opline; - op1_type = orig_op1_type = p->op1_type; - op2_type = orig_op2_type = p->op2_type; - op3_type = p->op3_type; + uint8_t op1_type = orig_op1_type, op2_type = orig_op2_type, op3_type = p->op3_type; if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) { op1_type = IS_UNKNOWN; } @@ -1672,14 +1622,19 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin op3_type = IS_UNKNOWN; } +// zend_class_entry *op1_ce = NULL; if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) { // op1_ce = (zend_class_entry*)(p+1)->ce; p++; } + + zend_class_entry *op2_ce = NULL; if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { op2_ce = (zend_class_entry*)(p+1)->ce; p++; } + + uint8_t val_type = IS_UNKNOWN; if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) { val_type = (p+1)->op1_type; p++; @@ -2127,7 +2082,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin default: break; } - len = zend_jit_trace_op_len(opline); + int len = zend_jit_trace_op_len(opline); if (ssa->var_info) { /* Add statically inferred ranges */ if (ssa_ops[idx].op1_def >= 0) { @@ -2310,11 +2265,11 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } else if (p->op == ZEND_JIT_TRACE_ENTER) { op_array = p->op_array; - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); ssa = &jit_extension->func_info.ssa; - call = frame->call; + zend_jit_trace_stack_frame *call = frame->call; if (!call) { /* Trace missed INIT_FCALL opcode */ call = top; @@ -2333,9 +2288,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin frame = call; level++; - i = 0; - v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); - while (i < op_array->last_var) { + for (int i = 0, v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); i < op_array->last_var; ++i, ++v) { ssa_vars[v].var = i; if (i < op_array->num_args) { if (ssa->var_info @@ -2376,17 +2329,15 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin /* Propagate argument type */ ssa_var_info[v].type &= STACK_INFO(frame->stack, i); } - i++; - v++; } } else if (p->op == ZEND_JIT_TRACE_BACK) { op_array = p->op_array; - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); ssa = &jit_extension->func_info.ssa; if (level == 0) { - i = 0; - v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); + int i = 0; + int v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); while (i < op_array->last_var) { ssa_vars[v].var = i; if (!ssa->var_info @@ -2456,7 +2407,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { - call = top; + zend_jit_trace_stack_frame *const call = top; TRACE_FRAME_INIT(call, p->func, 0, 0); call->prev = frame->call; call->used_stack = 0; @@ -2495,7 +2446,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { - call = frame->call; + zend_jit_trace_stack_frame *const call = frame->call; if (call) { top = call; frame->call = call->prev; @@ -2533,7 +2484,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin zend_ssa_phi *phi = tssa->blocks[1].phis; op_array = trace_buffer->op_array; - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); ssa = &jit_extension->func_info.ssa; @@ -2699,22 +2650,8 @@ static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, in static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num) { - const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes; - zend_jit_trace_rec *p; - const zend_op_array *op_array; - zend_jit_op_array_trace_extension *jit_extension; - const zend_ssa *op_array_ssa; - const zend_ssa_op *ssa_op; - int i, j, idx, count, level; - int last_idx = -1; - int *start, *end; - uint8_t *flags; - const zend_op_array **vars_op_array; - zend_lifetime_interval **intervals, *list, *ival; - void *checkpoint; - zend_jit_trace_stack_frame *frame; - zend_jit_trace_stack *stack; - uint32_t parent_vars_count = parent_trace ? + const zend_op **const ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes; + const uint32_t parent_vars_count = parent_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0; zend_jit_trace_stack *parent_stack = parent_trace ? zend_jit_traces[parent_trace].stack_map + @@ -2723,34 +2660,34 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace ZEND_ASSERT(ssa->var_info != NULL); - start = do_alloca(sizeof(int) * ssa->vars_count * 2 + + int *const start = do_alloca(sizeof(int) * ssa->vars_count * 2 + ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_op_array*) * ssa->vars_count), use_heap); if (!start) { return NULL; } - end = start + ssa->vars_count; - flags = (uint8_t*)(end + ssa->vars_count); - vars_op_array = (const zend_op_array**)(flags + ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count)); + int *const end = start + ssa->vars_count; + uint8_t *const flags = (uint8_t*)(end + ssa->vars_count); + const zend_op_array **const vars_op_array = (const zend_op_array**)(flags + ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count)); memset(start, -1, sizeof(int) * ssa->vars_count * 2); memset(flags, 0, sizeof(uint8_t) * ssa->vars_count); memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count); - op_array = trace_buffer->op_array; - jit_extension = + const zend_op_array *op_array = trace_buffer->op_array; + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - op_array_ssa = &jit_extension->func_info.ssa; - frame = JIT_G(current_frame); + const zend_ssa *op_array_ssa = &jit_extension->func_info.ssa; + zend_jit_trace_stack_frame *frame = JIT_G(current_frame); frame->prev = NULL; frame->func = (const zend_function*)op_array; - stack = frame->stack; + zend_jit_trace_stack *stack = frame->stack; - count = 0; + int count = 0; - i = 0; - j = op_array->last_var; + int i = 0; + int j = op_array->last_var; if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) { j += op_array->T; } @@ -2803,11 +2740,10 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace } } - p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; - level = 0; - ssa_op = ssa->ops; - idx = 0; - for (;;p++) { + int level = 0; + const zend_ssa_op *ssa_op = ssa->ops; + int idx = 0; + for (zend_jit_trace_rec *p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;; ++p) { if (p->op == ZEND_JIT_TRACE_VM) { const zend_op *opline = p->opline; int len; @@ -3030,7 +2966,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace frame->func = (const zend_function*)p->op_array; stack = frame->stack; op_array = p->op_array; - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); op_array_ssa = &jit_extension->func_info.ssa; j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); @@ -3056,7 +2992,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace zend_jit_close_var(stack, i, start, end, flags, idx-1); } op_array = p->op_array; - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); op_array_ssa = &jit_extension->func_info.ssa; frame = zend_jit_trace_ret_frame(frame, op_array); @@ -3086,6 +3022,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace } } + int last_idx = -1; if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { @@ -3130,10 +3067,10 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace return NULL; } - checkpoint = zend_arena_checkpoint(CG(arena)); - intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval)); + void *const checkpoint = zend_arena_checkpoint(CG(arena)); + zend_lifetime_interval **const intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval)); memset(intervals, 0, sizeof(zend_lifetime_interval*) * ssa->vars_count); - list = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval) * count); + zend_lifetime_interval *list = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval) * count); j = 0; for (i = 0; i < ssa->vars_count; i++) { if (start[i] >= 0 && end[i] >= 0) { @@ -3159,7 +3096,6 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace } count = j; free_alloca(start, use_heap); - start = end = NULL; if (!count) { zend_arena_release(&CG(arena), checkpoint); @@ -3307,22 +3243,19 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace list = zend_jit_sort_intervals(intervals, ssa->vars_count); if (list) { - ival = list; - while (ival) { + for (zend_lifetime_interval *ival = list; ival; ival = ival->list_next) { if (ival->hint) { ival->hint->used_as_hint = ival; } - ival = ival->list_next; + } } if (list) { if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { fprintf(stderr, "---- TRACE %d Live Ranges\n", ZEND_JIT_TRACE_NUM); - ival = list; - while (ival) { + for (zend_lifetime_interval *ival = list; ival; ival = ival->list_next) { zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival); - ival = ival->list_next; } } } @@ -3331,15 +3264,12 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace list = zend_jit_linear_scan(&dummy_op_array, ssa_opcodes, ssa, list); if (list) { - zend_lifetime_interval *ival, *next; - memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*)); - ival = list; count = 0; - while (ival != NULL) { + for (zend_lifetime_interval *ival = list; ival;) { ZEND_ASSERT(ival->reg != ZREG_NONE); count++; - next = ival->list_next; + zend_lifetime_interval *const next = ival->list_next; ival->list_next = intervals[ival->ssa_var]; intervals[ival->ssa_var] = ival; ival = next; @@ -3439,10 +3369,8 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { fprintf(stderr, "---- TRACE %d Allocated Live Ranges\n", ZEND_JIT_TRACE_NUM); for (i = 0; i < ssa->vars_count; i++) { - ival = intervals[i]; - while (ival) { + for (zend_lifetime_interval *ival = intervals[i]; ival; ival = ival->list_next) { zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival); - ival = ival->list_next; } } } @@ -3494,9 +3422,8 @@ static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offse static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op) { - int var = ssa_op->result_def; - int i; - int use = ssa->vars[var].use_chain; + const int var = ssa_op->result_def; + const int use = ssa->vars[var].use_chain; const zend_op *opline; if (use < 0 @@ -3548,7 +3475,7 @@ static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ss } } - for (i = ssa->vars[var].definition; i < use; i++) { + for (int i = ssa->vars[var].definition; i < use; i++) { if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME || ssa_opcodes[i]->opcode == ZEND_DO_FCALL @@ -3562,9 +3489,7 @@ static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ss static bool zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size) { - uint32_t i; - - for (i = 0; i < stack_size; i++) { + for (uint32_t i = 0; i < stack_size; i++) { if (STACK_REG(stack, i) != ZREG_NONE && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) { return true; @@ -3575,17 +3500,15 @@ static bool zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stac static bool zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num) { - const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; - uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags; - uint32_t stack_size; - zend_jit_trace_stack *stack; + const zend_op *const opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; + const uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags; if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) { return true; } - stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size; - stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset; + const uint32_t stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size; + zend_jit_trace_stack *const stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset; return zend_jit_trace_stack_needs_deoptimization(stack, stack_size); } @@ -3599,12 +3522,11 @@ static bool zend_jit_trace_deoptimization(dasm_State **Dst, zend_lifetime_interval **ra, bool polymorphic_side_trace) { - int i; bool has_constants = false; bool has_unsaved_vars = false; // TODO: Merge this loop with the following register LOAD loop to implement parallel move ??? - for (i = 0; i < parent_vars_count; i++) { + for (int i = 0; i < parent_vars_count; i++) { int8_t reg = STACK_REG(parent_stack, i); if (reg != ZREG_NONE) { @@ -3639,7 +3561,7 @@ static bool zend_jit_trace_deoptimization(dasm_State **Dst, if (has_unsaved_vars && (has_constants || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)))) { - for (i = 0; i < parent_vars_count; i++) { + for (int i = 0; i < parent_vars_count; i++) { int8_t reg = STACK_REG(parent_stack, i); if (reg != ZREG_NONE) { @@ -3664,7 +3586,7 @@ static bool zend_jit_trace_deoptimization(dasm_State **Dst, } if (has_constants) { - for (i = 0; i < parent_vars_count; i++) { + for (int i = 0; i < parent_vars_count; i++) { int8_t reg = STACK_REG(parent_stack, i); if (reg != ZREG_NONE) { @@ -3740,17 +3662,15 @@ static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true) { - zend_long op1_min, op1_max, op2_min, op2_max; - if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) { return; } - op1_min = OP1_MIN_RANGE(); - op1_max = OP1_MAX_RANGE(); - op2_min = OP2_MIN_RANGE(); - op2_max = OP2_MAX_RANGE(); + zend_long op1_min = OP1_MIN_RANGE(); + zend_long op1_max = OP1_MAX_RANGE(); + zend_long op2_min = OP2_MIN_RANGE(); + zend_long op2_max = OP2_MAX_RANGE(); switch (opline->opcode) { case ZEND_IS_EQUAL: @@ -3858,8 +3778,6 @@ static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op *const*ssa_opcodes, const zend_op_array *op_array) { - zend_uchar prev_opcode; - if (opline->op1_type == IS_CONST && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) { @@ -3867,7 +3785,7 @@ static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_o if ((ssa_op-1)->op1_def == ssa_op->op2_use) { ssa_op--; opline = ssa_opcodes[ssa_op - ssa->ops]; - prev_opcode = opline->opcode; + const zend_uchar prev_opcode = opline->opcode; if (prev_opcode == ZEND_PRE_INC || prev_opcode == ZEND_PRE_DEC || prev_opcode == ZEND_POST_INC @@ -3877,7 +3795,7 @@ static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_o } else if ((ssa_op-1)->result_def == ssa_op->op2_use) { ssa_op--; opline = ssa_opcodes[ssa_op - ssa->ops]; - prev_opcode = opline->opcode; + const zend_uchar prev_opcode = opline->opcode; if (prev_opcode == ZEND_ADD || prev_opcode == ZEND_SUB) { return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 && @@ -3892,7 +3810,7 @@ static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_o if ((ssa_op-1)->op1_def == ssa_op->op1_use) { ssa_op--; opline = ssa_opcodes[ssa_op - ssa->ops]; - prev_opcode = opline->opcode; + const zend_uchar prev_opcode = opline->opcode; if (prev_opcode == ZEND_PRE_INC || prev_opcode == ZEND_PRE_DEC || prev_opcode == ZEND_POST_INC @@ -3902,7 +3820,7 @@ static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_o } else if ((ssa_op-1)->result_def == ssa_op->op1_use) { ssa_op--; opline = ssa_opcodes[ssa_op - ssa->ops]; - prev_opcode = opline->opcode; + const zend_uchar prev_opcode = opline->opcode; if (prev_opcode == ZEND_ADD || prev_opcode == ZEND_SUB) { return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 && @@ -3912,7 +3830,7 @@ static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_o } } else { const zend_ssa_op *prev_ssa_op = ssa_op - 1; - prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode; + zend_uchar prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode; if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ) && prev_ssa_op != ssa->ops @@ -3985,84 +3903,62 @@ static bool zend_jit_trace_next_is_send_result(const zend_op *oplin static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num) { const void *handler = NULL; - dasm_State* dasm_state = NULL; - zend_script *script = NULL; - zend_lifetime_interval **ra = NULL; - zend_string *name = NULL; - void *checkpoint; - const zend_op_array *op_array; - zend_ssa *ssa, *op_array_ssa; - const zend_op **ssa_opcodes; - zend_jit_trace_rec *p; - zend_jit_op_array_trace_extension *jit_extension; - int num_op_arrays = 0; - zend_jit_trace_info *t; - const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS]; - zend_uchar smart_branch_opcode; - const void *exit_addr; - uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info; + zend_script *const script = NULL; bool send_result = false; - bool skip_comparison; - zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; - zend_class_entry *ce; - bool ce_is_instanceof; - bool on_this = false; bool delayed_fetch_this = false; - bool avoid_refcounting = false; bool polymorphic_side_trace = parent_trace && (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL); - uint32_t i; - zend_jit_trace_stack_frame *frame, *top, *call; - zend_jit_trace_stack *stack; zend_uchar res_type = IS_UNKNOWN; - const zend_op *opline, *orig_opline; - const zend_ssa_op *ssa_op, *orig_ssa_op; - int checked_stack; - int peek_checked_stack; uint32_t frame_flags = 0; JIT_G(current_trace) = trace_buffer; - checkpoint = zend_arena_checkpoint(CG(arena)); + void *const checkpoint = zend_arena_checkpoint(CG(arena)); - ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays); + const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS]; + int num_op_arrays = 0; + zend_ssa *const ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays); if (!ssa) { goto jit_cleanup; } - ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes; + const zend_op **const ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes; /* Register allocation */ + zend_lifetime_interval **ra = NULL; if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num); } - p = trace_buffer; + zend_jit_trace_rec *p = trace_buffer; ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START); - op_array = p->op_array; - frame = JIT_G(current_frame); - top = zend_jit_trace_call_frame(frame, op_array); + const zend_op_array *op_array = p->op_array; + zend_jit_trace_stack_frame *frame = JIT_G(current_frame); + zend_jit_trace_stack_frame *top = zend_jit_trace_call_frame(frame, op_array); TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1); + int checked_stack; + int peek_checked_stack; frame->used_stack = checked_stack = peek_checked_stack = 0; - stack = frame->stack; - for (i = 0; i < op_array->last_var + op_array->T; i++) { + zend_jit_trace_stack *stack = frame->stack; + for (uint32_t i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); } - opline = p[1].opline; - name = zend_jit_trace_name(op_array, opline->lineno); + const zend_op *opline = p[1].opline; + zend_string *const name = zend_jit_trace_name(op_array, opline->lineno); p += ZEND_JIT_TRACE_START_REC_SIZE; + dasm_State* dasm_state = NULL; dasm_init(&dasm_state, DASM_MAXSECTION); dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); dasm_setup(&dasm_state, dasm_actions); - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - op_array_ssa = &jit_extension->func_info.ssa; + zend_ssa *op_array_ssa = &jit_extension->func_info.ssa; dasm_growpc(&dasm_state, 2); /* =>0: loop header */ /* =>1: end of code */ @@ -4113,7 +4009,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par last_var += op_array->T; } - for (i = 0; i < last_var; i++) { + for (uint32_t i = 0; i < last_var; i++) { uint32_t info = ssa->var_info[i].type; if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) { @@ -4190,7 +4086,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (ra && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { - for (i = 0; i < last_var; i++) { + for (uint32_t i = 0; i < last_var; i++) { if (ra[i] && (ra[i]->flags & ZREG_LOAD) != 0 && ra[i]->reg != stack[i].reg) { @@ -4278,7 +4174,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) { last_var += op_array->T; } - for (i = 0; i < last_var; i++) { + for (uint32_t i = 0; i < last_var; i++) { if (ra && ra[i] && (ra[i]->flags & ZREG_LOAD) != 0) { SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD); if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) { @@ -4289,7 +4185,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } - ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL; + const zend_ssa_op *ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL; for (;;p++) { if (p->op == ZEND_JIT_TRACE_VM) { uint8_t op1_type = p->op1_type; @@ -4346,6 +4242,18 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { gen_handler = false; + + zend_uchar smart_branch_opcode; + const void *exit_addr; + uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info; + bool skip_comparison; + zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; + zend_class_entry *ce; + bool ce_is_instanceof; + bool on_this = false; + bool avoid_refcounting = false; + const zend_op *orig_opline; + const zend_ssa_op *orig_ssa_op; switch (opline->opcode) { case ZEND_PRE_INC: case ZEND_PRE_DEC: @@ -5234,7 +5142,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (!exit_addr) { goto jit_failure; } - smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; + const zend_uchar smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; if (!zend_jit_cmp(&dasm_state, opline, op1_info, OP1_RANGE(), OP1_REG_ADDR(), op2_info, OP2_RANGE(), OP2_REG_ADDR(), @@ -5245,7 +5153,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true); } else { - smart_branch_opcode = 0; + const zend_uchar smart_branch_opcode = 0; exit_addr = NULL; if (!zend_jit_cmp(&dasm_state, opline, op1_info, OP1_RANGE(), OP1_REG_ADDR(), @@ -5285,7 +5193,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (opline->opcode == ZEND_IS_NOT_IDENTICAL) { exit_if_true = !exit_if_true; } - smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; + const zend_uchar smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; if (!zend_jit_identical(&dasm_state, opline, op1_info, OP1_RANGE(), OP1_REG_ADDR(), op2_info, OP2_RANGE(), OP2_REG_ADDR(), @@ -5296,7 +5204,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true); } else { - smart_branch_opcode = 0; + const zend_uchar smart_branch_opcode = 0; exit_addr = NULL; if (!zend_jit_identical(&dasm_state, opline, op1_info, OP1_RANGE(), OP1_REG_ADDR(), @@ -6168,8 +6076,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) { gen_handler = true; - op1_info = OP1_INFO(); - op2_info = OP2_INFO(); + uint32_t op1_info = OP1_INFO(); + uint32_t op2_info = OP2_INFO(); if (op1_info & MAY_BE_GUARD) { op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } @@ -6500,7 +6408,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } } else if (p->op == ZEND_JIT_TRACE_ENTER) { - call = frame->call; + zend_jit_trace_stack_frame *const call = frame->call; assert(call && &call->func->op_array == p->op_array); if (opline->opcode == ZEND_DO_UCALL @@ -6560,7 +6468,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (ra) { int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); - for (i = 0; i < op_array->last_var; i++,j++) { + for (uint32_t i = 0; i < op_array->last_var; i++,j++) { if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) { SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD); if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) { @@ -6588,7 +6496,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); - for (i = 0; i < op_array->last_var + op_array->T; i++, j++) { + for (uint32_t i = 0; i < op_array->last_var + op_array->T; i++, j++) { /* Initialize abstract stack using SSA */ if (!(ssa->var_info[j].type & MAY_BE_GUARD) && has_concrete_type(ssa->var_info[j].type)) { @@ -6599,7 +6507,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (ra) { j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); - for (i = 0; i < op_array->last_var + op_array->T; i++, j++) { + for (uint32_t i = 0; i < op_array->last_var + op_array->T; i++, j++) { if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) { SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD); if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) { @@ -6609,7 +6517,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } } else { - for (i = 0; i < op_array->last_var + op_array->T; i++) { + for (uint32_t i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); } } @@ -6635,7 +6543,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par num_args = init_opline->extended_value; } - call = top; + zend_jit_trace_stack_frame *const call = top; TRACE_FRAME_INIT(call, p->func, frame_flags, num_args); call->prev = frame->call; if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { @@ -6674,7 +6582,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par zend_jit_op_array_trace_extension *jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array); - i = 0; + uint32_t i = 0; while (i < p->op_array->num_args) { /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */ SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); @@ -6694,13 +6602,13 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par i++; } } else { - for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) { + for (uint32_t i = 0; i < p->op_array->last_var + p->op_array->T; i++) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); } } } else { ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION); - for (i = 0; i < p->op_array->num_args; i++) { + for (uint32_t i = 0; i < p->op_array->num_args; i++) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); } } @@ -6708,9 +6616,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par bool skip_guard = false; if (init_opline) { - zend_call_info *call_info = jit_extension->func_info.callee_info; - - while (call_info) { + for (const zend_call_info *call_info = jit_extension->func_info.callee_info; call_info; call_info = call_info->next_callee) { if (call_info->caller_init_opline == init_opline && !call_info->is_prototype) { if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) { @@ -6724,7 +6630,6 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par skip_guard = true; break; } - call_info = call_info->next_callee; } if (!skip_guard && !zend_jit_may_be_polymorphic_call(init_opline) @@ -6764,7 +6669,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { - call = frame->call; + zend_jit_trace_stack_frame *const call = frame->call; if (call) { checked_stack -= call->used_stack; top = call; @@ -6777,7 +6682,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END); - t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; + zend_jit_trace_info *const t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; if (!parent_trace && zend_jit_trace_uses_initial_ip()) { t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP; @@ -6787,9 +6692,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { if (ra) { - zend_ssa_phi *phi = ssa->blocks[1].phis; - - while (phi) { + for (zend_ssa_phi *phi = ssa->blocks[1].phis; phi; phi = phi->next) { if (ra[phi->ssa_var] && ra[phi->sources[1]] && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var) @@ -6799,7 +6702,6 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par /* TODO: Alternatively, we may try to update alredy generated deoptimization info */ zend_jit_store_var_type(&dasm_state, phi->var, STACK_TYPE(stack, phi->var)); } - phi = phi->next; } } if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { @@ -6813,10 +6715,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT; } if (!(t->flags & ZEND_JIT_TRACE_LOOP)) { - const void *timeout_exit_addr = NULL; - t->flags |= ZEND_JIT_TRACE_LOOP; + const void *timeout_exit_addr = NULL; if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP) || (ra @@ -6841,8 +6742,6 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par goto jit_failure; } if (p->stop == ZEND_JIT_TRACE_STOP_LINK) { - const void *timeout_exit_addr = NULL; - t->link = zend_jit_find_trace(p->opline->handler); if (t->link == 0) { /* this can happen if ZEND_JIT_EXIT_INVALIDATE was handled @@ -6860,13 +6759,15 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (!parent_trace && zend_jit_trace_uses_initial_ip()) { t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP; } + + const void *timeout_exit_addr = NULL; if (parent_trace && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT) && zend_jit_traces[parent_trace].root == t->link) { if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) { uint32_t exit_point; - for (i = 0; i < op_array->last_var + op_array->T; i++) { + for (uint32_t i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); } exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); @@ -6903,12 +6804,11 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (handler) { if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) { - const zend_op_array *rec_op_array; - - rec_op_array = op_array = trace_buffer->op_array; + const zend_op_array *const rec_op_array = trace_buffer->op_array; + const zend_op_array *op_array = rec_op_array; size_t offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); - p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; - for (;;p++) { + + for (const zend_jit_trace_rec *p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;; p++) { if (p->op == ZEND_JIT_TRACE_VM) { opline = p->opline; } else if (p->op == ZEND_JIT_TRACE_ENTER) { @@ -6940,10 +6840,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par frame = JIT_G(current_frame)->prev; do { if (frame->call_opline) { - op_array = &frame->func->op_array; + const zend_op_array *const op_array = &frame->func->op_array; const size_t offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); zend_jit_trace_setup_ret_counter(frame->call_opline, offset); } frame = frame->prev; @@ -6962,8 +6860,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par jit_cleanup: /* Clean up used op_arrays */ while (num_op_arrays > 0) { - op_array = op_arrays[--num_op_arrays]; - jit_extension = + const zend_op_array *const op_array = op_arrays[--num_op_arrays]; + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); jit_extension->func_info.num = 0; @@ -6985,22 +6883,17 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num) { const void *handler = NULL; - dasm_State* dasm_state = NULL; - void *checkpoint; - char name[32]; - const zend_op *opline; - uint32_t stack_size; - zend_jit_trace_stack *stack; - bool original_handler = false; if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) { return dasm_labels[zend_lbtrace_escape]; } - checkpoint = zend_arena_checkpoint(CG(arena));; + void *const checkpoint = zend_arena_checkpoint(CG(arena));; + char name[32]; sprintf(name, "ESCAPE-%d-%d", trace_num, exit_num); + dasm_State* dasm_state = NULL; dasm_init(&dasm_state, DASM_MAXSECTION); dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); dasm_setup(&dasm_state, dasm_actions); @@ -7008,8 +6901,8 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n zend_jit_align_func(&dasm_state); /* Deoptimization */ - stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size; - stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset; + const uint32_t stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size; + zend_jit_trace_stack *const stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset; if (!zend_jit_trace_deoptimization(&dasm_state, zend_jit_traces[trace_num].exit_info[exit_num].flags, @@ -7018,7 +6911,9 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n goto jit_failure; } - opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; + bool original_handler = false; + + const zend_op *const opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; if (opline) { if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) { /* prevent endless loop */ @@ -7161,13 +7056,10 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace /* Set counting handler back to original VM handler. */ static void zend_jit_stop_hot_trace_counters(zend_op_array *op_array) { - zend_jit_op_array_trace_extension *jit_extension; - uint32_t i; - - jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); zend_shared_alloc_lock(); SHM_UNPROTECT(); - for (i = 0; i < op_array->last; i++) { + for (uint32_t i = 0; i < op_array->last; i++) { /* Opline with Jit-ed code handler is skipped. */ if (jit_extension->trace_info[i].trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) { @@ -7257,9 +7149,8 @@ static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_sto uint8_t *cache_count = JIT_G(bad_root_cache_count); uint8_t *cache_stop = JIT_G(bad_root_cache_stop); uint32_t cache_slot = JIT_G(bad_root_slot); - uint32_t i; - for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { + for (uint32_t i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { if (cache_opline[i] == opline) { if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) { cache_opline[i] = NULL; @@ -7278,7 +7169,7 @@ static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_sto } } } - i = cache_slot; + const uint32_t i = cache_slot; cache_opline[i] = opline; cache_count[i] = 1; cache_stop[i] = stop; @@ -7290,24 +7181,20 @@ static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_sto static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa) { zend_jit_trace_rec *p = trace_buffer; - const zend_op_array *op_array; - const zend_op *opline; uint32_t level = 1 + trace_buffer[0].level; - int idx, len, i, v, vars_count, call_level; ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START); - op_array = p->op_array; + const zend_op_array *op_array = p->op_array; p += ZEND_JIT_TRACE_START_REC_SIZE; - idx = 0; - call_level = 0; if (tssa && tssa->var_info) { + int vars_count; if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { vars_count = op_array->last_var; } else { vars_count = op_array->last_var + op_array->T; } - for (i = 0; i < vars_count; i++) { + for (int i = 0; i < vars_count; i++) { if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) { fprintf(stderr, " %*c;", level, ' '); zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE); @@ -7334,19 +7221,19 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa } } + int idx = 0; + int call_level = 0; while (1) { if (p->op == ZEND_JIT_TRACE_VM) { - uint8_t op1_type, op2_type, op3_type; - - opline = p->opline; + const zend_op *opline = p->opline; fprintf(stderr, "%04d%*c", (int)(opline - op_array->opcodes), level, ' '); zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL); - op1_type = p->op1_type; - op2_type = p->op2_type; - op3_type = p->op3_type; + const uint8_t op1_type = p->op1_type; + const uint8_t op2_type = p->op2_type; + const uint8_t op3_type = p->op3_type; if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) { fprintf(stderr, " ;"); if (op1_type != IS_UNKNOWN) { @@ -7384,15 +7271,13 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa } } if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) { - uint8_t val_type; - const char *type; - if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) { fprintf(stderr, " ;"); } p++; - val_type = p->op1_type; + const uint8_t val_type = p->op1_type; + const char *type; if (val_type == IS_UNDEF) { type = "undef"; } else if (val_type == IS_REFERENCE) { @@ -7405,7 +7290,7 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa fprintf(stderr, "\n"); idx++; - len = zend_jit_trace_op_len(opline); + int len = zend_jit_trace_op_len(opline); while (len > 1) { opline++; fprintf(stderr, "%04d%*c;", @@ -7428,9 +7313,9 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa level++; if (tssa && tssa->var_info) { call_level++; - v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); - vars_count = op_array->last_var; - for (i = 0; i < vars_count; i++, v++) { + int v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); + const int vars_count = op_array->last_var; + for (int i = 0; i < vars_count; i++, v++) { if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) { fprintf(stderr, " %*c;", level, ' '); zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE); @@ -7450,9 +7335,9 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa ZSTR_VAL(op_array->filename)); if (tssa && tssa->var_info) { if (call_level == 0) { - v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); - vars_count = op_array->last_var + op_array->T; - for (i = 0; i < vars_count; i++, v++) { + int v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); + const int vars_count = op_array->last_var + op_array->T; + for (int i = 0; i < vars_count; i++, v++) { if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) { fprintf(stderr, " %*c;", level, ' '); zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE); @@ -7494,10 +7379,8 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa static void zend_jit_dump_exit_info(zend_jit_trace_info *t) { - int i, j; - fprintf(stderr, "---- TRACE %d exit info\n", t->id); - for (i = 0; i < t->exit_count; i++) { + for (int i = 0; i < t->exit_count; i++) { const zend_op_array *op_array = t->exit_info[i].op_array; uint32_t stack_size = t->exit_info[i].stack_size; zend_jit_trace_stack *stack = t->stack_map + t->exit_info[i].stack_offset; @@ -7528,7 +7411,7 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t) if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) { fprintf(stderr, "/FREE_OP2"); } - for (j = 0; j < stack_size; j++) { + for (int j = 0; j < stack_size; j++) { zend_uchar type = STACK_TYPE(stack, j); if (type != IS_UNKNOWN) { fprintf(stderr, " "); @@ -7569,8 +7452,6 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const const zend_op *orig_opline; zend_jit_trace_stop stop; int ret = 0; - zend_op_array *op_array; - uint32_t trace_num; zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH]; ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION); @@ -7578,10 +7459,9 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const opline < EX(func)->op_array.opcodes + EX(func)->op_array.last); repeat: - trace_num = ZEND_JIT_TRACE_NUM; orig_opline = opline; - op_array = &EX(func)->op_array; - const size_t offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); + uint32_t trace_num = ZEND_JIT_TRACE_NUM; + const size_t offset = ZEND_OP_TRACE_INFO_OFFSET(&EX(func)->op_array); EX(opline) = opline; @@ -7703,7 +7583,6 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num) { - const void *handler; bool do_bailout = false; zend_shared_alloc_lock(); @@ -7713,7 +7592,7 @@ static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num) zend_jit_unprotect(); zend_try { - handler = zend_jit_trace_exit_to_vm(trace_num, exit_num); + const void *const handler = zend_jit_trace_exit_to_vm(trace_num, exit_num); if (handler) { zend_jit_link_side_trace( @@ -7893,12 +7772,11 @@ static int zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t par { zend_jit_trace_stop stop; int ret = 0; - uint32_t trace_num; zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH]; uint32_t is_megamorphic = 0; uint32_t polymorphism = 0; - trace_num = ZEND_JIT_TRACE_NUM; + const uint32_t trace_num = ZEND_JIT_TRACE_NUM; /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */ if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) { @@ -8038,12 +7916,10 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe uint32_t trace_num = EG(jit_trace_num); zend_execute_data *execute_data = EG(current_execute_data); const zend_op *orig_opline = EX(opline); - const zend_op *opline; zend_jit_trace_info *t = &zend_jit_traces[trace_num]; bool repeat_last_opline = false; /* Deoptimization of VM stack state */ - uint32_t i; uint32_t stack_size = t->exit_info[exit_num].stack_size; zend_jit_trace_stack *stack = t->stack_map + t->exit_info[exit_num].stack_offset; @@ -8053,7 +7929,7 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe EX(call) = call; } - for (i = 0; i < stack_size; i++) { + for (uint32_t i = 0; i < stack_size; i++) { if (STACK_REG(stack, i) != ZREG_NONE) { if (STACK_TYPE(stack, i) == IS_LONG) { zend_long val; @@ -8115,7 +7991,7 @@ static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registe return true; } - opline = t->exit_info[exit_num].opline; + const zend_op *const opline = t->exit_info[exit_num].opline; if (opline) { if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) { @@ -8250,11 +8126,8 @@ static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array) { - zend_jit_op_array_trace_extension *jit_extension; - uint32_t i; - - jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - for (i = 0; i < op_array->last; i++) { + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + for (uint32_t i = 0; i < op_array->last; i++) { jit_extension->trace_info[i].trace_flags &= ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED; if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) { @@ -8281,13 +8154,9 @@ static int16_t *zend_jit_hot_counter_allocate(void) static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) { - zend_op *opline; - zend_jit_op_array_trace_extension *jit_extension; - uint32_t i; - ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op)); - jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info)); + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info)); if (!jit_extension) { return FAILURE; } @@ -8295,7 +8164,7 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE; jit_extension->op_array = op_array; jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes; - for (i = 0; i < op_array->last; i++) { + for (uint32_t i = 0; i < op_array->last; i++) { jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler; jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]); jit_extension->trace_info[i].counter = NULL; @@ -8313,11 +8182,11 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) return FAILURE; } - for (i = 0; i < cfg.blocks_count; i++) { + for (uint32_t i = 0; i < cfg.blocks_count; i++) { if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) { if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) { /* loop header */ - opline = op_array->opcodes + cfg.blocks[i].start; + zend_op *const opline = op_array->opcodes + cfg.blocks[i].start; zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, jit_extension->offset); if (!(trace_info->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { opline->handler = (const void*)zend_jit_loop_trace_counter_handler; From 5a5ba9dbe70edde8391723ae34b400543c39040c Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Mon, 2 Jan 2023 12:48:29 +0100 Subject: [PATCH 31/34] ext/opcache/jit/zend_jit_vm_helpers: declare variables where they are used (C99) --- ext/opcache/jit/zend_jit_vm_helpers.c | 73 ++++++++++----------------- 1 file changed, 26 insertions(+), 47 deletions(-) diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 959ccb05609c..3ba6b1b6331b 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -53,8 +53,6 @@ register const zend_op* volatile opline __asm__("x28"); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC) { - zend_execute_data *old_execute_data; - if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { zend_clean_and_cache_symbol_table(EX(symbol_table)); } @@ -69,7 +67,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t zend_free_extra_named_params(EX(extra_named_params)); } - old_execute_data = execute_data; + zend_execute_data *const old_execute_data = execute_data; execute_data = EX(prev_execute_data); zend_vm_stack_free_call_frame_ex(call_info, old_execute_data); @@ -132,7 +130,6 @@ void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D) if (EXPECTED(!(op_array->fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))) { uint32_t first_extra_arg = op_array->num_args; uint32_t num_args = EX_NUM_ARGS(); - zval *end, *src, *dst; uint32_t type_flags = 0; if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) { @@ -145,9 +142,9 @@ void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D) } /* move extra args into separate array after all CV and TMP vars */ - end = EX_VAR_NUM(first_extra_arg - 1); - src = end + (num_args - first_extra_arg); - dst = src + (op_array->last_var + op_array->T - first_extra_arg); + zval *const end = EX_VAR_NUM(first_extra_arg - 1); + zval *src = end + (num_args - first_extra_arg); + zval *dst = src + (op_array->last_var + op_array->T - first_extra_arg); if (EXPECTED(src != dst)) { do { type_flags |= Z_TYPE_INFO_P(src); @@ -256,11 +253,10 @@ static zend_always_inline zend_constant* _zend_quick_get_constant( zend_execute_data *execute_data = EG(current_execute_data); #endif const zend_op *opline = EX(opline); - zval *zv; zend_constant *c = NULL; /* null/true/false are resolved during compilation, so don't check for them here. */ - zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key)); + zval *zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key)); if (zv) { c = (zend_constant*)Z_PTR_P(zv); } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) { @@ -395,10 +391,9 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HAN static int zend_jit_trace_recursive_call_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level, int level) { - int i; int count = 0; - for (i = ret_level; i < level; i++) { + for (int i = ret_level; i < level; i++) { count += (unrolled_calls[i] == op_array); } return count; @@ -406,10 +401,9 @@ static int zend_jit_trace_recursive_call_count(const zend_op_array *op_array, co static int zend_jit_trace_recursive_ret_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level) { - int i; int count = 0; - for (i = 0; i < ret_level; i++) { + for (int i = 0; i < ret_level; i++) { count += (unrolled_calls[i] == op_array); } return count; @@ -432,12 +426,11 @@ static uint8_t zend_jit_trace_bad_stop_event(const zend_op *opline, int count) const zend_op **cache_opline = JIT_G(bad_root_cache_opline); uint8_t *cache_count = JIT_G(bad_root_cache_count); uint8_t *cache_stop = JIT_G(bad_root_cache_stop); - uint32_t i; if (count < 0) { count = 0; } - for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { + for (uint32_t i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { if (cache_opline[i] == opline) { if (cache_count[i] >= count) { return cache_stop[i]; @@ -455,9 +448,6 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR; do { - zend_function *func; - zend_jit_op_array_trace_extension *jit_extension; - if (call->prev_execute_data) { idx = zend_jit_trace_record_fake_init_call_ex(call->prev_execute_data, trace_buffer, idx, is_megamorphic, init_level + 1); if (idx < 0) { @@ -465,7 +455,7 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend } } - func = call->func; + zend_function *func = call->func; if (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE|ZEND_ACC_NEVER_CACHE)) { /* TODO: Can we continue recording ??? */ return -1; @@ -476,7 +466,7 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend } if (func->type == ZEND_USER_FUNCTION && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array); if (UNEXPECTED(!jit_extension || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE) @@ -548,18 +538,11 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, zend_execute_data *save_execute_data = execute_data; const zend_op *save_opline = opline; #endif - const zend_op *orig_opline, *end_opline; zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR; zend_jit_trace_stop halt = 0; int level = 0; int ret_level = 0; - zend_vm_opcode_handler_t handler; - const zend_op_array *op_array; - zend_jit_op_array_trace_extension *jit_extension; - size_t offset; - int idx, count; - uint8_t trace_flags, op1_type, op2_type, op3_type; - zend_class_entry *ce1, *ce2; + int idx; const zend_op *link_to_enter_opline = NULL; int backtrack_link_to_enter = -1; int backtrack_recursion = -1; @@ -576,18 +559,17 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, execute_data = ex; opline = EX(opline) = op; #else - int rc; zend_execute_data *execute_data = ex; const zend_op *opline = EX(opline); #endif zend_execute_data *prev_call = EX(call); - orig_opline = opline; + const zend_op *const orig_opline = opline; - op_array = &EX(func)->op_array; - jit_extension = + const zend_op_array *op_array = &EX(func)->op_array; + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - offset = jit_extension->offset; + size_t offset = jit_extension->offset; if (!op_array->function_name || (op_array->fn_flags & ZEND_ACC_CLOSURE)) { op_array = jit_extension->op_array; @@ -619,8 +601,8 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, } while (1) { - ce1 = ce2 = NULL; - op1_type = op2_type = op3_type = IS_UNKNOWN; + zend_class_entry *ce1 = NULL, *ce2 = NULL; + uint8_t op1_type = IS_UNKNOWN, op2_type = IS_UNKNOWN, op3_type = IS_UNKNOWN; if ((opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && opline->opcode != ZEND_ROPE_ADD && opline->opcode != ZEND_ROPE_END @@ -846,7 +828,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, break; } - handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; + const zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; #ifdef HAVE_GCC_GLOBAL_REGS handler(); if (UNEXPECTED(opline == zend_jit_halt_op)) { @@ -857,7 +839,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, } if (UNEXPECTED(execute_data != prev_execute_data)) { #else - rc = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); + int rc = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); if (rc != 0) { if (rc < 0) { stop = ZEND_JIT_TRACE_STOP_RETURN; @@ -873,7 +855,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, #endif op_array = &EX(func)->op_array; - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); if (UNEXPECTED(!jit_extension) || UNEXPECTED(!(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))) { @@ -910,7 +892,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, EX(return_value) != NULL ? ZEND_JIT_TRACE_RETURN_VALUE_USED : 0, op_array); - count = zend_jit_trace_recursive_call_count(&EX(func)->op_array, unrolled_calls, ret_level, level); + const int count = zend_jit_trace_recursive_call_count(&EX(func)->op_array, unrolled_calls, ret_level, level); if (opline == orig_opline) { if (count + 1 >= JIT_G(max_recursive_calls)) { @@ -940,7 +922,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, break; } TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array); - count = zend_jit_trace_recursive_ret_count(&EX(func)->op_array, unrolled_calls, ret_level); + const int count = zend_jit_trace_recursive_ret_count(&EX(func)->op_array, unrolled_calls, ret_level); if (opline == orig_opline) { if (count + 1 >= JIT_G(max_recursive_returns)) { stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET; @@ -998,9 +980,6 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, if (EX(call) != prev_call) { if (EX(call) && EX(call)->prev_execute_data == prev_call) { - zend_function *func; - zend_jit_op_array_trace_extension *jit_extension; - if (EX(call)->func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { /* TODO: Can we continue recording ??? */ stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE; @@ -1010,7 +989,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; break; } - func = EX(call)->func; + zend_function *func = EX(call)->func; if (func->type == ZEND_INTERNAL_FUNCTION && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) { stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; @@ -1018,7 +997,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, } if (func->type == ZEND_USER_FUNCTION && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array); if (UNEXPECTED(!jit_extension) || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE) @@ -1059,7 +1038,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, break; } - trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags; + const uint8_t trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags; if (trace_flags) { if (trace_flags & ZEND_JIT_TRACE_JITED) { if (trace_flags & ZEND_JIT_TRACE_START_LOOP) { @@ -1144,7 +1123,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, } } - end_opline = opline; + const zend_op *end_opline = opline; if (!ZEND_JIT_TRACE_STOP_OK(stop)) { if (backtrack_recursion > 0) { idx = backtrack_recursion; From afa8f848655b772b40090e8375708569db9d7561 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Mon, 2 Jan 2023 12:59:39 +0100 Subject: [PATCH 32/34] ext/opcache/jit/zend_jit_{x86,arm64}: declare variables where they are used (C99) --- ext/opcache/jit/zend_jit_arm64.dasc | 280 ++++++++++------------------ ext/opcache/jit/zend_jit_x86.dasc | 86 +++------ 2 files changed, 126 insertions(+), 240 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 7c2839587b93..c003edd7b784 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2571,11 +2571,9 @@ static int zend_jit_trace_escape_stub(dasm_State **Dst) static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) { - uint32_t i; - | bl >2 |1: - for (i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { + for (uint32_t i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { | bl >2 } |2: @@ -2990,13 +2988,10 @@ static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_t static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) { - uint32_t i; - const void *exit_addr; - /* Emit veneers table for exit points (B instruction for each exit number) */ |.cold_code - for (i = 0; i < t->exit_count; i++) { - exit_addr = zend_jit_trace_get_exit_addr(i); + for (uint32_t i = 0; i < t->exit_count; i++) { + const void *const exit_addr = zend_jit_trace_get_exit_addr(i); if (!exit_addr) { return 0; } @@ -3010,9 +3005,7 @@ static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) { int ret = 0; - uint8_t *p, *end; const void *veneer = NULL; - ptrdiff_t delta; if (jmp_table_size) { const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*))); @@ -3026,8 +3019,8 @@ static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size } while (--jmp_table_size); } - end = (uint8_t*)code; - p = end + size; + uint8_t *const end = (uint8_t*)code; + uint8_t *p = end + size; while (p > end) { uint32_t *ins_ptr; uint32_t ins; @@ -3037,7 +3030,7 @@ static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size ins = *ins_ptr; if ((ins & 0xfc000000u) == 0x14000000u) { // B (imm26:0..25) - delta = (uint32_t*)from_addr - ins_ptr; + ptrdiff_t delta = (uint32_t*)from_addr - ins_ptr; if (((ins ^ (uint32_t)delta) & 0x01ffffffu) == 0) { delta = (uint32_t*)to_addr - ins_ptr; if (((delta + 0x02000000) >> 26) != 0) { @@ -3052,7 +3045,7 @@ static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size } else if ((ins & 0xff000000u) == 0x54000000u || (ins & 0x7e000000u) == 0x34000000u) { // B.cond, CBZ, CBNZ (imm19:5..23) - delta = (uint32_t*)from_addr - ins_ptr; + ptrdiff_t delta = (uint32_t*)from_addr - ins_ptr; if (((ins ^ ((uint32_t)delta << 5)) & 0x00ffffe0u) == 0) { delta = (uint32_t*)to_addr - ins_ptr; if (((delta + 0x40000) >> 19) != 0) { @@ -3070,7 +3063,7 @@ static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size } } else if ((ins & 0x7e000000u) == 0x36000000u) { // TBZ, TBNZ (imm14:5..18) - delta = (uint32_t*)from_addr - ins_ptr; + ptrdiff_t delta = (uint32_t*)from_addr - ins_ptr; if (((ins ^ ((uint32_t)delta << 5)) & 0x0007ffe0u) == 0) { delta = (uint32_t*)to_addr - ins_ptr; if (((delta + 0x2000) >> 14) != 0) { @@ -3105,7 +3098,6 @@ static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) { - const void *link_addr; size_t prologue_size; /* Skip prologue. */ @@ -3126,7 +3118,7 @@ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, // mov FP, FCARG1x prologue_size = 12; } - link_addr = (const void*)((const char*)t->code_start + prologue_size); + const void *const link_addr = (const void*)((const char*)t->code_start + prologue_size); if (timeout_exit_addr) { /* Check timeout for links to LOOP */ @@ -3306,8 +3298,6 @@ static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_arra const zend_op *next_opline = trace->opline; const zend_op *exit_opline = NULL; - uint32_t exit_point; - const void *exit_addr; uint32_t old_info = 0; uint32_t old_res_info = 0; zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; @@ -3354,8 +3344,8 @@ static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_arra old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); @@ -3778,13 +3768,10 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op if (may_overflow && (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { - int32_t exit_point; - const void *exit_addr; - zend_jit_trace_stack *stack; - uint32_t old_op1_info, old_res_info = 0; + uint32_t old_res_info = 0; - stack = JIT_G(current_frame)->stack; - old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); + zend_jit_trace_stack *const stack = JIT_G(current_frame)->stack; + const uint32_t old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); @@ -3808,8 +3795,8 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op } } - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -5078,13 +5065,11 @@ static int zend_jit_concat_helper(dasm_State **Dst, static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) { - zend_jit_addr op1_addr, op2_addr; - ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); - op1_addr = OP1_ADDR(); - op2_addr = OP2_ADDR(); + const zend_jit_addr op1_addr = OP1_ADDR(); + const zend_jit_addr op2_addr = OP2_ADDR(); return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); } @@ -6014,10 +5999,10 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) { - zend_jit_addr op2_addr, op3_addr, res_addr; + zend_jit_addr res_addr; - op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; - op3_addr = OP1_DATA_ADDR(); + const zend_jit_addr op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; + const zend_jit_addr op3_addr = OP1_DATA_ADDR(); if (opline->result_type == IS_UNUSED) { res_addr = 0; } else { @@ -6225,14 +6210,13 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) { - zend_jit_addr op2_addr, op3_addr, var_addr; const void *not_found_exit_addr = NULL; uint32_t var_info = MAY_BE_NULL; ZEND_ASSERT(opline->result_type == IS_UNUSED); - op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; - op3_addr = OP1_DATA_ADDR(); + const zend_jit_addr op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; + const zend_jit_addr op3_addr = OP1_DATA_ADDR(); | SET_EX_OPLINE opline, REG0 if (op1_info & MAY_BE_REF) { @@ -6371,7 +6355,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint3 } } - var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + const zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); switch (opline->extended_value) { case ZEND_ADD: case ZEND_SUB: @@ -6463,13 +6447,11 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint3 static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) { - zend_jit_addr op1_addr, op2_addr; - ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); - op1_addr = OP1_ADDR(); - op2_addr = OP2_ADDR(); + zend_jit_addr op1_addr = OP1_ADDR(); + const zend_jit_addr op2_addr = OP2_ADDR(); if (op1_info & MAY_BE_REF) { binary_op_type binary_op = get_binary_op(opline->extended_value); @@ -8638,9 +8620,6 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) { - int32_t exit_point; - const void *exit_addr; - if (func->type == ZEND_INTERNAL_FUNCTION) { #ifdef ZEND_WIN32 // TODO: ASLR may cause different addresses in different workers ??? @@ -8656,8 +8635,8 @@ static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zen return 0; } - exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -8834,12 +8813,11 @@ static int zend_jit_init_method_call(dasm_State **Dst, zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; zend_function *func = NULL; - zval *function_name; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); - function_name = RT_CONSTANT(opline, opline->op2); + zval *const function_name = RT_CONSTANT(opline, opline->op2); if (info) { call_info = info->callee_info; @@ -8954,11 +8932,8 @@ static int zend_jit_init_method_call(dasm_State **Dst, && trace->op == ZEND_JIT_TRACE_INIT_CALL && trace->func ) { - int32_t exit_point; - const void *exit_addr; - - exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -9073,14 +9048,10 @@ static int zend_jit_init_closure_call(dasm_State **Dst, && trace->op == ZEND_JIT_TRACE_INIT_CALL && trace->func && trace->func->type == ZEND_USER_FUNCTION) { - const zend_op *opcodes; - int32_t exit_point; - const void *exit_addr; - func = (zend_function*)trace->func; - opcodes = func->op_array.opcodes; - exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const zend_op *const opcodes = func->op_array.opcodes; + const int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -9126,12 +9097,10 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; const zend_function *func = NULL; - uint32_t i; zend_jit_addr res_addr; uint32_t call_num_args = 0; bool unknown_num_args = 0; const void *exit_addr = NULL; - const zend_op *prev_opline; if (RETURN_VALUE_USED(opline)) { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); @@ -9140,7 +9109,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RSP, TMP_ZVAL_OFFSET); } - prev_opline = opline - 1; + const zend_op *prev_opline = opline - 1; while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { prev_opline--; } @@ -9350,7 +9319,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // opline = op_array->opcodes; if (func && !unknown_num_args) { | ADD_SUB_64_WITH_CONST_32 add, TMP1, RX, (EX_NUM_TO_VAR(call_num_args) + offsetof(zval, u1.type_info)), TMP1 // induction variable - for (i = call_num_args; i < func->op_array.last_var; i++) { + for (uint32_t i = call_num_args; i < func->op_array.last_var; i++) { | // ZVAL_UNDEF(EX_VAR(n)) | str wzr, [TMP1], #16 } @@ -9601,7 +9570,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // zend_vm_stack_free_args(call); if (func && !unknown_num_args) { - for (i = 0; i < call_num_args; i++ ) { + for (uint32_t i = 0; i < call_num_args; i++ ) { if (zend_jit_needs_arg_dtor(func, i, call_info)) { uint32_t offset = EX_NUM_TO_VAR(i); zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset); @@ -9719,7 +9688,6 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) { uint32_t arg_num = opline->op2.num; - zend_jit_addr arg_addr; ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); @@ -9768,7 +9736,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o } } - arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + const zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); @@ -9804,10 +9772,8 @@ static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) { - zend_jit_addr op1_addr, arg_addr, ref_addr; - - op1_addr = OP1_ADDR(); - arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + zend_jit_addr op1_addr = OP1_ADDR(); + const zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (!zend_jit_reuse_ip(Dst)) { return 0; @@ -9862,7 +9828,7 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend | movz TMP1w, #GC_REFERENCE | str TMP1w, [REG0, #offsetof(zend_reference, gc.u.type_info)] | str xzr, [REG0, #offsetof(zend_reference, sources.ptr)] - ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); + const zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); if (opline->op1_type == IS_VAR) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); @@ -9889,13 +9855,12 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) { uint32_t arg_num = opline->op2.num; - zend_jit_addr arg_addr; ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || arg_num <= MAX_ARG_FLAG_NUM); - arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + const zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (!zend_jit_reuse_ip(Dst)) { return 0; @@ -10323,7 +10288,6 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { - uint32_t mask; zend_jit_addr op1_addr = OP1_ADDR(); // TODO: support for is_resource() ??? @@ -10366,7 +10330,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t } if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - mask = opline->extended_value; + const uint32_t mask = opline->extended_value; if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 if (exit_addr) { @@ -11618,13 +11582,11 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, uint32_t target_label2, const void *exit_addr) { - zend_jit_addr op2_addr, res_addr; - // TODO: support for empty() ??? ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); - op2_addr = OP2_ADDR(); - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + const zend_jit_addr op2_addr = OP2_ADDR(); + const zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr @@ -12115,7 +12077,6 @@ static int zend_jit_fetch_obj(dasm_State **Dst, int may_throw) { zval *member; - zend_property_info *prop_info; bool may_be_dynamic = 1; zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); @@ -12128,7 +12089,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); - prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); + zend_property_info *prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 @@ -12364,11 +12325,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { uint32_t flags = 0; - uint32_t old_info; zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; - int32_t exit_point; - const void *exit_addr; - uint32_t type; zend_jit_addr val_addr = prop_addr; if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) @@ -12386,13 +12343,15 @@ static int zend_jit_fetch_obj(dasm_State **Dst, ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; } - type = concrete_type(res_info); + const uint32_t type = concrete_type(res_info); + + const void *exit_addr; if (prop_type != IS_UNKNOWN && prop_type != IS_UNDEF && prop_type != IS_REFERENCE && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) { - exit_point = zend_jit_trace_get_exit_point(opline, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -12404,10 +12363,10 @@ static int zend_jit_fetch_obj(dasm_State **Dst, SET_STACK_REG(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); } - old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + const uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); - exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline+1, flags); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -12587,9 +12546,6 @@ static int zend_jit_incdec_obj(dasm_State **Dst, zend_class_entry *trace_ce, uint8_t prop_type) { - zval *member; - zend_string *name; - zend_property_info *prop_info; zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr res_addr = 0; zend_jit_addr prop_addr; @@ -12605,10 +12561,10 @@ static int zend_jit_incdec_obj(dasm_State **Dst, res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } - member = RT_CONSTANT(opline, opline->op2); + zval *const member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); - name = Z_STR_P(member); - prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); + zend_string *const name = Z_STR_P(member); + zend_property_info *prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 @@ -12923,12 +12879,10 @@ static int zend_jit_incdec_obj(dasm_State **Dst, && (res_info & MAY_BE_LONG)) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); - int32_t exit_point; - const void *exit_addr; SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -13014,9 +12968,6 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, zend_class_entry *trace_ce, uint8_t prop_type) { - zval *member; - zend_string *name; - zend_property_info *prop_info; zend_jit_addr val_addr = OP1_DATA_ADDR(); zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr prop_addr; @@ -13029,10 +12980,10 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, ZEND_ASSERT(op1_info & MAY_BE_OBJECT); ZEND_ASSERT(opline->result_type == IS_UNUSED); - member = RT_CONSTANT(opline, opline->op2); + zval *const member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); - name = Z_STR_P(member); - prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); + zend_string *const name = Z_STR_P(member); + zend_property_info *prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 @@ -13396,9 +13347,6 @@ static int zend_jit_assign_obj(dasm_State **Dst, uint8_t prop_type, int may_throw) { - zval *member; - zend_string *name; - zend_property_info *prop_info; zend_jit_addr val_addr = OP1_DATA_ADDR(); zend_jit_addr res_addr = 0; zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); @@ -13413,10 +13361,10 @@ static int zend_jit_assign_obj(dasm_State **Dst, ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); - member = RT_CONSTANT(opline, opline->op2); + zval *const member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); - name = Z_STR_P(member); - prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); + zend_string *const name = Z_STR_P(member); + zend_property_info *prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 @@ -13698,12 +13646,9 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_i static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) { if (opline->op1_type == IS_CONST) { - zval *zv; - size_t len; - - zv = RT_CONSTANT(opline, opline->op1); + zval *const zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); - len = Z_STRLEN_P(zv); + const size_t len = Z_STRLEN_P(zv); if (len > 0) { const char *str = Z_STRVAL_P(zv); @@ -13739,12 +13684,9 @@ static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_i static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) { if (opline->op1_type == IS_CONST) { - zval *zv; - size_t len; - - zv = RT_CONSTANT(opline, opline->op1); + zval *const zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); - len = Z_STRLEN_P(zv); + const size_t len = Z_STRLEN_P(zv); | SET_ZVAL_LVAL res_addr, len, TMP1, TMP2 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { @@ -13775,12 +13717,9 @@ static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1 static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) { if (opline->op1_type == IS_CONST) { - zval *zv; - zend_long count; - - zv = RT_CONSTANT(opline, opline->op1); + zval *const zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); - count = zend_hash_num_elements(Z_ARRVAL_P(zv)); + const zend_long count = zend_hash_num_elements(Z_ARRVAL_P(zv)); | SET_ZVAL_LVAL res_addr, count, TMP1, TMP2 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { @@ -13871,13 +13810,6 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) { - uint32_t count; - Bucket *p; - const zend_op *target; - int b; - int32_t exit_point; - const void *exit_addr; - if (default_label) { | cbz REG0, &default_label } else if (next_opline) { @@ -13905,8 +13837,8 @@ static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend trace_info->jmp_table_size += zend_hash_num_elements(jumptable); } - count = jumptable->nNumUsed; - p = jumptable->arData; + uint32_t count = jumptable->nNumUsed; + Bucket *p = jumptable->arData; do { if (Z_TYPE(p->val) == IS_UNDEF) { if (default_label) { @@ -13917,15 +13849,15 @@ static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend | .addr =>default_b } } else { - target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); + const zend_op *const target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); if (!next_opline) { - b = ssa->cfg.map[target - op_array->opcodes]; + const int b = ssa->cfg.map[target - op_array->opcodes]; | .addr =>b } else if (next_opline == target) { | .addr >3 } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(target, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -13999,24 +13931,20 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o uint32_t op1_info = OP1_INFO(); zend_jit_addr op1_addr = OP1_ADDR(); const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); - const zend_op *target; int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; - int b; - int32_t exit_point; const void *fallback_label = NULL; const void *default_label = NULL; - const void *exit_addr; if (next_opline) { if (next_opline != opline + 1) { - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); fallback_label = zend_jit_trace_get_exit_addr(exit_point); if (!fallback_label) { return 0; } } if (next_opline != default_opline) { - exit_point = zend_jit_trace_get_exit_point(default_opline, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(default_opline, 0); default_label = zend_jit_trace_get_exit_addr(exit_point); if (!default_label) { return 0; @@ -14091,15 +14019,15 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o | .addr =>default_b } } else { - target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); + const zend_op *const target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); if (!next_opline) { - b = ssa->cfg.map[target - op_array->opcodes]; + const int b = ssa->cfg.map[target - op_array->opcodes]; | .addr =>b } else if (next_opline == target) { | .addr >3 } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(target, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -14509,7 +14437,6 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o if (!exit_addr || exit_opcode == ZEND_JMP) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); - uint32_t val_info; if (RETURN_VALUE_USED(opline)) { zend_jit_addr res_addr = RES_ADDR(); @@ -14569,7 +14496,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o | MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 } - val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); + uint32_t val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); if (val_info & MAY_BE_ARRAY) { val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } @@ -14625,14 +14552,12 @@ static int zend_jit_fetch_constant(dasm_State **Dst, if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); - int32_t exit_point; - const void *exit_addr = NULL; SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); - exit_point = zend_jit_trace_get_exit_point(opline+1, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline+1, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -14723,18 +14648,15 @@ static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t o static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) { - uint32_t offset; - - offset = (opline->opcode == ZEND_ROPE_INIT) ? + const uint32_t offset = (opline->opcode == ZEND_ROPE_INIT) ? opline->result.var : opline->op1.var + opline->extended_value * sizeof(zend_string*); if (opline->op2_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op2); - zend_string *str; ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); - str = Z_STR_P(zv); + zend_string *const str = Z_STR_P(zv); | LOAD_ADDR REG0, str | MEM_ACCESS_64_WITH_UOFFSET str, REG0, FP, offset, TMP1 } else { @@ -14837,8 +14759,6 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, { zend_jit_addr var_addr = *var_addr_ptr; uint32_t var_info = *var_info_ptr; - int32_t exit_point; - const void *exit_addr; if (add_indirect_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); @@ -14871,8 +14791,8 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, if (!(var_type & IS_TRACE_REFERENCE) && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { - exit_point = zend_jit_trace_get_exit_point(opline, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -15368,18 +15288,14 @@ void **dasm_labels_veneers = NULL; static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset) { - void *veneer; - ptrdiff_t na; - int n, m; - /* try to reuse veneers for global labels */ if ((ins >> 16) == DASM_REL_LG && *(b-1) < 0 && dasm_labels_veneers[-*(b-1)]) { - veneer = dasm_labels_veneers[-*(b-1)]; - na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; - n = (int)na; + void *const veneer = dasm_labels_veneers[-*(b-1)]; + const ptrdiff_t na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; + const int n = (int)na; /* check if we can jump to veneer */ if ((ptrdiff_t)n != na) { @@ -15414,9 +15330,9 @@ static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int ZEND_ASSERT(exit_point < t->exit_count); - veneer = (char*)buffer + dasm_getpclabel(&Dst, 1) - (t->exit_count - exit_point) * 4; - na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; - n = (int)na; + void *const veneer = (char*)buffer + dasm_getpclabel(&Dst, 1) - (t->exit_count - exit_point) * 4; + const ptrdiff_t na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; + const int n = (int)na; /* check if we can jump to veneer */ if ((ptrdiff_t)n != na) { @@ -15452,14 +15368,14 @@ static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int } } - veneer = (char*)buffer + (Dst->codesize + dasm_venners_size); + void *const veneer = (char*)buffer + (Dst->codesize + dasm_venners_size); if (veneer > dasm_end) { return 0; /* jit_buffer_size overflow */ } - na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; - n = (int)na; + ptrdiff_t na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; + const int n = (int)na; /* check if we can jump to veneer */ if ((ptrdiff_t)n != na) { @@ -15498,7 +15414,7 @@ static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int /* check if we can use B to jump from veneer */ na = (ptrdiff_t)cp + offset - (ptrdiff_t)veneer - 4; - m = (int)na; + const int m = (int)na; if ((ptrdiff_t)m != na) { ZEND_ASSERT(0); return 0; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 489da1c99fdf..5c06277c6174 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -14230,9 +14230,6 @@ static int zend_jit_assign_obj(dasm_State **Dst, uint8_t prop_type, int may_throw) { - zval *member; - zend_string *name; - zend_property_info *prop_info; zend_jit_addr val_addr = OP1_DATA_ADDR(); zend_jit_addr res_addr = 0; zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); @@ -14247,10 +14244,10 @@ static int zend_jit_assign_obj(dasm_State **Dst, ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); - member = RT_CONSTANT(opline, opline->op2); + zval *const member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); - name = Z_STR_P(member); - prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); + zend_string *const name = Z_STR_P(member); + zend_property_info *prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1a, this_addr @@ -14581,12 +14578,9 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_i static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) { if (opline->op1_type == IS_CONST) { - zval *zv; - size_t len; - - zv = RT_CONSTANT(opline, opline->op1); + zval *const zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); - len = Z_STRLEN_P(zv); + const size_t len = Z_STRLEN_P(zv); if (len > 0) { const char *str = Z_STRVAL_P(zv); @@ -14636,12 +14630,9 @@ static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_i static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) { if (opline->op1_type == IS_CONST) { - zval *zv; - size_t len; - - zv = RT_CONSTANT(opline, opline->op1); + zval *const zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); - len = Z_STRLEN_P(zv); + const size_t len = Z_STRLEN_P(zv); | SET_ZVAL_LVAL res_addr, len if (Z_MODE(res_addr) == IS_MEM_ZVAL) { @@ -14672,12 +14663,9 @@ static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1 static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) { if (opline->op1_type == IS_CONST) { - zval *zv; - zend_long count; - - zv = RT_CONSTANT(opline, opline->op1); + zval *const zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); - count = zend_hash_num_elements(Z_ARRVAL_P(zv)); + const zend_long count = zend_hash_num_elements(Z_ARRVAL_P(zv)); | SET_ZVAL_LVAL res_addr, count if (Z_MODE(res_addr) == IS_MEM_ZVAL) { @@ -14768,13 +14756,6 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) { - uint32_t count; - Bucket *p; - const zend_op *target; - int b; - int32_t exit_point; - const void *exit_addr; - | test r0, r0 if (default_label) { | jz &default_label @@ -14813,8 +14794,8 @@ static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend trace_info->jmp_table_size += zend_hash_num_elements(jumptable); } - count = jumptable->nNumUsed; - p = jumptable->arData; + uint32_t count = jumptable->nNumUsed; + Bucket *p = jumptable->arData; do { if (Z_TYPE(p->val) == IS_UNDEF) { if (default_label) { @@ -14825,15 +14806,15 @@ static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend | .aword =>default_b } } else { - target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); + const zend_op *const target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); if (!next_opline) { - b = ssa->cfg.map[target - op_array->opcodes]; + const int b = ssa->cfg.map[target - op_array->opcodes]; | .aword =>b } else if (next_opline == target) { | .aword >3 } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(target, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -14866,7 +14847,6 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); zval *jump_zv = NULL; - int b; if (opline->opcode == ZEND_SWITCH_LONG) { if (Z_TYPE_P(zv) == IS_LONG) { @@ -14895,6 +14875,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o } ZEND_ASSERT(target == next_opline); } else { + int b; if (jump_zv != NULL) { b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; } else { @@ -14907,24 +14888,20 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o uint32_t op1_info = OP1_INFO(); zend_jit_addr op1_addr = OP1_ADDR(); const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); - const zend_op *target; int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; - int b; - int32_t exit_point; const void *fallback_label = NULL; const void *default_label = NULL; - const void *exit_addr; if (next_opline) { if (next_opline != opline + 1) { - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); fallback_label = zend_jit_trace_get_exit_addr(exit_point); if (!fallback_label) { return 0; } } if (next_opline != default_opline) { - exit_point = zend_jit_trace_get_exit_point(default_opline, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(default_opline, 0); default_label = zend_jit_trace_get_exit_addr(exit_point); if (!default_label) { return 0; @@ -15003,15 +14980,15 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o | .aword =>default_b } } else { - target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); + const zend_op *const target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); if (!next_opline) { - b = ssa->cfg.map[target - op_array->opcodes]; + const int b = ssa->cfg.map[target - op_array->opcodes]; | .aword =>b } else if (next_opline == target) { | .aword >3 } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(target, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -15542,14 +15519,12 @@ static int zend_jit_fetch_constant(dasm_State **Dst, if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); - int32_t exit_point; - const void *exit_addr = NULL; SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); - exit_point = zend_jit_trace_get_exit_point(opline+1, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline+1, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -15643,18 +15618,15 @@ static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t o static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) { - uint32_t offset; - - offset = (opline->opcode == ZEND_ROPE_INIT) ? + const uint32_t offset = (opline->opcode == ZEND_ROPE_INIT) ? opline->result.var : opline->op1.var + opline->extended_value * sizeof(zend_string*); if (opline->op2_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op2); - zend_string *str; ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); - str = Z_STR_P(zv); + zend_string *const str = Z_STR_P(zv); | ADDR_STORE aword [FP + offset], str, r0 } else { zend_jit_addr op2_addr = OP2_ADDR(); @@ -15756,8 +15728,6 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, { zend_jit_addr var_addr = *var_addr_ptr; uint32_t var_info = *var_info_ptr; - int32_t exit_point; - const void *exit_addr; if (add_indirect_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); @@ -15790,8 +15760,8 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, if (!(var_type & IS_TRACE_REFERENCE) && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { - exit_point = zend_jit_trace_get_exit_point(opline, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; From 3f28de67f53393e5487b3520372e8eea3e2b381c Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Mon, 2 Jan 2023 23:24:13 +0100 Subject: [PATCH 33/34] ext/opcache/jit/trace: split zend_jit_trace_frame_size() --- ext/opcache/jit/zend_jit_trace.c | 81 +++++++++++++++++++------------- 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index ff287a12088f..21263ad1a15e 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -451,25 +451,40 @@ static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa #define CHECK_OP1_DATA_TRACE_TYPE() \ CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type) -static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array) +static zend_always_inline size_t zend_jit_trace_op_array_frame_size(const zend_op_array *op_array) { - if (op_array && op_array->type == ZEND_USER_FUNCTION) { + if (op_array) { + assert(op_array->type == ZEND_USER_FUNCTION); return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack))); - } else if (op_array) { - return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack))); } else { return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack)); } } -static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) +static zend_always_inline size_t zend_jit_trace_function_frame_size(const zend_function *func) { - return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array)); + if (func && func->type == ZEND_USER_FUNCTION) { + return zend_jit_trace_op_array_frame_size(&func->op_array); + } else if (func) { + return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(func->common.num_args * sizeof(zend_jit_trace_stack))); + } else { + return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack)); + } +} + +static zend_jit_trace_stack_frame* zend_jit_trace_op_array_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) +{ + return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_op_array_frame_size(op_array)); +} + +static zend_jit_trace_stack_frame* zend_jit_trace_function_call_frame(zend_jit_trace_stack_frame *frame, const zend_function *func) +{ + return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_function_frame_size(func)); } -static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) +static zend_jit_trace_stack_frame* zend_jit_trace_op_array_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) { - return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array)); + return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_op_array_frame_size(op_array)); } static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, zend_uchar type) @@ -1160,7 +1175,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin * Calculate size of abstract stack; * Construct regular SSA for involved op_array */ const zend_op_array *op_array = trace_buffer->op_array; - size_t stack_size = zend_jit_trace_frame_size(op_array); + size_t stack_size = zend_jit_trace_op_array_frame_size(op_array); size_t stack_top = stack_size; size_t stack_bottom = 0; int ssa_ops_count = 0; @@ -1200,7 +1215,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa_ops_count += zend_jit_trace_op_len(p->opline); } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { call_level++; - stack_top += zend_jit_trace_frame_size(p->op_array); + stack_top += zend_jit_trace_function_frame_size(p->func); if (stack_top > stack_size) { stack_size = stack_top; } @@ -1213,7 +1228,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS; } } - const size_t frame_size = zend_jit_trace_frame_size(p->op_array); + const size_t frame_size = zend_jit_trace_function_frame_size(p->func); if (call_level == 0) { if (stack_top + frame_size > stack_size) { stack_size = stack_top + frame_size; @@ -1225,7 +1240,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } else if (p->op == ZEND_JIT_TRACE_ENTER) { op_array = p->op_array; if (call_level == 0) { - stack_top += zend_jit_trace_frame_size(op_array); + stack_top += zend_jit_trace_op_array_frame_size(op_array); if (stack_top > stack_size) { stack_size = stack_top; } @@ -1250,7 +1265,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } else if (p->op == ZEND_JIT_TRACE_BACK) { if (level == 0) { - stack_bottom += zend_jit_trace_frame_size(p->op_array); + stack_bottom += zend_jit_trace_op_array_frame_size(p->op_array); zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); ssa = &jit_extension->func_info.ssa; @@ -1267,7 +1282,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa = zend_jit_trace_build_ssa(op_array, script); } } else { - stack_top -= zend_jit_trace_frame_size(op_array); + stack_top -= zend_jit_trace_op_array_frame_size(op_array); level--; } op_array = p->op_array; @@ -1366,7 +1381,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin idx++; } } else if (p->op == ZEND_JIT_TRACE_ENTER) { - frame = zend_jit_trace_call_frame(frame, op_array); + frame = zend_jit_trace_op_array_call_frame(frame, op_array); stack = frame->stack; op_array = p->op_array; level++; @@ -1379,7 +1394,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } else if (p->op == ZEND_JIT_TRACE_BACK) { op_array = p->op_array; - frame = zend_jit_trace_ret_frame(frame, op_array); + frame = zend_jit_trace_op_array_ret_frame(frame, op_array); stack = frame->stack; if (level == 0) { if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) { @@ -1582,7 +1597,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } frame = JIT_G(current_frame); - zend_jit_trace_stack_frame *top = zend_jit_trace_call_frame(frame, op_array); + zend_jit_trace_stack_frame *top = zend_jit_trace_op_array_call_frame(frame, op_array); TRACE_FRAME_INIT(frame, op_array, 0, 0); TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); frame->used_stack = 0; @@ -2275,7 +2290,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin call = top; TRACE_FRAME_INIT(call, op_array, 0, 0); call->used_stack = 0; - top = zend_jit_trace_call_frame(top, op_array); + top = zend_jit_trace_op_array_call_frame(top, op_array); for (i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); } @@ -2397,7 +2412,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ZEND_ASSERT(&frame->func->op_array == op_array); } else { max_used_stack = used_stack = -1; - frame = zend_jit_trace_ret_frame(frame, op_array); + frame = zend_jit_trace_op_array_ret_frame(frame, op_array); TRACE_FRAME_INIT(frame, op_array, 0, 0); TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); frame->used_stack = 0; @@ -2412,9 +2427,9 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin call->prev = frame->call; call->used_stack = 0; frame->call = call; - top = zend_jit_trace_call_frame(top, p->op_array); + top = zend_jit_trace_function_call_frame(top, p->func); if (p->func && p->func->type == ZEND_USER_FUNCTION) { - for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) { + for (i = 0; i < p->func->op_array.last_var + p->func->op_array.T; i++) { SET_STACK_INFO(call->stack, i, -1); } } @@ -2961,7 +2976,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace /* New call frames */ zend_jit_trace_stack_frame *prev_frame = frame; - frame = zend_jit_trace_call_frame(frame, op_array); + frame = zend_jit_trace_op_array_call_frame(frame, op_array); frame->prev = prev_frame; frame->func = (const zend_function*)p->op_array; stack = frame->stack; @@ -2995,7 +3010,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); op_array_ssa = &jit_extension->func_info.ssa; - frame = zend_jit_trace_ret_frame(frame, op_array); + frame = zend_jit_trace_op_array_ret_frame(frame, op_array); stack = frame->stack; if (level == 0) { /* New return frames */ @@ -3937,7 +3952,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START); const zend_op_array *op_array = p->op_array; zend_jit_trace_stack_frame *frame = JIT_G(current_frame); - zend_jit_trace_stack_frame *top = zend_jit_trace_call_frame(frame, op_array); + zend_jit_trace_stack_frame *top = zend_jit_trace_op_array_call_frame(frame, op_array); TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1); int checked_stack; int peek_checked_stack; @@ -6489,7 +6504,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par stack = frame->stack; ZEND_ASSERT(&frame->func->op_array == op_array); } else { - frame = zend_jit_trace_ret_frame(frame, op_array); + frame = zend_jit_trace_op_array_ret_frame(frame, op_array); TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1); frame->used_stack = checked_stack = peek_checked_stack = 0; stack = frame->stack; @@ -6575,40 +6590,40 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } frame->call = call; - top = zend_jit_trace_call_frame(top, p->op_array); + top = zend_jit_trace_function_call_frame(top, p->func); if (p->func) { if (p->func->type == ZEND_USER_FUNCTION) { if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array); + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&p->func->op_array); uint32_t i = 0; - while (i < p->op_array->num_args) { + while (i < p->func->op_array.num_args) { /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */ SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); i++; } - while (i < p->op_array->last_var) { + while (i < p->func->op_array.last_var) { if (jit_extension - && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) { + && zend_jit_var_may_alias(&p->func->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); } else { SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1); } i++; } - while (i < p->op_array->last_var + p->op_array->T) { + while (i < p->func->op_array.last_var + p->func->op_array.T) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); i++; } } else { - for (uint32_t i = 0; i < p->op_array->last_var + p->op_array->T; i++) { + for (uint32_t i = 0; i < p->func->op_array.last_var + p->func->op_array.T; i++) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); } } } else { ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION); - for (uint32_t i = 0; i < p->op_array->num_args; i++) { + for (uint32_t i = 0; i < p->func->common.num_args; i++) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); } } From 3333ab917d237b9392145980fc1fb9cf08d108f4 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 3 Jan 2023 13:37:16 +0100 Subject: [PATCH 34/34] ext/opcache/jit/internal: add zend_jit_op_array_trace_ssa() --- ext/opcache/jit/zend_jit_internal.h | 11 ++++++++ ext/opcache/jit/zend_jit_trace.c | 44 ++++++++--------------------- 2 files changed, 22 insertions(+), 33 deletions(-) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 081ea615326f..c35829ef4703 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -527,6 +527,17 @@ static zend_always_inline zend_op_trace_info *ZEND_OP_TRACE_INFO2(const zend_op_ return ZEND_OP_TRACE_INFO(opline, jit_extension->offset); } +/** + * Access the #zend_ssa of an #zend_op_array. + */ +static zend_always_inline zend_ssa *zend_jit_op_array_trace_ssa(const zend_op_array *op_array) +{ + zend_jit_op_array_trace_extension *jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + ZEND_ASSERT(jit_extension != NULL); + return &jit_extension->func_info.ssa; +} + /* Recorder */ typedef enum _zend_jit_trace_op { ZEND_JIT_TRACE_VM, diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 21263ad1a15e..506c7b8c2e6e 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -1248,9 +1248,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin call_level--; } level++; - zend_jit_op_array_trace_extension *const jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - ssa = &jit_extension->func_info.ssa; + ssa = zend_jit_op_array_trace_ssa(op_array); if (ssa->cfg.blocks_count) { /* pass */ } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) { @@ -1266,9 +1264,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } else if (p->op == ZEND_JIT_TRACE_BACK) { if (level == 0) { stack_bottom += zend_jit_trace_op_array_frame_size(p->op_array); - zend_jit_op_array_trace_extension *const jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - ssa = &jit_extension->func_info.ssa; + ssa = zend_jit_op_array_trace_ssa(op_array); if (ssa->cfg.blocks_count) { /* pass */ } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) { @@ -1488,9 +1484,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin /* 4. Type inference */ op_array = trace_buffer->op_array; - zend_jit_op_array_trace_extension *const jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - ssa = &jit_extension->func_info.ssa; + ssa = zend_jit_op_array_trace_ssa(op_array); zend_ssa_var_info *const ssa_var_info = tssa->var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info)); @@ -2280,9 +2274,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } else if (p->op == ZEND_JIT_TRACE_ENTER) { op_array = p->op_array; - zend_jit_op_array_trace_extension *const jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - ssa = &jit_extension->func_info.ssa; + ssa = zend_jit_op_array_trace_ssa(op_array); zend_jit_trace_stack_frame *call = frame->call; if (!call) { @@ -2347,9 +2339,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } else if (p->op == ZEND_JIT_TRACE_BACK) { op_array = p->op_array; - zend_jit_op_array_trace_extension *const jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - ssa = &jit_extension->func_info.ssa; + ssa = zend_jit_op_array_trace_ssa(op_array); if (level == 0) { int i = 0; int v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); @@ -2499,9 +2489,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin zend_ssa_phi *phi = tssa->blocks[1].phis; op_array = trace_buffer->op_array; - zend_jit_op_array_trace_extension *const jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - ssa = &jit_extension->func_info.ssa; + ssa = zend_jit_op_array_trace_ssa(op_array); while (phi) { uint32_t t = ssa_var_info[phi->ssa_var].type; @@ -2691,9 +2679,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count); const zend_op_array *op_array = trace_buffer->op_array; - zend_jit_op_array_trace_extension *const jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - const zend_ssa *op_array_ssa = &jit_extension->func_info.ssa; + const zend_ssa *op_array_ssa = zend_jit_op_array_trace_ssa(op_array); zend_jit_trace_stack_frame *frame = JIT_G(current_frame); frame->prev = NULL; frame->func = (const zend_function*)op_array; @@ -2981,9 +2967,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace frame->func = (const zend_function*)p->op_array; stack = frame->stack; op_array = p->op_array; - zend_jit_op_array_trace_extension *const jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - op_array_ssa = &jit_extension->func_info.ssa; + op_array_ssa = zend_jit_op_array_trace_ssa(op_array); j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); for (i = 0; i < op_array->last_var; i++) { SET_STACK_VAR(stack, i, j); @@ -3007,9 +2991,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace zend_jit_close_var(stack, i, start, end, flags, idx-1); } op_array = p->op_array; - zend_jit_op_array_trace_extension *const jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - op_array_ssa = &jit_extension->func_info.ssa; + op_array_ssa = zend_jit_op_array_trace_ssa(op_array); frame = zend_jit_trace_op_array_ret_frame(frame, op_array); stack = frame->stack; if (level == 0) { @@ -6468,9 +6450,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par TRACE_FRAME_SET_THIS_CHECKED(call); } op_array = (zend_op_array*)p->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - op_array_ssa = &jit_extension->func_info.ssa; + op_array_ssa = zend_jit_op_array_trace_ssa(op_array); frame->call = call->prev; call->prev = frame; if (p->info & ZEND_JIT_TRACE_RETURN_VALUE_USED) { @@ -6494,9 +6474,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } else if (p->op == ZEND_JIT_TRACE_BACK) { op_array = (zend_op_array*)p->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - op_array_ssa = &jit_extension->func_info.ssa; + op_array_ssa = zend_jit_op_array_trace_ssa(op_array); top = frame; if (frame->prev) { checked_stack -= frame->used_stack;