diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 429d8dbd49f5a..f4ae712ce571f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -106,20 +106,57 @@ typedef struct _zend_jit_stub { zend_ulong zend_jit_profile_counter = 0; int zend_jit_profile_counter_rid = -1; -int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; +#define ZEND_HOT_COUNTERS_COUNT 128 +static int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; const zend_op *zend_jit_halt_op = NULL; static int zend_jit_vm_kind = 0; #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP -static int zend_write_protect = 1; +static bool zend_write_protect = true; #endif +/** + * Start of the shared memory area to contain JIT machine code. Call + * zend_jit_unprotect() before writing to it, and call + * zend_jit_protect() when done. + * + * #dasmn_end is the end of the area and #dasm_ptr points to the end + * of the portion that is really used currently. + * + * This area is allocated during startup and is never changed. + * Allocations are linear by incrementing #dasm_ptr. + */ static void *dasm_buf = NULL; + +/** + * End of the shared memory area to contain JIT machine code. + */ static void *dasm_end = NULL; + +/** + * Pointer to pointer to end of the used portion of the shared memory + * area to contain JIT machine code. + * + * This "pointer to pointer" points to inside the shared memory area, + * because it needs to be shared across all processes/threads. All + * accesses to the pointed-to pointer need to be protected using + * zend_shared_alloc_lock(). + * + * Additionally, dasm_ptr[1] contains a pointer to the end of the + * stubs and veneers. This is used by zend_jit_restart() to free all + * JIT-generated functions, but keep those stubs and veneers. + */ static void **dasm_ptr = NULL; +/** + * The total size of the #dasm_buf, including the trailer which + * #dasm_ptr points to (after #dasm_end). + */ static size_t dasm_size = 0; +/** + * Counter for checking the opcache.jit_bisect_limit setting. + */ static zend_long jit_bisect_pos = 0; static const void *zend_jit_runtime_jit_handler = NULL; @@ -133,7 +170,7 @@ static const void *zend_jit_loop_trace_counter_handler = NULL; static int ZEND_FASTCALL zend_runtime_jit(void); static int zend_jit_trace_op_len(const zend_op *opline); -static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline); +static bool zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline); static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags); static const void *zend_jit_trace_get_exit_addr(uint32_t n); static void zend_jit_trace_add_code(const void *start, uint32_t size); @@ -165,13 +202,11 @@ static bool dominates(const zend_basic_block *blocks, int a, int b) { static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use) { - int next_use; - if (ssa->vars[var].phi_use_chain) { zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; do { if (!ssa->vars[phi->ssa_var].no_val) { - return 0; + return false; } phi = zend_ssa_next_use_phi(ssa, var, phi); } while (phi); @@ -186,19 +221,19 @@ static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa * if (b != ssa->cfg.map[prev_use] && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use]) && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) { - return 0; + return false; } prev_use = zend_ssa_next_use(ssa->ops, var, prev_use); } } - next_use = zend_ssa_next_use(ssa->ops, var, use); + const int next_use = zend_ssa_next_use(ssa->ops, var, use); if (next_use < 0) { - return 1; + return true; } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) { - return 1; + return true; } - return 0; + return false; } static bool zend_ival_is_last_use(const zend_lifetime_interval *ival, int use) @@ -211,7 +246,7 @@ static bool zend_ival_is_last_use(const zend_lifetime_interval *ival, int use) } return range->end == use; } - return 0; + return false; } static bool zend_is_commutative(zend_uchar opcode) @@ -224,12 +259,12 @@ static bool zend_is_commutative(zend_uchar opcode) opcode == ZEND_BW_XOR; } -static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, - zend_ssa_range *op1_range, - zend_jit_addr op1_addr, - zend_ssa_range *op2_range, - zend_jit_addr op2_addr, - bool *result) +static bool zend_jit_is_constant_cmp_long_long(const zend_op *opline, + zend_ssa_range *op1_range, + zend_jit_addr op1_addr, + zend_ssa_range *op2_range, + zend_jit_addr op2_addr, + bool *result) { zend_long op1_min; zend_long op1_max; @@ -243,7 +278,7 @@ static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG); op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr)); } else { - return 0; + return false; } if (op2_range) { @@ -253,7 +288,7 @@ static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG); op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr)); } else { - return 0; + return false; } switch (opline->opcode) { @@ -262,56 +297,54 @@ static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, case ZEND_CASE: case ZEND_CASE_STRICT: if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { - *result = 1; - return 1; + *result = true; + return true; } else if (op1_max < op2_min || op1_min > op2_max) { - *result = 0; - return 1; + *result = false; + return true; } - return 0; + return false; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { - *result = 0; - return 1; + *result = false; + return true; } else if (op1_max < op2_min || op1_min > op2_max) { - *result = 1; - return 1; + *result = true; + return true; } - return 0; + return false; case ZEND_IS_SMALLER: if (op1_max < op2_min) { - *result = 1; - return 1; + *result = true; + return true; } else if (op1_min >= op2_max) { - *result = 0; - return 1; + *result = false; + return true; } - return 0; + return false; case ZEND_IS_SMALLER_OR_EQUAL: if (op1_max <= op2_min) { - *result = 1; - return 1; + *result = true; + return true; } else if (op1_min > op2_max) { - *result = 0; - return 1; + *result = false; + return true; } - return 0; + return false; default: ZEND_UNREACHABLE(); } - return 0; + return false; } -static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace) +static bool zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, const zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace) { - int skip; - if (trace) { zend_jit_trace_rec *p = trace; ssa_op++; - while (1) { + while (true) { if (p->op == ZEND_JIT_TRACE_VM) { switch (p->opline->opcode) { case ZEND_SEND_ARRAY: @@ -341,13 +374,13 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons case ZEND_DECLARE_ANON_CLASS: case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: - return 1; + return true; case ZEND_DO_ICALL: case ZEND_DO_UCALL: case ZEND_DO_FCALL_BY_NAME: case ZEND_DO_FCALL: case ZEND_CALLABLE_CONVERT: - return 0; + return false; case ZEND_SEND_VAL: case ZEND_SEND_VAR: case ZEND_SEND_VAL_EX: @@ -360,14 +393,14 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons break; default: if (zend_may_throw(opline, ssa_op, op_array, ssa)) { - return 1; + return true; } } ssa_op += zend_jit_trace_op_len(opline); } else if (p->op == ZEND_JIT_TRACE_ENTER || p->op == ZEND_JIT_TRACE_BACK || p->op == ZEND_JIT_TRACE_END) { - return 1; + return true; } p++; } @@ -378,11 +411,11 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons opline++; ssa_op++; - skip = (call_level == 1); + bool skip = (call_level == 1); while (opline != end) { if (!skip) { if (zend_may_throw(opline, ssa_op, op_array, ssa)) { - return 1; + return true; } } switch (opline->opcode) { @@ -394,7 +427,7 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons case ZEND_SEND_REF: case ZEND_SEND_VAR_NO_REF: case ZEND_SEND_VAR_NO_REF_EX: - skip = 0; + skip = false; break; case ZEND_SEND_ARRAY: case ZEND_SEND_USER: @@ -423,7 +456,7 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons case ZEND_DECLARE_ANON_CLASS: case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: - return 1; + return true; case ZEND_DO_ICALL: case ZEND_DO_UCALL: case ZEND_DO_FCALL_BY_NAME: @@ -432,27 +465,27 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons end = opline; if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { /* INIT_FCALL and DO_FCALL in different BasicBlocks */ - return 1; + return true; } - return 0; + return false; } opline++; ssa_op++; } - return 1; + return true; } else { const zend_op *end = call_info->caller_call_opline; /* end may be null if an opcode like EXIT is part of the argument list. */ if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { /* INIT_FCALL and DO_FCALL in different BasicBlocks */ - return 1; + return true; } opline++; ssa_op++; - skip = (call_level == 1); + bool skip = (call_level == 1); while (opline != end) { if (skip) { switch (opline->opcode) { @@ -464,27 +497,27 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons case ZEND_SEND_REF: case ZEND_SEND_VAR_NO_REF: case ZEND_SEND_VAR_NO_REF_EX: - skip = 0; + skip = false; break; case ZEND_SEND_ARRAY: case ZEND_SEND_USER: case ZEND_SEND_UNPACK: - return 1; + return true; } } else { if (zend_may_throw(opline, ssa_op, op_array, ssa)) { - return 1; + return true; } } opline++; ssa_op++; } - return 0; + return false; } } -static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info) +static uint32_t skip_valid_arguments(const zend_op_array *op_array, const zend_ssa *ssa, const zend_call_info *call_info) { uint32_t num_args = 0; zend_function *func = call_info->callee_func; @@ -496,8 +529,8 @@ static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ss if (ZEND_TYPE_IS_SET(arg_info->type)) { if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) { - zend_op *opline = call_info->arg_info[num_args].opline; - zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; + const zend_op *opline = call_info->arg_info[num_args].opline; + const zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type); if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) { break; @@ -511,13 +544,13 @@ static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ss return num_args; } -static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var) +static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var) { - uint32_t j, info; + uint32_t info; if (ssa->vars && ssa->var_info) { info = ssa->var_info[var].type; - for (j = op_array->last_var; j < ssa->vars_count; j++) { + for (uint32_t j = op_array->last_var; j < ssa->vars_count; j++) { if (ssa->vars[j].var == var) { info |= ssa->var_info[j].type; } @@ -530,7 +563,7 @@ static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, u #ifdef ZEND_JIT_USE_RC_INFERENCE /* Refcount may be increased by RETURN opcode */ if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) { - for (j = 0; j < ssa->cfg.blocks_count; j++) { + for (uint32_t j = 0; j < ssa->cfg.blocks_count; j++) { if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) && ssa->cfg.blocks[j].len > 0) { const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1; @@ -556,7 +589,7 @@ static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_i if (!JIT_G(current_frame) || !JIT_G(current_frame)->call->func || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { - return 0; + return false; } /* break missing intentionally */ case ZEND_FETCH_OBJ_R: @@ -572,7 +605,7 @@ static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_i if (!JIT_G(current_frame) || !JIT_G(current_frame)->call->func || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { - return 0; + return false; } /* break missing intentionally */ case ZEND_FETCH_DIM_R: @@ -584,16 +617,15 @@ static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_i } break; } - return 0; + return false; } static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags) { - zval *zv; zend_constant *c = NULL; /* null/true/false are resolved during compilation, so don't check for them here. */ - zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key)); + zval *zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key)); if (zv) { c = (zend_constant*)Z_PTR_P(zv); } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) { @@ -608,8 +640,6 @@ static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags) static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename) { - zend_property_info *info = NULL; - if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) || !ce || !(ce->ce_flags & ZEND_ACC_LINKED) || @@ -644,7 +674,7 @@ static zend_property_info* zend_get_known_property_info(const zend_op_array *op_ } } - info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); + zend_property_info *const info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); if (info == NULL || !IS_VALID_PROPERTY_OFFSET(info->offset) || (info->flags & ZEND_ACC_STATIC)) { @@ -667,8 +697,6 @@ static zend_property_info* zend_get_known_property_info(const zend_op_array *op_ static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename) { - zend_property_info *info; - if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) { return 1; } @@ -680,7 +708,7 @@ static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *memb } } - info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); + zend_property_info *const info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); if (info == NULL || !IS_VALID_PROPERTY_OFFSET(info->offset) || (info->flags & ZEND_ACC_STATIC)) { @@ -692,7 +720,7 @@ static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *memb return 1; } - return 0; + return false; } #define OP_RANGE(ssa_op, opN) \ @@ -756,6 +784,8 @@ typedef enum _sp_adj_kind { static int sp_adj[SP_ADJ_LAST]; +static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs); + /* The generated code may contain tautological comparisons, ignore them. */ #if defined(__clang__) # pragma clang diagnostic push @@ -802,6 +832,10 @@ ZEND_EXT_API void zend_jit_status(zval *ret) add_assoc_zval(ret, "jit", &stats); } +/** + * Generate a name for a JIT-generated native function (for the + * disassembler and for external debuggers/profilers/tracers). + */ static zend_string *zend_jit_func_name(const zend_op_array *op_array) { smart_str buf = {0}; @@ -895,12 +929,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, uint32_t sp_offset, uint32_t sp_adjustment) { - size_t size; int ret; - void *entry; -#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE) - zend_string *str = NULL; -#endif if (rt_opline && ssa && ssa->cfg.map) { /* Create additional entry point, to switch from interpreter to JIT-ed @@ -941,6 +970,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, } } + size_t size; ret = dasm_link(dasm_state, &size); if (ret != DASM_S_OK) { #if ZEND_DEBUG @@ -971,7 +1001,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, size += dasm_venners_size; #endif - entry = *dasm_ptr; + void *const entry = *dasm_ptr; *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT)); /* flush the hardware I-cache */ @@ -1010,6 +1040,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, } #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE) + zend_string *str = NULL; if (!name) { if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) { str = zend_jit_func_name(op_array); @@ -1105,13 +1136,12 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, return entry; } -static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa) +static bool zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, const zend_ssa *ssa) { int res; - zend_long op1_min, op1_max, op2_min, op2_max; if (!ssa->ops || !ssa->var_info) { - return 1; + return true; } switch (opline->opcode) { case ZEND_PRE_INC: @@ -1121,14 +1151,14 @@ static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, c || !ssa->var_info[res].has_range || ssa->var_info[res].range.overflow) { if (!OP1_HAS_RANGE()) { - return 1; + return true; } - op1_max = OP1_MAX_RANGE(); + const zend_long op1_max = OP1_MAX_RANGE(); if (op1_max == ZEND_LONG_MAX) { - return 1; + return true; } } - return 0; + return false; case ZEND_PRE_DEC: case ZEND_POST_DEC: res = ssa_op->op1_def; @@ -1136,68 +1166,68 @@ static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, c || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow) { if (!OP1_HAS_RANGE()) { - return 1; + return true; } - op1_min = OP1_MIN_RANGE(); + const zend_long op1_min = OP1_MIN_RANGE(); if (op1_min == ZEND_LONG_MIN) { - return 1; + return true; } } - return 0; + return false; case ZEND_ADD: res = ssa_op->result_def; if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } - op1_min = OP1_MIN_RANGE(); - op2_min = OP2_MIN_RANGE(); + const zend_long op1_min = OP1_MIN_RANGE(); + const zend_long op2_min = OP2_MIN_RANGE(); if (zend_add_will_overflow(op1_min, op2_min)) { - return 1; + return true; } } if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.overflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } - op1_max = OP1_MAX_RANGE(); - op2_max = OP2_MAX_RANGE(); + const zend_long op1_max = OP1_MAX_RANGE(); + const zend_long op2_max = OP2_MAX_RANGE(); if (zend_add_will_overflow(op1_max, op2_max)) { - return 1; + return true; } } - return 0; + return false; case ZEND_SUB: res = ssa_op->result_def; if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } - op1_min = OP1_MIN_RANGE(); - op2_max = OP2_MAX_RANGE(); + const zend_long op1_min = OP1_MIN_RANGE(); + const zend_long op2_max = OP2_MAX_RANGE(); if (zend_sub_will_overflow(op1_min, op2_max)) { - return 1; + return true; } } if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.overflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } - op1_max = OP1_MAX_RANGE(); - op2_min = OP2_MIN_RANGE(); + const zend_long op1_max = OP1_MAX_RANGE(); + const zend_long op2_min = OP2_MIN_RANGE(); if (zend_sub_will_overflow(op1_max, op2_min)) { - return 1; + return true; } } - return 0; + return false; case ZEND_MUL: res = ssa_op->result_def; return (res < 0 || @@ -1211,54 +1241,54 @@ static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, c || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } - op1_min = OP1_MIN_RANGE(); - op2_min = OP2_MIN_RANGE(); + const zend_long op1_min = OP1_MIN_RANGE(); + const zend_long op2_min = OP2_MIN_RANGE(); if (zend_add_will_overflow(op1_min, op2_min)) { - return 1; + return true; } } if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.overflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } - op1_max = OP1_MAX_RANGE(); - op2_max = OP2_MAX_RANGE(); + const zend_long op1_max = OP1_MAX_RANGE(); + const zend_long op2_max = OP2_MAX_RANGE(); if (zend_add_will_overflow(op1_max, op2_max)) { - return 1; + return true; } } - return 0; + return false; } else if (opline->extended_value == ZEND_SUB) { res = ssa_op->op1_def; if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.underflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } - op1_min = OP1_MIN_RANGE(); - op2_max = OP2_MAX_RANGE(); + const zend_long op1_min = OP1_MIN_RANGE(); + const zend_long op2_max = OP2_MAX_RANGE(); if (zend_sub_will_overflow(op1_min, op2_max)) { - return 1; + return true; } } if (res < 0 || !ssa->var_info[res].has_range || ssa->var_info[res].range.overflow) { if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) { - return 1; + return true; } - op1_max = OP1_MAX_RANGE(); - op2_min = OP2_MIN_RANGE(); + const zend_long op1_max = OP1_MAX_RANGE(); + const zend_long op2_min = OP2_MIN_RANGE(); if (zend_sub_will_overflow(op1_max, op2_min)) { - return 1; + return true; } } - return 0; + return false; } else if (opline->extended_value == ZEND_MUL) { res = ssa_op->op1_def; return (res < 0 || @@ -1268,15 +1298,13 @@ static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, c } ZEND_FALLTHROUGH; default: - return 1; + return true; } } static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg) { - uint32_t flags; - - flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY; + const uint32_t flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY; zend_build_cfg(&CG(arena), op_array, flags, cfg); @@ -1526,10 +1554,9 @@ static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval **intervals, int count) { zend_lifetime_interval *list, *last; - int i; list = NULL; - i = 0; + int i = 0; while (i < count) { list = intervals[i]; i++; @@ -1580,11 +1607,11 @@ static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval ** static ZEND_ATTRIBUTE_UNUSED void zend_jit_print_regset(zend_regset regset) { zend_reg reg; - int first = 1; + bool first = true; ZEND_REGSET_FOREACH(regset, reg) { if (first) { - first = 0; + first = false; fprintf(stderr, "%s", zend_reg_name[reg]); } else { fprintf(stderr, ", %s", zend_reg_name[reg]); @@ -1628,17 +1655,16 @@ static bool zend_jit_in_loop(zend_ssa *ssa, int header, zend_basic_block *b) } b = ssa->cfg.blocks + b->loop_header; } - return 0; + return false; } static void zend_jit_compute_loop_body(zend_ssa *ssa, int header, int n, zend_bitset loop_body) { zend_basic_block *b = ssa->cfg.blocks + n; - uint32_t i; tail_call: if (b->len) { - for (i = b->start; i < b->start + b->len; i++) { + for (uint32_t i = b->start; i < b->start + b->len; i++) { zend_bitset_incl(loop_body, i); } } @@ -1681,34 +1707,27 @@ static void zend_jit_add_hint(zend_lifetime_interval **intervals, int dst, int s Michael Franz, CGO'10 (2010), Figure 4. */ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list) { - int set_size, i, j, k, l; - uint32_t n; - zend_bitset live, live_in, pi_vars, loop_body; - int *block_order; - zend_ssa_phi *phi; - zend_lifetime_interval **intervals; - size_t mem_size; ALLOCA_FLAG(use_heap); - set_size = zend_bitset_len(ssa->vars_count); - mem_size = + const int set_size = zend_bitset_len(ssa->vars_count); + const size_t mem_size = ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE) + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE) + ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int)); - intervals = do_alloca(mem_size, use_heap); + zend_lifetime_interval **const intervals = do_alloca(mem_size, use_heap); if (!intervals) { *list = NULL; return FAILURE; } - live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*))); - live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE)); - pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); - loop_body = (zend_bitset)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); - block_order = (int*)((char*)loop_body + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE)); + const zend_bitset live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*))); + zend_bitset live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE)); + const zend_bitset pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); + const zend_bitset loop_body = (zend_bitset)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE)); + int *const block_order = (int*)((char*)loop_body + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE)); memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*)); zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count); @@ -1717,21 +1736,19 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss * are before this block, and where all blocks belonging to the same loop * are contiguous ??? */ - for (l = zend_jit_compute_block_order(ssa, block_order) - 1; l >= 0; l--) { - zend_basic_block *b; - - i = block_order[l]; - b = ssa->cfg.blocks + i; + for (int l = zend_jit_compute_block_order(ssa, block_order) - 1; l >= 0; l--) { + int i = block_order[l]; + zend_basic_block *const b = ssa->cfg.blocks + i; /* live = UNION of successor.liveIn for each successor of b */ /* live.add(phi.inputOf(b)) for each phi of successors of b */ zend_bitset_clear(live, set_size); - for (j = 0; j < b->successors_count; j++) { + for (int j = 0; j < b->successors_count; j++) { int succ = b->successors[j]; zend_bitset_union(live, live_in + set_size * succ, set_size); zend_bitset_clear(pi_vars, set_size); - for (phi = ssa->blocks[succ].phis; phi; phi = phi->next) { + for (zend_ssa_phi *phi = ssa->blocks[succ].phis; phi; phi = phi->next) { if (ssa->vars[phi->ssa_var].no_val) { /* skip */ } else if (phi->pi >= 0) { @@ -1742,7 +1759,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss zend_bitset_incl(pi_vars, phi->var); } } else if (!zend_bitset_in(pi_vars, phi->var)) { - for (k = 0; k < ssa->cfg.blocks[succ].predecessors_count; k++) { + for (int k = 0; k < ssa->cfg.blocks[succ].predecessors_count; k++) { if (ssa->cfg.predecessors[ssa->cfg.blocks[succ].predecessor_offset + k] == i) { if (phi->sources[k] >= 0 && zend_bitset_in(candidates, phi->sources[k])) { zend_bitset_incl(live, phi->sources[k]); @@ -1755,6 +1772,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss } /* addRange(var, b.from, b.to) for each var in live */ + int j; ZEND_BITSET_FOREACH(live, set_size, j) { if (zend_bitset_in(candidates, j)) { if (zend_jit_add_range(intervals, j, b->start, b->start + b->len - 1) != SUCCESS) { @@ -1764,15 +1782,12 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss } ZEND_BITSET_FOREACH_END(); /* for each operation op of b in reverse order */ - for (n = b->start + b->len; n > b->start;) { - zend_ssa_op *op; - const zend_op *opline; - uint32_t num; - + for (uint32_t n = b->start + b->len; n > b->start;) { n--; - op = ssa->ops + n; - opline = op_array->opcodes + n; + zend_ssa_op *const op = ssa->ops + n; + const zend_op *const opline = op_array->opcodes + n; + uint32_t num; if (UNEXPECTED(opline->opcode == ZEND_OP_DATA)) { num = n - 1; } else { @@ -1831,7 +1846,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss } /* live.remove(phi.output) for each phi of b */ - for (phi = ssa->blocks[i].phis; phi; phi = phi->next) { + for (zend_ssa_phi *phi = ssa->blocks[i].phis; phi; phi = phi->next) { zend_bitset_excl(live, phi->ssa_var); } @@ -1839,7 +1854,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss zend_bitset_copy(live_in + set_size * i, live, set_size); } - for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) { + for (int i = ssa->cfg.blocks_count - 1; i >= 0; i--) { zend_basic_block *b = ssa->cfg.blocks + i; /* if b is loop header */ @@ -1861,6 +1876,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss } while (zend_bitset_in(loop_body, to)); to--; + int j; ZEND_BITSET_FOREACH(live, set_size, j) { if (zend_jit_add_range(intervals, j, from, to) != SUCCESS) { goto failure; @@ -1874,21 +1890,19 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) { /* Register hinting (a cheap way for register coalescing) */ - for (i = 0; i < ssa->vars_count; i++) { + for (int i = 0; i < ssa->vars_count; i++) { if (intervals[i]) { - int src; - if (ssa->vars[i].definition_phi) { zend_ssa_phi *phi = ssa->vars[i].definition_phi; if (phi->pi >= 0) { - src = phi->sources[0]; + const int src = phi->sources[0]; if (intervals[src]) { zend_jit_add_hint(intervals, i, src); } } else { - for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) { - src = phi->sources[k]; + for (int k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) { + int src = phi->sources[k]; if (src >= 0) { if (ssa->vars[src].definition_phi && ssa->vars[src].definition_phi->pi >= 0 @@ -1905,7 +1919,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss } } } - for (i = 0; i < ssa->vars_count; i++) { + for (int i = 0; i < ssa->vars_count; i++) { if (intervals[i] && !intervals[i]->hint) { if (ssa->vars[i].definition >= 0) { @@ -2017,12 +2031,12 @@ static bool zend_interval_covers(zend_lifetime_interval *ival, uint32_t position do { if (position >= range->start && position <= range->end) { - return 1; + return true; } range = range->next; } while (range); - return 0; + return false; } static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_lifetime_interval *ival2) @@ -2047,15 +2061,12 @@ static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_l /* See "Optimized Interval Splitting in a Linear Scan Register Allocator", Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling */ -static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_regset *hints, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free) +static bool zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_regset *hints, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free) { zend_lifetime_interval *it; uint32_t freeUntilPos[ZREG_NUM]; - uint32_t pos, pos2; - zend_reg i, reg, reg2; + zend_reg i; zend_reg hint = ZREG_NONE; - zend_regset low_priority_regs; - zend_life_range *range; if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_FP); @@ -2108,7 +2119,7 @@ static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const z } if (hint == ZREG_NONE && ZEND_REGSET_IS_EMPTY(available)) { - return 0; + return false; } /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and @@ -2130,7 +2141,7 @@ static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const z /* Handle Scratch Registers */ /* TODO: Optimize ??? */ - range = ¤t->range; + zend_life_range *range = ¤t->range; do { uint32_t line = range->start; uint32_t last_use_line = (uint32_t)-1; @@ -2204,16 +2215,16 @@ static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const z if (current->used_as_hint) { ZEND_REGSET_INCL(*hints, hint); } - return 1; + return true; } if (ZEND_REGSET_IS_EMPTY(available)) { - return 0; + return false; } - pos = 0; reg = ZREG_NONE; - pos2 = 0; reg2 = ZREG_NONE; - low_priority_regs = *hints; + uint32_t pos = 0; zend_reg reg = ZREG_NONE; + uint32_t pos2 = 0; zend_reg reg2 = ZREG_NONE; + zend_regset low_priority_regs = *hints; if (current->used_as_hint) { /* TODO: Avoid most often scratch registers. Find a better way ??? */ low_priority_regs = ZEND_REGSET_UNION(low_priority_regs, ZEND_REGSET_LOW_PRIORITY); @@ -2241,14 +2252,14 @@ static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const z if (reg == ZREG_NONE) { /* no register available without spilling */ - return 0; + return false; } else if (zend_interval_end(current) < pos) { /* register available for the whole interval */ current->reg = reg; if (current->used_as_hint) { ZEND_REGSET_INCL(*hints, reg); } - return 1; + return true; #if 0 // TODO: allow low priority register usage } else if (reg2 != ZREG_NONE && zend_interval_end(current) < pos2) { @@ -2257,19 +2268,19 @@ static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const z if (current->used_as_hint) { ZEND_REGSET_INCL(*hints, reg2); } - return 1; + return true; #endif } else { /* TODO: enable interval splitting ??? */ /* register available for the first part of the interval */ if (1 || zend_jit_split_interval(current, pos, list, free) != SUCCESS) { - return 0; + return false; } current->reg = reg; if (current->used_as_hint) { ZEND_REGSET_INCL(*hints, reg); } - return 1; + return true; } } @@ -2287,25 +2298,22 @@ static int zend_jit_allocate_blocked_reg(void) Christian Wimmer VEE'10 (2005), Figure 2. */ static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *list) { - zend_lifetime_interval *unhandled, *active, *inactive, *handled, *free; - zend_lifetime_interval *current, **p, *q; - uint32_t position; zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); zend_regset hints = ZEND_REGSET_EMPTY; - unhandled = list; + zend_lifetime_interval *unhandled = list; /* active = inactive = handled = free = {} */ - active = inactive = handled = free = NULL; + zend_lifetime_interval *active = NULL, *inactive = NULL, *handled = NULL, *free = NULL; while (unhandled != NULL) { - current = unhandled; + zend_lifetime_interval *current = unhandled; unhandled = unhandled->list_next; - position = current->range.start; + const uint32_t position = current->range.start; - p = &active; + zend_lifetime_interval **p = &active; while (*p) { uint32_t end = zend_interval_end(*p); - q = *p; + zend_lifetime_interval *const q = *p; if (end < position) { /* move ival from active to handled */ ZEND_REGSET_INCL(available, q->reg); @@ -2327,7 +2335,7 @@ static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_arra while (*p) { uint32_t end = zend_interval_end(*p); - q = *p; + zend_lifetime_interval *const q = *p; if (end < position) { /* move ival from inactive to handled */ *p = q->list_next; @@ -2357,7 +2365,7 @@ static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_arra /* move active to handled */ while (active) { - current = active; + zend_lifetime_interval *current = active; active = active->list_next; current->list_next = handled; handled = current; @@ -2365,7 +2373,7 @@ static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_arra /* move inactive to handled */ while (inactive) { - current = inactive; + zend_lifetime_interval *current = inactive; inactive = inactive->list_next; current->list_next = handled; handled = current; @@ -2415,11 +2423,6 @@ static void zend_jit_dump_lifetime_interval(const zend_op_array *op_array, const static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array *op_array, zend_ssa *ssa) { - void *checkpoint; - int set_size, candidates_count, i; - zend_bitset candidates = NULL; - zend_lifetime_interval *list, *ival; - zend_lifetime_interval **intervals; ALLOCA_FLAG(use_heap); if (!ssa->var_info) { @@ -2427,14 +2430,14 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array } /* Identify SSA variables suitable for register allocation */ - set_size = zend_bitset_len(ssa->vars_count); - candidates = ZEND_BITSET_ALLOCA(set_size, use_heap); + const int set_size = zend_bitset_len(ssa->vars_count); + const zend_bitset candidates = ZEND_BITSET_ALLOCA(set_size, use_heap); if (!candidates) { return NULL; } - candidates_count = 0; + int candidates_count = 0; zend_bitset_clear(candidates, set_size); - for (i = 0; i < ssa->vars_count; i++) { + for (int i = 0; i < ssa->vars_count; i++) { if (zend_jit_may_be_in_reg(op_array, ssa, i)) { zend_bitset_incl(candidates, i); candidates_count++; @@ -2445,17 +2448,17 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array return NULL; } - checkpoint = zend_arena_checkpoint(CG(arena)); + void *const checkpoint = zend_arena_checkpoint(CG(arena)); /* Find life-time intervals */ + zend_lifetime_interval *list; if (zend_jit_compute_liveness(op_array, ssa, candidates, &list) != SUCCESS) { goto failure; } if (list) { /* Set ZREG_LAST_USE flags */ - ival = list; - while (ival) { + for (zend_lifetime_interval *ival = list; ival; ival = ival->list_next) { zend_life_range *range = &ival->range; while (range->next) { @@ -2464,17 +2467,14 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array if (zend_ssa_is_last_use(op_array, ssa, ival->ssa_var, range->end)) { ival->flags |= ZREG_LAST_USE; } - ival = ival->list_next; } } if (list) { if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); - ival = list; - while (ival) { + for (zend_lifetime_interval *ival = list; ival; ival = ival->list_next) { zend_jit_dump_lifetime_interval(op_array, ssa, ival); - ival = ival->list_next; } fprintf(stderr, "\n"); } @@ -2483,13 +2483,12 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array list = zend_jit_linear_scan(op_array, NULL, ssa, list); if (list) { - intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*)); + zend_lifetime_interval **const intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*)); if (!intervals) { goto failure; } - ival = list; - while (ival != NULL) { + for (zend_lifetime_interval *ival = list; ival;) { zend_lifetime_interval *next = ival->list_next; ival->list_next = intervals[ival->ssa_var]; @@ -2499,7 +2498,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) { /* Naive SSA resolution */ - for (i = 0; i < ssa->vars_count; i++) { + for (int i = 0; i < ssa->vars_count; i++) { if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) { zend_ssa_phi *phi = ssa->vars[i].definition_phi; int k, src; @@ -2520,7 +2519,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array } } } else { - int need_move = 0; + bool need_move = false; for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) { src = phi->sources[k]; @@ -2533,12 +2532,12 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array } if (intervals[i]) { if (!intervals[src]) { - need_move = 1; + need_move = true; } else if (intervals[i]->reg != intervals[src]->reg) { - need_move = 1; + need_move = true; } } else if (intervals[src]) { - need_move = 1; + need_move = true; } } } @@ -2565,18 +2564,18 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array } } /* Remove useless register allocation */ - for (i = 0; i < ssa->vars_count; i++) { + for (int i = 0; i < ssa->vars_count; i++) { if (intervals[i] && ((intervals[i]->flags & ZREG_LOAD) || ((intervals[i]->flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) && ssa->vars[i].use_chain < 0) { - bool may_remove = 1; + bool may_remove = true; zend_ssa_phi *phi = ssa->vars[i].phi_use_chain; while (phi) { if (intervals[phi->ssa_var] && !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) { - may_remove = 0; + may_remove = false; break; } phi = zend_ssa_next_use_phi(ssa, i, phi); @@ -2587,19 +2586,19 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array } } /* Remove intervals used once */ - for (i = 0; i < ssa->vars_count; i++) { + for (int i = 0; i < ssa->vars_count; i++) { if (intervals[i] && (intervals[i]->flags & ZREG_LOAD) && (intervals[i]->flags & ZREG_STORE) && (ssa->vars[i].use_chain < 0 || zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) { - bool may_remove = 1; + bool may_remove = true; zend_ssa_phi *phi = ssa->vars[i].phi_use_chain; while (phi) { if (intervals[phi->ssa_var] && !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) { - may_remove = 0; + may_remove = false; break; } phi = zend_ssa_next_use_phi(ssa, i, phi); @@ -2613,11 +2612,9 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); - for (i = 0; i < ssa->vars_count; i++) { - ival = intervals[i]; - while (ival) { + for (int i = 0; i < ssa->vars_count; i++) { + for (zend_lifetime_interval *ival = list; ival; ival = ival->list_next) { zend_jit_dump_lifetime_interval(op_array, ssa, ival); - ival = ival->list_next; } } fprintf(stderr, "\n"); @@ -2641,9 +2638,9 @@ static bool zend_jit_next_is_send_result(const zend_op *opline) && (opline+1)->op1_type == IS_TMP_VAR && (opline+1)->op2_type != IS_CONST && (opline+1)->op1.var == opline->result.var) { - return 1; + return true; } - return 0; + return false; } static bool zend_jit_supported_binary_op(zend_uchar op, uint32_t op1_info, uint32_t op2_info) @@ -2674,24 +2671,15 @@ static bool zend_jit_supported_binary_op(zend_uchar op, uint32_t op1_info, uint3 } } +/** + * Caller must have called zend_shared_alloc_lock() and + * zend_jit_unprotect(). + */ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline) { - int b, i, end; - zend_op *opline; - dasm_State* dasm_state = NULL; - void *handler; int call_level = 0; - void *checkpoint = NULL; - zend_lifetime_interval **ra = NULL; - bool is_terminated = 1; /* previous basic block is terminated by jump */ - bool recv_emitted = 0; /* emitted at least one RECV opcode */ - zend_uchar smart_branch_opcode; - uint32_t target_label, target_label2; - uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info; - zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; - zend_class_entry *ce; - bool ce_is_instanceof; - bool on_this; + bool is_terminated = true; /* previous basic block is terminated by jump */ + bool recv_emitted = false; /* emitted at least one RECV opcode */ if (JIT_G(bisect_limit)) { jit_bisect_pos++; @@ -2707,17 +2695,19 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } } + void *checkpoint = NULL; + zend_lifetime_interval **ra = NULL; if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) { checkpoint = zend_arena_checkpoint(CG(arena)); ra = zend_jit_allocate_registers(op_array, ssa); } /* mark hidden branch targets */ - for (b = 0; b < ssa->cfg.blocks_count; b++) { + for (int b = 0; b < ssa->cfg.blocks_count; b++) { if (ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE && ssa->cfg.blocks[b].len > 1) { - opline = op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1; + zend_op *const opline = op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1; if (opline->opcode == ZEND_DO_FCALL && (opline-1)->opcode == ZEND_NEW) { ssa->cfg.blocks[ssa->cfg.blocks[b].successors[0]].flags |= ZEND_BB_TARGET; @@ -2725,6 +2715,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } } + dasm_State* dasm_state = NULL; dasm_init(&dasm_state, DASM_MAXSECTION); dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); dasm_setup(&dasm_state, dasm_actions); @@ -2732,7 +2723,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op dasm_growpc(&dasm_state, ssa->cfg.blocks_count * 2 + 1); zend_jit_align_func(&dasm_state); - for (b = 0; b < ssa->cfg.blocks_count; b++) { + for (int b = 0; b < ssa->cfg.blocks_count; b++) { if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) { continue; } @@ -2757,7 +2748,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } else //#endif if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) { - opline = op_array->opcodes + ssa->cfg.blocks[b].start; + zend_op *const opline = op_array->opcodes + ssa->cfg.blocks[b].start; if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) { if (opline->opcode == ZEND_RECV_INIT) { if (opline == op_array->opcodes || @@ -2766,12 +2757,12 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op zend_jit_jmp(&dasm_state, b); } zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); - for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) { + for (int i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) { zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b + i); } zend_jit_prologue(&dasm_state); } - recv_emitted = 1; + recv_emitted = true; } else if (opline->opcode == ZEND_RECV) { if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { /* skip */ @@ -2797,7 +2788,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); zend_jit_prologue(&dasm_state); - recv_emitted = 1; + recv_emitted = true; } } else { if (recv_emitted) { @@ -2815,7 +2806,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b); zend_jit_prologue(&dasm_state); - recv_emitted = 1; + recv_emitted = true; } } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE && ssa->cfg.blocks[b].len == 1 && @@ -2833,7 +2824,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } } - is_terminated = 0; + is_terminated = false; zend_jit_label(&dasm_state, b); if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) { @@ -2887,10 +2878,10 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op phi = phi->next; } } - end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1; - for (i = ssa->cfg.blocks[b].start; i <= end; i++) { + const int end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1; + for (int i = ssa->cfg.blocks[b].start; i <= end; i++) { zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL; - opline = op_array->opcodes + i; + zend_op *const opline = op_array->opcodes + i; switch (opline->opcode) { case ZEND_INIT_FCALL: case ZEND_INIT_FCALL_BY_NAME: @@ -2904,6 +2895,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { + zend_uchar smart_branch_opcode; + uint32_t target_label, target_label2; + uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info; + zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; + zend_class_entry *ce; + bool ce_is_instanceof; + bool on_this; + switch (opline->opcode) { case ZEND_PRE_INC: case ZEND_PRE_DEC: @@ -3160,14 +3159,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op break; } ce = NULL; - ce_is_instanceof = 0; - on_this = 0; + ce_is_instanceof = false; + on_this = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_OBJECT)) { @@ -3208,14 +3207,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op break; } ce = NULL; - ce_is_instanceof = 0; - on_this = 0; + ce_is_instanceof = false; + on_this = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_OBJECT)) { @@ -3249,14 +3248,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op break; } ce = NULL; - ce_is_instanceof = 0; - on_this = 0; + ce_is_instanceof = false; + on_this = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_OBJECT)) { @@ -3418,7 +3417,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } goto done; case ZEND_DO_UCALL: - is_terminated = 1; + is_terminated = true; ZEND_FALLTHROUGH; case ZEND_DO_ICALL: case ZEND_DO_FCALL_BY_NAME: @@ -3547,7 +3546,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } } else { int j; - bool left_frame = 0; + bool left_frame = false; if (!zend_jit_return(&dasm_state, opline, op_array, op1_info, OP1_REG_ADDR())) { @@ -3568,7 +3567,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_free_cvs(&dasm_state)) { goto jit_failure; } - left_frame = 1; + left_frame = true; } if (!left_frame) { for (j = 0 ; j < op_array->last_var; j++) { @@ -3576,7 +3575,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (!left_frame) { - left_frame = 1; + left_frame = true; if (!zend_jit_leave_frame(&dasm_state)) { goto jit_failure; } @@ -3751,14 +3750,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op break; } ce = NULL; - ce_is_instanceof = 0; - on_this = 0; + ce_is_instanceof = false; + on_this = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; op1_addr = 0; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_OBJECT)) { @@ -3902,14 +3901,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op break; } ce = NULL; - ce_is_instanceof = 0; - on_this = 0; + ce_is_instanceof = false; + on_this = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; op1_addr = 0; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_OBJECT)) { @@ -3979,7 +3978,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) { goto jit_failure; } - is_terminated = 1; + is_terminated = true; break; case ZEND_CATCH: case ZEND_FAST_CALL: @@ -3996,7 +3995,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_tail_handler(&dasm_state, opline)) { goto jit_failure; } - is_terminated = 1; + is_terminated = true; break; /* stackless execution */ case ZEND_INCLUDE_OR_EVAL: @@ -4006,7 +4005,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_call(&dasm_state, opline, b + 1)) { goto jit_failure; } - is_terminated = 1; + is_terminated = true; break; case ZEND_JMPZ: case ZEND_JMPNZ: @@ -4067,7 +4066,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op zend_jit_cond_jmp(&dasm_state, next_opline, ssa->cfg.blocks[b].successors[0]); if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) { zend_jit_call(&dasm_state, next_opline, b + 1); - is_terminated = 1; + is_terminated = true; } else { zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1, NULL); } @@ -4091,7 +4090,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b+1].successors[1])) { goto jit_failure; } - is_terminated = 1; + is_terminated = true; } } done: @@ -4106,7 +4105,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } } - handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0, + void *const handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0, (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET, SP_ADJ_JIT); if (!handler) { goto jit_failure; @@ -4146,12 +4145,8 @@ static void zend_jit_collect_calls(zend_op_array *op_array, zend_script *script) static void zend_jit_cleanup_func_info(zend_op_array *op_array) { zend_func_info *func_info = ZEND_FUNC_INFO(op_array); - zend_call_info *caller_info, *callee_info; if (func_info) { - caller_info = func_info->caller_info; - callee_info = func_info->callee_info; - if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) { @@ -4165,34 +4160,30 @@ static void zend_jit_cleanup_func_info(zend_op_array *op_array) ZEND_SET_FUNC_INFO(op_array, NULL); } - while (caller_info) { + for (zend_call_info *caller_info = func_info->caller_info; caller_info; caller_info = caller_info->next_caller) { if (caller_info->caller_op_array) { zend_jit_cleanup_func_info(caller_info->caller_op_array); } - caller_info = caller_info->next_caller; } - while (callee_info) { + + for (zend_call_info *callee_info = func_info->callee_info; callee_info; callee_info = callee_info->next_callee) { if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) { zend_jit_cleanup_func_info(&callee_info->callee_func->op_array); } - callee_info = callee_info->next_callee; } } } static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline) { - zend_ssa ssa; - void *checkpoint; - zend_func_info *func_info; - if (*dasm_ptr == dasm_end) { return FAILURE; } - checkpoint = zend_arena_checkpoint(CG(arena)); + void *const checkpoint = zend_arena_checkpoint(CG(arena)); /* Build SSA */ + zend_ssa ssa; memset(&ssa, 0, sizeof(zend_ssa)); if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) { @@ -4201,7 +4192,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) { zend_jit_collect_calls(op_array, script); - func_info = ZEND_FUNC_INFO(op_array); + zend_func_info *const func_info = ZEND_FUNC_INFO(op_array); func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array); if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { zend_init_func_return_info(op_array, script, &func_info->return_info); @@ -4235,25 +4226,21 @@ static int ZEND_FASTCALL zend_runtime_jit(void) { zend_execute_data *execute_data = EG(current_execute_data); zend_op_array *op_array = &EX(func)->op_array; - zend_op *opline = op_array->opcodes; - zend_jit_op_array_extension *jit_extension; - bool do_bailout = 0; + bool do_bailout = false; zend_shared_alloc_lock(); - if (ZEND_FUNC_INFO(op_array)) { + zend_jit_op_array_extension *const jit_extension = + (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); + + if (jit_extension) { SHM_UNPROTECT(); zend_jit_unprotect(); zend_try { /* restore original opcode handlers */ - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } - jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); + zend_op *const opline = zend_find_function_entry(op_array); opline->handler = jit_extension->orig_handler; /* perform real JIT for this function */ @@ -4276,31 +4263,22 @@ static int ZEND_FASTCALL zend_runtime_jit(void) return 0; } -void zend_jit_check_funcs(HashTable *function_table, bool is_method) { - zend_op *opline; +static void zend_jit_check_funcs(HashTable *function_table, bool is_method) { zend_function *func; - zend_op_array *op_array; - uintptr_t counter; - zend_jit_op_array_extension *jit_extension; ZEND_HASH_MAP_REVERSE_FOREACH_PTR(function_table, func) { if (func->type == ZEND_INTERNAL_FUNCTION) { break; } - op_array = &func->op_array; - opline = op_array->opcodes; - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } + zend_op_array *const op_array = &func->op_array; + zend_op *const opline = zend_find_function_entry(op_array); if (opline->handler == zend_jit_profile_jit_handler) { if (!RUN_TIME_CACHE(op_array)) { continue; } - counter = (uintptr_t)ZEND_COUNTER_INFO(op_array); + const uintptr_t counter = (uintptr_t)ZEND_COUNTER_INFO(op_array); ZEND_COUNTER_INFO(op_array) = 0; - jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); + zend_jit_op_array_extension *const jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array); opline->handler = jit_extension->orig_handler; if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) { zend_real_jit_func(op_array, NULL, NULL); @@ -4312,26 +4290,24 @@ void zend_jit_check_funcs(HashTable *function_table, bool is_method) { void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline) { zend_op_array *op_array = &EX(func)->op_array; - zend_jit_op_array_hot_extension *jit_extension; - uint32_t i; - bool do_bailout = 0; + bool do_bailout = false; zend_shared_alloc_lock(); - jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array); + zend_jit_op_array_hot_extension *const jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array); if (jit_extension) { SHM_UNPROTECT(); zend_jit_unprotect(); zend_try { - for (i = 0; i < op_array->last; i++) { + for (uint32_t i = 0; i < op_array->last; i++) { op_array->opcodes[i].handler = jit_extension->orig_handlers[i]; } /* perform real JIT for this function */ zend_real_jit_func(op_array, NULL, opline); } zend_catch { - do_bailout = 1; + do_bailout = true; } zend_end_try(); zend_jit_protect(); @@ -4349,14 +4325,7 @@ void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg) { if (JIT_G(hot_func)) { - zend_op *opline = op_array->opcodes; - - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } - + zend_op *const opline = zend_find_function_entry(op_array); opline->handler = (const void*)zend_jit_func_hot_counter_handler; } @@ -4375,15 +4344,12 @@ static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cf static int zend_jit_restart_hot_counters(zend_op_array *op_array) { - zend_jit_op_array_hot_extension *jit_extension; - zend_cfg cfg; - uint32_t i; - - jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array); - for (i = 0; i < op_array->last; i++) { + zend_jit_op_array_hot_extension *const jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array); + for (uint32_t i = 0; i < op_array->last; i++) { op_array->opcodes[i].handler = jit_extension->orig_handlers[i]; } + zend_cfg cfg; if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) { return FAILURE; } @@ -4395,25 +4361,22 @@ static int zend_jit_restart_hot_counters(zend_op_array *op_array) static int zend_jit_setup_hot_counters(zend_op_array *op_array) { - zend_jit_op_array_hot_extension *jit_extension; - zend_cfg cfg; - uint32_t i; - ZEND_ASSERT(zend_jit_func_hot_counter_handler != NULL); ZEND_ASSERT(zend_jit_loop_hot_counter_handler != NULL); + zend_cfg cfg; if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) { return FAILURE; } - jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*)); + zend_jit_op_array_hot_extension *const jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*)); if (!jit_extension) { return FAILURE; } memset(&jit_extension->func_info, 0, sizeof(zend_func_info)); jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS; - jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)]; - for (i = 0; i < op_array->last; i++) { + jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) % ZEND_HOT_COUNTERS_COUNT]; + for (uint32_t i = 0; i < op_array->last; i++) { jit_extension->orig_handlers[i] = op_array->opcodes[i].handler; } ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); @@ -4427,6 +4390,23 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array) #include "jit/zend_jit_trace.c" +/** + * Allocate and initialize a new #zend_jit_op_array_extension for the + * given #zend_op_array. + */ +static zend_jit_op_array_extension *zend_jit_new_op_array_extension(zend_op_array *op_array) +{ + zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension)); + if (!jit_extension) { + return NULL; + } + + memset(&jit_extension->func_info, 0, sizeof(zend_func_info)); + ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); + zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension); + return jit_extension; +} + ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) { if (dasm_ptr == NULL) { @@ -4435,7 +4415,6 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) { zend_jit_op_array_extension *jit_extension; - zend_op *opline = op_array->opcodes; if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) { ZEND_SET_FUNC_INFO(op_array, NULL); @@ -4445,27 +4424,17 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) /* Set run-time JIT handler */ ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL); - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } - jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension)); + zend_op *const opline = zend_find_function_entry(op_array); + jit_extension = zend_jit_new_op_array_extension(op_array); if (!jit_extension) { return FAILURE; } - memset(&jit_extension->func_info, 0, sizeof(zend_func_info)); jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC; jit_extension->orig_handler = (void*)opline->handler; - ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); opline->handler = (const void*)zend_jit_runtime_jit_handler; - zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension); return SUCCESS; } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) { - zend_jit_op_array_extension *jit_extension; - zend_op *opline = op_array->opcodes; - if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) { ZEND_SET_FUNC_INFO(op_array, NULL); zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT"); @@ -4474,21 +4443,14 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) ZEND_ASSERT(zend_jit_profile_jit_handler != NULL); if (op_array->function_name) { - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } - jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension)); + zend_op *const opline = zend_find_function_entry(op_array); + zend_jit_op_array_extension *const jit_extension = zend_jit_new_op_array_extension(op_array); if (!jit_extension) { return FAILURE; } - memset(&jit_extension->func_info, 0, sizeof(zend_func_info)); jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST; jit_extension->orig_handler = (void*)opline->handler; - ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); opline->handler = (const void*)zend_jit_profile_jit_handler; - zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension); } return SUCCESS; @@ -4505,17 +4467,13 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) ZEND_EXT_API int zend_jit_script(zend_script *script) { - void *checkpoint; - zend_call_graph call_graph; - zend_func_info *info; - int i; - if (dasm_ptr == NULL || *dasm_ptr == dasm_end) { return FAILURE; } - checkpoint = zend_arena_checkpoint(CG(arena)); + void *const checkpoint = zend_arena_checkpoint(CG(arena)); + zend_call_graph call_graph; call_graph.op_arrays_count = 0; zend_build_call_graph(&CG(arena), script, &call_graph); @@ -4525,14 +4483,14 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - for (i = 0; i < call_graph.op_arrays_count; i++) { + for (int i = 0; i < call_graph.op_arrays_count; i++) { if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) { goto jit_failure; } } } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) { - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + for (int i = 0; i < call_graph.op_arrays_count; i++) { + zend_func_info *const info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) { goto jit_failure; @@ -4541,8 +4499,8 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) } } - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + for (int i = 0; i < call_graph.op_arrays_count; i++) { + zend_func_info *const info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]); if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { @@ -4551,8 +4509,8 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) } } - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + for (int i = 0; i < call_graph.op_arrays_count; i++) { + zend_func_info *const info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) { goto jit_failure; @@ -4562,16 +4520,16 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) } if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) { - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + for (int i = 0; i < call_graph.op_arrays_count; i++) { + zend_func_info *const info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa); } } } - for (i = 0; i < call_graph.op_arrays_count; i++) { - info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + for (int i = 0; i < call_graph.op_arrays_count; i++) { + zend_func_info *const info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); if (info) { if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) { goto jit_failure; @@ -4579,7 +4537,7 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) } } - for (i = 0; i < call_graph.op_arrays_count; i++) { + for (int i = 0; i < call_graph.op_arrays_count; i++) { ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); } } else { @@ -4593,9 +4551,9 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { zend_class_entry *ce; - zend_op_array *op_array; ZEND_HASH_MAP_FOREACH_PTR(&script->class_table, ce) { + zend_op_array *op_array; ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) { if (!ZEND_FUNC_INFO(op_array)) { void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes); @@ -4612,7 +4570,7 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) jit_failure: if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) { - for (i = 0; i < call_graph.op_arrays_count; i++) { + for (int i = 0; i < call_graph.op_arrays_count; i++) { ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); } } @@ -4703,29 +4661,30 @@ static void zend_jit_init_handlers(void) } } -static int zend_jit_make_stubs(void) +/** + * Generate all JIT stub functions. + */ +static bool zend_jit_make_stubs(void) { dasm_State* dasm_state = NULL; - uint32_t i; - dasm_init(&dasm_state, DASM_MAXSECTION); dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); - for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) { + for (uint32_t i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) { dasm_setup(&dasm_state, dasm_actions); if (!zend_jit_stubs[i].stub(&dasm_state)) { - return 0; + return false; } if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name, 0, zend_jit_stubs[i].offset, zend_jit_stubs[i].adjustment)) { - return 0; + return false; } } zend_jit_init_handlers(); dasm_free(&dasm_state); - return 1; + return true; } static void zend_jit_globals_ctor(zend_jit_globals *jit_globals) @@ -4737,7 +4696,7 @@ static void zend_jit_globals_ctor(zend_jit_globals *jit_globals) static int zend_jit_parse_config_num(zend_long jit) { if (jit == 0) { - JIT_G(on) = 0; + JIT_G(on) = false; return SUCCESS; } @@ -4760,7 +4719,7 @@ static int zend_jit_parse_config_num(zend_long jit) if (jit / 10 != 0) return FAILURE; - JIT_G(on) = 1; + JIT_G(on) = true; return SUCCESS; } @@ -4776,30 +4735,30 @@ ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage) if (ZSTR_LEN(jit) == 0 || zend_string_equals_literal_ci(jit, "disable")) { - JIT_G(enabled) = 0; - JIT_G(on) = 0; + JIT_G(enabled) = false; + JIT_G(on) = false; return SUCCESS; } else if (zend_string_equals_literal_ci(jit, "0") || zend_string_equals_literal_ci(jit, "off") || zend_string_equals_literal_ci(jit, "no") || zend_string_equals_literal_ci(jit, "false")) { - JIT_G(enabled) = 1; - JIT_G(on) = 0; + JIT_G(enabled) = true; + JIT_G(on) = false; return SUCCESS; } else if (zend_string_equals_literal_ci(jit, "1") || zend_string_equals_literal_ci(jit, "on") || zend_string_equals_literal_ci(jit, "yes") || zend_string_equals_literal_ci(jit, "true") || zend_string_equals_literal_ci(jit, "tracing")) { - JIT_G(enabled) = 1; - JIT_G(on) = 1; + JIT_G(enabled) = true; + JIT_G(on) = true; JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS; JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE; JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX; return SUCCESS; } else if (zend_string_equals_literal_ci(jit, "function")) { - JIT_G(enabled) = 1; - JIT_G(on) = 1; + JIT_G(enabled) = true; + JIT_G(on) = true; JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT; JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD; JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX; @@ -4810,14 +4769,14 @@ ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage) if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) { goto failure; } - JIT_G(enabled) = 1; + JIT_G(enabled) = true; return SUCCESS; } failure: zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number"); - JIT_G(enabled) = 0; - JIT_G(on) = 0; + JIT_G(enabled) = false; + JIT_G(on) = false; return FAILURE; } @@ -4860,8 +4819,8 @@ ZEND_EXT_API int zend_jit_check_support(void) if (zend_jit_vm_kind != ZEND_VM_KIND_CALL && zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled."); - JIT_G(enabled) = 0; - JIT_G(on) = 0; + JIT_G(enabled) = false; + JIT_G(on) = false; return FAILURE; } @@ -4869,8 +4828,8 @@ ZEND_EXT_API int zend_jit_check_support(void) if (strcmp(sapi_module.name, "phpdbg") != 0) { zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled."); } - JIT_G(enabled) = 0; - JIT_G(on) = 0; + JIT_G(enabled) = false; + JIT_G(on) = false; return FAILURE; } @@ -4884,8 +4843,8 @@ ZEND_EXT_API int zend_jit_check_support(void) default: if (zend_get_user_opcode_handler(i) != NULL) { zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled."); - JIT_G(enabled) = 0; - JIT_G(on) = 0; + JIT_G(enabled) = false; + JIT_G(on) = false; return FAILURE; } } @@ -5061,9 +5020,7 @@ ZEND_EXT_API void zend_jit_shutdown(void) static void zend_jit_reset_counters(void) { - int i; - - for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) { + for (int i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) { zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT; } } @@ -5121,15 +5078,7 @@ static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array) #if 0 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ??? } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) { - zend_op *opline = op_array->opcodes; - zend_jit_op_array_extension *jit_extension = - (zend_jit_op_array_extension*)func_info; - - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } + zend_op *const opline = zend_find_function_entry(op_array); if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) { opline->handler = (const void*)zend_jit_runtime_jit_handler; } else { diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index d22422181af9c..73ebfd9fbcdd4 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -119,6 +119,10 @@ typedef struct _zend_jit_globals { zend_sym_node *symbols; /* symbols for disassembler */ + /** + * True while zend_jit_trace_execute() runs, to avoid + * recursively calling it twice. + */ bool tracing; zend_jit_trace_rec *current_trace; @@ -140,10 +144,36 @@ extern int jit_globals_id; extern zend_jit_globals jit_globals; #endif +/** + * Activate the JIT on the given #zend_op_array. + * + * @return SUCCESS or FAILURE + */ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); + +/** + * Activate the JIT on the given #zend_script. + * + * @return SUCCESS or FAILURE + */ ZEND_EXT_API int zend_jit_script(zend_script *script); + +/** + * Make the JIT machine code area (i.e. #dasm_buf) writable (but not + * executable). Call this before generating new machine code. Call + * zend_jit_protect() when done. + * + * (These write/execute permissions affect only the current process.) + */ ZEND_EXT_API void zend_jit_unprotect(void); + +/** + * Make the JIT machine code area (i.e. #dasm_buf) executable (but not + * writable). Call this after generating new machine code to undo the + * effect of zend_jit_unprotect(). + */ ZEND_EXT_API void zend_jit_protect(void); + ZEND_EXT_API void zend_jit_init(void); ZEND_EXT_API int zend_jit_config(zend_string *jit_options, int stage); ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage); diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 712e3ba113606..c003edd7b7849 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2571,11 +2571,9 @@ static int zend_jit_trace_escape_stub(dasm_State **Dst) static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) { - uint32_t i; - | bl >2 |1: - for (i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { + for (uint32_t i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { | bl >2 } |2: @@ -2990,13 +2988,10 @@ static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_t static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) { - uint32_t i; - const void *exit_addr; - /* Emit veneers table for exit points (B instruction for each exit number) */ |.cold_code - for (i = 0; i < t->exit_count; i++) { - exit_addr = zend_jit_trace_get_exit_addr(i); + for (uint32_t i = 0; i < t->exit_count; i++) { + const void *const exit_addr = zend_jit_trace_get_exit_addr(i); if (!exit_addr) { return 0; } @@ -3010,9 +3005,7 @@ static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) { int ret = 0; - uint8_t *p, *end; const void *veneer = NULL; - ptrdiff_t delta; if (jmp_table_size) { const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*))); @@ -3026,8 +3019,8 @@ static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size } while (--jmp_table_size); } - end = (uint8_t*)code; - p = end + size; + uint8_t *const end = (uint8_t*)code; + uint8_t *p = end + size; while (p > end) { uint32_t *ins_ptr; uint32_t ins; @@ -3037,7 +3030,7 @@ static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size ins = *ins_ptr; if ((ins & 0xfc000000u) == 0x14000000u) { // B (imm26:0..25) - delta = (uint32_t*)from_addr - ins_ptr; + ptrdiff_t delta = (uint32_t*)from_addr - ins_ptr; if (((ins ^ (uint32_t)delta) & 0x01ffffffu) == 0) { delta = (uint32_t*)to_addr - ins_ptr; if (((delta + 0x02000000) >> 26) != 0) { @@ -3052,7 +3045,7 @@ static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size } else if ((ins & 0xff000000u) == 0x54000000u || (ins & 0x7e000000u) == 0x34000000u) { // B.cond, CBZ, CBNZ (imm19:5..23) - delta = (uint32_t*)from_addr - ins_ptr; + ptrdiff_t delta = (uint32_t*)from_addr - ins_ptr; if (((ins ^ ((uint32_t)delta << 5)) & 0x00ffffe0u) == 0) { delta = (uint32_t*)to_addr - ins_ptr; if (((delta + 0x40000) >> 19) != 0) { @@ -3070,7 +3063,7 @@ static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size } } else if ((ins & 0x7e000000u) == 0x36000000u) { // TBZ, TBNZ (imm14:5..18) - delta = (uint32_t*)from_addr - ins_ptr; + ptrdiff_t delta = (uint32_t*)from_addr - ins_ptr; if (((ins ^ ((uint32_t)delta << 5)) & 0x0007ffe0u) == 0) { delta = (uint32_t*)to_addr - ins_ptr; if (((delta + 0x2000) >> 14) != 0) { @@ -3105,7 +3098,6 @@ static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) { - const void *link_addr; size_t prologue_size; /* Skip prologue. */ @@ -3126,7 +3118,7 @@ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, // mov FP, FCARG1x prologue_size = 12; } - link_addr = (const void*)((const char*)t->code_start + prologue_size); + const void *const link_addr = (const void*)((const char*)t->code_start + prologue_size); if (timeout_exit_addr) { /* Check timeout for links to LOOP */ @@ -3243,11 +3235,8 @@ static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32 static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) { - zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - size_t offset = jit_extension->offset; const void *handler = - (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; + (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO2(op_array, opline)->call_handler; if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; @@ -3309,8 +3298,6 @@ static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_arra const zend_op *next_opline = trace->opline; const zend_op *exit_opline = NULL; - uint32_t exit_point; - const void *exit_addr; uint32_t old_info = 0; uint32_t old_res_info = 0; zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; @@ -3357,8 +3344,8 @@ static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_arra old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); @@ -3781,13 +3768,10 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op if (may_overflow && (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { - int32_t exit_point; - const void *exit_addr; - zend_jit_trace_stack *stack; - uint32_t old_op1_info, old_res_info = 0; + uint32_t old_res_info = 0; - stack = JIT_G(current_frame)->stack; - old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); + zend_jit_trace_stack *const stack = JIT_G(current_frame)->stack; + const uint32_t old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); @@ -3811,8 +3795,8 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op } } - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -5081,13 +5065,11 @@ static int zend_jit_concat_helper(dasm_State **Dst, static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) { - zend_jit_addr op1_addr, op2_addr; - ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); - op1_addr = OP1_ADDR(); - op2_addr = OP2_ADDR(); + const zend_jit_addr op1_addr = OP1_ADDR(); + const zend_jit_addr op2_addr = OP2_ADDR(); return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); } @@ -6017,10 +5999,10 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) { - zend_jit_addr op2_addr, op3_addr, res_addr; + zend_jit_addr res_addr; - op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; - op3_addr = OP1_DATA_ADDR(); + const zend_jit_addr op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; + const zend_jit_addr op3_addr = OP1_DATA_ADDR(); if (opline->result_type == IS_UNUSED) { res_addr = 0; } else { @@ -6228,14 +6210,13 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) { - zend_jit_addr op2_addr, op3_addr, var_addr; const void *not_found_exit_addr = NULL; uint32_t var_info = MAY_BE_NULL; ZEND_ASSERT(opline->result_type == IS_UNUSED); - op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; - op3_addr = OP1_DATA_ADDR(); + const zend_jit_addr op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; + const zend_jit_addr op3_addr = OP1_DATA_ADDR(); | SET_EX_OPLINE opline, REG0 if (op1_info & MAY_BE_REF) { @@ -6374,7 +6355,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint3 } } - var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + const zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); switch (opline->extended_value) { case ZEND_ADD: case ZEND_SUB: @@ -6466,13 +6447,11 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint3 static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) { - zend_jit_addr op1_addr, op2_addr; - ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); - op1_addr = OP1_ADDR(); - op2_addr = OP2_ADDR(); + zend_jit_addr op1_addr = OP1_ADDR(); + const zend_jit_addr op2_addr = OP2_ADDR(); if (op1_info & MAY_BE_REF) { binary_op_type binary_op = get_binary_op(opline->extended_value); @@ -8641,9 +8620,6 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) { - int32_t exit_point; - const void *exit_addr; - if (func->type == ZEND_INTERNAL_FUNCTION) { #ifdef ZEND_WIN32 // TODO: ASLR may cause different addresses in different workers ??? @@ -8659,8 +8635,8 @@ static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zen return 0; } - exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -8837,12 +8813,11 @@ static int zend_jit_init_method_call(dasm_State **Dst, zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; zend_function *func = NULL; - zval *function_name; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); - function_name = RT_CONSTANT(opline, opline->op2); + zval *const function_name = RT_CONSTANT(opline, opline->op2); if (info) { call_info = info->callee_info; @@ -8957,11 +8932,8 @@ static int zend_jit_init_method_call(dasm_State **Dst, && trace->op == ZEND_JIT_TRACE_INIT_CALL && trace->func ) { - int32_t exit_point; - const void *exit_addr; - - exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -9076,14 +9048,10 @@ static int zend_jit_init_closure_call(dasm_State **Dst, && trace->op == ZEND_JIT_TRACE_INIT_CALL && trace->func && trace->func->type == ZEND_USER_FUNCTION) { - const zend_op *opcodes; - int32_t exit_point; - const void *exit_addr; - func = (zend_function*)trace->func; - opcodes = func->op_array.opcodes; - exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const zend_op *const opcodes = func->op_array.opcodes; + const int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -9129,12 +9097,10 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; const zend_function *func = NULL; - uint32_t i; zend_jit_addr res_addr; uint32_t call_num_args = 0; bool unknown_num_args = 0; const void *exit_addr = NULL; - const zend_op *prev_opline; if (RETURN_VALUE_USED(opline)) { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); @@ -9143,7 +9109,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RSP, TMP_ZVAL_OFFSET); } - prev_opline = opline - 1; + const zend_op *prev_opline = opline - 1; while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { prev_opline--; } @@ -9353,7 +9319,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // opline = op_array->opcodes; if (func && !unknown_num_args) { | ADD_SUB_64_WITH_CONST_32 add, TMP1, RX, (EX_NUM_TO_VAR(call_num_args) + offsetof(zval, u1.type_info)), TMP1 // induction variable - for (i = call_num_args; i < func->op_array.last_var; i++) { + for (uint32_t i = call_num_args; i < func->op_array.last_var; i++) { | // ZVAL_UNDEF(EX_VAR(n)) | str wzr, [TMP1], #16 } @@ -9604,7 +9570,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // zend_vm_stack_free_args(call); if (func && !unknown_num_args) { - for (i = 0; i < call_num_args; i++ ) { + for (uint32_t i = 0; i < call_num_args; i++ ) { if (zend_jit_needs_arg_dtor(func, i, call_info)) { uint32_t offset = EX_NUM_TO_VAR(i); zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset); @@ -9722,7 +9688,6 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) { uint32_t arg_num = opline->op2.num; - zend_jit_addr arg_addr; ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); @@ -9771,7 +9736,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o } } - arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + const zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); @@ -9807,10 +9772,8 @@ static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) { - zend_jit_addr op1_addr, arg_addr, ref_addr; - - op1_addr = OP1_ADDR(); - arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + zend_jit_addr op1_addr = OP1_ADDR(); + const zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (!zend_jit_reuse_ip(Dst)) { return 0; @@ -9865,7 +9828,7 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend | movz TMP1w, #GC_REFERENCE | str TMP1w, [REG0, #offsetof(zend_reference, gc.u.type_info)] | str xzr, [REG0, #offsetof(zend_reference, sources.ptr)] - ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); + const zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); if (opline->op1_type == IS_VAR) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); @@ -9892,13 +9855,12 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) { uint32_t arg_num = opline->op2.num; - zend_jit_addr arg_addr; ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || arg_num <= MAX_ARG_FLAG_NUM); - arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + const zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (!zend_jit_reuse_ip(Dst)) { return 0; @@ -10326,7 +10288,6 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { - uint32_t mask; zend_jit_addr op1_addr = OP1_ADDR(); // TODO: support for is_resource() ??? @@ -10369,7 +10330,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t } if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - mask = opline->extended_value; + const uint32_t mask = opline->extended_value; if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 if (exit_addr) { @@ -11621,13 +11582,11 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, uint32_t target_label2, const void *exit_addr) { - zend_jit_addr op2_addr, res_addr; - // TODO: support for empty() ??? ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); - op2_addr = OP2_ADDR(); - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + const zend_jit_addr op2_addr = OP2_ADDR(); + const zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr @@ -12118,7 +12077,6 @@ static int zend_jit_fetch_obj(dasm_State **Dst, int may_throw) { zval *member; - zend_property_info *prop_info; bool may_be_dynamic = 1; zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); @@ -12131,7 +12089,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); - prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); + zend_property_info *prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 @@ -12367,11 +12325,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { uint32_t flags = 0; - uint32_t old_info; zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; - int32_t exit_point; - const void *exit_addr; - uint32_t type; zend_jit_addr val_addr = prop_addr; if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) @@ -12389,13 +12343,15 @@ static int zend_jit_fetch_obj(dasm_State **Dst, ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; } - type = concrete_type(res_info); + const uint32_t type = concrete_type(res_info); + + const void *exit_addr; if (prop_type != IS_UNKNOWN && prop_type != IS_UNDEF && prop_type != IS_REFERENCE && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) { - exit_point = zend_jit_trace_get_exit_point(opline, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -12407,10 +12363,10 @@ static int zend_jit_fetch_obj(dasm_State **Dst, SET_STACK_REG(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); } - old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + const uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); - exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline+1, flags); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -12590,9 +12546,6 @@ static int zend_jit_incdec_obj(dasm_State **Dst, zend_class_entry *trace_ce, uint8_t prop_type) { - zval *member; - zend_string *name; - zend_property_info *prop_info; zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr res_addr = 0; zend_jit_addr prop_addr; @@ -12608,10 +12561,10 @@ static int zend_jit_incdec_obj(dasm_State **Dst, res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } - member = RT_CONSTANT(opline, opline->op2); + zval *const member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); - name = Z_STR_P(member); - prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); + zend_string *const name = Z_STR_P(member); + zend_property_info *prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 @@ -12926,12 +12879,10 @@ static int zend_jit_incdec_obj(dasm_State **Dst, && (res_info & MAY_BE_LONG)) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); - int32_t exit_point; - const void *exit_addr; SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -13017,9 +12968,6 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, zend_class_entry *trace_ce, uint8_t prop_type) { - zval *member; - zend_string *name; - zend_property_info *prop_info; zend_jit_addr val_addr = OP1_DATA_ADDR(); zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr prop_addr; @@ -13032,10 +12980,10 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, ZEND_ASSERT(op1_info & MAY_BE_OBJECT); ZEND_ASSERT(opline->result_type == IS_UNUSED); - member = RT_CONSTANT(opline, opline->op2); + zval *const member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); - name = Z_STR_P(member); - prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); + zend_string *const name = Z_STR_P(member); + zend_property_info *prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 @@ -13399,9 +13347,6 @@ static int zend_jit_assign_obj(dasm_State **Dst, uint8_t prop_type, int may_throw) { - zval *member; - zend_string *name; - zend_property_info *prop_info; zend_jit_addr val_addr = OP1_DATA_ADDR(); zend_jit_addr res_addr = 0; zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); @@ -13416,10 +13361,10 @@ static int zend_jit_assign_obj(dasm_State **Dst, ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); - member = RT_CONSTANT(opline, opline->op2); + zval *const member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); - name = Z_STR_P(member); - prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); + zend_string *const name = Z_STR_P(member); + zend_property_info *prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 @@ -13701,12 +13646,9 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_i static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) { if (opline->op1_type == IS_CONST) { - zval *zv; - size_t len; - - zv = RT_CONSTANT(opline, opline->op1); + zval *const zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); - len = Z_STRLEN_P(zv); + const size_t len = Z_STRLEN_P(zv); if (len > 0) { const char *str = Z_STRVAL_P(zv); @@ -13742,12 +13684,9 @@ static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_i static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) { if (opline->op1_type == IS_CONST) { - zval *zv; - size_t len; - - zv = RT_CONSTANT(opline, opline->op1); + zval *const zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); - len = Z_STRLEN_P(zv); + const size_t len = Z_STRLEN_P(zv); | SET_ZVAL_LVAL res_addr, len, TMP1, TMP2 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { @@ -13778,12 +13717,9 @@ static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1 static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) { if (opline->op1_type == IS_CONST) { - zval *zv; - zend_long count; - - zv = RT_CONSTANT(opline, opline->op1); + zval *const zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); - count = zend_hash_num_elements(Z_ARRVAL_P(zv)); + const zend_long count = zend_hash_num_elements(Z_ARRVAL_P(zv)); | SET_ZVAL_LVAL res_addr, count, TMP1, TMP2 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { @@ -13874,13 +13810,6 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) { - uint32_t count; - Bucket *p; - const zend_op *target; - int b; - int32_t exit_point; - const void *exit_addr; - if (default_label) { | cbz REG0, &default_label } else if (next_opline) { @@ -13908,8 +13837,8 @@ static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend trace_info->jmp_table_size += zend_hash_num_elements(jumptable); } - count = jumptable->nNumUsed; - p = jumptable->arData; + uint32_t count = jumptable->nNumUsed; + Bucket *p = jumptable->arData; do { if (Z_TYPE(p->val) == IS_UNDEF) { if (default_label) { @@ -13920,15 +13849,15 @@ static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend | .addr =>default_b } } else { - target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); + const zend_op *const target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); if (!next_opline) { - b = ssa->cfg.map[target - op_array->opcodes]; + const int b = ssa->cfg.map[target - op_array->opcodes]; | .addr =>b } else if (next_opline == target) { | .addr >3 } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(target, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -14002,24 +13931,20 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o uint32_t op1_info = OP1_INFO(); zend_jit_addr op1_addr = OP1_ADDR(); const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); - const zend_op *target; int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; - int b; - int32_t exit_point; const void *fallback_label = NULL; const void *default_label = NULL; - const void *exit_addr; if (next_opline) { if (next_opline != opline + 1) { - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); fallback_label = zend_jit_trace_get_exit_addr(exit_point); if (!fallback_label) { return 0; } } if (next_opline != default_opline) { - exit_point = zend_jit_trace_get_exit_point(default_opline, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(default_opline, 0); default_label = zend_jit_trace_get_exit_addr(exit_point); if (!default_label) { return 0; @@ -14094,15 +14019,15 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o | .addr =>default_b } } else { - target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); + const zend_op *const target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); if (!next_opline) { - b = ssa->cfg.map[target - op_array->opcodes]; + const int b = ssa->cfg.map[target - op_array->opcodes]; | .addr =>b } else if (next_opline == target) { | .addr >3 } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(target, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -14512,7 +14437,6 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o if (!exit_addr || exit_opcode == ZEND_JMP) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); - uint32_t val_info; if (RETURN_VALUE_USED(opline)) { zend_jit_addr res_addr = RES_ADDR(); @@ -14572,7 +14496,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o | MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 } - val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); + uint32_t val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); if (val_info & MAY_BE_ARRAY) { val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } @@ -14628,14 +14552,12 @@ static int zend_jit_fetch_constant(dasm_State **Dst, if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); - int32_t exit_point; - const void *exit_addr = NULL; SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); - exit_point = zend_jit_trace_get_exit_point(opline+1, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline+1, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -14726,18 +14648,15 @@ static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t o static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) { - uint32_t offset; - - offset = (opline->opcode == ZEND_ROPE_INIT) ? + const uint32_t offset = (opline->opcode == ZEND_ROPE_INIT) ? opline->result.var : opline->op1.var + opline->extended_value * sizeof(zend_string*); if (opline->op2_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op2); - zend_string *str; ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); - str = Z_STR_P(zv); + zend_string *const str = Z_STR_P(zv); | LOAD_ADDR REG0, str | MEM_ACCESS_64_WITH_UOFFSET str, REG0, FP, offset, TMP1 } else { @@ -14840,8 +14759,6 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, { zend_jit_addr var_addr = *var_addr_ptr; uint32_t var_info = *var_info_ptr; - int32_t exit_point; - const void *exit_addr; if (add_indirect_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); @@ -14874,8 +14791,8 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, if (!(var_type & IS_TRACE_REFERENCE) && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { - exit_point = zend_jit_trace_get_exit_point(opline, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -15371,18 +15288,14 @@ void **dasm_labels_veneers = NULL; static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset) { - void *veneer; - ptrdiff_t na; - int n, m; - /* try to reuse veneers for global labels */ if ((ins >> 16) == DASM_REL_LG && *(b-1) < 0 && dasm_labels_veneers[-*(b-1)]) { - veneer = dasm_labels_veneers[-*(b-1)]; - na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; - n = (int)na; + void *const veneer = dasm_labels_veneers[-*(b-1)]; + const ptrdiff_t na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; + const int n = (int)na; /* check if we can jump to veneer */ if ((ptrdiff_t)n != na) { @@ -15417,9 +15330,9 @@ static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int ZEND_ASSERT(exit_point < t->exit_count); - veneer = (char*)buffer + dasm_getpclabel(&Dst, 1) - (t->exit_count - exit_point) * 4; - na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; - n = (int)na; + void *const veneer = (char*)buffer + dasm_getpclabel(&Dst, 1) - (t->exit_count - exit_point) * 4; + const ptrdiff_t na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; + const int n = (int)na; /* check if we can jump to veneer */ if ((ptrdiff_t)n != na) { @@ -15455,14 +15368,14 @@ static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int } } - veneer = (char*)buffer + (Dst->codesize + dasm_venners_size); + void *const veneer = (char*)buffer + (Dst->codesize + dasm_venners_size); if (veneer > dasm_end) { return 0; /* jit_buffer_size overflow */ } - na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; - n = (int)na; + ptrdiff_t na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; + const int n = (int)na; /* check if we can jump to veneer */ if ((ptrdiff_t)n != na) { @@ -15501,7 +15414,7 @@ static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int /* check if we can use B to jump from veneer */ na = (ptrdiff_t)cp + offset - (ptrdiff_t)veneer - 4; - m = (int)na; + const int m = (int)na; if ((ptrdiff_t)m != na) { ZEND_ASSERT(0); return 0; diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index fb640ff9b97ca..c35829ef47030 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -219,8 +219,18 @@ static zend_always_inline bool zend_jit_same_addr(zend_jit_addr addr1, zend_jit_ return 0; } +/** + * #ZEND_FUNC_INFO for functions handled by the JIT with trigger + * #ZEND_JIT_ON_FIRST_EXEC or #ZEND_JIT_ON_PROF_REQUEST. The handler + * #override is zend_jit_runtime_jit_handler(). + */ typedef struct _zend_jit_op_array_extension { zend_func_info func_info; + + /** + * The original opcode handler, just in case we need to + * restore it. + */ const void *orig_handler; } zend_jit_op_array_extension; @@ -233,11 +243,7 @@ extern int zend_jit_profile_counter_rid; /* Hot Counters */ -#define ZEND_HOT_COUNTERS_COUNT 128 - -extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT]; - -static zend_always_inline zend_long zend_jit_hash(const void *ptr) +static zend_always_inline zend_ulong zend_jit_hash(const void *ptr) { uintptr_t x; @@ -256,14 +262,31 @@ static zend_always_inline zend_long zend_jit_hash(const void *ptr) void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline); +/** + * #ZEND_FUNC_INFO for functions being handled by the JIT with trigger + * #ZEND_JIT_ON_HOT_COUNTERS. The handler override is + * #zend_jit_func_hot_counter_handler(). + */ typedef struct _zend_jit_op_array_hot_extension { zend_func_info func_info; + + /** + * Pointer to the counter. Points to inside global variable + * #zend_jit_hot_counters. + */ int16_t *counter; + + /** + * The original handler for each opline, just in case we need + * to restore it. + */ const void *orig_handlers[1]; } zend_jit_op_array_hot_extension; -#define zend_jit_op_array_hash(op_array) \ - zend_jit_hash((op_array)->opcodes) +static zend_always_inline zend_ulong zend_jit_op_array_hash(const zend_op_array *op_array) +{ + return zend_jit_hash(op_array->opcodes); +} extern const zend_op *zend_jit_halt_op; @@ -333,8 +356,6 @@ zend_constant* ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t fla zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key); /* Tracer */ -#define zend_jit_opline_hash(opline) \ - zend_jit_hash(opline) #define ZEND_JIT_TRACE_STOP(_) \ _(LOOP, "loop") \ @@ -396,10 +417,29 @@ typedef enum _zend_jit_trace_stop { #define ZEND_JIT_TRACE_START_RETURN (1<<2) #define ZEND_JIT_TRACE_START_SIDE (1<<3) /* used for side traces */ +/** + * Was this opline already processed by the JIT? This is used to skip + * further JIT calls. + */ #define ZEND_JIT_TRACE_JITED (1<<4) + +/** + * Was this opline blacklisted by the JIT? Blacklisted oplines will + * never be inspected again by the JIT. + */ #define ZEND_JIT_TRACE_BLACKLISTED (1<<5) + +/** + * Is this opline supported by the JIT? This is determined by + * zend_jit_trace_supported(). + */ #define ZEND_JIT_TRACE_UNSUPPORTED (1<<6) +/** + * Not an actual trace_flag; just a descriptive macro indicating the + * absence of #ZEND_JIT_TRACE_UNSUPPORTED for the + * zend_jit_trace_supported() return value. + */ #define ZEND_JIT_TRACE_SUPPORTED 0 #define ZEND_JIT_EXIT_JITED (1<<0) @@ -414,25 +454,89 @@ typedef enum _zend_jit_trace_stop { #define ZEND_JIT_EXIT_METHOD_CALL (1<<9) /* exit because of polymorphic INIT_METHOD_CALL call */ #define ZEND_JIT_EXIT_INVALIDATE (1<<10) /* invalidate current trace */ +/** + * Per-opline information for #zend_jit_op_array_trace_extension. + */ typedef union _zend_op_trace_info { zend_op dummy; /* the size of this structure must be the same as zend_op */ struct { + /** + * The original opcode handler, just in case we need + * to restore it. + */ const void *orig_handler; const void *call_handler; + + /** + * Pointer to the counter. Points to inside global variable + * #zend_jit_hot_counters. + */ int16_t *counter; + + /** + * Bit mask of the ZEND_JIT_TRACE_* flags. + */ uint8_t trace_flags; }; } zend_op_trace_info; +/** + * #ZEND_FUNC_INFO for functions handled by the JIT with trigger + * #ZEND_JIT_ON_HOT_TRACE. The handler override is + * #zend_jit_loop_trace_counter_handler() or + * #zend_jit_func_trace_counter_handler(), configured by + * #zend_jit_setup_hot_trace_counters(). + */ typedef struct _zend_jit_op_array_trace_extension { zend_func_info func_info; const zend_op_array *op_array; size_t offset; /* offset from "zend_op" to corresponding "op_info" */ + + /** + * Information for each opline. + */ zend_op_trace_info trace_info[1]; } zend_jit_op_array_trace_extension; -#define ZEND_OP_TRACE_INFO(opline, offset) \ - ((zend_op_trace_info*)(((char*)opline) + offset)) +static zend_always_inline size_t ZEND_OP_TRACE_INFO_OFFSET(const zend_op_array *op_array) +{ + const zend_jit_op_array_trace_extension *jit_extension = + (const zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + ZEND_ASSERT(jit_extension != NULL); + return jit_extension->offset; +} + +static zend_always_inline zend_op_trace_info *ZEND_OP_TRACE_INFO(const zend_op *opline, size_t offset) +{ + return (zend_op_trace_info*)((char*)opline + offset); +} + +/** + * Access the #zend_op_trace_info of an #zend_op by looking up the + * offset from the specified #zend_op_array. + */ +static zend_always_inline zend_op_trace_info *ZEND_OP_TRACE_INFO2(const zend_op_array *op_array, const zend_op *opline) +{ + /* the opline must be from the specified function */ + ZEND_ASSERT(opline >= op_array->opcodes); + ZEND_ASSERT(opline < op_array->opcodes + op_array->last); + + const zend_jit_op_array_trace_extension *jit_extension = + (const zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + ZEND_ASSERT(jit_extension != NULL); + return ZEND_OP_TRACE_INFO(opline, jit_extension->offset); +} + +/** + * Access the #zend_ssa of an #zend_op_array. + */ +static zend_always_inline zend_ssa *zend_jit_op_array_trace_ssa(const zend_op_array *op_array) +{ + zend_jit_op_array_trace_extension *jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + ZEND_ASSERT(jit_extension != NULL); + return &jit_extension->func_info.ssa; +} /* Recorder */ typedef enum _zend_jit_trace_op { @@ -707,7 +811,6 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HAND ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS); int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline); -int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs); zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start, uint32_t is_megamorphc); static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit_trace_rec *trace, const zend_op *opline, bool *exit_if_true) @@ -772,6 +875,25 @@ static zend_always_inline bool zend_jit_may_be_polymorphic_call(const zend_op *o } } +/** + * Look up the function entry, i.e. the first #zend_op that is not + * #ZEND_RECV or #ZEND_RECV_INIT. + */ +static inline zend_op *zend_find_function_entry(zend_op_array *op_array) +{ + zend_op *opline = op_array->opcodes; + + if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { + opline++; + + ZEND_ASSERT(opline < op_array->opcodes + op_array->last); + } + } + + return opline; +} + /* Instruction cache flush */ #ifndef JIT_CACHE_FLUSH # if ZEND_JIT_TARGET_ARM64 diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 55db634ec8825..506c7b8c2e6e4 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -88,7 +88,6 @@ static int zend_jit_trace_startup(zend_bool reattached) static const void *zend_jit_trace_allocate_exit_group(uint32_t n) { dasm_State* dasm_state = NULL; - const void *entry; char name[32]; dasm_init(&dasm_state, DASM_MAXSECTION); @@ -97,14 +96,12 @@ static const void *zend_jit_trace_allocate_exit_group(uint32_t n) zend_jit_trace_exit_group_stub(&dasm_state, n); sprintf(name, "jit$$trace_exit_%d", n); - entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0, SP_ADJ_JIT, SP_ADJ_NONE); + const void *const entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0, SP_ADJ_JIT, SP_ADJ_NONE); dasm_free(&dasm_state); #ifdef HAVE_DISASM if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { - uint32_t i; - - for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { + for (uint32_t i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { sprintf(name, "jit$$trace_exit_%d", n + i); zend_jit_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING); } @@ -154,9 +151,8 @@ static zend_jit_trace_info *zend_jit_get_current_trace_info(void) static uint32_t zend_jit_trace_find_exit_point(const void* addr) { uint32_t n = ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP; - uint32_t i; - for (i = 0; i < n; i++) { + for (uint32_t i = 0; i < n; i++) { if ((const char*)addr >= (const char*)zend_jit_exit_groups[i] && (const char*)addr < (const char*)zend_jit_exit_groups[i] + (ZEND_JIT_EXIT_POINTS_PER_GROUP * ZEND_JIT_EXIT_POINTS_SPACING)) { @@ -171,7 +167,6 @@ static uint32_t zend_jit_trace_find_exit_point(const void* addr) static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags) { zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; - uint32_t exit_point; const zend_op_array *op_array; uint32_t stack_offset = (uint32_t)-1; uint32_t stack_size; @@ -219,7 +214,7 @@ static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t } while (i > 0); } - exit_point = t->exit_count; + const uint32_t exit_point = t->exit_count; if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) { if (stack_size != 0 && stack_offset == (uint32_t)-1) { stack_offset = t->stack_map_size; @@ -256,9 +251,7 @@ static void zend_jit_trace_add_code(const void *start, uint32_t size) */ static uint32_t zend_jit_find_trace(const void *addr) { - uint32_t i; - - for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) { + for (uint32_t i = 1; i < ZEND_JIT_TRACE_NUM; i++) { if (zend_jit_traces[i].code_start == addr) { return i; } @@ -290,7 +283,7 @@ static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t return buf.s; } -static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline) +static bool zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline) { switch (opline->opcode) { case ZEND_IS_IDENTICAL: @@ -313,7 +306,7 @@ static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op case ZEND_ARRAY_KEY_EXISTS: if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) { /* smart branch */ - return 1; + return true; } break; case ZEND_JMPZ: @@ -332,11 +325,11 @@ static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op case ZEND_SWITCH_STRING: case ZEND_MATCH: /* branch opcodes */ - return 1; + return true; case ZEND_NEW: if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) { /* NEW may skip constructor without arguments */ - return 1; + return true; } break; case ZEND_CATCH: @@ -350,16 +343,16 @@ static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op case ZEND_INCLUDE_OR_EVAL: case ZEND_MATCH_ERROR: /* unsupported */ - return 1; + return true; case ZEND_DO_FCALL: /* potentially polymorphic call */ - return 1; + return true; #if 0 case ZEND_DO_UCALL: case ZEND_DO_FCALL_BY_NAME: /* monomorphic call */ // TODO: recompilation may change target ??? - return 0; + return false; #endif case ZEND_RETURN_BY_REF: case ZEND_RETURN: @@ -368,7 +361,7 @@ static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op default: break; } - return 0; + return false; } static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(zend_uchar type, uint32_t info) @@ -458,25 +451,40 @@ static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa #define CHECK_OP1_DATA_TRACE_TYPE() \ CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type) -static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array) +static zend_always_inline size_t zend_jit_trace_op_array_frame_size(const zend_op_array *op_array) { - if (op_array && op_array->type == ZEND_USER_FUNCTION) { + if (op_array) { + assert(op_array->type == ZEND_USER_FUNCTION); return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack))); - } else if (op_array) { - return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack))); } else { return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack)); } } -static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) +static zend_always_inline size_t zend_jit_trace_function_frame_size(const zend_function *func) { - return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array)); + if (func && func->type == ZEND_USER_FUNCTION) { + return zend_jit_trace_op_array_frame_size(&func->op_array); + } else if (func) { + return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(func->common.num_args * sizeof(zend_jit_trace_stack))); + } else { + return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack)); + } } -static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) +static zend_jit_trace_stack_frame* zend_jit_trace_op_array_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) { - return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array)); + return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_op_array_frame_size(op_array)); +} + +static zend_jit_trace_stack_frame* zend_jit_trace_function_call_frame(zend_jit_trace_stack_frame *frame, const zend_function *func) +{ + return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_function_frame_size(func)); +} + +static zend_jit_trace_stack_frame* zend_jit_trace_op_array_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) +{ + return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_op_array_frame_size(op_array)); } static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, zend_uchar type) @@ -489,7 +497,7 @@ static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack return; } if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { - zend_arg_info *arg_info; + const zend_arg_info *arg_info; ZEND_ASSERT(arg_num <= op_array->num_args); arg_info = &op_array->arg_info[arg_num-1]; @@ -523,7 +531,7 @@ static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, if (type != IS_UNKNOWN && type < IS_STRING && ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type)) { - return 0; + return false; } } if (call_info && arg_num < call_info->num_args && call_info->arg_info[arg_num].opline) { @@ -538,7 +546,7 @@ static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, // TODO: few functions (e.g. pcntl_exec) modify arrays in-place ??? if (type != IS_ARRAY && (ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) { - return 0; + return false; } } } @@ -546,15 +554,12 @@ static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, } } - return 1; + return true; } static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script) { - zend_jit_op_array_trace_extension *jit_extension; - zend_ssa *ssa; - - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); jit_extension->func_info.num = 0; jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC @@ -562,7 +567,7 @@ static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_sc | ZEND_FUNC_JIT_ON_HOT_COUNTERS | ZEND_FUNC_JIT_ON_HOT_TRACE; memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa)); - ssa = &jit_extension->func_info.ssa; + zend_ssa *const ssa = &jit_extension->func_info.ssa; if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) { do { @@ -673,29 +678,21 @@ static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline) static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack) { - const zend_op_array *op_array; - zend_jit_trace_rec *p; - int k, vars_count; - zend_bitset use, def; uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS; - uint32_t set_size; - zend_ssa_phi *prev = NULL; int level = 0; ALLOCA_FLAG(use_heap); - op_array = trace_buffer->op_array; - set_size = zend_bitset_len(op_array->last_var + op_array->T); - use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap); + const zend_op_array *const op_array = trace_buffer->op_array; + const uint32_t set_size = zend_bitset_len(op_array->last_var + op_array->T); + const zend_bitset use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap); memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE); - def = use + set_size; - p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; - for (;;p++) { + const zend_bitset def = use + set_size; + for (zend_jit_trace_rec *p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;;p++) { if (p->op == ZEND_JIT_TRACE_VM && level == 0) { const zend_op *opline = p->opline; - int len; zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def); - len = zend_jit_trace_op_len(opline); + int len = zend_jit_trace_op_len(opline); while (len > 1) { opline++; if (opline->opcode != ZEND_OP_DATA) { @@ -721,12 +718,14 @@ static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ss zend_bitset_intersection(use, def, set_size); + int vars_count; if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { vars_count = op_array->last_var; } else { vars_count = op_array->last_var + op_array->T; } - for (k = 0; k < vars_count; k++) { + zend_ssa_phi *prev = NULL; + for (int k = 0; k < vars_count; k++) { if (zend_bitset_in(use, k)) { zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1, ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) + @@ -762,9 +761,8 @@ static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32 const zend_op_array *op_array = trace_buffer->op_array; const zend_op *opline = trace_buffer[1].opline; int count = opline - op_array->opcodes; - int i; - for(i = 0; i < count; i++) { + for(int i = 0; i < count; i++) { zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1, ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) + @@ -792,7 +790,6 @@ static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32 static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack) { const zend_op *opline = trace_buffer[1].opline - 1; - int i; if (RETURN_VALUE_USED(opline)) { zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1, @@ -800,7 +797,7 @@ static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_ ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) + sizeof(void*) * 2); - i = EX_VAR_TO_NUM(opline->result.var); + const int i = EX_VAR_TO_NUM(opline->result.var); phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi))); phi->sources[0] = STACK_VAR(stack, i); phi->sources[1] = -1; @@ -816,13 +813,9 @@ static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_ return ssa_vars_count; } -static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +static bool zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *const *tssa_opcodes, zend_ssa *tssa, int ssa_var) { - int var, use; - zend_ssa_op *op; - zend_ssa_var_info *info; - unsigned int no_val; - zend_ssa_alias_kind alias; + int var; if (tssa->vars[ssa_var].phi_use_chain) { // TODO: this may be incorrect ??? @@ -830,10 +823,13 @@ static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const } else { var = ssa_var; } - use = tssa->vars[var].use_chain; + const int use = tssa->vars[var].use_chain; if (use >= 0) { ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last); - op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes); + zend_ssa_op *const op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes); + unsigned int no_val; + zend_ssa_alias_kind alias; + zend_ssa_var_info *info; if (tssa->ops[use].op1_use == var) { no_val = ssa->vars[op->op1_use].no_val; alias = ssa->vars[op->op1_use].alias; @@ -848,17 +844,17 @@ static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const info = ssa->var_info + op->result_use; } else { assert(0); - return 0; + return false; } tssa->vars[ssa_var].no_val = no_val; tssa->vars[ssa_var].alias = alias; memcpy(&tssa->var_info[ssa_var], info, sizeof(zend_ssa_var_info)); - return 1; + return true; } - return 0; + return false; } -static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op *const *tssa_opcodes, zend_ssa *tssa, int ssa_var) { zend_ssa_range tmp; int def = tssa->vars[ssa_var].definition; @@ -869,22 +865,19 @@ static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const tssa->var_info[ssa_var].range.max = tmp.max; tssa->var_info[ssa_var].range.underflow = tmp.underflow; tssa->var_info[ssa_var].range.overflow = tmp.overflow; - tssa->var_info[ssa_var].has_range = 1; + tssa->var_info[ssa_var].has_range = true; } } -static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *const*tssa_opcodes, zend_ssa *tssa, int ssa_var) { - int def; - zend_ssa_op *op; - zend_ssa_var_info *info; - unsigned int no_val; - zend_ssa_alias_kind alias; - - def = tssa->vars[ssa_var].definition; + const int def = tssa->vars[ssa_var].definition; if (def >= 0) { ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last); - op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes); + zend_ssa_op *const op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes); + zend_ssa_var_info *info; + unsigned int no_val; + zend_ssa_alias_kind alias; if (tssa->ops[def].op1_def == ssa_var) { no_val = ssa->vars[op->op1_def].no_val; alias = ssa->vars[op->op1_def].alias; @@ -916,23 +909,20 @@ static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, con tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow; tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow; } else { - tssa->var_info[ssa_var].has_range = 1; + tssa->var_info[ssa_var].has_range = true; tssa->var_info[ssa_var].range = info->range; } } } } -static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) +static bool zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op *const*tssa_opcodes, zend_ssa *tssa, int ssa_var) { - int def; - zend_ssa_op *op; - zend_ssa_var_info *info; - - def = tssa->vars[ssa_var].definition; + const int def = tssa->vars[ssa_var].definition; if (def >= 0) { ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last); - op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes); + zend_ssa_op *const op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes); + zend_ssa_var_info *info; if (tssa->ops[def].op1_def == ssa_var) { info = ssa->var_info + op->op1_def; } else if (tssa->ops[def].op2_def == ssa_var) { @@ -941,7 +931,7 @@ static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, c info = ssa->var_info + op->result_def; } else { assert(0); - return 0; + return false; } tssa->var_info[ssa_var].type &= info->type; if (info->ce) { @@ -970,13 +960,13 @@ static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, c tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow; tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow; } else { - tssa->var_info[ssa_var].has_range = 1; + tssa->var_info[ssa_var].has_range = true; tssa->var_info[ssa_var].range = info->range; } } - return 1; + return true; } - return 0; + return false; } static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op) @@ -1071,7 +1061,7 @@ static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, c return NULL; } -static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var) +static bool is_checked_guard(const zend_ssa *tssa, const zend_op *const*ssa_opcodes, uint32_t var, uint32_t phi_var) { if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG && !(tssa->var_info[var].type & MAY_BE_REF)) { @@ -1086,12 +1076,12 @@ static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, u || opline->opcode == ZEND_POST_INC) { if (tssa->ops[idx].op1_use >= 0 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) { - return 0; + return false; } if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - return 0; + return false; } - return 1; + return true; } else if (opline->opcode == ZEND_ASSIGN_OP && (opline->extended_value == ZEND_ADD || opline->extended_value == ZEND_SUB @@ -1099,20 +1089,20 @@ static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, u if ((opline->op2_type & (IS_VAR|IS_CV)) && tssa->ops[idx].op2_use >= 0 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) { - return 0; + return false; } if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - return 0; + return false; } if (opline->op2_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op2); if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) { - return 0; + return false; } } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - return 0; + return false; } - return 1; + return true; } } if (tssa->ops[idx].result_def == var) { @@ -1123,28 +1113,28 @@ static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, u if ((opline->op1_type & (IS_VAR|IS_CV)) && tssa->ops[idx].op1_use >= 0 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) { - return 0; + return false; } if ((opline->op2_type & (IS_VAR|IS_CV)) && tssa->ops[idx].op2_use >= 0 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) { - return 0; + return false; } if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) { - return 0; + return false; } } else if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - return 0; + return false; } if (opline->op2_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op2); if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) { - return 0; + return false; } } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - return 0; + return false; } return 1; } else if (opline->opcode == ZEND_PRE_DEC @@ -1154,17 +1144,17 @@ static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, u if ((opline->op1_type & (IS_VAR|IS_CV)) && tssa->ops[idx].op1_use >= 0 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) { - return 0; + return false; } if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - return 0; + return false; } - return 1; + return true; } } } } - return 0; + return false; } typedef struct _zend_tssa { @@ -1177,42 +1167,26 @@ static const zend_op _nop_opcode = {0}; static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr) { - zend_ssa *tssa; - zend_ssa_op *ssa_ops, *op; - zend_ssa_var *ssa_vars; - zend_ssa_var_info *ssa_var_info; - const zend_op_array *op_array; - const zend_op *opline; - const zend_op **ssa_opcodes; - zend_jit_trace_rec *p; - int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count; - zend_jit_trace_stack *stack; - uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS; - uint32_t optimization_level = 0; - int call_level, level, num_op_arrays, used_stack, max_used_stack; - size_t frame_size, stack_top, stack_size, stack_bottom; - zend_jit_op_array_trace_extension *jit_extension; - zend_ssa *ssa; - zend_jit_trace_stack_frame *frame, *top, *call; - zend_ssa_var_info return_value_info; + const uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS; + const uint32_t optimization_level = 0; /* 1. Count number of TSSA opcodes; * Count number of activation frames; * Calculate size of abstract stack; * Construct regular SSA for involved op_array */ - op_array = trace_buffer->op_array; - stack_top = stack_size = zend_jit_trace_frame_size(op_array); - stack_bottom = 0; - p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; - ssa_ops_count = 0; - call_level = 0; - level = 0; - num_op_arrays = 0; + const zend_op_array *op_array = trace_buffer->op_array; + size_t stack_size = zend_jit_trace_op_array_frame_size(op_array); + size_t stack_top = stack_size; + size_t stack_bottom = 0; + int ssa_ops_count = 0; + int call_level = 0; + int level = 0; + int num_op_arrays = 0; /* Remember op_array to cleanup */ op_arrays[num_op_arrays++] = op_array; /* Build SSA */ - ssa = zend_jit_trace_build_ssa(op_array, script); - for (;;p++) { + zend_ssa *ssa = zend_jit_trace_build_ssa(op_array, script); + for (zend_jit_trace_rec *p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;; p++) { if (p->op == ZEND_JIT_TRACE_VM) { if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) { const zend_op *opline = p->opline; @@ -1241,7 +1215,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa_ops_count += zend_jit_trace_op_len(p->opline); } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { call_level++; - stack_top += zend_jit_trace_frame_size(p->op_array); + stack_top += zend_jit_trace_function_frame_size(p->func); if (stack_top > stack_size) { stack_size = stack_top; } @@ -1254,7 +1228,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS; } } - frame_size = zend_jit_trace_frame_size(p->op_array); + const size_t frame_size = zend_jit_trace_function_frame_size(p->func); if (call_level == 0) { if (stack_top + frame_size > stack_size) { stack_size = stack_top + frame_size; @@ -1266,7 +1240,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } else if (p->op == ZEND_JIT_TRACE_ENTER) { op_array = p->op_array; if (call_level == 0) { - stack_top += zend_jit_trace_frame_size(op_array); + stack_top += zend_jit_trace_op_array_frame_size(op_array); if (stack_top > stack_size) { stack_size = stack_top; } @@ -1274,9 +1248,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin call_level--; } level++; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - ssa = &jit_extension->func_info.ssa; + ssa = zend_jit_op_array_trace_ssa(op_array); if (ssa->cfg.blocks_count) { /* pass */ } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) { @@ -1291,10 +1263,8 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } else if (p->op == ZEND_JIT_TRACE_BACK) { if (level == 0) { - stack_bottom += zend_jit_trace_frame_size(p->op_array); - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - ssa = &jit_extension->func_info.ssa; + stack_bottom += zend_jit_trace_op_array_frame_size(p->op_array); + ssa = zend_jit_op_array_trace_ssa(op_array); if (ssa->cfg.blocks_count) { /* pass */ } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) { @@ -1308,7 +1278,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa = zend_jit_trace_build_ssa(op_array, script); } } else { - stack_top -= zend_jit_trace_frame_size(op_array); + stack_top -= zend_jit_trace_op_array_frame_size(op_array); level--; } op_array = p->op_array; @@ -1319,10 +1289,10 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin *num_op_arrays_ptr = num_op_arrays; /* Allocate space for abstract stack */ - JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom); + zend_jit_trace_stack_frame *frame = JIT_G(current_frame) = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom); /* 2. Construct TSSA */ - tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa)); + zend_ssa *const tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa)); tssa->cfg.flags = ZEND_SSA_TSSA; tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block)); tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block)); @@ -1362,20 +1332,21 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin return tssa; } - tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op)); + zend_ssa_op *const ssa_ops = tssa->ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op)); memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op)); - ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*)); + const zend_op **const ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*)); ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes; ssa_opcodes[ssa_ops_count] = &_nop_opcode; op_array = trace_buffer->op_array; + int ssa_vars_count; if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { ssa_vars_count = op_array->last_var; } else { ssa_vars_count = op_array->last_var + op_array->T; } - stack = frame->stack; - for (i = 0; i < ssa_vars_count; i++) { + zend_jit_trace_stack *stack = frame->stack; + for (int i = 0; i < ssa_vars_count; i++) { SET_STACK_VAR(stack, i, i); } @@ -1389,27 +1360,24 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack); } - p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; - idx = 0; + int idx = 0; level = 0; - for (;;p++) { + for (zend_jit_trace_rec *p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;; p++) { if (p->op == ZEND_JIT_TRACE_VM) { - opline = p->opline; + const zend_op *opline = p->opline; ssa_opcodes[idx] = opline; ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack); idx++; - len = zend_jit_trace_op_len(p->opline); - while (len > 1) { + for (int len = zend_jit_trace_op_len(p->opline); len > 1; --len) { opline++; ssa_opcodes[idx] = opline; if (opline->opcode != ZEND_OP_DATA) { ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack); } idx++; - len--; } } else if (p->op == ZEND_JIT_TRACE_ENTER) { - frame = zend_jit_trace_call_frame(frame, op_array); + frame = zend_jit_trace_op_array_call_frame(frame, op_array); stack = frame->stack; op_array = p->op_array; level++; @@ -1417,19 +1385,19 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin return NULL; } ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count); - for (i = 0; i < op_array->last_var; i++) { + for (int i = 0; i < op_array->last_var; i++) { SET_STACK_VAR(stack, i, ssa_vars_count++); } } else if (p->op == ZEND_JIT_TRACE_BACK) { op_array = p->op_array; - frame = zend_jit_trace_ret_frame(frame, op_array); + frame = zend_jit_trace_op_array_ret_frame(frame, op_array); stack = frame->stack; if (level == 0) { if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) { return NULL; } ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count); - for (i = 0; i < op_array->last_var + op_array->T; i++) { + for (int i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_VAR(stack, i, ssa_vars_count++); } } else { @@ -1442,13 +1410,14 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin op_array = trace_buffer->op_array; tssa->vars_count = ssa_vars_count; - tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var)); + zend_ssa_var *const ssa_vars = tssa->vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var)); + int vars_count; if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { vars_count = op_array->last_var; } else { vars_count = op_array->last_var + op_array->T; } - i = 0; + int i = 0; while (i < vars_count) { ssa_vars[i].var = i; ssa_vars[i].scc = -1; @@ -1482,9 +1451,9 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin /* 3. Compute use-def chains */ idx = (ssa_ops_count - 1); - op = ssa_ops + idx; + zend_ssa_op *op = ssa_ops + idx; while (idx >= 0) { - opline = ssa_opcodes[idx]; + const zend_op *const opline = ssa_opcodes[idx]; if (op->op1_use >= 0) { op->op1_use_chain = ssa_vars[op->op1_use].use_chain; ssa_vars[op->op1_use].use_chain = idx; @@ -1515,11 +1484,9 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin /* 4. Type inference */ op_array = trace_buffer->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - ssa = &jit_extension->func_info.ssa; + ssa = zend_jit_op_array_trace_ssa(op_array); - tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info)); + zend_ssa_var_info *const ssa_var_info = tssa->var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info)); if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { i = 0; @@ -1536,7 +1503,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i); } if (op_array->arg_info) { - zend_arg_info *arg_info = &op_array->arg_info[i]; + const zend_arg_info *arg_info = &op_array->arg_info[i]; zend_class_entry *ce; uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce); @@ -1545,7 +1512,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } ssa_var_info[i].type = tmp; ssa_var_info[i].ce = ce; - ssa_var_info[i].is_instanceof = 1; + ssa_var_info[i].is_instanceof = true; } else { ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } @@ -1569,7 +1536,6 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin int parent_vars_count = 0; zend_jit_trace_stack *parent_stack = NULL; - i = 0; if (parent_trace) { parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size, op_array->last_var + op_array->T); @@ -1579,7 +1545,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset; } } - while (i < op_array->last_var + op_array->T) { + for (int i = 0; i < op_array->last_var + op_array->T; ++i) { if (!ssa->var_info || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) { if (ssa->vars && i < ssa->vars_count) { @@ -1607,7 +1573,6 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } } - i++; } } @@ -1615,50 +1580,44 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { /* Propagate initial value through Phi functions */ - zend_ssa_phi *phi = tssa->blocks[1].phis; - while (phi) { + for (zend_ssa_phi *phi = tssa->blocks[1].phis; phi; phi = phi->next) { if (!ssa->var_info || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var)) { ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias; ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type; } - phi = phi->next; } } frame = JIT_G(current_frame); - top = zend_jit_trace_call_frame(frame, op_array); + zend_jit_trace_stack_frame *top = zend_jit_trace_op_array_call_frame(frame, op_array); TRACE_FRAME_INIT(frame, op_array, 0, 0); TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); frame->used_stack = 0; for (i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1); } + zend_ssa_var_info return_value_info; memset(&return_value_info, 0, sizeof(return_value_info)); + int used_stack, max_used_stack; if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { max_used_stack = used_stack = 0; } else { max_used_stack = used_stack = -1; } - p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; idx = 0; level = 0; - opline = NULL; - for (;;p++) { + const zend_op *opline = NULL; + for (zend_jit_trace_rec *p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;; p++) { if (p->op == ZEND_JIT_TRACE_VM) { - uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type; - uint8_t val_type = IS_UNKNOWN; -// zend_class_entry *op1_ce = NULL; - zend_class_entry *op2_ce = NULL; + const uint8_t orig_op1_type = p->op1_type, orig_op2_type = p->op2_type; opline = p->opline; - op1_type = orig_op1_type = p->op1_type; - op2_type = orig_op2_type = p->op2_type; - op3_type = p->op3_type; + uint8_t op1_type = orig_op1_type, op2_type = orig_op2_type, op3_type = p->op3_type; if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) { op1_type = IS_UNKNOWN; } @@ -1672,14 +1631,19 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin op3_type = IS_UNKNOWN; } +// zend_class_entry *op1_ce = NULL; if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) { // op1_ce = (zend_class_entry*)(p+1)->ce; p++; } + + zend_class_entry *op2_ce = NULL; if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { op2_ce = (zend_class_entry*)(p+1)->ce; p++; } + + uint8_t val_type = IS_UNKNOWN; if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) { val_type = (p+1)->op1_type; p++; @@ -2001,7 +1965,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD; } if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { - zend_arg_info *arg_info; + const zend_arg_info *arg_info; ZEND_ASSERT(frame->call->func->op_array.arg_info); arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1]; @@ -2127,7 +2091,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin default: break; } - len = zend_jit_trace_op_len(opline); + int len = zend_jit_trace_op_len(opline); if (ssa->var_info) { /* Add statically inferred ranges */ if (ssa_ops[idx].op1_def >= 0) { @@ -2310,17 +2274,15 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } else if (p->op == ZEND_JIT_TRACE_ENTER) { op_array = p->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - ssa = &jit_extension->func_info.ssa; + ssa = zend_jit_op_array_trace_ssa(op_array); - call = frame->call; + zend_jit_trace_stack_frame *call = frame->call; if (!call) { /* Trace missed INIT_FCALL opcode */ call = top; TRACE_FRAME_INIT(call, op_array, 0, 0); call->used_stack = 0; - top = zend_jit_trace_call_frame(top, op_array); + top = zend_jit_trace_op_array_call_frame(top, op_array); for (i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); } @@ -2333,9 +2295,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin frame = call; level++; - i = 0; - v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); - while (i < op_array->last_var) { + for (int i = 0, v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); i < op_array->last_var; ++i, ++v) { ssa_vars[v].var = i; if (i < op_array->num_args) { if (ssa->var_info @@ -2344,7 +2304,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } else { ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i); if (op_array->arg_info) { - zend_arg_info *arg_info = &op_array->arg_info[i]; + const zend_arg_info *arg_info = &op_array->arg_info[i]; zend_class_entry *ce; uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce); @@ -2353,7 +2313,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } ssa_var_info[v].type = tmp; ssa_var_info[v].ce = ce; - ssa_var_info[v].is_instanceof = 1; + ssa_var_info[v].is_instanceof = true; } else { ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } @@ -2376,17 +2336,13 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin /* Propagate argument type */ ssa_var_info[v].type &= STACK_INFO(frame->stack, i); } - i++; - v++; } } else if (p->op == ZEND_JIT_TRACE_BACK) { op_array = p->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - ssa = &jit_extension->func_info.ssa; + ssa = zend_jit_op_array_trace_ssa(op_array); if (level == 0) { - i = 0; - v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); + int i = 0; + int v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); while (i < op_array->last_var) { ssa_vars[v].var = i; if (!ssa->var_info @@ -2446,7 +2402,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ZEND_ASSERT(&frame->func->op_array == op_array); } else { max_used_stack = used_stack = -1; - frame = zend_jit_trace_ret_frame(frame, op_array); + frame = zend_jit_trace_op_array_ret_frame(frame, op_array); TRACE_FRAME_INIT(frame, op_array, 0, 0); TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); frame->used_stack = 0; @@ -2456,14 +2412,14 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { - call = top; + zend_jit_trace_stack_frame *const call = top; TRACE_FRAME_INIT(call, p->func, 0, 0); call->prev = frame->call; call->used_stack = 0; frame->call = call; - top = zend_jit_trace_call_frame(top, p->op_array); + top = zend_jit_trace_function_call_frame(top, p->func); if (p->func && p->func->type == ZEND_USER_FUNCTION) { - for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) { + for (i = 0; i < p->func->op_array.last_var + p->func->op_array.T; i++) { SET_STACK_INFO(call->stack, i, -1); } } @@ -2495,7 +2451,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { - call = frame->call; + zend_jit_trace_stack_frame *const call = frame->call; if (call) { top = call; frame->call = call->prev; @@ -2513,7 +2469,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin if (opline->result_type != IS_UNDEF) { zend_class_entry *ce; const zend_function *func = p->func; - zend_arg_info *ret_info = func->common.arg_info - 1; + const zend_arg_info *ret_info = func->common.arg_info - 1; uint32_t ret_type = zend_fetch_arg_info_type(NULL, ret_info, &ce); ssa_var_info[ssa_ops[idx-1].result_def].type &= ret_type; @@ -2533,9 +2489,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin zend_ssa_phi *phi = tssa->blocks[1].phis; op_array = trace_buffer->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - ssa = &jit_extension->func_info.ssa; + ssa = zend_jit_op_array_trace_ssa(op_array); while (phi) { uint32_t t = ssa_var_info[phi->ssa_var].type; @@ -2699,22 +2653,8 @@ static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, in static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num) { - const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes; - zend_jit_trace_rec *p; - const zend_op_array *op_array; - zend_jit_op_array_trace_extension *jit_extension; - const zend_ssa *op_array_ssa; - const zend_ssa_op *ssa_op; - int i, j, idx, count, level; - int last_idx = -1; - int *start, *end; - uint8_t *flags; - const zend_op_array **vars_op_array; - zend_lifetime_interval **intervals, *list, *ival; - void *checkpoint; - zend_jit_trace_stack_frame *frame; - zend_jit_trace_stack *stack; - uint32_t parent_vars_count = parent_trace ? + const zend_op **const ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes; + const uint32_t parent_vars_count = parent_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0; zend_jit_trace_stack *parent_stack = parent_trace ? zend_jit_traces[parent_trace].stack_map + @@ -2723,34 +2663,32 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace ZEND_ASSERT(ssa->var_info != NULL); - start = do_alloca(sizeof(int) * ssa->vars_count * 2 + + int *const start = do_alloca(sizeof(int) * ssa->vars_count * 2 + ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_op_array*) * ssa->vars_count), use_heap); if (!start) { return NULL; } - end = start + ssa->vars_count; - flags = (uint8_t*)(end + ssa->vars_count); - vars_op_array = (const zend_op_array**)(flags + ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count)); + int *const end = start + ssa->vars_count; + uint8_t *const flags = (uint8_t*)(end + ssa->vars_count); + const zend_op_array **const vars_op_array = (const zend_op_array**)(flags + ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count)); memset(start, -1, sizeof(int) * ssa->vars_count * 2); memset(flags, 0, sizeof(uint8_t) * ssa->vars_count); memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count); - op_array = trace_buffer->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - op_array_ssa = &jit_extension->func_info.ssa; - frame = JIT_G(current_frame); + const zend_op_array *op_array = trace_buffer->op_array; + const zend_ssa *op_array_ssa = zend_jit_op_array_trace_ssa(op_array); + zend_jit_trace_stack_frame *frame = JIT_G(current_frame); frame->prev = NULL; frame->func = (const zend_function*)op_array; - stack = frame->stack; + zend_jit_trace_stack *stack = frame->stack; - count = 0; + int count = 0; - i = 0; - j = op_array->last_var; + int i = 0; + int j = op_array->last_var; if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) { j += op_array->T; } @@ -2803,11 +2741,10 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace } } - p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; - level = 0; - ssa_op = ssa->ops; - idx = 0; - for (;;p++) { + int level = 0; + const zend_ssa_op *ssa_op = ssa->ops; + int idx = 0; + for (zend_jit_trace_rec *p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;; ++p) { if (p->op == ZEND_JIT_TRACE_VM) { const zend_op *opline = p->opline; int len; @@ -2822,7 +2759,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace && ssa_op->op1_def >= 0 && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) { /* avoid register allocation in case of possibility of indirect modification*/ - support_opline = 0; + support_opline = false; } if (ssa_op->op1_use >= 0 @@ -3025,14 +2962,12 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace /* New call frames */ zend_jit_trace_stack_frame *prev_frame = frame; - frame = zend_jit_trace_call_frame(frame, op_array); + frame = zend_jit_trace_op_array_call_frame(frame, op_array); frame->prev = prev_frame; frame->func = (const zend_function*)p->op_array; stack = frame->stack; op_array = p->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - op_array_ssa = &jit_extension->func_info.ssa; + op_array_ssa = zend_jit_op_array_trace_ssa(op_array); j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); for (i = 0; i < op_array->last_var; i++) { SET_STACK_VAR(stack, i, j); @@ -3056,10 +2991,8 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace zend_jit_close_var(stack, i, start, end, flags, idx-1); } op_array = p->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - op_array_ssa = &jit_extension->func_info.ssa; - frame = zend_jit_trace_ret_frame(frame, op_array); + op_array_ssa = zend_jit_op_array_trace_ssa(op_array); + frame = zend_jit_trace_op_array_ret_frame(frame, op_array); stack = frame->stack; if (level == 0) { /* New return frames */ @@ -3086,6 +3019,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace } } + int last_idx = -1; if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { @@ -3130,10 +3064,10 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace return NULL; } - checkpoint = zend_arena_checkpoint(CG(arena)); - intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval)); + void *const checkpoint = zend_arena_checkpoint(CG(arena)); + zend_lifetime_interval **const intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval)); memset(intervals, 0, sizeof(zend_lifetime_interval*) * ssa->vars_count); - list = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval) * count); + zend_lifetime_interval *list = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval) * count); j = 0; for (i = 0; i < ssa->vars_count; i++) { if (start[i] >= 0 && end[i] >= 0) { @@ -3159,7 +3093,6 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace } count = j; free_alloca(start, use_heap); - start = end = NULL; if (!count) { zend_arena_release(&CG(arena), checkpoint); @@ -3307,22 +3240,19 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace list = zend_jit_sort_intervals(intervals, ssa->vars_count); if (list) { - ival = list; - while (ival) { + for (zend_lifetime_interval *ival = list; ival; ival = ival->list_next) { if (ival->hint) { ival->hint->used_as_hint = ival; } - ival = ival->list_next; + } } if (list) { if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { fprintf(stderr, "---- TRACE %d Live Ranges\n", ZEND_JIT_TRACE_NUM); - ival = list; - while (ival) { + for (zend_lifetime_interval *ival = list; ival; ival = ival->list_next) { zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival); - ival = ival->list_next; } } } @@ -3331,15 +3261,12 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace list = zend_jit_linear_scan(&dummy_op_array, ssa_opcodes, ssa, list); if (list) { - zend_lifetime_interval *ival, *next; - memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*)); - ival = list; count = 0; - while (ival != NULL) { + for (zend_lifetime_interval *ival = list; ival;) { ZEND_ASSERT(ival->reg != ZREG_NONE); count++; - next = ival->list_next; + zend_lifetime_interval *const next = ival->list_next; ival->list_next = intervals[ival->ssa_var]; intervals[ival->ssa_var] = ival; ival = next; @@ -3439,10 +3366,8 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { fprintf(stderr, "---- TRACE %d Allocated Live Ranges\n", ZEND_JIT_TRACE_NUM); for (i = 0; i < ssa->vars_count; i++) { - ival = intervals[i]; - while (ival) { + for (zend_lifetime_interval *ival = intervals[i]; ival; ival = ival->list_next) { zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival); - ival = ival->list_next; } } } @@ -3478,31 +3403,31 @@ static void zend_jit_trace_clenup_stack(zend_jit_trace_stack *stack, const zend_ static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset) { zend_op *next_opline = (zend_op*)(opline + 1); + zend_op_trace_info *const next_trace_info = ZEND_OP_TRACE_INFO(next_opline, offset); - if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) { + if (JIT_G(hot_return) && !next_trace_info->trace_flags) { ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL); - if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) { - ZEND_OP_TRACE_INFO(next_opline, offset)->counter = + if (!next_trace_info->counter) { + next_trace_info->counter = &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; } - ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN; + next_trace_info->trace_flags = ZEND_JIT_TRACE_START_RETURN; next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler; } } static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op) { - int var = ssa_op->result_def; - int i; - int use = ssa->vars[var].use_chain; + const int var = ssa_op->result_def; + const int use = ssa->vars[var].use_chain; const zend_op *opline; if (use < 0 || ssa->vars[var].phi_use_chain || ssa->ops[use].op1_use != var || ssa->ops[use].op1_use_chain != -1) { - return 0; + return false; } opline = ssa_opcodes[use]; @@ -3514,7 +3439,7 @@ static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ss || !JIT_G(current_frame)->call || !JIT_G(current_frame)->call->func || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { - return 0; + return false; } } else if (opline->opcode != ZEND_FETCH_OBJ_R && opline->opcode != ZEND_FETCH_OBJ_IS @@ -3525,13 +3450,13 @@ static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ss && opline->opcode != ZEND_PRE_DEC_OBJ && opline->opcode != ZEND_POST_INC_OBJ && opline->opcode != ZEND_POST_DEC_OBJ) { - return 0; + return false; } if (opline->op2_type != IS_CONST || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { - return 0; + return false; } if (opline->opcode == ZEND_ASSIGN_OBJ_OP) { @@ -3539,71 +3464,66 @@ static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ss && (opline+1)->op1_type == IS_CV && (opline+1)->op1.var == opline->op1.var) { /* skip $a->prop += $a; */ - return 0; + return false; } if (!zend_jit_supported_binary_op( opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) { - return 0; + return false; } } - for (i = ssa->vars[var].definition; i < use; i++) { + for (int i = ssa->vars[var].definition; i < use; i++) { if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME || ssa_opcodes[i]->opcode == ZEND_DO_FCALL || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) { - return 0; + return false; } } - return 1; + return true; } -static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size) +static bool zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size) { - uint32_t i; - - for (i = 0; i < stack_size; i++) { + for (uint32_t i = 0; i < stack_size; i++) { if (STACK_REG(stack, i) != ZREG_NONE && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) { - return 1; + return true; } } - return 0; + return false; } -static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num) +static bool zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num) { - const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; - uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags; - uint32_t stack_size; - zend_jit_trace_stack *stack; + const zend_op *const opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; + const uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags; if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) { - return 1; + return true; } - stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size; - stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset; + const uint32_t stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size; + zend_jit_trace_stack *const stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset; return zend_jit_trace_stack_needs_deoptimization(stack, stack_size); } -static int zend_jit_trace_deoptimization(dasm_State **Dst, - uint32_t flags, - const zend_op *opline, - zend_jit_trace_stack *parent_stack, - int parent_vars_count, - zend_ssa *ssa, - zend_jit_trace_stack *stack, - zend_lifetime_interval **ra, - bool polymorphic_side_trace) +static bool zend_jit_trace_deoptimization(dasm_State **Dst, + uint32_t flags, + const zend_op *opline, + zend_jit_trace_stack *parent_stack, + int parent_vars_count, + zend_ssa *ssa, + zend_jit_trace_stack *stack, + zend_lifetime_interval **ra, + bool polymorphic_side_trace) { - int i; - bool has_constants = 0; - bool has_unsaved_vars = 0; + bool has_constants = false; + bool has_unsaved_vars = false; // TODO: Merge this loop with the following register LOAD loop to implement parallel move ??? - for (i = 0; i < parent_vars_count; i++) { + for (int i = 0; i < parent_vars_count; i++) { int8_t reg = STACK_REG(parent_stack, i); if (reg != ZREG_NONE) { @@ -3615,14 +3535,14 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, if (stack) { SET_STACK_REG_EX(stack, i, reg, STACK_FLAGS(parent_stack, i)); } - has_unsaved_vars = 1; + has_unsaved_vars = true; } else { uint8_t type = STACK_TYPE(parent_stack, i); if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) && !zend_jit_store_var(Dst, 1 << type, i, reg, STACK_MEM_TYPE(parent_stack, i) != type)) { - return 0; + return false; } if (stack) { SET_STACK_TYPE(stack, i, type, 1); @@ -3630,7 +3550,7 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, } } else { /* delay custom deoptimization instructions to prevent register clobbering */ - has_constants = 1; + has_constants = true; } } } @@ -3638,7 +3558,7 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, if (has_unsaved_vars && (has_constants || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)))) { - for (i = 0; i < parent_vars_count; i++) { + for (int i = 0; i < parent_vars_count; i++) { int8_t reg = STACK_REG(parent_stack, i); if (reg != ZREG_NONE) { @@ -3654,7 +3574,7 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) && !zend_jit_store_var(Dst, 1 << type, i, reg, STACK_MEM_TYPE(parent_stack, i) != type)) { - return 0; + return false; } } } @@ -3663,7 +3583,7 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, } if (has_constants) { - for (i = 0; i < parent_vars_count; i++) { + for (int i = 0; i < parent_vars_count; i++) { int8_t reg = STACK_REG(parent_stack, i); if (reg != ZREG_NONE) { @@ -3671,20 +3591,20 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, /* pass */ } else if (reg == ZREG_THIS) { if (polymorphic_side_trace) { - ssa->var_info[i].delayed_fetch_this = 1; + ssa->var_info[i].delayed_fetch_this = true; if (stack) { SET_STACK_REG(stack, i, ZREG_THIS); } } else if (!zend_jit_load_this(Dst, EX_NUM_TO_VAR(i))) { - return 0; + return false; } } else { if (reg == ZREG_ZVAL_COPY_GPR0 &&!zend_jit_escape_if_undef_r0(Dst, i, flags, opline)) { - return 0; + return false; } if (!zend_jit_store_const(Dst, i, reg)) { - return 0; + return false; } } } @@ -3693,7 +3613,7 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { if (!zend_jit_save_call_chain(Dst, -1)) { - return 0; + return false; } } @@ -3701,7 +3621,7 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, const zend_op *op = opline - 1; if (!zend_jit_free_op(Dst, op, -1, op->op2.var)) { - return 0; + return false; } } @@ -3709,47 +3629,45 @@ static int zend_jit_trace_deoptimization(dasm_State **Dst, const zend_op *op = opline - 1; if (!zend_jit_free_op(Dst, op, -1, op->op1.var)) { - return 0; + return false; } } if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) { if (!zend_jit_check_exception(Dst)) { - return 0; + return false; } } if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) { if (!zend_jit_free_trampoline(Dst)) { - return 0; + return false; } } - return 1; + return true; } static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max) { - info->has_range = 1; + info->has_range = true; info->range.min = min; info->range.max = max; - info->range.underflow = 0; - info->range.overflow = 0; + info->range.underflow = false; + info->range.overflow = false; } static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true) { - zend_long op1_min, op1_max, op2_min, op2_max; - if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) { return; } - op1_min = OP1_MIN_RANGE(); - op1_max = OP1_MAX_RANGE(); - op2_min = OP2_MIN_RANGE(); - op2_max = OP2_MAX_RANGE(); + zend_long op1_min = OP1_MIN_RANGE(); + zend_long op1_max = OP1_MAX_RANGE(); + zend_long op2_min = OP2_MIN_RANGE(); + zend_long op2_max = OP2_MAX_RANGE(); switch (opline->opcode) { case ZEND_IS_EQUAL: @@ -3855,10 +3773,8 @@ static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const } } -static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array) +static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op *const*ssa_opcodes, const zend_op_array *op_array) { - zend_uchar prev_opcode; - if (opline->op1_type == IS_CONST && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) { @@ -3866,7 +3782,7 @@ static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_o if ((ssa_op-1)->op1_def == ssa_op->op2_use) { ssa_op--; opline = ssa_opcodes[ssa_op - ssa->ops]; - prev_opcode = opline->opcode; + const zend_uchar prev_opcode = opline->opcode; if (prev_opcode == ZEND_PRE_INC || prev_opcode == ZEND_PRE_DEC || prev_opcode == ZEND_POST_INC @@ -3876,7 +3792,7 @@ static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_o } else if ((ssa_op-1)->result_def == ssa_op->op2_use) { ssa_op--; opline = ssa_opcodes[ssa_op - ssa->ops]; - prev_opcode = opline->opcode; + const zend_uchar prev_opcode = opline->opcode; if (prev_opcode == ZEND_ADD || prev_opcode == ZEND_SUB) { return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 && @@ -3891,7 +3807,7 @@ static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_o if ((ssa_op-1)->op1_def == ssa_op->op1_use) { ssa_op--; opline = ssa_opcodes[ssa_op - ssa->ops]; - prev_opcode = opline->opcode; + const zend_uchar prev_opcode = opline->opcode; if (prev_opcode == ZEND_PRE_INC || prev_opcode == ZEND_PRE_DEC || prev_opcode == ZEND_POST_INC @@ -3901,7 +3817,7 @@ static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_o } else if ((ssa_op-1)->result_def == ssa_op->op1_use) { ssa_op--; opline = ssa_opcodes[ssa_op - ssa->ops]; - prev_opcode = opline->opcode; + const zend_uchar prev_opcode = opline->opcode; if (prev_opcode == ZEND_ADD || prev_opcode == ZEND_SUB) { return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 && @@ -3911,7 +3827,7 @@ static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_o } } else { const zend_ssa_op *prev_ssa_op = ssa_op - 1; - prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode; + zend_uchar prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode; if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ) && prev_ssa_op != ssa->ops @@ -3933,19 +3849,19 @@ static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_o || prev_opcode == ZEND_CASE_STRICT) { if (ssa_op->op1_use < 0) { if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) { - return 0; + return false; } } if (ssa_op->op2_use < 0) { if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) { - return 0; + return false; } } - return 1; + return true; } } } - return 0; + return false; } static bool zend_jit_trace_next_is_send_result(const zend_op *opline, @@ -3972,92 +3888,74 @@ static bool zend_jit_trace_next_is_send_result(const zend_op *oplin zend_jit_trace_send_type(opline+1, frame->call, res_type); } } - return 1; + return true; } - return 0; + return false; } +/** + * Caller must have called zend_shared_alloc_lock() and + * zend_jit_unprotect(). + */ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num) { const void *handler = NULL; - dasm_State* dasm_state = NULL; - zend_script *script = NULL; - zend_lifetime_interval **ra = NULL; - zend_string *name = NULL; - void *checkpoint; - const zend_op_array *op_array; - zend_ssa *ssa, *op_array_ssa; - const zend_op **ssa_opcodes; - zend_jit_trace_rec *p; - zend_jit_op_array_trace_extension *jit_extension; - int num_op_arrays = 0; - zend_jit_trace_info *t; - const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS]; - zend_uchar smart_branch_opcode; - const void *exit_addr; - uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info; - bool send_result = 0; - bool skip_comparison; - zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; - zend_class_entry *ce; - bool ce_is_instanceof; - bool on_this = 0; - bool delayed_fetch_this = 0; - bool avoid_refcounting = 0; + zend_script *const script = NULL; + bool send_result = false; + bool delayed_fetch_this = false; bool polymorphic_side_trace = parent_trace && (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL); - uint32_t i; - zend_jit_trace_stack_frame *frame, *top, *call; - zend_jit_trace_stack *stack; zend_uchar res_type = IS_UNKNOWN; - const zend_op *opline, *orig_opline; - const zend_ssa_op *ssa_op, *orig_ssa_op; - int checked_stack; - int peek_checked_stack; uint32_t frame_flags = 0; JIT_G(current_trace) = trace_buffer; - checkpoint = zend_arena_checkpoint(CG(arena)); + void *const checkpoint = zend_arena_checkpoint(CG(arena)); - ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays); + const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS]; + int num_op_arrays = 0; + zend_ssa *const ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays); if (!ssa) { goto jit_cleanup; } - ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes; + const zend_op **const ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes; /* Register allocation */ + zend_lifetime_interval **ra = NULL; if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num); } - p = trace_buffer; + zend_jit_trace_rec *p = trace_buffer; ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START); - op_array = p->op_array; - frame = JIT_G(current_frame); - top = zend_jit_trace_call_frame(frame, op_array); + const zend_op_array *op_array = p->op_array; + zend_jit_trace_stack_frame *frame = JIT_G(current_frame); + zend_jit_trace_stack_frame *top = zend_jit_trace_op_array_call_frame(frame, op_array); TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1); + int checked_stack; + int peek_checked_stack; frame->used_stack = checked_stack = peek_checked_stack = 0; - stack = frame->stack; - for (i = 0; i < op_array->last_var + op_array->T; i++) { + zend_jit_trace_stack *stack = frame->stack; + for (uint32_t i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); } - opline = p[1].opline; - name = zend_jit_trace_name(op_array, opline->lineno); + const zend_op *opline = p[1].opline; + zend_string *const name = zend_jit_trace_name(op_array, opline->lineno); p += ZEND_JIT_TRACE_START_REC_SIZE; + dasm_State* dasm_state = NULL; dasm_init(&dasm_state, DASM_MAXSECTION); dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); dasm_setup(&dasm_state, dasm_actions); - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - op_array_ssa = &jit_extension->func_info.ssa; + zend_ssa *op_array_ssa = &jit_extension->func_info.ssa; dasm_growpc(&dasm_state, 2); /* =>0: loop header */ /* =>1: end of code */ @@ -4108,7 +4006,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par last_var += op_array->T; } - for (i = 0; i < last_var; i++) { + for (uint32_t i = 0; i < last_var; i++) { uint32_t info = ssa->var_info[i].type; if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) { @@ -4185,7 +4083,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (ra && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { - for (i = 0; i < last_var; i++) { + for (uint32_t i = 0; i < last_var; i++) { if (ra[i] && (ra[i]->flags & ZREG_LOAD) != 0 && ra[i]->reg != stack[i].reg) { @@ -4273,7 +4171,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) { last_var += op_array->T; } - for (i = 0; i < last_var; i++) { + for (uint32_t i = 0; i < last_var; i++) { if (ra && ra[i] && (ra[i]->flags & ZREG_LOAD) != 0) { SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD); if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) { @@ -4284,7 +4182,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } - ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL; + const zend_ssa_op *ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL; for (;;p++) { if (p->op == ZEND_JIT_TRACE_VM) { uint8_t op1_type = p->op1_type; @@ -4340,7 +4238,19 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { - gen_handler = 0; + gen_handler = false; + + zend_uchar smart_branch_opcode; + const void *exit_addr; + uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info; + bool skip_comparison; + zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; + zend_class_entry *ce; + bool ce_is_instanceof; + bool on_this = false; + bool avoid_refcounting = false; + const zend_op *orig_opline; + const zend_ssa_op *orig_ssa_op; switch (opline->opcode) { case ZEND_PRE_INC: case ZEND_PRE_DEC: @@ -4412,7 +4322,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par res_addr = RES_REG_ADDR(); if (Z_MODE(res_addr) != IS_REG && zend_jit_trace_next_is_send_result(opline, p, frame)) { - send_result = 1; + send_result = true; res_use_info = -1; res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); if (!zend_jit_reuse_ip(&dasm_state)) { @@ -4450,7 +4360,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par goto jit_failure; } if (ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -4465,7 +4375,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par goto jit_failure; } if (ssa->vars[ssa_op->op2_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op2_use].guarded_reference = 1; + ssa->var_info[ssa_op->op2_use].guarded_reference = true; } } else { CHECK_OP2_TRACE_TYPE(); @@ -4484,7 +4394,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par res_addr = RES_REG_ADDR(); if (Z_MODE(res_addr) != IS_REG && zend_jit_trace_next_is_send_result(opline, p, frame)) { - send_result = 1; + send_result = true; res_use_info = -1; res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); if (!zend_jit_reuse_ip(&dasm_state)) { @@ -4534,7 +4444,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } res_addr = RES_REG_ADDR(); if (zend_jit_trace_next_is_send_result(opline, p, frame)) { - send_result = 1; + send_result = true; res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); if (!zend_jit_reuse_ip(&dasm_state)) { goto jit_failure; @@ -4614,7 +4524,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -4641,15 +4551,15 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par break; } ce = NULL; - ce_is_instanceof = 0; - on_this = delayed_fetch_this = 0; - op1_indirect = 0; + ce_is_instanceof = false; + on_this = delayed_fetch_this = false; + op1_indirect = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { if (ssa_op->op1_use >= 0) { delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; @@ -4662,7 +4572,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (opline->op1_type == IS_VAR) { if (orig_op1_type != IS_UNKNOWN && (orig_op1_type & IS_TRACE_INDIRECT)) { - op1_indirect = 1; + op1_indirect = true; if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { goto jit_failure; @@ -4677,7 +4587,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -4695,7 +4605,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } if (delayed_fetch_this) { - on_this = 1; + on_this = true; } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; } else if (op_array_ssa->ops @@ -4732,15 +4642,15 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par break; } ce = NULL; - ce_is_instanceof = 0; - on_this = delayed_fetch_this = 0; - op1_indirect = 0; + ce_is_instanceof = false; + on_this = delayed_fetch_this = false; + op1_indirect = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { if (ssa_op->op1_use >= 0) { delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; @@ -4753,7 +4663,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (opline->op1_type == IS_VAR) { if (orig_op1_type != IS_UNKNOWN && (orig_op1_type & IS_TRACE_INDIRECT)) { - op1_indirect = 1; + op1_indirect = true; if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { goto jit_failure; @@ -4768,7 +4678,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -4786,7 +4696,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } if (delayed_fetch_this) { - on_this = 1; + on_this = true; } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; } else if (op_array_ssa->ops @@ -4812,15 +4722,15 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par break; } ce = NULL; - ce_is_instanceof = 0; - on_this = delayed_fetch_this = 0; - op1_indirect = 0; + ce_is_instanceof = false; + on_this = delayed_fetch_this = false; + op1_indirect = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { if (ssa_op->op1_use >= 0) { delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; @@ -4833,7 +4743,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (opline->op1_type == IS_VAR) { if (orig_op1_type != IS_UNKNOWN && (orig_op1_type & IS_TRACE_INDIRECT)) { - op1_indirect = 1; + op1_indirect = true; if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { goto jit_failure; @@ -4848,7 +4758,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -4866,7 +4776,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } if (delayed_fetch_this) { - on_this = 1; + on_this = true; } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; } else if (op_array_ssa->ops @@ -4920,7 +4830,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -4985,7 +4895,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } if (opline->result_type == IS_UNUSED) { res_addr = 0; @@ -5013,7 +4923,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par res_info = RES_INFO(); if (Z_MODE(res_addr) != IS_REG && zend_jit_trace_next_is_send_result(opline, p, frame)) { - send_result = 1; + send_result = true; res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); if (!zend_jit_reuse_ip(&dasm_state)) { goto jit_failure; @@ -5119,7 +5029,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } goto done; case ZEND_SEND_VAR: @@ -5217,7 +5127,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par CHECK_OP1_TRACE_TYPE(); CHECK_OP2_TRACE_TYPE(); if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point; @@ -5229,7 +5139,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (!exit_addr) { goto jit_failure; } - smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; + const zend_uchar smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; if (!zend_jit_cmp(&dasm_state, opline, op1_info, OP1_RANGE(), OP1_REG_ADDR(), op2_info, OP2_RANGE(), OP2_REG_ADDR(), @@ -5240,7 +5150,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true); } else { - smart_branch_opcode = 0; + const zend_uchar smart_branch_opcode = 0; exit_addr = NULL; if (!zend_jit_cmp(&dasm_state, opline, op1_info, OP1_RANGE(), OP1_REG_ADDR(), @@ -5265,7 +5175,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par CHECK_OP1_TRACE_TYPE(); CHECK_OP2_TRACE_TYPE(); if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point; @@ -5280,7 +5190,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (opline->opcode == ZEND_IS_NOT_IDENTICAL) { exit_if_true = !exit_if_true; } - smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; + const zend_uchar smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; if (!zend_jit_identical(&dasm_state, opline, op1_info, OP1_RANGE(), OP1_REG_ADDR(), op2_info, OP2_RANGE(), OP2_REG_ADDR(), @@ -5291,7 +5201,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true); } else { - smart_branch_opcode = 0; + const zend_uchar smart_branch_opcode = 0; exit_addr = NULL; if (!zend_jit_identical(&dasm_state, opline, op1_info, OP1_RANGE(), OP1_REG_ADDR(), @@ -5305,7 +5215,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par goto done; case ZEND_DEFINED: if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); @@ -5330,7 +5240,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op1_info = OP1_INFO(); CHECK_OP1_TRACE_TYPE(); if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point; @@ -5369,8 +5279,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } else { int j; - int may_throw = 0; - bool left_frame = 0; + bool may_throw = false; + bool left_frame = false; if (!zend_jit_return(&dasm_state, opline, op_array, op1_info, OP1_REG_ADDR())) { @@ -5381,7 +5291,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (!zend_jit_free_cvs(&dasm_state)) { goto jit_failure; } - left_frame = 1; + left_frame = true; } if (!left_frame) { for (j = 0 ; j < op_array->last_var; j++) { @@ -5403,7 +5313,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (!left_frame) { - left_frame = 1; + left_frame = true; if (!zend_jit_leave_frame(&dasm_state)) { goto jit_failure; } @@ -5413,7 +5323,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) { if (info & MAY_BE_RC1) { - may_throw = 1; + may_throw = true; } } } @@ -5524,13 +5434,13 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); } if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); @@ -5560,7 +5470,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par break; } if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); @@ -5600,9 +5510,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; if (ssa_op->op1_def >= 0) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } } else { @@ -5673,7 +5583,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -5689,7 +5599,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W) && !(op1_info & (MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { - ssa->var_info[ssa_op->result_def].indirect_reference = 1; + ssa->var_info[ssa_op->result_def].indirect_reference = true; } goto done; case ZEND_ISSET_ISEMPTY_DIM_OBJ: @@ -5707,7 +5617,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -5715,7 +5625,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op2_info = OP2_INFO(); CHECK_OP2_TRACE_TYPE(); if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { - bool exit_if_true = 0; + bool exit_if_true = false; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); uint32_t exit_point; @@ -5780,22 +5690,22 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par case ZEND_FETCH_OBJ_R: case ZEND_FETCH_OBJ_IS: case ZEND_FETCH_OBJ_W: - on_this = delayed_fetch_this = 0; - avoid_refcounting = 0; + on_this = delayed_fetch_this = false; + avoid_refcounting = false; if (opline->op2_type != IS_CONST || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { break; } ce = NULL; - ce_is_instanceof = 0; - op1_indirect = 0; + ce_is_instanceof = false; + op1_indirect = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_OBJECT)) { @@ -5806,7 +5716,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && opline->opcode == ZEND_FETCH_OBJ_W) { if (orig_op1_type != IS_UNKNOWN && (orig_op1_type & IS_TRACE_INDIRECT)) { - op1_indirect = 1; + op1_indirect = true; if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { goto jit_failure; @@ -5821,7 +5731,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -5843,7 +5753,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting; } if (delayed_fetch_this) { - on_this = 1; + on_this = true; } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; } else if (op_array_ssa->ops @@ -5870,7 +5780,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op1_info = OP1_INFO(); } if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_def].guarded_reference = 1; + ssa->var_info[ssa_op->op1_def].guarded_reference = true; } if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) { goto jit_failure; @@ -5937,7 +5847,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -5959,7 +5869,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -5972,11 +5882,11 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } goto done; case ZEND_FETCH_THIS: - delayed_fetch_this = 0; + delayed_fetch_this = false; if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) { if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) { - ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1; - delayed_fetch_this = 1; + ssa->var_info[ssa_op->result_def].delayed_fetch_this = true; + delayed_fetch_this = true; } } if (!zend_jit_fetch_this(&dasm_state, opline, op_array, delayed_fetch_this)) { @@ -6066,15 +5976,15 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) { break; } - on_this = delayed_fetch_this = 0; + on_this = delayed_fetch_this = false; ce = NULL; - ce_is_instanceof = 0; + ce_is_instanceof = false; if (opline->op1_type == IS_UNUSED) { op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; ce = op_array->scope; ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; op1_addr = 0; - on_this = 1; + on_this = true; } else { op1_info = OP1_INFO(); op1_addr = OP1_REG_ADDR(); @@ -6089,7 +5999,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->op1_type == IS_CV && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { - ssa->var_info[ssa_op->op1_use].guarded_reference = 1; + ssa->var_info[ssa_op->op1_use].guarded_reference = true; } } else { CHECK_OP1_TRACE_TYPE(); @@ -6107,7 +6017,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; } if (delayed_fetch_this) { - on_this = 1; + on_this = true; } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; } else if (op_array_ssa->ops @@ -6162,9 +6072,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) { - gen_handler = 1; - op1_info = OP1_INFO(); - op2_info = OP2_INFO(); + gen_handler = true; + uint32_t op1_info = OP1_INFO(); + uint32_t op2_info = OP2_INFO(); if (op1_info & MAY_BE_GUARD) { op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } @@ -6284,9 +6194,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->result_def, &tmp)) { ssa->var_info[ssa_op->result_def].range.min = tmp.min; ssa->var_info[ssa_op->result_def].range.max = tmp.max; - ssa->var_info[ssa_op->result_def].range.underflow = 0; - ssa->var_info[ssa_op->result_def].range.overflow = 0; - ssa->var_info[ssa_op->result_def].has_range = 1; + ssa->var_info[ssa_op->result_def].range.underflow = false; + ssa->var_info[ssa_op->result_def].range.overflow = false; + ssa->var_info[ssa_op->result_def].has_range = true; } } if (ssa_op->op1_def >= 0 @@ -6345,9 +6255,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) { ssa->var_info[ssa_op->op1_def].range.min = tmp.min; ssa->var_info[ssa_op->op1_def].range.max = tmp.max; - ssa->var_info[ssa_op->op1_def].range.underflow = 0; - ssa->var_info[ssa_op->op1_def].range.overflow = 0; - ssa->var_info[ssa_op->op1_def].has_range = 1; + ssa->var_info[ssa_op->op1_def].range.underflow = false; + ssa->var_info[ssa_op->op1_def].range.overflow = false; + ssa->var_info[ssa_op->op1_def].has_range = true; } } if (ssa_op->op2_def >= 0 @@ -6386,9 +6296,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op2_def, &tmp)) { ssa->var_info[ssa_op->op2_def].range.min = tmp.min; ssa->var_info[ssa_op->op2_def].range.max = tmp.max; - ssa->var_info[ssa_op->op2_def].range.underflow = 0; - ssa->var_info[ssa_op->op2_def].range.overflow = 0; - ssa->var_info[ssa_op->op2_def].has_range = 1; + ssa->var_info[ssa_op->op2_def].range.underflow = false; + ssa->var_info[ssa_op->op2_def].range.overflow = false; + ssa->var_info[ssa_op->op2_def].has_range = true; } } @@ -6429,9 +6339,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) { ssa->var_info[ssa_op->op1_def].range.min = tmp.min; ssa->var_info[ssa_op->op1_def].range.max = tmp.max; - ssa->var_info[ssa_op->op1_def].range.underflow = 0; - ssa->var_info[ssa_op->op1_def].range.overflow = 0; - ssa->var_info[ssa_op->op1_def].has_range = 1; + ssa->var_info[ssa_op->op1_def].range.underflow = false; + ssa->var_info[ssa_op->op1_def].range.overflow = false; + ssa->var_info[ssa_op->op1_def].has_range = true; } } ssa_op++; @@ -6491,11 +6401,11 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) { p++; } - send_result = 0; + send_result = false; } } } else if (p->op == ZEND_JIT_TRACE_ENTER) { - call = frame->call; + zend_jit_trace_stack_frame *const call = frame->call; assert(call && &call->func->op_array == p->op_array); if (opline->opcode == ZEND_DO_UCALL @@ -6540,9 +6450,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par TRACE_FRAME_SET_THIS_CHECKED(call); } op_array = (zend_op_array*)p->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - op_array_ssa = &jit_extension->func_info.ssa; + op_array_ssa = zend_jit_op_array_trace_ssa(op_array); frame->call = call->prev; call->prev = frame; if (p->info & ZEND_JIT_TRACE_RETURN_VALUE_USED) { @@ -6555,7 +6463,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (ra) { int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); - for (i = 0; i < op_array->last_var; i++,j++) { + for (uint32_t i = 0; i < op_array->last_var; i++,j++) { if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) { SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD); if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) { @@ -6566,9 +6474,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } else if (p->op == ZEND_JIT_TRACE_BACK) { op_array = (zend_op_array*)p->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - op_array_ssa = &jit_extension->func_info.ssa; + op_array_ssa = zend_jit_op_array_trace_ssa(op_array); top = frame; if (frame->prev) { checked_stack -= frame->used_stack; @@ -6576,14 +6482,14 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par stack = frame->stack; ZEND_ASSERT(&frame->func->op_array == op_array); } else { - frame = zend_jit_trace_ret_frame(frame, op_array); + frame = zend_jit_trace_op_array_ret_frame(frame, op_array); TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1); frame->used_stack = checked_stack = peek_checked_stack = 0; stack = frame->stack; if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); - for (i = 0; i < op_array->last_var + op_array->T; i++, j++) { + for (uint32_t i = 0; i < op_array->last_var + op_array->T; i++, j++) { /* Initialize abstract stack using SSA */ if (!(ssa->var_info[j].type & MAY_BE_GUARD) && has_concrete_type(ssa->var_info[j].type)) { @@ -6594,7 +6500,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } if (ra) { j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); - for (i = 0; i < op_array->last_var + op_array->T; i++, j++) { + for (uint32_t i = 0; i < op_array->last_var + op_array->T; i++, j++) { if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) { SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD); if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) { @@ -6604,7 +6510,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } } else { - for (i = 0; i < op_array->last_var + op_array->T; i++) { + for (uint32_t i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); } } @@ -6630,7 +6536,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par num_args = init_opline->extended_value; } - call = top; + zend_jit_trace_stack_frame *const call = top; TRACE_FRAME_INIT(call, p->func, frame_flags, num_args); call->prev = frame->call; if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { @@ -6662,50 +6568,48 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } frame->call = call; - top = zend_jit_trace_call_frame(top, p->op_array); + top = zend_jit_trace_function_call_frame(top, p->func); if (p->func) { if (p->func->type == ZEND_USER_FUNCTION) { if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array); + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&p->func->op_array); - i = 0; - while (i < p->op_array->num_args) { + uint32_t i = 0; + while (i < p->func->op_array.num_args) { /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */ SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); i++; } - while (i < p->op_array->last_var) { + while (i < p->func->op_array.last_var) { if (jit_extension - && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) { + && zend_jit_var_may_alias(&p->func->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); } else { SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1); } i++; } - while (i < p->op_array->last_var + p->op_array->T) { + while (i < p->func->op_array.last_var + p->func->op_array.T) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); i++; } } else { - for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) { + for (uint32_t i = 0; i < p->func->op_array.last_var + p->func->op_array.T; i++) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); } } } else { ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION); - for (i = 0; i < p->op_array->num_args; i++) { + for (uint32_t i = 0; i < p->func->common.num_args; i++) { SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); } } if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) { - int skip_guard = 0; + bool skip_guard = false; if (init_opline) { - zend_call_info *call_info = jit_extension->func_info.callee_info; - - while (call_info) { + for (const zend_call_info *call_info = jit_extension->func_info.callee_info; call_info; call_info = call_info->next_callee) { if (call_info->caller_init_opline == init_opline && !call_info->is_prototype) { if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) { @@ -6716,15 +6620,14 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par break; } } - skip_guard = 1; + skip_guard = true; break; } - call_info = call_info->next_callee; } if (!skip_guard && !zend_jit_may_be_polymorphic_call(init_opline) && !zend_jit_may_be_modified(p->func, op_array)) { - skip_guard = 1; + skip_guard = true; } } @@ -6759,7 +6662,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { - call = frame->call; + zend_jit_trace_stack_frame *const call = frame->call; if (call) { checked_stack -= call->used_stack; top = call; @@ -6772,7 +6675,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END); - t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; + zend_jit_trace_info *const t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; if (!parent_trace && zend_jit_trace_uses_initial_ip()) { t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP; @@ -6782,9 +6685,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { if (ra) { - zend_ssa_phi *phi = ssa->blocks[1].phis; - - while (phi) { + for (zend_ssa_phi *phi = ssa->blocks[1].phis; phi; phi = phi->next) { if (ra[phi->ssa_var] && ra[phi->sources[1]] && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var) @@ -6794,7 +6695,6 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par /* TODO: Alternatively, we may try to update alredy generated deoptimization info */ zend_jit_store_var_type(&dasm_state, phi->var, STACK_TYPE(stack, phi->var)); } - phi = phi->next; } } if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { @@ -6808,10 +6708,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT; } if (!(t->flags & ZEND_JIT_TRACE_LOOP)) { - const void *timeout_exit_addr = NULL; - t->flags |= ZEND_JIT_TRACE_LOOP; + const void *timeout_exit_addr = NULL; if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP) || (ra @@ -6836,8 +6735,6 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par goto jit_failure; } if (p->stop == ZEND_JIT_TRACE_STOP_LINK) { - const void *timeout_exit_addr = NULL; - t->link = zend_jit_find_trace(p->opline->handler); if (t->link == 0) { /* this can happen if ZEND_JIT_EXIT_INVALIDATE was handled @@ -6855,13 +6752,15 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (!parent_trace && zend_jit_trace_uses_initial_ip()) { t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP; } + + const void *timeout_exit_addr = NULL; if (parent_trace && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT) && zend_jit_traces[parent_trace].root == t->link) { if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) { uint32_t exit_point; - for (i = 0; i < op_array->last_var + op_array->T; i++) { + for (uint32_t i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); } exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); @@ -6898,26 +6797,22 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (handler) { if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) { - const zend_op_array *rec_op_array; - - rec_op_array = op_array = trace_buffer->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; - for (;;p++) { + const zend_op_array *const rec_op_array = trace_buffer->op_array; + const zend_op_array *op_array = rec_op_array; + size_t offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); + + for (const zend_jit_trace_rec *p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;; p++) { if (p->op == ZEND_JIT_TRACE_VM) { opline = p->opline; } else if (p->op == ZEND_JIT_TRACE_ENTER) { if (p->op_array == rec_op_array) { - zend_jit_trace_setup_ret_counter(opline, jit_extension->offset); + zend_jit_trace_setup_ret_counter(opline, offset); } op_array = p->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); } else if (p->op == ZEND_JIT_TRACE_BACK) { op_array = p->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); } else if (p->op == ZEND_JIT_TRACE_END) { break; } @@ -6938,10 +6833,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par frame = JIT_G(current_frame)->prev; do { if (frame->call_opline) { - op_array = &frame->func->op_array; - jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset); + const zend_op_array *const op_array = &frame->func->op_array; + const size_t offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); + zend_jit_trace_setup_ret_counter(frame->call_opline, offset); } frame = frame->prev; } while (frame); @@ -6959,8 +6853,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par jit_cleanup: /* Clean up used op_arrays */ while (num_op_arrays > 0) { - op_array = op_arrays[--num_op_arrays]; - jit_extension = + const zend_op_array *const op_array = op_arrays[--num_op_arrays]; + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); jit_extension->func_info.num = 0; @@ -6982,22 +6876,17 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num) { const void *handler = NULL; - dasm_State* dasm_state = NULL; - void *checkpoint; - char name[32]; - const zend_op *opline; - uint32_t stack_size; - zend_jit_trace_stack *stack; - bool original_handler = 0; if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) { return dasm_labels[zend_lbtrace_escape]; } - checkpoint = zend_arena_checkpoint(CG(arena));; + void *const checkpoint = zend_arena_checkpoint(CG(arena));; + char name[32]; sprintf(name, "ESCAPE-%d-%d", trace_num, exit_num); + dasm_State* dasm_state = NULL; dasm_init(&dasm_state, DASM_MAXSECTION); dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); dasm_setup(&dasm_state, dasm_actions); @@ -7005,8 +6894,8 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n zend_jit_align_func(&dasm_state); /* Deoptimization */ - stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size; - stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset; + const uint32_t stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size; + zend_jit_trace_stack *const stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset; if (!zend_jit_trace_deoptimization(&dasm_state, zend_jit_traces[trace_num].exit_info[exit_num].flags, @@ -7015,11 +6904,13 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n goto jit_failure; } - opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; + bool original_handler = false; + + const zend_op *const opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; if (opline) { if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) { /* prevent endless loop */ - original_handler = 1; + original_handler = true; } zend_jit_set_ip_ex(&dasm_state, opline, original_handler); } @@ -7034,111 +6925,112 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n return handler; } -static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset) +/** + * Helper function for zend_jit_compile_root_trace(). + */ +static zend_jit_trace_stop _zend_jit_compile_root_trace(zend_jit_trace_rec *const trace_buffer, const zend_op *opline, zend_op_trace_info *const trace_info) { - zend_jit_trace_stop ret; - const void *handler; - uint8_t orig_trigger; - zend_jit_trace_info *t = NULL; zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; - bool do_bailout = 0; - - zend_shared_alloc_lock(); - - /* Checks under lock */ - if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) { - ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE; - } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) { - ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; - } else { - zend_try { - SHM_UNPROTECT(); - zend_jit_unprotect(); - - t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; + zend_jit_trace_info *const t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; + + t->id = ZEND_JIT_TRACE_NUM; + t->root = ZEND_JIT_TRACE_NUM; + t->parent = 0; + t->link = 0; + t->exit_count = 0; + t->child_count = 0; + t->stack_map_size = 0; + t->flags = 0; + t->polymorphism = 0; + t->jmp_table_size = 0; + t->op_array = trace_buffer[0].op_array; + t->opline = trace_buffer[1].opline; + t->exit_info = exit_info; + t->stack_map = NULL; + + const uint8_t orig_trigger = JIT_G(trigger); + JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE; + + const void *handler = zend_jit_trace(trace_buffer, 0, 0); + + JIT_G(trigger) = orig_trigger; - t->id = ZEND_JIT_TRACE_NUM; - t->root = ZEND_JIT_TRACE_NUM; - t->parent = 0; - t->link = 0; - t->exit_count = 0; - t->child_count = 0; - t->stack_map_size = 0; - t->flags = 0; - t->polymorphism = 0; - t->jmp_table_size = 0; - t->op_array = trace_buffer[0].op_array; - t->opline = trace_buffer[1].opline; - t->exit_info = exit_info; - t->stack_map = NULL; + if (handler) { + t->exit_info = NULL; + if (t->exit_count) { + /* reallocate exit_info into shared memory */ + t->exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc_copy(exit_info, + sizeof(zend_jit_trace_exit_info) * t->exit_count); - orig_trigger = JIT_G(trigger); - JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE; + if (!t->exit_info) { + if (t->stack_map) { + efree(t->stack_map); + } + return ZEND_JIT_TRACE_STOP_NO_SHM; + } + } - handler = zend_jit_trace(trace_buffer, 0, 0); + if (t->stack_map_size) { + zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc_copy( + t->stack_map, + t->stack_map_size * sizeof(zend_jit_trace_stack)); + efree(t->stack_map); + t->stack_map = shared_stack_map; + if (!shared_stack_map) { + return ZEND_JIT_TRACE_STOP_NO_SHM; + } + } - JIT_G(trigger) = orig_trigger; + t->exit_counters = ZEND_JIT_EXIT_COUNTERS; + ZEND_JIT_EXIT_COUNTERS += t->exit_count; - if (handler) { - zend_jit_trace_exit_info *shared_exit_info = NULL; + ((zend_op*)opline)->handler = handler; - t->exit_info = NULL; - if (t->exit_count) { - /* reallocate exit_info into shared memory */ - shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc( - sizeof(zend_jit_trace_exit_info) * t->exit_count); + ZEND_JIT_TRACE_NUM++; + trace_info->trace_flags |= ZEND_JIT_TRACE_JITED; - if (!shared_exit_info) { - if (t->stack_map) { - efree(t->stack_map); - t->stack_map = NULL; - } - ret = ZEND_JIT_TRACE_STOP_NO_SHM; - goto exit; - } - memcpy(shared_exit_info, exit_info, - sizeof(zend_jit_trace_exit_info) * t->exit_count); - t->exit_info = shared_exit_info; - } + if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 + && t->exit_count > 0) { + zend_jit_dump_exit_info(t); + } - if (t->stack_map_size) { - zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack)); - if (!shared_stack_map) { - ret = ZEND_JIT_TRACE_STOP_NO_SHM; - goto exit; - } - memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); - efree(t->stack_map); - t->stack_map = shared_stack_map; - } + return ZEND_JIT_TRACE_STOP_COMPILED; + } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || + ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { + if (t->stack_map) { + efree(t->stack_map); + } + return ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; + } else { + if (t->stack_map) { + efree(t->stack_map); + } + return ZEND_JIT_TRACE_STOP_COMPILER_ERROR; + } +} - t->exit_counters = ZEND_JIT_EXIT_COUNTERS; - ZEND_JIT_EXIT_COUNTERS += t->exit_count; +static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset) +{ + zend_jit_trace_stop ret; + bool do_bailout = false; - ((zend_op*)opline)->handler = handler; + zend_shared_alloc_lock(); - ZEND_JIT_TRACE_NUM++; - ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED; + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, offset); - ret = ZEND_JIT_TRACE_STOP_COMPILED; - } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || - ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { - if (t->stack_map) { - efree(t->stack_map); - t->stack_map = NULL; - } - ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; - } else { - if (t->stack_map) { - efree(t->stack_map); - t->stack_map = NULL; - } - ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR; - } + /* Checks under lock */ + if ((trace_info->trace_flags & ZEND_JIT_TRACE_JITED)) { + ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE; + } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) { + ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; + } else { + zend_try { + SHM_UNPROTECT(); + zend_jit_unprotect(); -exit:; + ret = _zend_jit_compile_root_trace(trace_buffer, opline, trace_info); } zend_catch { - do_bailout = 1; + do_bailout = true; } zend_end_try(); zend_jit_protect(); @@ -7151,25 +7043,16 @@ exit:; zend_bailout(); } - if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 - && ret == ZEND_JIT_TRACE_STOP_COMPILED - && t->exit_count > 0) { - zend_jit_dump_exit_info(t); - } - return ret; } /* Set counting handler back to original VM handler. */ static void zend_jit_stop_hot_trace_counters(zend_op_array *op_array) { - zend_jit_op_array_trace_extension *jit_extension; - uint32_t i; - - jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); zend_shared_alloc_lock(); SHM_UNPROTECT(); - for (i = 0; i < op_array->last; i++) { + for (uint32_t i = 0; i < op_array->last; i++) { /* Opline with Jit-ed code handler is skipped. */ if (jit_extension->trace_info[i].trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) { @@ -7235,14 +7118,16 @@ static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset) { zend_shared_alloc_lock(); - if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) { + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, offset); + + if (!(trace_info->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) { SHM_UNPROTECT(); zend_jit_unprotect(); ((zend_op*)opline)->handler = - ZEND_OP_TRACE_INFO(opline, offset)->orig_handler; + trace_info->orig_handler; - ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED; + trace_info->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED; zend_jit_protect(); SHM_PROTECT(); @@ -7257,56 +7142,52 @@ static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_sto uint8_t *cache_count = JIT_G(bad_root_cache_count); uint8_t *cache_stop = JIT_G(bad_root_cache_stop); uint32_t cache_slot = JIT_G(bad_root_slot); - uint32_t i; - for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { + for (uint32_t i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { if (cache_opline[i] == opline) { if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) { cache_opline[i] = NULL; - return 1; + return true; } else { #if 0 - if (ZEND_OP_TRACE_INFO(opline, offset)->counter) { - *ZEND_OP_TRACE_INFO(opline, offset)->counter = + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, offset); + if (trace_info->counter) { + *trace_info->counter = random() % ZEND_JIT_TRACE_COUNTER_MAX; } #endif cache_count[i]++; cache_stop[i] = stop; - return 0; + return false; } } } - i = cache_slot; + const uint32_t i = cache_slot; cache_opline[i] = opline; cache_count[i] = 1; cache_stop[i] = stop; cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS; JIT_G(bad_root_slot) = cache_slot; - return 0; + return false; } static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa) { zend_jit_trace_rec *p = trace_buffer; - const zend_op_array *op_array; - const zend_op *opline; uint32_t level = 1 + trace_buffer[0].level; - int idx, len, i, v, vars_count, call_level; ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START); - op_array = p->op_array; + const zend_op_array *op_array = p->op_array; p += ZEND_JIT_TRACE_START_REC_SIZE; - idx = 0; - call_level = 0; if (tssa && tssa->var_info) { + int vars_count; if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { vars_count = op_array->last_var; } else { vars_count = op_array->last_var + op_array->T; } - for (i = 0; i < vars_count; i++) { + for (int i = 0; i < vars_count; i++) { if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) { fprintf(stderr, " %*c;", level, ' '); zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE); @@ -7333,19 +7214,19 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa } } + int idx = 0; + int call_level = 0; while (1) { if (p->op == ZEND_JIT_TRACE_VM) { - uint8_t op1_type, op2_type, op3_type; - - opline = p->opline; + const zend_op *opline = p->opline; fprintf(stderr, "%04d%*c", (int)(opline - op_array->opcodes), level, ' '); zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL); - op1_type = p->op1_type; - op2_type = p->op2_type; - op3_type = p->op3_type; + const uint8_t op1_type = p->op1_type; + const uint8_t op2_type = p->op2_type; + const uint8_t op3_type = p->op3_type; if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) { fprintf(stderr, " ;"); if (op1_type != IS_UNKNOWN) { @@ -7383,15 +7264,13 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa } } if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) { - uint8_t val_type; - const char *type; - if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) { fprintf(stderr, " ;"); } p++; - val_type = p->op1_type; + const uint8_t val_type = p->op1_type; + const char *type; if (val_type == IS_UNDEF) { type = "undef"; } else if (val_type == IS_REFERENCE) { @@ -7404,7 +7283,7 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa fprintf(stderr, "\n"); idx++; - len = zend_jit_trace_op_len(opline); + int len = zend_jit_trace_op_len(opline); while (len > 1) { opline++; fprintf(stderr, "%04d%*c;", @@ -7427,9 +7306,9 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa level++; if (tssa && tssa->var_info) { call_level++; - v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); - vars_count = op_array->last_var; - for (i = 0; i < vars_count; i++, v++) { + int v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); + const int vars_count = op_array->last_var; + for (int i = 0; i < vars_count; i++, v++) { if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) { fprintf(stderr, " %*c;", level, ' '); zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE); @@ -7449,9 +7328,9 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa ZSTR_VAL(op_array->filename)); if (tssa && tssa->var_info) { if (call_level == 0) { - v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); - vars_count = op_array->last_var + op_array->T; - for (i = 0; i < vars_count; i++, v++) { + int v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); + const int vars_count = op_array->last_var + op_array->T; + for (int i = 0; i < vars_count; i++, v++) { if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) { fprintf(stderr, " %*c;", level, ' '); zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE); @@ -7493,10 +7372,8 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa static void zend_jit_dump_exit_info(zend_jit_trace_info *t) { - int i, j; - fprintf(stderr, "---- TRACE %d exit info\n", t->id); - for (i = 0; i < t->exit_count; i++) { + for (int i = 0; i < t->exit_count; i++) { const zend_op_array *op_array = t->exit_info[i].op_array; uint32_t stack_size = t->exit_info[i].stack_size; zend_jit_trace_stack *stack = t->stack_map + t->exit_info[i].stack_offset; @@ -7527,7 +7404,7 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t) if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) { fprintf(stderr, "/FREE_OP2"); } - for (j = 0; j < stack_size; j++) { + for (int j = 0; j < stack_size; j++) { zend_uchar type = STACK_TYPE(stack, j); if (type != IS_UNKNOWN) { fprintf(stderr, " "); @@ -7568,10 +7445,6 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const const zend_op *orig_opline; zend_jit_trace_stop stop; int ret = 0; - zend_op_array *op_array; - zend_jit_op_array_trace_extension *jit_extension; - size_t offset; - uint32_t trace_num; zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH]; ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION); @@ -7579,28 +7452,28 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const opline < EX(func)->op_array.opcodes + EX(func)->op_array.last); repeat: - trace_num = ZEND_JIT_TRACE_NUM; orig_opline = opline; - op_array = &EX(func)->op_array; - jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - offset = jit_extension->offset; + uint32_t trace_num = ZEND_JIT_TRACE_NUM; + const size_t offset = ZEND_OP_TRACE_INFO_OFFSET(&EX(func)->op_array); EX(opline) = opline; + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, offset); + /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */ - if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) { + if (trace_info->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) { return 0; } if (JIT_G(tracing)) { - ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter); + ++(*trace_info->counter); return 0; } if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n", trace_num, - zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags), + zend_jit_trace_star_desc(trace_info->trace_flags), EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "", EX(func)->op_array.scope ? "::" : "", EX(func)->op_array.function_name ? @@ -7615,10 +7488,10 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const goto abort; } - JIT_G(tracing) = 1; + JIT_G(tracing) = true; stop = zend_jit_trace_execute(execute_data, opline, trace_buffer, - ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0); - JIT_G(tracing) = 0; + trace_info->trace_flags & ZEND_JIT_TRACE_START_MASK, 0); + JIT_G(tracing) = false; if (stop & ZEND_JIT_TRACE_HALT) { ret = -1; @@ -7627,15 +7500,10 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) { orig_opline = trace_buffer[1].opline; - op_array = (zend_op_array*)trace_buffer[0].op_array; - jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - offset = jit_extension->offset; if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { const zend_op_array *op_array = trace_buffer[0].op_array; const zend_op *opline = trace_buffer[1].opline; - zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - size_t offset = jit_extension->offset; + const size_t offset = ZEND_OP_TRACE_INFO_OFFSET(op_array); fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n", trace_num, @@ -7708,8 +7576,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num) { - const void *handler; - bool do_bailout = 0; + bool do_bailout = false; zend_shared_alloc_lock(); @@ -7718,7 +7585,7 @@ static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num) zend_jit_unprotect(); zend_try { - handler = zend_jit_trace_exit_to_vm(trace_num, exit_num); + const void *const handler = zend_jit_trace_exit_to_vm(trace_num, exit_num); if (handler) { zend_jit_link_side_trace( @@ -7730,7 +7597,7 @@ static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num) } zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED; } zend_catch { - do_bailout = 1; + do_bailout = true; } zend_end_try(); zend_jit_protect(); @@ -7750,10 +7617,10 @@ static bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num) zend_jit_traces[trace_num].exit_counters + exit_num; if (*counter + 1 >= JIT_G(hot_side_exit) + JIT_G(blacklist_side_trace)) { - return 1; + return true; } (*counter)++; - return 0; + return false; } static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num) @@ -7762,125 +7629,123 @@ static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num) zend_jit_traces[trace_num].exit_counters + exit_num; if (*counter + 1 >= JIT_G(hot_side_exit)) { - return 1; + return true; } (*counter)++; - return 0; + return false; } -static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism) +/** + * Helper function for zend_jit_compile_side_trace(). + */ +static zend_jit_trace_stop _zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism) { - zend_jit_trace_stop ret; - const void *handler; - uint8_t orig_trigger; - zend_jit_trace_info *t; zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; - bool do_bailout = 0; + zend_jit_trace_info *const t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; - zend_shared_alloc_lock(); + t->id = ZEND_JIT_TRACE_NUM; + t->root = zend_jit_traces[parent_num].root; + t->parent = parent_num; + t->link = 0; + t->exit_count = 0; + t->child_count = 0; + t->stack_map_size = 0; + t->flags = 0; + t->polymorphism = polymorphism; + t->jmp_table_size = 0; + t->opline = NULL; + t->exit_info = exit_info; + t->stack_map = NULL; - /* Checks under lock */ - if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) { - ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE; - } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) { - ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; - } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) { - ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN; - } else { - SHM_UNPROTECT(); - zend_jit_unprotect(); + const uint8_t orig_trigger = JIT_G(trigger); + JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE; - zend_try { - t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; + const void *const handler = zend_jit_trace(trace_buffer, parent_num, exit_num); - t->id = ZEND_JIT_TRACE_NUM; - t->root = zend_jit_traces[parent_num].root; - t->parent = parent_num; - t->link = 0; - t->exit_count = 0; - t->child_count = 0; - t->stack_map_size = 0; - t->flags = 0; - t->polymorphism = polymorphism; - t->jmp_table_size = 0; - t->opline = NULL; - t->exit_info = exit_info; - t->stack_map = NULL; + JIT_G(trigger) = orig_trigger; - orig_trigger = JIT_G(trigger); - JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE; + if (handler) { + t->exit_info = NULL; + if (t->exit_count) { + /* reallocate exit_info into shared memory */ + t->exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc_copy(exit_info, + sizeof(zend_jit_trace_exit_info) * t->exit_count); - handler = zend_jit_trace(trace_buffer, parent_num, exit_num); + if (!t->exit_info) { + if (t->stack_map) { + efree(t->stack_map); + } + return ZEND_JIT_TRACE_STOP_NO_SHM; + } + } - JIT_G(trigger) = orig_trigger; + if (t->stack_map_size) { + zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc_copy( + t->stack_map, + t->stack_map_size * sizeof(zend_jit_trace_stack)); + efree(t->stack_map); + t->stack_map = shared_stack_map; + if (!shared_stack_map) { + return ZEND_JIT_TRACE_STOP_NO_SHM; + } + } - if (handler) { - zend_jit_trace_exit_info *shared_exit_info = NULL; + zend_jit_link_side_trace( + zend_jit_traces[parent_num].code_start, + zend_jit_traces[parent_num].code_size, + zend_jit_traces[parent_num].jmp_table_size, + exit_num, + handler); - t->exit_info = NULL; - if (t->exit_count) { - /* reallocate exit_info into shared memory */ - shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc( - sizeof(zend_jit_trace_exit_info) * t->exit_count); + t->exit_counters = ZEND_JIT_EXIT_COUNTERS; + ZEND_JIT_EXIT_COUNTERS += t->exit_count; - if (!shared_exit_info) { - if (t->stack_map) { - efree(t->stack_map); - t->stack_map = NULL; - } - ret = ZEND_JIT_TRACE_STOP_NO_SHM; - goto exit; - } - memcpy(shared_exit_info, exit_info, - sizeof(zend_jit_trace_exit_info) * t->exit_count); - t->exit_info = shared_exit_info; - } + zend_jit_traces[zend_jit_traces[parent_num].root].child_count++; + ZEND_JIT_TRACE_NUM++; + zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED; - if (t->stack_map_size) { - zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack)); - if (!shared_stack_map) { - efree(t->stack_map); - ret = ZEND_JIT_TRACE_STOP_NO_SHM; - goto exit; - } - memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); - efree(t->stack_map); - t->stack_map = shared_stack_map; - } + if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 + && t->exit_count > 0) { + zend_jit_dump_exit_info(t); + } - zend_jit_link_side_trace( - zend_jit_traces[parent_num].code_start, - zend_jit_traces[parent_num].code_size, - zend_jit_traces[parent_num].jmp_table_size, - exit_num, - handler); + return ZEND_JIT_TRACE_STOP_COMPILED; + } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || + ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { + if (t->stack_map) { + efree(t->stack_map); + } + return ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; + } else { + if (t->stack_map) { + efree(t->stack_map); + } + return ZEND_JIT_TRACE_STOP_COMPILER_ERROR; + } +} - t->exit_counters = ZEND_JIT_EXIT_COUNTERS; - ZEND_JIT_EXIT_COUNTERS += t->exit_count; +static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism) +{ + zend_jit_trace_stop ret; + bool do_bailout = false; - zend_jit_traces[zend_jit_traces[parent_num].root].child_count++; - ZEND_JIT_TRACE_NUM++; - zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED; + zend_shared_alloc_lock(); - ret = ZEND_JIT_TRACE_STOP_COMPILED; - } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || - ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { - if (t->stack_map) { - efree(t->stack_map); - t->stack_map = NULL; - } - ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; - } else { - if (t->stack_map) { - efree(t->stack_map); - t->stack_map = NULL; - } - ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR; - } + /* Checks under lock */ + if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) { + ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE; + } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) { + ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; + } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) { + ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN; + } else { + SHM_UNPROTECT(); + zend_jit_unprotect(); -exit:; + zend_try { + ret = _zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism); } zend_catch { - do_bailout = 1; + do_bailout = true; } zend_end_try(); zend_jit_protect(); @@ -7893,25 +7758,18 @@ exit:; zend_bailout(); } - if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 - && ret == ZEND_JIT_TRACE_STOP_COMPILED - && t->exit_count > 0) { - zend_jit_dump_exit_info(t); - } - return ret; } -int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num) +static int zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num) { zend_jit_trace_stop stop; int ret = 0; - uint32_t trace_num; zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH]; uint32_t is_megamorphic = 0; uint32_t polymorphism = 0; - trace_num = ZEND_JIT_TRACE_NUM; + const uint32_t trace_num = ZEND_JIT_TRACE_NUM; /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */ if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) { @@ -7954,9 +7812,9 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 } } - JIT_G(tracing) = 1; + JIT_G(tracing) = true; stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic); - JIT_G(tracing) = 0; + JIT_G(tracing) = false; if (stop & ZEND_JIT_TRACE_HALT) { ret = -1; @@ -7967,13 +7825,11 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { const zend_op_array *op_array = trace_buffer[0].op_array; const zend_op *opline = trace_buffer[1].opline; - zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - size_t offset = jit_extension->offset; + const zend_op_trace_info *trace_info = ZEND_OP_TRACE_INFO2(op_array, opline); fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n", trace_num, - zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags), + zend_jit_trace_star_desc(trace_info->trace_flags), op_array->scope ? ZSTR_VAL(op_array->scope->name) : "", op_array->scope ? "::" : "", op_array->function_name ? @@ -8048,17 +7904,15 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 return ret; } -int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs) +static int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs) { uint32_t trace_num = EG(jit_trace_num); zend_execute_data *execute_data = EG(current_execute_data); const zend_op *orig_opline = EX(opline); - const zend_op *opline; zend_jit_trace_info *t = &zend_jit_traces[trace_num]; - int repeat_last_opline = 0; + bool repeat_last_opline = false; /* Deoptimization of VM stack state */ - uint32_t i; uint32_t stack_size = t->exit_info[exit_num].stack_size; zend_jit_trace_stack *stack = t->stack_map + t->exit_info[exit_num].stack_offset; @@ -8068,7 +7922,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf EX(call) = call; } - for (i = 0; i < stack_size; i++) { + for (uint32_t i = 0; i < stack_size; i++) { if (STACK_REG(stack, i) != ZREG_NONE) { if (STACK_TYPE(stack, i) == IS_LONG) { zend_long val; @@ -8110,7 +7964,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) { /* Undefined array index or property */ - repeat_last_opline = 1; + repeat_last_opline = true; } else { ZVAL_COPY(EX_VAR_NUM(i), val); } @@ -8127,10 +7981,10 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf && EX(opline)->opcode != ZEND_FETCH_LIST_R) { Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var)); } - return 1; + return true; } - opline = t->exit_info[exit_num].opline; + const zend_op *const opline = t->exit_info[exit_num].opline; if (opline) { if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) { @@ -8153,7 +8007,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf } if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) { if (EG(exception)) { - return 1; + return true; } } if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { @@ -8163,7 +8017,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf zend_string_release_ex(func->common.function_name, 0); zend_free_trampoline(func); EX(opline) = opline; - return 1; + return true; } } @@ -8172,10 +8026,10 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf } if (zend_atomic_bool_load_ex(&EG(vm_interrupt)) || JIT_G(tracing)) { - return 1; + return true; /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */ } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) { - return 0; + return false; } ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION); @@ -8195,7 +8049,6 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf } if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_INVALIDATE) { - zend_jit_op_array_trace_extension *jit_extension; uint32_t num = trace_num; while (t->root != num) { @@ -8205,10 +8058,10 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf zend_shared_alloc_lock(); - jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array); + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO2(t->op_array, t->opline); /* Checks under lock, just in case something has changed while we were waiting for the lock */ - if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) { + if (!(trace_info->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) { /* skip: not JIT-ed nor blacklisted */ } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) { /* skip: too many root traces */ @@ -8216,14 +8069,14 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf SHM_UNPROTECT(); zend_jit_unprotect(); - if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) { + if (trace_info->trace_flags & ZEND_JIT_TRACE_START_LOOP) { ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler; - } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) { + } else if (trace_info->trace_flags & ZEND_JIT_TRACE_START_ENTER) { ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler; - } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) { + } else if (trace_info->trace_flags & ZEND_JIT_TRACE_START_RETURN) { ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler; } - ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &= + trace_info->trace_flags &= ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN; zend_jit_protect(); @@ -8232,7 +8085,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf zend_shared_alloc_unlock(); - return 0; + return false; } if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) { @@ -8242,7 +8095,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf fprintf(stderr, "---- EXIT %d/%d blacklisted\n", trace_num, exit_num); } - return 0; + return false; } } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) { return zend_jit_trace_hot_side(execute_data, trace_num, exit_num); @@ -8266,11 +8119,8 @@ static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array) { - zend_jit_op_array_trace_extension *jit_extension; - uint32_t i; - - jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - for (i = 0; i < op_array->last; i++) { + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + for (uint32_t i = 0; i < op_array->last; i++) { jit_extension->trace_info[i].trace_flags &= ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED; if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) { @@ -8284,15 +8134,22 @@ static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array) return SUCCESS; } -static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) +/** + * Allocate a new "hot counter" from #zend_jit_hot_counters. + */ +static int16_t *zend_jit_hot_counter_allocate(void) { - zend_op *opline; - zend_jit_op_array_trace_extension *jit_extension; - uint32_t i; + int16_t *counter = + &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; + ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; + return counter; +} +static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) +{ ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op)); - jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info)); + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info)); if (!jit_extension) { return FAILURE; } @@ -8300,7 +8157,7 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE; jit_extension->op_array = op_array; jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes; - for (i = 0; i < op_array->last; i++) { + for (uint32_t i = 0; i < op_array->last; i++) { jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler; jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]); jit_extension->trace_info[i].counter = NULL; @@ -8318,19 +8175,19 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) return FAILURE; } - for (i = 0; i < cfg.blocks_count; i++) { + for (uint32_t i = 0; i < cfg.blocks_count; i++) { if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) { if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) { /* loop header */ - opline = op_array->opcodes + cfg.blocks[i].start; - if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { + zend_op *const opline = op_array->opcodes + cfg.blocks[i].start; + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, jit_extension->offset); + if (!(trace_info->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { opline->handler = (const void*)zend_jit_loop_trace_counter_handler; - if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) { - ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = - &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; - ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; + if (!trace_info->counter) { + trace_info->counter = + zend_jit_hot_counter_allocate(); } - ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |= + trace_info->trace_flags |= ZEND_JIT_TRACE_START_LOOP; } } @@ -8340,20 +8197,14 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) if (JIT_G(hot_func)) { ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL); - opline = op_array->opcodes; - if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { - opline++; - } - } - - if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) { + zend_op *const opline = zend_find_function_entry(op_array); + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO(opline, jit_extension->offset); + if (!trace_info->trace_flags) { /* function entry */ opline->handler = (const void*)zend_jit_func_trace_counter_handler; - ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = - &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; - ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; - ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |= + trace_info->counter = + zend_jit_hot_counter_allocate(); + trace_info->trace_flags |= ZEND_JIT_TRACE_START_ENTER; } } @@ -8377,7 +8228,7 @@ static void zend_jit_trace_init_caches(void) static void zend_jit_trace_reset_caches(void) { - JIT_G(tracing) = 0; + JIT_G(tracing) = false; #ifdef ZTS if (!JIT_G(exit_counters)) { JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 8e02fbbbfeac2..3ba6b1b6331bf 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -53,8 +53,6 @@ register const zend_op* volatile opline __asm__("x28"); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC) { - zend_execute_data *old_execute_data; - if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) { zend_clean_and_cache_symbol_table(EX(symbol_table)); } @@ -69,7 +67,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t zend_free_extra_named_params(EX(extra_named_params)); } - old_execute_data = execute_data; + zend_execute_data *const old_execute_data = execute_data; execute_data = EX(prev_execute_data); zend_vm_stack_free_call_frame_ex(call_info, old_execute_data); @@ -132,7 +130,6 @@ void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D) if (EXPECTED(!(op_array->fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))) { uint32_t first_extra_arg = op_array->num_args; uint32_t num_args = EX_NUM_ARGS(); - zval *end, *src, *dst; uint32_t type_flags = 0; if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) { @@ -145,9 +142,9 @@ void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D) } /* move extra args into separate array after all CV and TMP vars */ - end = EX_VAR_NUM(first_extra_arg - 1); - src = end + (num_args - first_extra_arg); - dst = src + (op_array->last_var + op_array->T - first_extra_arg); + zval *const end = EX_VAR_NUM(first_extra_arg - 1); + zval *src = end + (num_args - first_extra_arg); + zval *dst = src + (op_array->last_var + op_array->T - first_extra_arg); if (EXPECTED(src != dst)) { do { type_flags |= Z_TYPE_INFO_P(src); @@ -256,11 +253,10 @@ static zend_always_inline zend_constant* _zend_quick_get_constant( zend_execute_data *execute_data = EG(current_execute_data); #endif const zend_op *opline = EX(opline); - zval *zv; zend_constant *c = NULL; /* null/true/false are resolved during compilation, so don't check for them here. */ - zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key)); + zval *zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key)); if (zv) { c = (zend_constant*)Z_PTR_P(zv); } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) { @@ -306,17 +302,16 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key) static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC) { - zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array); - size_t offset = jit_extension->offset; #ifndef HAVE_GCC_GLOBAL_REGS const zend_op *opline = EX(opline); #endif - *(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost; + zend_op_trace_info *const trace_info = ZEND_OP_TRACE_INFO2(&EX(func)->op_array, opline); - if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) { - *(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_COUNTER_INIT; + *(trace_info->counter) -= cost; + + if (UNEXPECTED(*(trace_info->counter) <= 0)) { + *(trace_info->counter) = ZEND_JIT_COUNTER_INIT; if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) { #ifdef HAVE_GCC_GLOBAL_REGS opline = NULL; @@ -333,7 +328,7 @@ static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_c return 1; #endif } else { - zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->orig_handler; + zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)trace_info->orig_handler; ZEND_OPCODE_TAIL_CALL(handler); } } @@ -396,10 +391,9 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HAN static int zend_jit_trace_recursive_call_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level, int level) { - int i; int count = 0; - for (i = ret_level; i < level; i++) { + for (int i = ret_level; i < level; i++) { count += (unrolled_calls[i] == op_array); } return count; @@ -407,10 +401,9 @@ static int zend_jit_trace_recursive_call_count(const zend_op_array *op_array, co static int zend_jit_trace_recursive_ret_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level) { - int i; int count = 0; - for (i = 0; i < ret_level; i++) { + for (int i = 0; i < ret_level; i++) { count += (unrolled_calls[i] == op_array); } return count; @@ -433,12 +426,11 @@ static uint8_t zend_jit_trace_bad_stop_event(const zend_op *opline, int count) const zend_op **cache_opline = JIT_G(bad_root_cache_opline); uint8_t *cache_count = JIT_G(bad_root_cache_count); uint8_t *cache_stop = JIT_G(bad_root_cache_stop); - uint32_t i; if (count < 0) { count = 0; } - for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { + for (uint32_t i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { if (cache_opline[i] == opline) { if (cache_count[i] >= count) { return cache_stop[i]; @@ -456,9 +448,6 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR; do { - zend_function *func; - zend_jit_op_array_trace_extension *jit_extension; - if (call->prev_execute_data) { idx = zend_jit_trace_record_fake_init_call_ex(call->prev_execute_data, trace_buffer, idx, is_megamorphic, init_level + 1); if (idx < 0) { @@ -466,7 +455,7 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend } } - func = call->func; + zend_function *func = call->func; if (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE|ZEND_ACC_NEVER_CACHE)) { /* TODO: Can we continue recording ??? */ return -1; @@ -477,7 +466,7 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend } if (func->type == ZEND_USER_FUNCTION && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array); if (UNEXPECTED(!jit_extension || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE) @@ -549,18 +538,11 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, zend_execute_data *save_execute_data = execute_data; const zend_op *save_opline = opline; #endif - const zend_op *orig_opline, *end_opline; zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR; zend_jit_trace_stop halt = 0; int level = 0; int ret_level = 0; - zend_vm_opcode_handler_t handler; - const zend_op_array *op_array; - zend_jit_op_array_trace_extension *jit_extension; - size_t offset; - int idx, count; - uint8_t trace_flags, op1_type, op2_type, op3_type; - zend_class_entry *ce1, *ce2; + int idx; const zend_op *link_to_enter_opline = NULL; int backtrack_link_to_enter = -1; int backtrack_recursion = -1; @@ -577,18 +559,17 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, execute_data = ex; opline = EX(opline) = op; #else - int rc; zend_execute_data *execute_data = ex; const zend_op *opline = EX(opline); #endif zend_execute_data *prev_call = EX(call); - orig_opline = opline; + const zend_op *const orig_opline = opline; - op_array = &EX(func)->op_array; - jit_extension = + const zend_op_array *op_array = &EX(func)->op_array; + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - offset = jit_extension->offset; + size_t offset = jit_extension->offset; if (!op_array->function_name || (op_array->fn_flags & ZEND_ACC_CLOSURE)) { op_array = jit_extension->op_array; @@ -620,8 +601,8 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, } while (1) { - ce1 = ce2 = NULL; - op1_type = op2_type = op3_type = IS_UNKNOWN; + zend_class_entry *ce1 = NULL, *ce2 = NULL; + uint8_t op1_type = IS_UNKNOWN, op2_type = IS_UNKNOWN, op3_type = IS_UNKNOWN; if ((opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && opline->opcode != ZEND_ROPE_ADD && opline->opcode != ZEND_ROPE_END @@ -847,7 +828,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, break; } - handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; + const zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; #ifdef HAVE_GCC_GLOBAL_REGS handler(); if (UNEXPECTED(opline == zend_jit_halt_op)) { @@ -858,7 +839,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, } if (UNEXPECTED(execute_data != prev_execute_data)) { #else - rc = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); + int rc = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); if (rc != 0) { if (rc < 0) { stop = ZEND_JIT_TRACE_STOP_RETURN; @@ -874,7 +855,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, #endif op_array = &EX(func)->op_array; - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); if (UNEXPECTED(!jit_extension) || UNEXPECTED(!(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))) { @@ -911,7 +892,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, EX(return_value) != NULL ? ZEND_JIT_TRACE_RETURN_VALUE_USED : 0, op_array); - count = zend_jit_trace_recursive_call_count(&EX(func)->op_array, unrolled_calls, ret_level, level); + const int count = zend_jit_trace_recursive_call_count(&EX(func)->op_array, unrolled_calls, ret_level, level); if (opline == orig_opline) { if (count + 1 >= JIT_G(max_recursive_calls)) { @@ -941,7 +922,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, break; } TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array); - count = zend_jit_trace_recursive_ret_count(&EX(func)->op_array, unrolled_calls, ret_level); + const int count = zend_jit_trace_recursive_ret_count(&EX(func)->op_array, unrolled_calls, ret_level); if (opline == orig_opline) { if (count + 1 >= JIT_G(max_recursive_returns)) { stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET; @@ -999,9 +980,6 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, if (EX(call) != prev_call) { if (EX(call) && EX(call)->prev_execute_data == prev_call) { - zend_function *func; - zend_jit_op_array_trace_extension *jit_extension; - if (EX(call)->func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { /* TODO: Can we continue recording ??? */ stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE; @@ -1011,7 +989,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; break; } - func = EX(call)->func; + zend_function *func = EX(call)->func; if (func->type == ZEND_INTERNAL_FUNCTION && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) { stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; @@ -1019,7 +997,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, } if (func->type == ZEND_USER_FUNCTION && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { - jit_extension = + zend_jit_op_array_trace_extension *const jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array); if (UNEXPECTED(!jit_extension) || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE) @@ -1060,7 +1038,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, break; } - trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags; + const uint8_t trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags; if (trace_flags) { if (trace_flags & ZEND_JIT_TRACE_JITED) { if (trace_flags & ZEND_JIT_TRACE_START_LOOP) { @@ -1145,7 +1123,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, } } - end_opline = opline; + const zend_op *end_opline = opline; if (!ZEND_JIT_TRACE_STOP_OK(stop)) { if (backtrack_recursion > 0) { idx = backtrack_recursion; diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index de2fa51526d0c..5c06277c61741 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -3594,11 +3594,8 @@ static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32 static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) { - zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - size_t offset = jit_extension->offset; const void *handler = - (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; + (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO2(op_array, opline)->call_handler; if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; @@ -14233,9 +14230,6 @@ static int zend_jit_assign_obj(dasm_State **Dst, uint8_t prop_type, int may_throw) { - zval *member; - zend_string *name; - zend_property_info *prop_info; zend_jit_addr val_addr = OP1_DATA_ADDR(); zend_jit_addr res_addr = 0; zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); @@ -14250,10 +14244,10 @@ static int zend_jit_assign_obj(dasm_State **Dst, ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); - member = RT_CONSTANT(opline, opline->op2); + zval *const member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); - name = Z_STR_P(member); - prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); + zend_string *const name = Z_STR_P(member); + zend_property_info *prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1a, this_addr @@ -14584,12 +14578,9 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_i static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) { if (opline->op1_type == IS_CONST) { - zval *zv; - size_t len; - - zv = RT_CONSTANT(opline, opline->op1); + zval *const zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); - len = Z_STRLEN_P(zv); + const size_t len = Z_STRLEN_P(zv); if (len > 0) { const char *str = Z_STRVAL_P(zv); @@ -14639,12 +14630,9 @@ static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_i static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) { if (opline->op1_type == IS_CONST) { - zval *zv; - size_t len; - - zv = RT_CONSTANT(opline, opline->op1); + zval *const zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); - len = Z_STRLEN_P(zv); + const size_t len = Z_STRLEN_P(zv); | SET_ZVAL_LVAL res_addr, len if (Z_MODE(res_addr) == IS_MEM_ZVAL) { @@ -14675,12 +14663,9 @@ static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1 static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) { if (opline->op1_type == IS_CONST) { - zval *zv; - zend_long count; - - zv = RT_CONSTANT(opline, opline->op1); + zval *const zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); - count = zend_hash_num_elements(Z_ARRVAL_P(zv)); + const zend_long count = zend_hash_num_elements(Z_ARRVAL_P(zv)); | SET_ZVAL_LVAL res_addr, count if (Z_MODE(res_addr) == IS_MEM_ZVAL) { @@ -14771,13 +14756,6 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) { - uint32_t count; - Bucket *p; - const zend_op *target; - int b; - int32_t exit_point; - const void *exit_addr; - | test r0, r0 if (default_label) { | jz &default_label @@ -14816,8 +14794,8 @@ static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend trace_info->jmp_table_size += zend_hash_num_elements(jumptable); } - count = jumptable->nNumUsed; - p = jumptable->arData; + uint32_t count = jumptable->nNumUsed; + Bucket *p = jumptable->arData; do { if (Z_TYPE(p->val) == IS_UNDEF) { if (default_label) { @@ -14828,15 +14806,15 @@ static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend | .aword =>default_b } } else { - target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); + const zend_op *const target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); if (!next_opline) { - b = ssa->cfg.map[target - op_array->opcodes]; + const int b = ssa->cfg.map[target - op_array->opcodes]; | .aword =>b } else if (next_opline == target) { | .aword >3 } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(target, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -14869,7 +14847,6 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); zval *jump_zv = NULL; - int b; if (opline->opcode == ZEND_SWITCH_LONG) { if (Z_TYPE_P(zv) == IS_LONG) { @@ -14898,6 +14875,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o } ZEND_ASSERT(target == next_opline); } else { + int b; if (jump_zv != NULL) { b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; } else { @@ -14910,24 +14888,20 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o uint32_t op1_info = OP1_INFO(); zend_jit_addr op1_addr = OP1_ADDR(); const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); - const zend_op *target; int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; - int b; - int32_t exit_point; const void *fallback_label = NULL; const void *default_label = NULL; - const void *exit_addr; if (next_opline) { if (next_opline != opline + 1) { - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); fallback_label = zend_jit_trace_get_exit_addr(exit_point); if (!fallback_label) { return 0; } } if (next_opline != default_opline) { - exit_point = zend_jit_trace_get_exit_point(default_opline, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(default_opline, 0); default_label = zend_jit_trace_get_exit_addr(exit_point); if (!default_label) { return 0; @@ -15006,15 +14980,15 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o | .aword =>default_b } } else { - target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); + const zend_op *const target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); if (!next_opline) { - b = ssa->cfg.map[target - op_array->opcodes]; + const int b = ssa->cfg.map[target - op_array->opcodes]; | .aword =>b } else if (next_opline == target) { | .aword >3 } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(target, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -15545,14 +15519,12 @@ static int zend_jit_fetch_constant(dasm_State **Dst, if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); - int32_t exit_point; - const void *exit_addr = NULL; SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); - exit_point = zend_jit_trace_get_exit_point(opline+1, 0); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline+1, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } @@ -15646,18 +15618,15 @@ static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t o static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) { - uint32_t offset; - - offset = (opline->opcode == ZEND_ROPE_INIT) ? + const uint32_t offset = (opline->opcode == ZEND_ROPE_INIT) ? opline->result.var : opline->op1.var + opline->extended_value * sizeof(zend_string*); if (opline->op2_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op2); - zend_string *str; ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); - str = Z_STR_P(zv); + zend_string *const str = Z_STR_P(zv); | ADDR_STORE aword [FP + offset], str, r0 } else { zend_jit_addr op2_addr = OP2_ADDR(); @@ -15759,8 +15728,6 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, { zend_jit_addr var_addr = *var_addr_ptr; uint32_t var_info = *var_info_ptr; - int32_t exit_point; - const void *exit_addr; if (add_indirect_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); @@ -15793,8 +15760,8 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, if (!(var_type & IS_TRACE_REFERENCE) && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { - exit_point = zend_jit_trace_get_exit_point(opline, 0); - exit_addr = zend_jit_trace_get_exit_addr(exit_point); + const int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *const exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; diff --git a/ext/opcache/zend_shared_alloc.c b/ext/opcache/zend_shared_alloc.c index 703796d51d85c..2c5fe381b7bed 100644 --- a/ext/opcache/zend_shared_alloc.c +++ b/ext/opcache/zend_shared_alloc.c @@ -363,6 +363,14 @@ void *zend_shared_alloc(size_t size) return NULL; } +void *zend_shared_alloc_copy(const void *src, size_t size) +{ + void *dest = zend_shared_alloc(size); + if (dest != NULL) + memcpy(dest, src, size); + return dest; +} + static zend_always_inline zend_ulong zend_rotr3(zend_ulong key) { return (key >> 3) | (key << ((sizeof(key) * 8) - 3)); diff --git a/ext/opcache/zend_shared_alloc.h b/ext/opcache/zend_shared_alloc.h index 1caf678e36fe4..09a0928f20d56 100644 --- a/ext/opcache/zend_shared_alloc.h +++ b/ext/opcache/zend_shared_alloc.h @@ -133,6 +133,14 @@ void zend_shared_alloc_shutdown(void); /* allocate shared memory block */ void *zend_shared_alloc(size_t size); +/** + * Allocate a shared memory block and copy raw data from the given + * source pointer into it. + * + * @return the new allocation or NULL if out of memory + */ +void *zend_shared_alloc_copy(const void *src, size_t size); + /** * Wrapper for zend_shared_alloc() which aligns at 64-byte boundary if * AVX or SSE2 are used.