Skip to content

Commit c9434a4

Browse files
dstogovdstogov
committed
Separate and fix deoptimization code generator
1 parent 7a3dcc3 commit c9434a4

File tree

1 file changed

+149
-126
lines changed

1 file changed

+149
-126
lines changed

‎ext/opcache/jit/zend_jit_trace.c

Lines changed: 149 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -2581,18 +2581,6 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace
25812581
return NULL;
25822582
}
25832583

2584-
static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size)
2585-
{
2586-
uint32_t i;
2587-
2588-
for (i = 0; i < stack_size; i++) {
2589-
if (STACK_REG(stack, i) != ZREG_NONE) {
2590-
return 1;
2591-
}
2592-
}
2593-
return 0;
2594-
}
2595-
25962584
static void zend_jit_trace_clenup_stack(zend_jit_trace_stack *stack, const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, zend_lifetime_interval **ra)
25972585
{
25982586
uint32_t line = ssa_op - ssa->ops;
@@ -2674,6 +2662,140 @@ static zend_bool zend_jit_may_delay_fetch_this(zend_ssa *ssa, const zend_op **ss
26742662
return 1;
26752663
}
26762664

2665+
static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size)
2666+
{
2667+
uint32_t i;
2668+
2669+
for (i = 0; i < stack_size; i++) {
2670+
if (STACK_REG(stack, i) != ZREG_NONE) {
2671+
return 1;
2672+
}
2673+
}
2674+
return 0;
2675+
}
2676+
2677+
static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
2678+
{
2679+
const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
2680+
uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags;
2681+
uint32_t stack_size;
2682+
zend_jit_trace_stack *stack;
2683+
2684+
if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) {
2685+
return 1;
2686+
}
2687+
2688+
stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
2689+
stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
2690+
return zend_jit_trace_stack_needs_deoptimization(stack, stack_size);
2691+
}
2692+
2693+
static int zend_jit_trace_deoptimization(dasm_State **Dst,
2694+
uint32_t flags,
2695+
const zend_op *opline,
2696+
zend_jit_trace_stack *parent_stack,
2697+
int parent_vars_count,
2698+
zend_jit_trace_stack *stack,
2699+
zend_lifetime_interval **ra)
2700+
{
2701+
int i;
2702+
zend_bool has_constants = 0;
2703+
zend_bool has_unsaved_vars = 0;
2704+
2705+
// TODO: Merge this loop with the following register LOAD loop to implement parallel move ???
2706+
for (i = 0; i < parent_vars_count; i++) {
2707+
int8_t reg = STACK_REG(parent_stack, i);
2708+
2709+
if (reg != ZREG_NONE) {
2710+
if (reg < ZREG_NUM) {
2711+
if (ra && ra[i] && ra[i]->reg == reg) {
2712+
/* register already loaded by parent trace */
2713+
if (stack) {
2714+
SET_STACK_REG(stack, i, reg);
2715+
}
2716+
has_unsaved_vars = 1;
2717+
} else if (!zend_jit_store_var(Dst, 1 << STACK_TYPE(parent_stack, i), i, reg)) {
2718+
return 0;
2719+
}
2720+
} else {
2721+
/* delay custom deoptimization instructions to prevent register cpobbering */
2722+
has_constants = 1;
2723+
}
2724+
}
2725+
}
2726+
2727+
if (has_unsaved_vars
2728+
&& (has_constants
2729+
|| (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)))) {
2730+
for (i = 0; i < parent_vars_count; i++) {
2731+
int8_t reg = STACK_REG(parent_stack, i);
2732+
2733+
if (reg != ZREG_NONE) {
2734+
if (reg < ZREG_NUM) {
2735+
if (ra && ra[i] && ra[i]->reg == reg) {
2736+
if (stack) {
2737+
SET_STACK_REG(stack, i, ZREG_NONE);
2738+
}
2739+
if (!zend_jit_store_var(Dst, 1 << STACK_TYPE(parent_stack, i), i, reg)) {
2740+
return 0;
2741+
}
2742+
}
2743+
}
2744+
}
2745+
}
2746+
}
2747+
2748+
if (has_constants) {
2749+
for (i = 0; i < parent_vars_count; i++) {
2750+
int8_t reg = STACK_REG(parent_stack, i);
2751+
2752+
if (reg != ZREG_NONE) {
2753+
if (reg < ZREG_NUM) {
2754+
/* pass */
2755+
} else if (reg == ZREG_THIS) {
2756+
if (!zend_jit_load_this(Dst, EX_NUM_TO_VAR(i))) {
2757+
return 0;
2758+
}
2759+
} else {
2760+
if (!zend_jit_store_const(Dst, i, reg)) {
2761+
return 0;
2762+
}
2763+
}
2764+
}
2765+
}
2766+
}
2767+
2768+
if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
2769+
if (!zend_jit_save_call_chain(Dst, -1)) {
2770+
return 0;
2771+
}
2772+
}
2773+
2774+
if (flags & ZEND_JIT_EXIT_FREE_OP2) {
2775+
const zend_op *op = opline - 1;
2776+
2777+
if (!zend_jit_free_op(Dst, op, -1, op->op2.var)) {
2778+
return 0;
2779+
}
2780+
}
2781+
2782+
if (flags & ZEND_JIT_EXIT_FREE_OP1) {
2783+
const zend_op *op = opline - 1;
2784+
2785+
if (!zend_jit_free_op(Dst, op, -1, op->op1.var)) {
2786+
return 0;
2787+
}
2788+
}
2789+
2790+
if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
2791+
if (!zend_jit_check_exception(Dst)) {
2792+
return 0;
2793+
}
2794+
}
2795+
2796+
return 1;
2797+
}
2798+
26772799
static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num)
26782800
{
26792801
const void *handler = NULL;
@@ -2824,49 +2946,11 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
28242946

28252947
if (parent_trace) {
28262948
/* Deoptimization */
2827-
2828-
// TODO: Merge this loop with the following LOAD loop to implement parallel move ???
2829-
for (i = 0; i < parent_vars_count; i++) {
2830-
if (STACK_REG(parent_stack, i) != ZREG_NONE) {
2831-
if (ra && ra[i] && ra[i]->reg == STACK_REG(parent_stack, i)) {
2832-
/* register already loaded by parent trace */
2833-
SET_STACK_REG(stack, i, ra[i]->reg);
2834-
} else if (STACK_REG(parent_stack, i) < ZREG_NUM) {
2835-
if (!zend_jit_store_var(&dasm_state, ssa->var_info[i].type, i, STACK_REG(parent_stack, i))) {
2836-
goto jit_failure;
2837-
}
2838-
} else if (STACK_REG(parent_stack, i) == ZREG_THIS) {
2839-
SET_STACK_REG(stack, i, ZREG_NONE);
2840-
if (!zend_jit_load_this(&dasm_state, EX_NUM_TO_VAR(i))) {
2841-
goto jit_failure;
2842-
}
2843-
} else {
2844-
SET_STACK_REG(stack, i, ZREG_NONE);
2845-
if (!zend_jit_store_const(&dasm_state, i, STACK_REG(parent_stack, i))) {
2846-
goto jit_failure;
2847-
}
2848-
}
2849-
}
2850-
}
2851-
if (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
2852-
zend_jit_save_call_chain(&dasm_state, -1);
2853-
}
2854-
if (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) {
2855-
const zend_op *op = zend_jit_traces[parent_trace].exit_info[exit_num].opline - 1;
2856-
if (!zend_jit_free_op(&dasm_state, op, -1, op->op2.var)) {
2857-
goto jit_failure;
2858-
}
2859-
}
2860-
if (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) {
2861-
const zend_op *op = zend_jit_traces[parent_trace].exit_info[exit_num].opline - 1;
2862-
if (!zend_jit_free_op(&dasm_state, op, -1, op->op1.var)) {
2863-
goto jit_failure;
2864-
}
2865-
}
2866-
if (zend_jit_traces[parent_trace].exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
2867-
if (!zend_jit_check_exception(&dasm_state)) {
2868-
goto jit_failure;
2869-
}
2949+
if (!zend_jit_trace_deoptimization(&dasm_state,
2950+
zend_jit_traces[parent_trace].exit_info[exit_num].flags,
2951+
zend_jit_traces[parent_trace].exit_info[exit_num].opline,
2952+
parent_stack, parent_vars_count, stack, ra)) {
2953+
goto jit_failure;
28702954
}
28712955
}
28722956

@@ -4457,26 +4541,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
44574541
}
44584542
} else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
44594543
|| p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
4460-
/* Generate code for trace deoptimization */
4461-
int i;
4462-
4463-
for (i = 0; i < op_array->last_var + op_array->T; i++) {
4464-
if (STACK_REG(stack, i) != ZREG_NONE) {
4465-
// TODO: optimize out useless stores ????
4466-
if (STACK_REG(stack, i) < ZREG_NUM) {
4467-
if (!zend_jit_store_var(&dasm_state, 1 << STACK_TYPE(stack, i), i, STACK_REG(stack, i))) {
4468-
goto jit_failure;
4469-
}
4470-
} else if (STACK_REG(stack, i) == ZREG_THIS) {
4471-
if (!zend_jit_load_this(&dasm_state, EX_NUM_TO_VAR(i))) {
4472-
goto jit_failure;
4473-
}
4474-
} else {
4475-
if (!zend_jit_store_const(&dasm_state, i, STACK_REG(stack, i))) {
4476-
goto jit_failure;
4477-
}
4478-
}
4479-
}
4544+
if (!zend_jit_trace_deoptimization(&dasm_state, 0, NULL,
4545+
stack, op_array->last_var + op_array->T, NULL, NULL)) {
4546+
goto jit_failure;
44804547
}
44814548
if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
44824549
if (!zend_jit_set_valid_ip(&dasm_state, p->opline)) {
@@ -4572,30 +4639,14 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
45724639
return handler;
45734640
}
45744641

4575-
static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
4576-
{
4577-
const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
4578-
uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags;
4579-
uint32_t stack_size;
4580-
zend_jit_trace_stack *stack;
4581-
4582-
if (opline || (flags & ZEND_JIT_EXIT_RESTORE_CALL)) {
4583-
return 1;
4584-
}
4585-
4586-
stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
4587-
stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
4588-
return zend_jit_trace_stack_needs_deoptimization(stack, stack_size);
4589-
}
4590-
45914642
static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num)
45924643
{
45934644
const void *handler = NULL;
45944645
dasm_State* dasm_state = NULL;
45954646
void *checkpoint;
45964647
char name[32];
45974648
const zend_op *opline;
4598-
uint32_t i, stack_size;
4649+
uint32_t stack_size;
45994650
zend_jit_trace_stack *stack;
46004651
zend_bool original_handler = 0;
46014652

@@ -4614,46 +4665,18 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n
46144665
zend_jit_align_func(&dasm_state);
46154666

46164667
/* Deoptimization */
4617-
if (zend_jit_traces[trace_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
4618-
zend_jit_save_call_chain(&dasm_state, -1);
4619-
}
46204668
stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
46214669
stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
4622-
for (i = 0; i < stack_size; i++) {
4623-
if (STACK_REG(stack, i) != ZREG_NONE) {
4624-
if (STACK_REG(stack, i) < ZREG_NUM) {
4625-
if (!zend_jit_store_var(&dasm_state, 1 << STACK_TYPE(stack, i), i, STACK_REG(stack, i))) {
4626-
goto jit_failure;
4627-
}
4628-
} else if (STACK_REG(stack, i) == ZREG_THIS) {
4629-
if (!zend_jit_load_this(&dasm_state, EX_NUM_TO_VAR(i))) {
4630-
goto jit_failure;
4631-
}
4632-
} else {
4633-
if (!zend_jit_store_const(&dasm_state, i, STACK_REG(stack, i))) {
4634-
goto jit_failure;
4635-
}
4636-
}
4637-
}
4670+
4671+
if (!zend_jit_trace_deoptimization(&dasm_state,
4672+
zend_jit_traces[trace_num].exit_info[exit_num].flags,
4673+
zend_jit_traces[trace_num].exit_info[exit_num].opline,
4674+
stack, stack_size, NULL, NULL)) {
4675+
goto jit_failure;
46384676
}
46394677

46404678
opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
46414679
if (opline) {
4642-
if (zend_jit_traces[trace_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) {
4643-
if (!zend_jit_free_op(&dasm_state, (opline-1), -1, (opline-1)->op2.var)) {
4644-
goto jit_failure;
4645-
}
4646-
}
4647-
if (zend_jit_traces[trace_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) {
4648-
if (!zend_jit_free_op(&dasm_state, (opline-1), -1, (opline-1)->op1.var)) {
4649-
goto jit_failure;
4650-
}
4651-
}
4652-
if (zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
4653-
if (!zend_jit_check_exception(&dasm_state)) {
4654-
goto jit_failure;
4655-
}
4656-
}
46574680
zend_jit_set_ip(&dasm_state, opline);
46584681
if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) {
46594682
/* prevent endless loop */

0 commit comments

Comments
 (0)