Skip to content

Commit b133183

Browse files
dstogovdstogov
committed
Create TSSA loops for recursive call/return traces and move invariant type guards out of loops.
1 parent 8957260 commit b133183

File tree

1 file changed

+117
-30
lines changed

1 file changed

+117
-30
lines changed

‎ext/opcache/jit/zend_jit_trace.c

Lines changed: 117 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -653,6 +653,66 @@ static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ss
653653
return ssa_vars_count;
654654
}
655655

656+
static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
657+
{
658+
zend_ssa_phi *prev = NULL;
659+
const zend_op_array *op_array = trace_buffer->op_array;
660+
const zend_op *opline = trace_buffer[1].opline;
661+
int count = opline - op_array->opcodes;
662+
int i;
663+
664+
for(i = 0; i < count; i++) {
665+
zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
666+
ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
667+
ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
668+
sizeof(void*) * 2);
669+
phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
670+
phi->sources[0] = STACK_VAR(stack, i);
671+
phi->sources[1] = -1;
672+
phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
673+
phi->pi = -1;
674+
phi->var = i;
675+
phi->ssa_var = ssa_vars_count;
676+
SET_STACK_VAR(stack, i, ssa_vars_count);
677+
ssa_vars_count++;
678+
phi->block = 1;
679+
if (prev) {
680+
prev->next = phi;
681+
} else {
682+
tssa->blocks[1].phis = phi;
683+
}
684+
prev = phi;
685+
}
686+
return ssa_vars_count;
687+
}
688+
689+
static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
690+
{
691+
const zend_op *opline = trace_buffer[1].opline - 1;
692+
int i;
693+
694+
if (RETURN_VALUE_USED(opline)) {
695+
zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
696+
ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
697+
ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
698+
sizeof(void*) * 2);
699+
700+
i = EX_VAR_TO_NUM(opline->result.var);
701+
phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
702+
phi->sources[0] = STACK_VAR(stack, i);
703+
phi->sources[1] = -1;
704+
phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
705+
phi->pi = -1;
706+
phi->var = i;
707+
phi->ssa_var = ssa_vars_count;
708+
SET_STACK_VAR(stack, i, ssa_vars_count);
709+
ssa_vars_count++;
710+
phi->block = 1;
711+
tssa->blocks[1].phis = phi;
712+
}
713+
return ssa_vars_count;
714+
}
715+
656716
static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
657717
{
658718
int var, use;
@@ -1084,7 +1144,9 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
10841144
tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block));
10851145
tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int));
10861146

1087-
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1147+
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1148+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1149+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
10881150
tssa->cfg.blocks_count = 2;
10891151
tssa->cfg.edges_count = 2;
10901152

@@ -1136,6 +1198,10 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
11361198
// TODO: For tracing, it's possible, to create pseudo Phi functions
11371199
// at the end of loop, without this additional pass (like LuaJIT) ???
11381200
ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, stack);
1201+
} else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
1202+
ssa_vars_count = zend_jit_trace_add_call_phis(trace_buffer, ssa_vars_count, tssa, stack);
1203+
} else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1204+
ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack);
11391205
}
11401206

11411207
p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
@@ -1213,7 +1279,9 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
12131279
i++;
12141280
}
12151281

1216-
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1282+
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1283+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1284+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
12171285
/* Update Phi sources */
12181286
zend_ssa_phi *phi = tssa->blocks[1].phis;
12191287

@@ -1352,7 +1420,9 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
13521420
}
13531421
}
13541422

1355-
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1423+
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1424+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1425+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
13561426
/* Propagate initial value through Phi functions */
13571427
zend_ssa_phi *phi = tssa->blocks[1].phis;
13581428

@@ -1978,7 +2048,9 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
19782048
while (q->op == ZEND_JIT_TRACE_INIT_CALL) {
19792049
q++;
19802050
}
1981-
if (q->op == ZEND_JIT_TRACE_VM) {
2051+
if (q->op == ZEND_JIT_TRACE_VM
2052+
|| (q->op == ZEND_JIT_TRACE_END
2053+
&& q->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)) {
19822054
const zend_op *opline = q->opline - 1;
19832055
if (opline->result_type != IS_UNUSED) {
19842056
ssa_var_info[
@@ -2038,7 +2110,9 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
20382110
}
20392111
}
20402112

2041-
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
2113+
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2114+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2115+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
20422116
/* Propagate guards through Phi sources */
20432117
zend_ssa_phi *phi = tssa->blocks[1].phis;
20442118

@@ -2248,7 +2322,9 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace
22482322
}
22492323
}
22502324

2251-
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
2325+
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2326+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2327+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
22522328
zend_ssa_phi *phi = ssa->blocks[1].phis;
22532329

22542330
while (phi) {
@@ -2546,7 +2622,9 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace
25462622
}
25472623
}
25482624

2549-
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
2625+
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2626+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2627+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
25502628
zend_ssa_phi *phi = ssa->blocks[1].phis;
25512629

25522630
while (phi) {
@@ -2558,12 +2636,14 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace
25582636
phi = phi->next;
25592637
}
25602638

2561-
for (i = 0; i < op_array->last_var; i++) {
2562-
if (start[i] >= 0 && !ssa->vars[i].phi_use_chain) {
2563-
end[i] = idx;
2564-
flags[i] &= ~ZREG_LAST_USE;
2565-
} else {
2566-
zend_jit_close_var(stack, i, start, end, flags, idx);
2639+
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
2640+
for (i = 0; i < op_array->last_var; i++) {
2641+
if (start[i] >= 0 && !ssa->vars[i].phi_use_chain) {
2642+
end[i] = idx;
2643+
flags[i] &= ~ZREG_LAST_USE;
2644+
} else {
2645+
zend_jit_close_var(stack, i, start, end, flags, idx);
2646+
}
25672647
}
25682648
}
25692649
} else {
@@ -2650,7 +2730,9 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace
26502730
}
26512731
}
26522732

2653-
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
2733+
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2734+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2735+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
26542736
zend_ssa_phi *phi = ssa->blocks[1].phis;
26552737

26562738
while (phi) {
@@ -2815,7 +2897,9 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace
28152897
}
28162898

28172899
/* SSA resolution */
2818-
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
2900+
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2901+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2902+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
28192903
zend_ssa_phi *phi = ssa->blocks[1].phis;
28202904

28212905
while (phi) {
@@ -3402,7 +3486,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
34023486
ZEND_UNREACHABLE();
34033487
// SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i));
34043488
} else if ((info & MAY_BE_GUARD) != 0
3405-
&& trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3489+
&& (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3490+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3491+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)
34063492
&& (ssa->vars[i].use_chain != -1
34073493
|| (ssa->vars[i].phi_use_chain
34083494
&& !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD)))) {
@@ -3422,7 +3508,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
34223508
}
34233509

34243510
if ((info & MAY_BE_PACKED_GUARD) != 0
3425-
&& trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3511+
&& (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3512+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3513+
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)
34263514
&& ssa->vars[i].use_chain != -1) {
34273515
if (!zend_jit_packed_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), info)) {
34283516
goto jit_failure;
@@ -6304,20 +6392,19 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa
63046392
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
63056393
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
63066394
|| trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6395+
zend_ssa_phi *p = tssa->blocks[1].phis;
6396+
63076397
fprintf(stderr, "LOOP:\n");
6308-
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
6309-
zend_ssa_phi *p = tssa->blocks[1].phis;
6310-
6311-
while (p) {
6312-
fprintf(stderr, " ;");
6313-
zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
6314-
fprintf(stderr, " = Phi(");
6315-
zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
6316-
fprintf(stderr, ", ");
6317-
zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
6318-
fprintf(stderr, ")\n");
6319-
p = p->next;
6320-
}
6398+
6399+
while (p) {
6400+
fprintf(stderr, " ;");
6401+
zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
6402+
fprintf(stderr, " = Phi(");
6403+
zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
6404+
fprintf(stderr, ", ");
6405+
zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
6406+
fprintf(stderr, ")\n");
6407+
p = p->next;
63216408
}
63226409
}
63236410
}

0 commit comments

Comments
 (0)