Skip to content

Commit 56a9eed

Browse files
dstogovdstogov
committed
Avoid unnecessary IP initiliaization on trace linking
1 parent 3f4a500 commit 56a9eed

File tree

3 files changed

+116
-41
lines changed

3 files changed

+116
-41
lines changed

‎ext/opcache/jit/zend_jit_internal.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,7 @@ typedef union _zend_jit_trace_stack {
348348
/* trace info flags */
349349
#define ZEND_JIT_TRACE_CHECK_INTERRUPT (1<<0)
350350
#define ZEND_JIT_TRACE_LOOP (1<<1)
351+
#define ZEND_JIT_TRACE_USES_INITIAL_IP (1<<2)
351352

352353
typedef struct _zend_jit_trace_info {
353354
uint32_t id; /* trace id */

‎ext/opcache/jit/zend_jit_trace.c

Lines changed: 63 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3088,7 +3088,6 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
30883088
zend_uchar res_type = IS_UNKNOWN;
30893089
const zend_op *opline, *orig_opline;
30903090
const zend_ssa_op *ssa_op, *orig_ssa_op;
3091-
const void *timeout_exit_addr = NULL;
30923091

30933092
JIT_G(current_trace) = trace_buffer;
30943093

@@ -3142,6 +3141,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
31423141

31433142
if (!parent_trace) {
31443143
zend_jit_set_last_valid_opline(opline);
3144+
zend_jit_track_last_valid_opline();
31453145
} else {
31463146
if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
31473147
zend_jit_trace_opline_guard(&dasm_state, opline);
@@ -3273,18 +3273,16 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
32733273
}
32743274
}
32753275

3276-
if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3277-
if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
3278-
uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, ZEND_JIT_EXIT_TO_VM);
3279-
3280-
timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
3281-
if (!timeout_exit_addr) {
3282-
goto jit_failure;
3283-
}
3284-
} else {
3285-
timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
3286-
}
3287-
}
3276+
// if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3277+
// if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
3278+
// uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
3279+
//
3280+
// timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
3281+
// if (!timeout_exit_addr) {
3282+
// goto jit_failure;
3283+
// }
3284+
// }
3285+
// }
32883286

32893287
if (ra && trace_buffer->stop != ZEND_JIT_TRACE_STOP_LOOP) {
32903288
int last_var = op_array->last_var;
@@ -5074,11 +5072,17 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
50745072

50755073
t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
50765074

5075+
if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
5076+
t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
5077+
}
5078+
50775079
if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
50785080
|| p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
50795081
|| p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
50805082
if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
5081-
if (!zend_jit_set_ip(&dasm_state, p->opline)) {
5083+
ZEND_ASSERT(!parent_trace);
5084+
if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
5085+
&& !zend_jit_set_ip(&dasm_state, p->opline)) {
50825086
goto jit_failure;
50835087
}
50845088
}
@@ -5087,7 +5091,25 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
50875091
t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
50885092
}
50895093
if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
5094+
const void *timeout_exit_addr = NULL;
5095+
50905096
t->flags |= ZEND_JIT_TRACE_LOOP;
5097+
5098+
if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
5099+
if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
5100+
|| (ra
5101+
&& zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
5102+
uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
5103+
5104+
timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5105+
if (!timeout_exit_addr) {
5106+
goto jit_failure;
5107+
}
5108+
} else {
5109+
timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
5110+
}
5111+
}
5112+
50915113
zend_jit_trace_end_loop(&dasm_state, 0, timeout_exit_addr); /* jump back to start of the trace loop */
50925114
}
50935115
} else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
@@ -5097,14 +5119,35 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
50975119
goto jit_failure;
50985120
}
50995121
if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
5100-
if (!zend_jit_set_ip(&dasm_state, p->opline)) {
5122+
const void *timeout_exit_addr = NULL;
5123+
5124+
t->link = zend_jit_find_trace(p->opline->handler);
5125+
if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
5126+
&& !zend_jit_set_ip(&dasm_state, p->opline)) {
51015127
goto jit_failure;
51025128
}
5103-
t->link = zend_jit_find_trace(p->opline->handler);
5104-
zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link],
5105-
parent_trace &&
5106-
(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT) &&
5107-
zend_jit_traces[parent_trace].root == t->link);
5129+
if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
5130+
t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
5131+
}
5132+
if (parent_trace
5133+
&& (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
5134+
&& zend_jit_traces[parent_trace].root == t->link) {
5135+
if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
5136+
uint32_t exit_point;
5137+
5138+
for (i = 0; i < op_array->last_var + op_array->T; i++) {
5139+
SET_STACK_TYPE(stack, i, IS_UNKNOWN);
5140+
}
5141+
exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
5142+
timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5143+
if (!timeout_exit_addr) {
5144+
goto jit_failure;
5145+
}
5146+
} else {
5147+
timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
5148+
}
5149+
}
5150+
zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link], timeout_exit_addr);
51085151
} else {
51095152
zend_jit_trace_return(&dasm_state, 0);
51105153
}

‎ext/opcache/jit/zend_jit_x86.dasc

Lines changed: 52 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1340,11 +1340,12 @@ static void* dasm_labels[zend_lb_MAX];
13401340

13411341
|.macro SET_EX_OPLINE, op, tmp_reg
13421342
|| if (op == last_valid_opline) {
1343+
|| zend_jit_use_last_valid_opline();
13431344
| SAVE_IP
13441345
|| } else {
13451346
| ADDR_OP2_2 mov, aword EX->opline, op, tmp_reg
13461347
|| if (!GCC_GLOBAL_REGS) {
1347-
|| last_valid_opline = NULL;
1348+
|| zend_jit_reset_last_valid_opline();
13481349
|| }
13491350
|| }
13501351
|.endmacro
@@ -1578,6 +1579,7 @@ static void* dasm_labels[zend_lb_MAX];
15781579

15791580
|.macro UNDEFINED_OFFSET, opline
15801581
|| if (opline == last_valid_opline) {
1582+
|| zend_jit_use_last_valid_opline();
15811583
| call ->undefined_offset_ex
15821584
|| } else {
15831585
| SET_EX_OPLINE opline, r0
@@ -1587,6 +1589,7 @@ static void* dasm_labels[zend_lb_MAX];
15871589

15881590
|.macro UNDEFINED_INDEX, opline
15891591
|| if (opline == last_valid_opline) {
1592+
|| zend_jit_use_last_valid_opline();
15901593
| call ->undefined_index_ex
15911594
|| } else {
15921595
| SET_EX_OPLINE opline, r0
@@ -1596,38 +1599,64 @@ static void* dasm_labels[zend_lb_MAX];
15961599

15971600
|.macro CANNOT_ADD_ELEMENT, opline
15981601
|| if (opline == last_valid_opline) {
1602+
|| zend_jit_use_last_valid_opline();
15991603
| call ->cannot_add_element_ex
16001604
|| } else {
16011605
| SET_EX_OPLINE opline, r0
16021606
| call ->cannot_add_element
16031607
|| }
16041608
|.endmacro
16051609

1606-
static zend_bool reuse_ip;
1607-
static zend_bool delayed_call_chain;
1608-
static uint32_t delayed_call_level;
1609-
static const zend_op *last_valid_opline;
1610-
static int jit_return_label;
1611-
static uint32_t current_trace_num;
1610+
static zend_bool reuse_ip = 0;
1611+
static zend_bool delayed_call_chain = 0;
1612+
static uint32_t delayed_call_level = 0;
1613+
static const zend_op *last_valid_opline = NULL;
1614+
static zend_bool use_last_vald_opline = 0;
1615+
static zend_bool track_last_valid_opline = 0;
1616+
static int jit_return_label = -1;
1617+
static uint32_t current_trace_num = 0;
1618+
1619+
static void zend_jit_track_last_valid_opline(void)
1620+
{
1621+
use_last_vald_opline = 0;
1622+
track_last_valid_opline = 1;
1623+
}
1624+
1625+
static void zend_jit_use_last_valid_opline(void)
1626+
{
1627+
if (track_last_valid_opline) {
1628+
use_last_vald_opline = 1;
1629+
track_last_valid_opline = 0;
1630+
}
1631+
}
1632+
1633+
static zend_bool zend_jit_trace_uses_initial_ip(void)
1634+
{
1635+
return use_last_vald_opline;
1636+
}
16121637

16131638
static void zend_jit_set_last_valid_opline(const zend_op *target_opline)
16141639
{
16151640
if (!reuse_ip) {
1641+
track_last_valid_opline = 0;
16161642
last_valid_opline = target_opline;
16171643
}
16181644
}
16191645

16201646
static void zend_jit_reset_last_valid_opline(void)
16211647
{
1648+
track_last_valid_opline = 0;
16221649
last_valid_opline = NULL;
16231650
}
16241651

1625-
static void zend_jit_start_reuse_ip(void) {
1626-
last_valid_opline = NULL;
1652+
static void zend_jit_start_reuse_ip(void)
1653+
{
1654+
zend_jit_reset_last_valid_opline();
16271655
reuse_ip = 1;
16281656
}
16291657

1630-
static void zend_jit_stop_reuse_ip(void) {
1658+
static void zend_jit_stop_reuse_ip(void)
1659+
{
16311660
reuse_ip = 0;
16321661
}
16331662

@@ -2804,6 +2833,8 @@ static int zend_jit_align_func(dasm_State **Dst)
28042833
reuse_ip = 0;
28052834
delayed_call_chain = 0;
28062835
last_valid_opline = NULL;
2836+
use_last_vald_opline = 0;
2837+
track_last_valid_opline = 0;
28072838
jit_return_label = -1;
28082839
|.align 16
28092840
return 1;
@@ -2849,16 +2880,15 @@ static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level)
28492880

28502881
static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline)
28512882
{
2852-
if (!last_valid_opline) {
2883+
if (last_valid_opline == opline) {
2884+
zend_jit_use_last_valid_opline();
2885+
} else if (GCC_GLOBAL_REGS && last_valid_opline) {
2886+
zend_jit_use_last_valid_opline();
2887+
| ADD_IP (opline - last_valid_opline) * sizeof(zend_op);
2888+
} else {
28532889
| LOAD_IP_ADDR opline
2854-
} else if (last_valid_opline != opline) {
2855-
if (GCC_GLOBAL_REGS) {
2856-
| ADD_IP (opline - last_valid_opline) * sizeof(zend_op);
2857-
} else {
2858-
| LOAD_IP_ADDR opline
2859-
}
28602890
}
2861-
last_valid_opline = opline;
2891+
zend_jit_set_last_valid_opline(opline);
28622892

28632893
return 1;
28642894
}
@@ -2890,6 +2920,7 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const
28902920
if (exit_addr) {
28912921
| jne &exit_addr
28922922
} else if (last_valid_opline == opline) {
2923+
|| zend_jit_use_last_valid_opline();
28932924
| jne ->interrupt_handler
28942925
} else {
28952926
| jne >1
@@ -3154,7 +3185,7 @@ static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_
31543185
return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
31553186
}
31563187

3157-
static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, zend_bool check_interrupt)
3188+
static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr)
31583189
{
31593190
const void *link_addr;
31603191
size_t prologue_size;
@@ -3188,11 +3219,11 @@ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t,
31883219
}
31893220
link_addr = (const void*)((const char*)t->code_start + prologue_size);
31903221

3191-
if (check_interrupt) {
3222+
if (timeout_exit_addr) {
31923223
/* Check timeout for links to LOOP */
31933224
| MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0
31943225
| je &link_addr
3195-
| jmp ->interrupt_handler
3226+
| jmp &timeout_exit_addr
31963227
} else {
31973228
| jmp &link_addr
31983229
}

0 commit comments

Comments
 (0)