Skip to content

Commit 98d8bcc

Browse files
dstogovdstogov
committed
Better trace_buffer packing
1 parent 555489d commit 98d8bcc

File tree

4 files changed

+95
-119
lines changed

4 files changed

+95
-119
lines changed

‎ext/opcache/jit/zend_jit_internal.h

Lines changed: 30 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -254,34 +254,47 @@ typedef enum _zend_jit_trace_op {
254254
ZEND_JIT_TRACE_DO_ICALL,
255255
ZEND_JIT_TRACE_ENTER,
256256
ZEND_JIT_TRACE_BACK,
257-
ZEND_JIT_TRACE_START,
258257
ZEND_JIT_TRACE_END,
258+
ZEND_JIT_TRACE_START,
259259
} zend_jit_trace_op;
260260

261261
#define IS_UNKNOWN 255 /* may be used for zend_jit_trace_rec.op?_type */
262262
#define IS_TRACE_REFERENCE (1<<5)
263263
#define IS_TRACE_INDIRECT (1<<6)
264264

265+
#define ZEND_JIT_TRACE_FAKE_INIT_CALL 0x00000100
266+
#define ZEND_JIT_TRACE_RETRUN_VALUE_USED 0x00000100
267+
268+
#define ZEND_JIT_TRACE_MAX_SSA_VAR 0x7ffffe
269+
#define ZEND_JIT_TRACE_SSA_VAR_SHIFT 9
270+
271+
#define ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(_info, var) do { \
272+
_info |= (var << ZEND_JIT_TRACE_SSA_VAR_SHIFT); \
273+
} while (0)
274+
#define ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(_info) \
275+
(_info >> ZEND_JIT_TRACE_SSA_VAR_SHIFT)
276+
265277
typedef struct _zend_jit_trace_rec {
266-
uint8_t op; /* zend_jit_trace_op */
267278
union {
268-
struct {
269-
uint8_t op1_type;/* recorded zval op1_type for ZEND_JIT_TRACE_VM */
270-
uint8_t op2_type;/* recorded zval op2_type for ZEND_JIT_TRACE_VM */
271-
uint8_t op3_type;/* recorded zval for op_data.op1_type for ZEND_JIT_TRACE_VM */
272-
};
273-
struct {
279+
struct { ZEND_ENDIAN_LOHI(
280+
uint8_t op, /* zend_jit_trace_op */
274281
union {
275-
int8_t return_value_used; /* for ZEND_JIT_TRACE_ENTER */
276-
uint8_t fake; /* for ZEND_JIT_TRACE_INIT_CALL */
277-
};
278-
uint8_t first_ssa_var; /* may be used for ZEND_JIT_TRACE_ENTER and ZEND_JIT_TRACE_BACK */
279-
};
280-
struct {
281-
uint8_t start; /* ZEND_JIT_TRACE_START_MASK for ZEND_JIT_TRACE_START/END */
282-
uint8_t stop; /* zend_jit_trace_stop for ZEND_JIT_TRACE_START/END */
283-
uint8_t level; /* recursive return level for ZEND_JIT_TRACE_START */
282+
struct {
283+
uint8_t op1_type;/* recorded zval op1_type for ZEND_JIT_TRACE_VM */
284+
uint8_t op2_type;/* recorded zval op2_type for ZEND_JIT_TRACE_VM */
285+
uint8_t op3_type;/* recorded zval for op_data.op1_type for ZEND_JIT_TRACE_VM */
286+
};
287+
struct {
288+
uint8_t start; /* ZEND_JIT_TRACE_START_MASK for ZEND_JIT_TRACE_START/END */
289+
uint8_t stop; /* zend_jit_trace_stop for ZEND_JIT_TRACE_START/END */
290+
uint8_t level; /* recursive return level for ZEND_JIT_TRACE_START */
291+
};
292+
})
284293
};
294+
uint32_t last;
295+
uint32_t info; /* "first_ssa_var" for ZEND_JIT_TRACE_ENTER and ZEND_JIT_TRACE_BACK,
296+
* "return_value_used" for ZEND_JIT_TRACE_ENTER,
297+
* "fake" for ZEND_JIT_TRACE_INIT_CALL */
285298
};
286299
union {
287300
const void *ptr;
@@ -292,15 +305,6 @@ typedef struct _zend_jit_trace_rec {
292305
};
293306
} zend_jit_trace_rec;
294307

295-
typedef struct _zend_jit_trace_start_rec {
296-
uint8_t op; /* zend_jit_trace_op */
297-
uint8_t start; /* ZEND_JIT_TRACE_START_MASK for ZEND_JIT_TRACE_START/END */
298-
uint8_t stop; /* zend_jit_trace_stop for ZEND_JIT_TRACE_START/END */
299-
uint8_t level; /* recursive return level for ZEND_JIT_TRACE_START */
300-
const zend_op_array *op_array;
301-
const zend_op *opline;
302-
} zend_jit_trace_start_rec;
303-
304308
#define ZEND_JIT_TRACE_START_REC_SIZE 2
305309

306310
typedef struct _zend_jit_trace_exit_info {

‎ext/opcache/jit/zend_jit_trace.c

Lines changed: 43 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -761,7 +761,7 @@ static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op)
761761

762762
static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, const zend_op_array *op_array)
763763
{
764-
if (!p->fake) {
764+
if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
765765
p--;
766766
while (1) {
767767
if (p->op == ZEND_JIT_TRACE_VM) {
@@ -1058,11 +1058,10 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
10581058
stack = frame->stack;
10591059
op_array = p->op_array;
10601060
level++;
1061-
// TODO: remove this restriction ???
1062-
if (ssa_vars_count >= 0xff) {
1061+
if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
10631062
return NULL;
10641063
}
1065-
p->first_ssa_var = ssa_vars_count;
1064+
ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
10661065
for (i = 0; i < op_array->last_var; i++) {
10671066
SET_STACK_VAR(stack, i, ssa_vars_count++);
10681067
}
@@ -1071,11 +1070,10 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
10711070
frame = zend_jit_trace_ret_frame(frame, op_array);
10721071
stack = frame->stack;
10731072
if (level == 0) {
1074-
// TODO: remove this restriction ???
1075-
if (ssa_vars_count >= 0xff) {
1073+
if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
10761074
return NULL;
10771075
}
1078-
p->first_ssa_var = ssa_vars_count;
1076+
ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
10791077
for (i = 0; i < op_array->last_var + op_array->T; i++) {
10801078
SET_STACK_VAR(stack, i, ssa_vars_count++);
10811079
}
@@ -1561,7 +1559,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
15611559

15621560
level++;
15631561
i = 0;
1564-
v = p->first_ssa_var;
1562+
v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
15651563
while (i < op_array->last_var) {
15661564
ssa_vars[v].var = i;
15671565
if (i < op_array->num_args) {
@@ -1601,7 +1599,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
16011599
ssa = &jit_extension->func_info.ssa;
16021600
if (level == 0) {
16031601
i = 0;
1604-
v = p->first_ssa_var;
1602+
v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
16051603
while (i < op_array->last_var) {
16061604
ssa_vars[v].var = i;
16071605
if (!ssa->var_info
@@ -1629,7 +1627,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
16291627
const zend_op *opline = q->opline - 1;
16301628
if (opline->result_type != IS_UNUSED) {
16311629
ssa_var_info[
1632-
p->first_ssa_var +
1630+
ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info) +
16331631
EX_VAR_TO_NUM(opline->result.var)] = return_value_info;
16341632
}
16351633
}
@@ -1729,19 +1727,20 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
17291727
trace_buffer->op_array->function_name ?
17301728
ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
17311729
ZSTR_VAL(trace_buffer->op_array->filename),
1732-
((zend_jit_trace_start_rec*)trace_buffer)->opline->lineno);
1730+
trace_buffer[1].opline->lineno);
17331731
} else {
17341732
fprintf(stderr, "---- TRACE %d TSSA start (%s) %s() %s:%d\n",
17351733
ZEND_JIT_TRACE_NUM,
17361734
zend_jit_trace_star_desc(trace_buffer->start),
17371735
trace_buffer->op_array->function_name ?
17381736
ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
17391737
ZSTR_VAL(trace_buffer->op_array->filename),
1740-
((zend_jit_trace_start_rec*)trace_buffer)->opline->lineno);
1738+
trace_buffer[1].opline->lineno);
17411739
}
17421740
zend_jit_dump_trace(trace_buffer, tssa);
17431741
if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LINK) {
1744-
uint32_t link_to = zend_jit_find_trace(EG(current_execute_data)->opline->handler);;
1742+
uint32_t idx = trace_buffer[1].last;
1743+
uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
17451744
fprintf(stderr, "---- TRACE %d TSSA stop (link to %d)\n",
17461745
ZEND_JIT_TRACE_NUM,
17471746
link_to);
@@ -2062,7 +2061,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace
20622061
jit_extension =
20632062
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
20642063
op_array_ssa = &jit_extension->func_info.ssa;
2065-
j = p->first_ssa_var;
2064+
j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
20662065
for (i = 0; i < op_array->last_var; i++) {
20672066
SET_STACK_VAR(stack, i, j);
20682067
vars_op_array[j] = op_array;
@@ -2091,7 +2090,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace
20912090
stack = frame->stack;
20922091
if (level == 0) {
20932092
/* New return frames */
2094-
j = p->first_ssa_var;
2093+
j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
20952094
for (i = 0; i < op_array->last_var + op_array->T; i++) {
20962095
SET_STACK_VAR(stack, i, j);
20972096
vars_op_array[j] = op_array;
@@ -2481,7 +2480,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
24812480
TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
24822481
stack = frame->stack;
24832482

2484-
opline = ((zend_jit_trace_start_rec*)p)->opline;
2483+
opline = p[1].opline;
24852484
name = zend_jit_trace_name(op_array, opline->lineno);
24862485
p += ZEND_JIT_TRACE_START_REC_SIZE;
24872486

@@ -3809,17 +3808,20 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
38093808
op_array_ssa = &jit_extension->func_info.ssa;
38103809
call = frame->call;
38113810
if (!call) {
3811+
uint32_t v;
3812+
38123813
assert(0); // This should be handled by "fake" ZEND_JIT_TRACE_INIT_CALL
38133814
/* Trace missed INIT_FCALL opcode */
38143815
call = top;
38153816
TRACE_FRAME_INIT(call, op_array, 0, -1); // TODO: should be possible to get the real number af arguments ???
38163817
top = zend_jit_trace_call_frame(top, op_array);
38173818
i = 0;
3819+
v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
38183820
while (i < p->op_array->num_args) {
38193821
/* Initialize abstract stack using SSA */
3820-
if (!(ssa->var_info[p->first_ssa_var + i].type & MAY_BE_GUARD)
3821-
&& has_concrete_type(ssa->var_info[p->first_ssa_var + i].type)) {
3822-
SET_STACK_TYPE(call->stack, i, concrete_type(ssa->var_info[p->first_ssa_var + i].type));
3822+
if (!(ssa->var_info[v + i].type & MAY_BE_GUARD)
3823+
&& has_concrete_type(ssa->var_info[v + i].type)) {
3824+
SET_STACK_TYPE(call->stack, i, concrete_type(ssa->var_info[v + i].type));
38233825
} else {
38243826
SET_STACK_TYPE(call->stack, i, IS_UNKNOWN);
38253827
}
@@ -3838,17 +3840,17 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
38383840
}
38393841
frame->call = call->prev;
38403842
call->prev = frame;
3841-
if (p->return_value_used) {
3843+
if (p->info & ZEND_JIT_TRACE_RETRUN_VALUE_USED) {
38423844
TRACE_FRAME_SET_RETURN_VALUE_USED(call);
38433845
} else {
38443846
TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call);
38453847
}
38463848
JIT_G(current_frame) = frame = call;
38473849
stack = frame->stack;
38483850
if (ra) {
3849-
for (i = 0; i < op_array->last_var; i++) {
3850-
int j = p->first_ssa_var + i;
3851+
int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
38513852

3853+
for (i = 0; i < op_array->last_var; i++,j++) {
38523854
if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
38533855
//SET_STACK_REG(stack, i, ra[j]->reg);
38543856
if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
@@ -3869,22 +3871,23 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
38693871
stack = frame->stack;
38703872
ZEND_ASSERT(&frame->func->op_array == op_array);
38713873
} else {
3874+
uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3875+
38723876
frame = zend_jit_trace_ret_frame(frame, op_array);
38733877
TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
38743878
stack = frame->stack;
3875-
for (i = 0; i < op_array->last_var + op_array->T; i++) {
3879+
for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
38763880
/* Initialize abstract stack using SSA */
3877-
if (!(ssa->var_info[p->first_ssa_var + i].type & MAY_BE_GUARD)
3878-
&& has_concrete_type(ssa->var_info[p->first_ssa_var + i].type)) {
3879-
SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[p->first_ssa_var + i].type));
3881+
if (!(ssa->var_info[j].type & MAY_BE_GUARD)
3882+
&& has_concrete_type(ssa->var_info[j].type)) {
3883+
SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[j].type));
38803884
} else {
38813885
SET_STACK_TYPE(stack, i, IS_UNKNOWN);
38823886
}
38833887
}
38843888
if (ra) {
3885-
for (i = 0; i < op_array->last_var + op_array->T; i++) {
3886-
int j = p->first_ssa_var + i;
3887-
3889+
j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3890+
for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
38883891
if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
38893892
//SET_STACK_REG(stack, i, ra[j]->reg);
38903893
if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
@@ -3917,7 +3920,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
39173920
call = top;
39183921
TRACE_FRAME_INIT(call, p->func, TRACE_FRAME_MASK_NESTED, num_args);
39193922
call->prev = frame->call;
3920-
if (!p->fake) {
3923+
if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
39213924
TRACE_FRAME_SET_LAST_SEND_BY_VAL(call);
39223925
}
39233926
frame->call = call;
@@ -3938,7 +3941,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
39383941
i++;
39393942
}
39403943
}
3941-
if (p->fake) {
3944+
if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
39423945
int skip_guard = 0;
39433946

39443947
if (init_opline) {
@@ -3952,7 +3955,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
39523955
call_info = call_info->next_callee;
39533956
}
39543957
}
3955-
if (!skip_guard && !zend_jit_init_fcall_guard(&dasm_state, NULL, p->func, ((zend_jit_trace_start_rec*)trace_buffer)->opline)) {
3958+
if (!skip_guard && !zend_jit_init_fcall_guard(&dasm_state, NULL, p->func, trace_buffer[1].opline)) {
39563959
goto jit_failure;
39573960
}
39583961
frame->call_level++;
@@ -4143,7 +4146,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace
41434146
t->child_count = 0;
41444147
t->stack_map_size = 0;
41454148
t->flags = 0;
4146-
t->opline = ((zend_jit_trace_start_rec*)trace_buffer)->opline;
4149+
t->opline = trace_buffer[1].opline;
41474150
t->exit_info = exit_info;
41484151
t->stack_map = NULL;
41494152

@@ -4400,7 +4403,7 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa
44004403
level++;
44014404
if (tssa && tssa->var_info) {
44024405
call_level++;
4403-
v = p->first_ssa_var;
4406+
v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
44044407
vars_count = op_array->last_var;
44054408
for (i = 0; i < vars_count; i++, v++) {
44064409
if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
@@ -4422,7 +4425,7 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa
44224425
ZSTR_VAL(op_array->filename));
44234426
if (tssa && tssa->var_info) {
44244427
if (call_level == 0) {
4425-
v = p->first_ssa_var;
4428+
v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
44264429
vars_count = op_array->last_var + op_array->T;
44274430
for (i = 0; i < vars_count; i++, v++) {
44284431
if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
@@ -4437,7 +4440,7 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa
44374440
}
44384441
} else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
44394442
if (p->func != (zend_function*)&zend_pass_function) {
4440-
fprintf(stderr, p->fake ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
4443+
fprintf(stderr, (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
44414444
level, ' ',
44424445
p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
44434446
p->func->common.scope ? "::" : "",
@@ -4571,7 +4574,8 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const
45714574
if (ZEND_JIT_TRACE_STOP_OK(stop)) {
45724575
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_STOP) {
45734576
if (stop == ZEND_JIT_TRACE_STOP_LINK) {
4574-
uint32_t link_to = zend_jit_find_trace(EG(current_execute_data)->opline->handler);;
4577+
uint32_t idx = trace_buffer[1].last;
4578+
uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
45754579
fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
45764580
trace_num,
45774581
link_to);
@@ -4840,7 +4844,8 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
48404844
if (ZEND_JIT_TRACE_STOP_OK(stop)) {
48414845
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_STOP) {
48424846
if (stop == ZEND_JIT_TRACE_STOP_LINK) {
4843-
uint32_t link_to = zend_jit_find_trace(EG(current_execute_data)->opline->handler);;
4847+
uint32_t idx = trace_buffer[1].last;
4848+
uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);;
48444849
fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
48454850
trace_num,
48464851
link_to);

0 commit comments

Comments
 (0)