Skip to content

Commit 8b12ea0

Browse files
dstogovdstogov
committed
Improved JIT for RECV and RECV_INIT instructions
1 parent 6079bdb commit 8b12ea0

File tree

2 files changed

+135
-156
lines changed

2 files changed

+135
-156
lines changed

‎ext/opcache/jit/zend_jit_helpers.c

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1169,7 +1169,7 @@ static zend_reference* ZEND_FASTCALL zend_jit_fetch_global_helper(zend_string *v
11691169
return ref;
11701170
}
11711171

1172-
static zend_always_inline zend_bool zend_jit_verify_type_common(zval *arg, const zend_op_array *op_array, zend_arg_info *arg_info, void **cache_slot)
1172+
static zend_always_inline zend_bool zend_jit_verify_type_common(zval *arg, zend_arg_info *arg_info, void **cache_slot)
11731173
{
11741174
uint32_t type_mask;
11751175

@@ -1227,16 +1227,23 @@ static zend_always_inline zend_bool zend_jit_verify_type_common(zval *arg, const
12271227
return 0;
12281228
}
12291229

1230-
static void ZEND_FASTCALL zend_jit_verify_arg_slow(zval *arg, const zend_op_array *op_array, uint32_t arg_num, zend_arg_info *arg_info, void **cache_slot)
1230+
//static void ZEND_FASTCALL zend_jit_verify_arg_slow(zval *arg, const zend_op_array *op_array, uint32_t arg_num, zend_arg_info *arg_info, void **cache_slot)
1231+
static int ZEND_FASTCALL zend_jit_verify_arg_slow(zval *arg, zend_arg_info *arg_info)
12311232
{
1232-
if (UNEXPECTED(!zend_jit_verify_type_common(arg, op_array, arg_info, cache_slot))) {
1233-
zend_verify_arg_error((zend_function*)op_array, arg_info, arg_num, cache_slot, arg);
1233+
zend_execute_data *execute_data = EG(current_execute_data);
1234+
const zend_op *opline = EX(opline);
1235+
void **cache_slot = CACHE_ADDR(opline->extended_value);
1236+
1237+
if (UNEXPECTED(!zend_jit_verify_type_common(arg, arg_info, cache_slot))) {
1238+
zend_verify_arg_error(EX(func), arg_info, opline->op1.num, cache_slot, arg);
1239+
return 0;
12341240
}
1241+
return 1;
12351242
}
12361243

12371244
static void ZEND_FASTCALL zend_jit_verify_return_slow(zval *arg, const zend_op_array *op_array, zend_arg_info *arg_info, void **cache_slot)
12381245
{
1239-
if (UNEXPECTED(!zend_jit_verify_type_common(arg, op_array, arg_info, cache_slot))) {
1246+
if (UNEXPECTED(!zend_jit_verify_type_common(arg, arg_info, cache_slot))) {
12401247
zend_verify_return_error((zend_function*)op_array, cache_slot, arg);
12411248
}
12421249
}

‎ext/opcache/jit/zend_jit_x86.dasc

Lines changed: 123 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -1180,11 +1180,11 @@ static void* dasm_labels[zend_lb_MAX];
11801180
|.endmacro
11811181

11821182
|.macro IF_Z_TYPE, zv, val, label
1183-
| IF_TYPE byte [zv + offsetof(zval, u1.v.type)], val, label
1183+
| IF_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label
11841184
|.endmacro
11851185

11861186
|.macro IF_NOT_Z_TYPE, zv, val, label
1187-
| IF_NOT_TYPE byte [zv + offsetof(zval, u1.v.type)], val, label
1187+
| IF_NOT_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label
11881188
|.endmacro
11891189

11901190
|.macro CMP_ZVAL_TYPE, addr, val
@@ -8511,9 +8511,14 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend
85118511
&& call_num_args <= func->op_array.num_args) {
85128512
uint32_t num_args;
85138513

8514-
if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0
8515-
&& call_info) {
8516-
num_args = skip_valid_arguments(op_array, ssa, call_info);
8514+
if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
8515+
if (trace) {
8516+
num_args = 0;
8517+
} else if (call_info) {
8518+
num_args = skip_valid_arguments(op_array, ssa, call_info);
8519+
} else {
8520+
num_args = call_num_args;
8521+
}
85178522
} else {
85188523
num_args = call_num_args;
85198524
}
@@ -10498,6 +10503,67 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, const z
1049810503
return 1;
1049910504
}
1050010505

10506+
static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, zend_bool check_exception)
10507+
{
10508+
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
10509+
zend_bool in_cold = 0;
10510+
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
10511+
zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1a : ZREG_R0;
10512+
10513+
if (ZEND_ARG_SEND_MODE(arg_info)) {
10514+
if (opline->opcode == ZEND_RECV_INIT) {
10515+
| GET_ZVAL_PTR Ra(tmp_reg), res_addr
10516+
| ZVAL_DEREF Ra(tmp_reg), MAY_BE_REF
10517+
res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0);
10518+
} else {
10519+
| GET_ZVAL_PTR Ra(tmp_reg), res_addr
10520+
res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val));
10521+
}
10522+
}
10523+
10524+
if (type_mask != 0) {
10525+
if (is_power_of_two(type_mask)) {
10526+
uint32_t type_code = concrete_type(type_mask);
10527+
| IF_NOT_ZVAL_TYPE res_addr, type_code, >1
10528+
} else {
10529+
| mov edx, 1
10530+
| mov cl, byte [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)+offsetof(zval, u1.v.type)]
10531+
| shl edx, cl
10532+
| test edx, type_mask
10533+
| je >1
10534+
}
10535+
10536+
|.cold_code
10537+
|1:
10538+
10539+
in_cold = 1;
10540+
}
10541+
10542+
if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) {
10543+
| LOAD_ZVAL_ADDR FCARG1a, res_addr
10544+
}
10545+
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
10546+
| SAVE_VALID_OPLINE opline, r0
10547+
} else {
10548+
| ADDR_OP2_2 mov, aword EX->opline, opline, r0
10549+
}
10550+
| LOAD_ADDR FCARG2a, (ptrdiff_t)arg_info
10551+
| EXT_CALL zend_jit_verify_arg_slow, r0
10552+
10553+
if (check_exception) {
10554+
| test eax, eax
10555+
| jz ->exception_handler
10556+
}
10557+
10558+
if (in_cold) {
10559+
| jmp >1
10560+
|.code
10561+
|1:
10562+
}
10563+
10564+
return 1;
10565+
}
10566+
1050110567
static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array)
1050210568
{
1050310569
uint32_t arg_num = opline->op1.num;
@@ -10528,7 +10594,11 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_
1052810594
| jb >1
1052910595
|.cold_code
1053010596
|1:
10531-
| SAVE_VALID_OPLINE opline, r0
10597+
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
10598+
| SAVE_VALID_OPLINE opline, r0
10599+
} else {
10600+
| ADDR_OP2_2 mov, aword EX->opline, opline, r0
10601+
}
1053210602
| mov FCARG1a, FP
1053310603
| EXT_CALL zend_missing_arg_error, r0
1053410604
| jmp ->exception_handler
@@ -10537,70 +10607,17 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_
1053710607
}
1053810608

1053910609
if (arg_info) {
10540-
// Type check
10541-
zend_type type = arg_info->type;
10542-
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
10543-
uint32_t type_mask;
10544-
10545-
| LOAD_ZVAL_ADDR r0, res_addr
10546-
if (ZEND_ARG_SEND_MODE(arg_info)) {
10547-
| GET_Z_PTR r0, r0
10548-
| add r0, offsetof(zend_reference, val)
10549-
}
10550-
10551-
type_mask = ZEND_TYPE_PURE_MASK(type);
10552-
if (type_mask == 0) {
10553-
| jmp >8
10554-
} else if (is_power_of_two(type_mask)) {
10555-
uint32_t type_code = concrete_type(type_mask);
10556-
| cmp byte [r0 + 8], type_code
10557-
| jne >8
10558-
} else {
10559-
| mov edx, 1
10560-
| mov cl, byte [r0 + 8]
10561-
| shl edx, cl
10562-
| test edx, type_mask
10563-
| je >8
10564-
}
10565-
10566-
|.cold_code
10567-
|8:
10568-
| SAVE_VALID_OPLINE opline, r0
10569-
| mov FCARG1a, r0
10570-
| mov r0, EX->run_time_cache
10571-
| add r0, opline->extended_value
10572-
| mov FCARG2a, EX->func
10573-
|.if X64WIN
10574-
| mov CARG3, arg_num
10575-
| LOAD_ADDR CARG4, (ptrdiff_t)arg_info
10576-
| mov aword A5, r0
10577-
| EXT_CALL zend_jit_verify_arg_slow, r0
10578-
|.elif X64
10579-
| mov CARG3, arg_num
10580-
| LOAD_ADDR CARG4, (ptrdiff_t)arg_info
10581-
| mov CARG5, r0
10582-
| EXT_CALL zend_jit_verify_arg_slow, r0
10583-
|.else
10584-
| sub r4, 4
10585-
| push r0
10586-
| push (ptrdiff_t)arg_info
10587-
| push arg_num
10588-
| EXT_CALL zend_jit_verify_arg_slow, r0
10589-
| add r4, 4
10590-
|.endif
10591-
10592-
if (!zend_jit_check_exception(Dst)) {
10610+
if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) {
1059310611
return 0;
1059410612
}
10595-
| jmp >1
10596-
|.code
10597-
|1:
1059810613
}
1059910614

10600-
if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) {
10601-
last_valid_opline = NULL;
10602-
if (!zend_jit_set_valid_ip(Dst, opline + 1)) {
10603-
return 0;
10615+
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
10616+
if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) {
10617+
last_valid_opline = NULL;
10618+
if (!zend_jit_set_valid_ip(Dst, opline + 1)) {
10619+
return 0;
10620+
}
1060410621
}
1060510622
}
1060610623

@@ -10609,8 +10626,6 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_
1060910626

1061010627
static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_bool is_last, int may_throw)
1061110628
{
10612-
zend_arg_info *arg_info = NULL;
10613-
uint8_t has_slow = 0;
1061410629
uint32_t arg_num = opline->op1.num;
1061510630
zval *zv = RT_CONSTANT(opline, opline->op2);
1061610631
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
@@ -10622,31 +10637,45 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zen
1062210637
}
1062310638
| ZVAL_COPY_CONST res_addr, -1, -1, zv, r0
1062410639
if (Z_REFCOUNTED_P(zv)) {
10625-
| ADDREF_CONST zv, r0
10640+
| ADDREF_CONST zv, r0
1062610641
}
10642+
1062710643
if (Z_CONSTANT_P(zv)) {
10628-
has_slow = 1;
10629-
| SAVE_VALID_OPLINE opline, r0
10630-
|.if X64
10631-
| LOAD_ZVAL_ADDR CARG1, res_addr
10632-
| mov r0, EX->func
10633-
| mov CARG2, [r0 + offsetof(zend_op_array, scope)]
10634-
| EXT_CALL zval_update_constant_ex, r0
10635-
|.else
10636-
| sub r4, 8
10637-
| mov r0, EX->func
10638-
| push dword [r0 + offsetof(zend_op_array, scope)]
10639-
| LOAD_ZVAL_ADDR r0, res_addr
10640-
| push r0
10641-
| EXT_CALL zval_update_constant_ex, r0
10642-
| add r4, 16
10643-
|.endif
10644-
| test al, al
10645-
| jnz >7
10644+
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
10645+
| SAVE_VALID_OPLINE opline, r0
10646+
} else {
10647+
| ADDR_OP2_2 mov, aword EX->opline, opline, r0
10648+
}
10649+
|.if X64
10650+
| LOAD_ZVAL_ADDR CARG1, res_addr
10651+
| mov r0, EX->func
10652+
| mov CARG2, [r0 + offsetof(zend_op_array, scope)]
10653+
| EXT_CALL zval_update_constant_ex, r0
10654+
|.else
10655+
| sub r4, 8
10656+
| mov r0, EX->func
10657+
| push dword [r0 + offsetof(zend_op_array, scope)]
10658+
| LOAD_ZVAL_ADDR r0, res_addr
10659+
| push r0
10660+
| EXT_CALL zval_update_constant_ex, r0
10661+
| add r4, 16
10662+
|.endif
10663+
| test al, al
10664+
| jnz >1
10665+
|.cold_code
10666+
|1:
10667+
| ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline
10668+
| SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF
10669+
| jmp ->exception_handler
10670+
|.code
1064610671
}
10672+
1064710673
|5:
10674+
1064810675
if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
1064910676
do {
10677+
zend_arg_info *arg_info;
10678+
1065010679
if (arg_num <= op_array->num_args) {
1065110680
arg_info = &op_array->arg_info[arg_num-1];
1065210681
} else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
@@ -10657,81 +10686,24 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zen
1065710686
if (!ZEND_TYPE_IS_SET(arg_info->type)) {
1065810687
break;
1065910688
}
10660-
has_slow += 2;
10661-
| LOAD_ZVAL_ADDR r0, res_addr
10662-
| ZVAL_DEREF r0, MAY_BE_REF
10663-
10664-
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
10665-
if (type_mask == 0) {
10666-
| jmp >8
10667-
} else if (is_power_of_two(type_mask)) {
10668-
uint32_t type_code = concrete_type(type_mask);
10669-
| cmp byte [r0 + 8], type_code
10670-
| jne >8
10671-
} else {
10672-
| mov edx, 1
10673-
| mov cl, byte [r0 + 8]
10674-
| shl edx, cl
10675-
| test edx, type_mask
10676-
| je >8
10689+
if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 0)) {
10690+
return 0;
1067710691
}
1067810692
} while (0);
1067910693
}
10680-
|9:
10694+
1068110695
if (may_throw) {
1068210696
if (!zend_jit_check_exception(Dst)) {
1068310697
return 0;
1068410698
}
1068510699
}
10686-
if (is_last) {
10687-
| LOAD_IP_ADDR (opline + 1)
10688-
last_valid_opline = (opline + 1);
10689-
}
1069010700

10691-
if (has_slow) {
10692-
|.cold_code
10693-
if (has_slow & 1) {
10694-
|7:
10695-
| ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline
10696-
| SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF
10697-
if (may_throw) {
10698-
if (!zend_jit_check_exception(Dst)) {
10699-
return 0;
10701+
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
10702+
if (is_last) {
10703+
| LOAD_IP_ADDR (opline + 1)
10704+
last_valid_opline = (opline + 1);
1070010705
}
1070110706
}
10702-
| jmp <5
10703-
}
10704-
if (has_slow & 2) {
10705-
|8:
10706-
| mov FCARG1a, r0
10707-
| mov r0, EX->run_time_cache
10708-
| lea r0, [r0 + opline->extended_value]
10709-
| mov FCARG2a, EX->func
10710-
|.if X64WIN
10711-
| mov CARG3, arg_num
10712-
| LOAD_ADDR CARG4, (ptrdiff_t)arg_info
10713-
| mov aword A5, r0
10714-
| SAVE_VALID_OPLINE opline, r0
10715-
| EXT_CALL zend_jit_verify_arg_slow, r0
10716-
|.elif X64
10717-
| mov CARG3, arg_num
10718-
| LOAD_ADDR CARG4, (ptrdiff_t)arg_info
10719-
| mov CARG5, r0
10720-
| SAVE_VALID_OPLINE opline, r0
10721-
| EXT_CALL zend_jit_verify_arg_slow, r0
10722-
|.else
10723-
| sub r4, 4
10724-
| push r0
10725-
| push (ptrdiff_t)arg_info
10726-
| push arg_num
10727-
| SAVE_VALID_OPLINE opline, r0
10728-
| EXT_CALL zend_jit_verify_arg_slow, r0
10729-
| add r4, 4
10730-
|.endif
10731-
| jmp <9
10732-
}
10733-
|.code
10734-
}
1073510707

1073610708
return 1;
1073710709
}

0 commit comments

Comments
 (0)