Skip to content

Commit 92448d9

Browse files
dstogovdstogov
committed
Merge branch 'jit-jumptable-x64'
* jit-jumptable-x64: Use RIP-relative addressing Align jumptables Fix jumtable index calculation Support JIT jumptables on x64
2 parents e33dd6a + 5872bfc commit 92448d9

File tree

4 files changed

+48
-33
lines changed

4 files changed

+48
-33
lines changed

‎ext/opcache/jit/dynasm/dasm_x86.h

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717

1818
/* Action definitions. DASM_STOP must be 255. */
1919
enum {
20-
DASM_DISP = 233,
20+
DASM_DISP = 232,
2121
DASM_IMM_S, DASM_IMM_B, DASM_IMM_W, DASM_IMM_D, DASM_IMM_WB, DASM_IMM_DB,
2222
DASM_VREG, DASM_SPACE, DASM_SETLABEL, DASM_REL_A, DASM_REL_LG, DASM_REL_PC,
23-
DASM_IMM_LG, DASM_IMM_PC, DASM_LABEL_LG, DASM_LABEL_PC, DASM_ALIGN,
24-
DASM_EXTERN, DASM_ESC, DASM_MARK, DASM_SECTION, DASM_STOP
23+
DASM_IMM_LG, DASM_IMM_PC, DASM_IMM_PC64, DASM_LABEL_LG, DASM_LABEL_PC,
24+
DASM_ALIGN, DASM_EXTERN, DASM_ESC, DASM_MARK, DASM_SECTION, DASM_STOP
2525
};
2626

2727
/* Maximum number of section buffer positions for a single dasm_put() call. */
@@ -228,6 +228,7 @@ void dasm_put(Dst_DECL, int start, ...)
228228
pl -= 246; n = *pl;
229229
if (n < 0) n = 0; /* Start new chain for fwd rel if label exists. */
230230
goto linkrel;
231+
case DASM_IMM_PC64: ofs += 4;
231232
case DASM_REL_PC:
232233
case DASM_IMM_PC: pl = D->pclabels + va_arg(ap, int); CKPL(pc, PC);
233234
putrel:
@@ -335,7 +336,8 @@ int dasm_link(Dst_DECL, size_t *szp)
335336
case DASM_SPACE: case DASM_IMM_LG: case DASM_VREG: p++;
336337
case DASM_DISP: case DASM_IMM_S: case DASM_IMM_B: case DASM_IMM_W:
337338
case DASM_IMM_D: case DASM_IMM_WB: case DASM_IMM_DB:
338-
case DASM_SETLABEL: case DASM_REL_A: case DASM_IMM_PC: pos++; break;
339+
case DASM_SETLABEL: case DASM_REL_A: case DASM_IMM_PC: case DASM_IMM_PC64:
340+
pos++; break;
339341
case DASM_LABEL_LG: p++;
340342
case DASM_LABEL_PC: b[pos++] += ofs; break; /* Fix label offset. */
341343
case DASM_ALIGN: ofs -= (b[pos++]+ofs)&*p++; break; /* Adjust ofs. */
@@ -359,10 +361,13 @@ int dasm_link(Dst_DECL, size_t *szp)
359361
#ifndef DASM_ALIGNED_WRITES
360362
typedef ZEND_SET_ALIGNED(1, unsigned short unaligned_short);
361363
typedef ZEND_SET_ALIGNED(1, unsigned int unaligned_int);
364+
typedef ZEND_SET_ALIGNED(1, uint64_t unaligned_uint64_t);
362365
#define dasmw(x) \
363366
do { *((unaligned_short *)cp) = (unsigned short)(x); cp+=2; } while (0)
364367
#define dasmd(x) \
365368
do { *((unaligned_int *)cp) = (unsigned int)(x); cp+=4; } while (0)
369+
#define dasmq(x) \
370+
do { *((unaligned_uint64_t *)cp) = (uint64_t)(x); cp+=8; } while (0)
366371
#else
367372
#define dasmw(x) do { dasmb(x); dasmb((x)>>8); } while (0)
368373
#define dasmd(x) do { dasmw(x); dasmw((x)>>16); } while (0)
@@ -442,6 +447,11 @@ int dasm_encode(Dst_DECL, void *buffer)
442447
n = *pb < 0 ? pb[1] : (*pb + (int)(ptrdiff_t)base);
443448
goto wd;
444449
}
450+
case DASM_IMM_PC64: {
451+
int *pb = DASM_POS2PTR(D, n);
452+
dasmq(*pb < 0 ? pb[1] : (*pb + (ptrdiff_t)base));
453+
break;
454+
}
445455
case DASM_LABEL_LG: {
446456
int idx = *p++;
447457
if (idx >= 10)

‎ext/opcache/jit/dynasm/dasm_x86.lua

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ local action_names = {
4747
-- action arg (1 byte) or int arg, 2 buffer pos (link, offset):
4848
"REL_LG", "REL_PC",
4949
-- action arg (1 byte) or int arg, 1 buffer pos (link):
50-
"IMM_LG", "IMM_PC",
50+
"IMM_LG", "IMM_PC", "IMM_PC64",
5151
-- action arg (1 byte) or int arg, 1 buffer pos (offset):
5252
"LABEL_LG", "LABEL_PC",
5353
-- action arg (1 byte), 1 buffer pos (offset):
@@ -434,7 +434,11 @@ local function wputlabel(aprefix, imm, num)
434434
end
435435
wputxb(imm)
436436
else
437-
waction(aprefix.."PC", imm, num)
437+
if aprefix == "IMM_" and x64 then
438+
waction("IMM_PC64", imm, num)
439+
else
440+
waction(aprefix.."PC", imm, num)
441+
end
438442
end
439443
end
440444

‎ext/opcache/jit/zend_jit.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,9 @@ static void *dasm_link_and_encode(dasm_State **dasm_state,
309309

310310
if (ret != DASM_S_OK) {
311311
// TODO: dasm_encode() failed ???
312+
#if ZEND_DEBUG
313+
ZEND_UNREACHABLE();
314+
#endif
312315
return NULL;
313316
}
314317

‎ext/opcache/jit/zend_jit_x86.dasc

Lines changed: 25 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -11377,10 +11377,6 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
1137711377
{
1137811378
HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
1137911379

11380-
if (sizeof(void*) == 8 && !IS_32BIT(dasm_end)) {
11381-
// TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
11382-
return 1;
11383-
}
1138411380
if (opline->op1_type == IS_CONST) {
1138511381
zval *zv = RT_CONSTANT(opline, opline->op1);
1138611382
zval *jump_zv;
@@ -11444,23 +11440,25 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
1144411440
| cmp FCARG2a, jumptable->nNumUsed
1144511441
| jae >3
1144611442
|.if X64
11447-
| // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
11448-
| movsxd r0, dword [FCARG2a * 4 + >4]
11449-
| jmp r0
11443+
if (!IS_32BIT(dasm_end)) {
11444+
| lea r0, aword [>4]
11445+
| jmp aword [r0 + FCARG2a * 8]
11446+
} else {
11447+
| jmp aword [FCARG2a * 8 + >4]
11448+
}
1145011449
|.else
1145111450
| jmp aword [FCARG2a * 4 + >4]
1145211451
|.endif
1145311452
|3:
1145411453
|.cold_code
11454+
|.align aword
1145511455
|4:
1145611456
p = jumptable->arData;
1145711457
do {
1145811458
if (Z_TYPE(p->val) == IS_UNDEF) {
11459-
| // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
1146011459
| .aword =>b
1146111460
} else {
1146211461
int b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)) - op_array->opcodes];
11463-
| // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
1146411462
| .aword =>b
1146511463
}
1146611464
p++;
@@ -11474,27 +11472,29 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
1147411472
| jz =>b
1147511473
| LOAD_ADDR FCARG1a, jumptable
1147611474
| sub r0, aword [FCARG1a + offsetof(HashTable, arData)]
11477-
| // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
11478-
| mov FCARG1a, (sizeof(Bucket) / sizeof(uint32_t))
11475+
| mov FCARG1a, (sizeof(Bucket) / sizeof(void*))
1147911476
|.if X64
1148011477
| cqo
1148111478
|.else
1148211479
| cdq
1148311480
|.endif
1148411481
| idiv FCARG1a
1148511482
|.if X64
11486-
| // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
11487-
| movsxd r0, dword [r0 + >4]
11488-
| jmp r0
11483+
if (!IS_32BIT(dasm_end)) {
11484+
| lea FCARG1a, aword [>4]
11485+
| jmp aword [FCARG1a + r0]
11486+
} else {
11487+
| jmp aword [r0 + >4]
11488+
}
1148911489
|.else
11490-
| jmp dword [r0 + >4]
11490+
| jmp aword [r0 + >4]
1149111491
|.endif
1149211492
|3:
1149311493
|.cold_code
11494+
|.align aword
1149411495
|4:
1149511496
ZEND_HASH_FOREACH_VAL(jumptable, val) {
1149611497
b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes];
11497-
| // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
1149811498
| .aword =>b
1149911499
} ZEND_HASH_FOREACH_END();
1150011500
|.code
@@ -11527,32 +11527,30 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
1152711527
| jz =>b
1152811528
| LOAD_ADDR FCARG1a, jumptable
1152911529
| sub r0, aword [FCARG1a + offsetof(HashTable, arData)]
11530-
| // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
11531-
| mov FCARG1a, (sizeof(Bucket) / sizeof(uint32_t))
11530+
| mov FCARG1a, (sizeof(Bucket) / sizeof(void*))
1153211531
|.if X64
1153311532
| cqo
1153411533
|.else
1153511534
| cdq
1153611535
|.endif
1153711536
| idiv FCARG1a
1153811537
|.if X64
11539-
| // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
11540-
| movsxd r0, dword [r0 + >4]
11541-
| jmp r0
11538+
if (!IS_32BIT(dasm_end)) {
11539+
| lea FCARG1a, aword [>4]
11540+
| jmp aword [FCARG1a + r0]
11541+
} else {
11542+
| jmp aword [r0 + >4]
11543+
}
1154211544
|.else
11543-
| jmp dword [r0 + >4]
11545+
| jmp aword [r0 + >4]
1154411546
|.endif
1154511547
|3:
1154611548
|.cold_code
11549+
|.align aword
1154711550
|4:
1154811551
ZEND_HASH_FOREACH_VAL(jumptable, val) {
1154911552
b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes];
11550-
| // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
11551-
|.if X64
1155211553
| .aword =>b
11553-
|.else
11554-
| .aword =>b
11555-
|.endif
1155611554
} ZEND_HASH_FOREACH_END();
1155711555
|.code
1155811556
}

0 commit comments

Comments
 (0)