diff --git a/TSRM/TSRM.c b/TSRM/TSRM.c index 2d60b6a9d6eee..a39564b8930d0 100644 --- a/TSRM/TSRM.c +++ b/TSRM/TSRM.c @@ -741,6 +741,14 @@ TSRM_API size_t tsrm_get_ls_cache_tcb_offset(void) asm ("leal _tsrm_ls_cache@ntpoff,%0" : "=r" (ret)); return ret; +#elif defined(__aarch64__) + size_t ret; + + asm("mov %0, xzr\n\t" + "add %0, %0, #:tprel_hi12:_tsrm_ls_cache, lsl #12\n\t" + "add %0, %0, #:tprel_lo12_nc:_tsrm_ls_cache" + : "=r" (ret)); + return ret; #else return 0; #endif diff --git a/build/Makefile.global b/build/Makefile.global index 2ff838cb3318d..6941bab6ad412 100644 --- a/build/Makefile.global +++ b/build/Makefile.global @@ -117,6 +117,7 @@ clean: find . -name .libs -a -type d|xargs rm -rf rm -f libphp.la $(SAPI_CLI_PATH) $(SAPI_CGI_PATH) $(SAPI_LITESPEED_PATH) $(SAPI_FPM_PATH) $(OVERALL_TARGET) modules/* libs/* rm -f ext/opcache/jit/zend_jit_x86.c + rm -f ext/opcache/jit/zend_jit_arm64.c distclean: clean rm -f Makefile config.cache config.log config.status Makefile.objects Makefile.fragments libtool main/php_config.h main/internal_functions_cli.c main/internal_functions.c Zend/zend_dtrace_gen.h Zend/zend_dtrace_gen.h.bak Zend/zend_config.h diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 633618e885498..f49480117ba2e 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -29,7 +29,7 @@ if test "$PHP_OPCACHE" != "no"; then if test "$PHP_OPCACHE_JIT" = "yes"; then case $host_cpu in - i[[34567]]86*|x86*) + i[[34567]]86*|x86*|aarch64) ;; *) AC_MSG_WARN([JIT not supported by host architecture]) @@ -77,6 +77,7 @@ if test "$PHP_OPCACHE" != "no"; then fi PHP_SUBST(DASM_FLAGS) + PHP_SUBST(DASM_ARCH) AC_MSG_CHECKING(for opagent in default path) for i in /usr/local /usr; do diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32 index a7f292ee7625f..764a2edaab146 100644 --- a/ext/opcache/config.w32 +++ b/ext/opcache/config.w32 @@ -25,6 +25,7 @@ if (PHP_OPCACHE != "no") { dasm_flags += " -D ZTS=1"; } DEFINE("DASM_FLAGS", dasm_flags); + DEFINE("DASM_ARCH", "x86"); AC_DEFINE('HAVE_JIT', 1, 'Define to enable JIT'); /* XXX read this dynamically */ diff --git a/ext/opcache/jit/Makefile.frag b/ext/opcache/jit/Makefile.frag index d44e06a3ad91b..98c5cdaea2494 100644 --- a/ext/opcache/jit/Makefile.frag +++ b/ext/opcache/jit/Makefile.frag @@ -2,11 +2,11 @@ $(builddir)/minilua: $(srcdir)/jit/dynasm/minilua.c $(BUILD_CC) $(srcdir)/jit/dynasm/minilua.c -lm -o $@ -$(builddir)/jit/zend_jit_x86.c: $(srcdir)/jit/zend_jit_x86.dasc $(srcdir)/jit/dynasm/*.lua $(builddir)/minilua - $(builddir)/minilua $(srcdir)/jit/dynasm/dynasm.lua $(DASM_FLAGS) -o $@ $(srcdir)/jit/zend_jit_x86.dasc +$(builddir)/jit/zend_jit_$(DASM_ARCH).c: $(srcdir)/jit/zend_jit_$(DASM_ARCH).dasc $(srcdir)/jit/dynasm/*.lua $(builddir)/minilua + $(builddir)/minilua $(srcdir)/jit/dynasm/dynasm.lua $(DASM_FLAGS) -o $@ $(srcdir)/jit/zend_jit_$(DASM_ARCH).dasc $(builddir)/jit/zend_jit.lo: \ - $(builddir)/jit/zend_jit_x86.c \ + $(builddir)/jit/zend_jit_$(DASM_ARCH).c \ $(srcdir)/jit/zend_jit_helpers.c \ $(srcdir)/jit/zend_jit_disasm.c \ $(srcdir)/jit/zend_jit_gdb.c \ diff --git a/ext/opcache/jit/dynasm/dasm_arm64.h b/ext/opcache/jit/dynasm/dasm_arm64.h index 8d1d9a9654247..909b51f808a80 100644 --- a/ext/opcache/jit/dynasm/dasm_arm64.h +++ b/ext/opcache/jit/dynasm/dasm_arm64.h @@ -404,6 +404,15 @@ int dasm_link(Dst_DECL, size_t *szp) return DASM_S_OK; } +#ifdef DASM_ADD_VENEER +#define CK_REL(x, o) \ + do { if (!(x) && !(n = DASM_ADD_VENEER(D, buffer, ins, b, cp, o))) \ + return DASM_S_RANGE_REL|(p-D->actionlist-1); \ + } while (0) +#else +#define CK_REL(x, o) CK(x, RANGE_REL) +#endif + #ifdef DASM_CHECKS #define CK(x, st) \ do { if (!(x)) return DASM_S_##st|(p-D->actionlist-1); } while (0) @@ -444,7 +453,7 @@ int dasm_encode(Dst_DECL, void *buffer) if (n < 0) { ptrdiff_t na = (ptrdiff_t)D->globals[-n] - (ptrdiff_t)cp + 4; n = (int)na; - CK((ptrdiff_t)n == na, RANGE_REL); + CK_REL((ptrdiff_t)n == na, na); goto patchrel; } /* fallthrough */ @@ -453,18 +462,18 @@ int dasm_encode(Dst_DECL, void *buffer) n = *DASM_POS2PTR(D, n) - (int)((char *)cp - base) + 4; patchrel: if (!(ins & 0xf800)) { /* B, BL */ - CK((n & 3) == 0 && ((n+0x08000000) >> 28) == 0, RANGE_REL); + CK_REL((n & 3) == 0 && ((n+0x08000000) >> 28) == 0, n); cp[-1] |= ((n >> 2) & 0x03ffffff); } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ - CK((n & 3) == 0 && ((n+0x00100000) >> 21) == 0, RANGE_REL); + CK_REL((n & 3) == 0 && ((n+0x00100000) >> 21) == 0, n); cp[-1] |= ((n << 3) & 0x00ffffe0); } else if ((ins & 0x3000) == 0x2000) { /* ADR */ - CK(((n+0x00100000) >> 21) == 0, RANGE_REL); + CK_REL(((n+0x00100000) >> 21) == 0, n); cp[-1] |= ((n << 3) & 0x00ffffe0) | ((n & 3) << 29); } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ cp[-1] |= ((n >> 9) & 0x00ffffe0) | (((n >> 12) & 3) << 29); } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ - CK((n & 3) == 0 && ((n+0x00008000) >> 16) == 0, RANGE_REL); + CK_REL((n & 3) == 0 && ((n+0x00008000) >> 16) == 0, n); cp[-1] |= ((n << 3) & 0x0007ffe0); } else if ((ins & 0x8000)) { /* absolute */ cp[0] = (unsigned int)((ptrdiff_t)cp - 4 + n); @@ -475,7 +484,7 @@ int dasm_encode(Dst_DECL, void *buffer) case DASM_REL_A: { ptrdiff_t na = (((ptrdiff_t)(*b++) << 32) | (unsigned int)n) - (ptrdiff_t)cp + 4; n = (int)na; - CK((ptrdiff_t)n == na, RANGE_REL); + CK_REL((ptrdiff_t)n == na, na); goto patchrel; } case DASM_LABEL_LG: diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 0cb9fc946df5c..8cd14fc987bbc 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -39,7 +39,14 @@ #include "Optimizer/zend_call_graph.h" #include "Optimizer/zend_dump.h" +#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32) #include "jit/zend_jit_x86.h" +#elif defined (__aarch64__) +#include "jit/zend_jit_arm64.h" +#else +#error "JIT not supported on this platform" +#endif + #include "jit/zend_jit_internal.h" #ifdef ZTS @@ -188,11 +195,6 @@ static bool zend_is_commutative(zend_uchar opcode) opcode == ZEND_BW_XOR; } -static bool zend_long_is_power_of_two(zend_long x) -{ - return (x > 0) && !(x & (x - 1)); -} - #define OP_RANGE(ssa_op, opN) \ (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \ ssa->var_info && \ @@ -204,19 +206,30 @@ static bool zend_long_is_power_of_two(zend_long x) #define OP2_RANGE() OP_RANGE(ssa_op, op2) #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1) +#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32) #include "dynasm/dasm_x86.h" +#elif defined(__aarch64__) +static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset); +#define DASM_ADD_VENEER zend_jit_add_veneer +#include "dynasm/dasm_arm64.h" +#endif + #include "jit/zend_jit_helpers.c" #include "jit/zend_jit_disasm.c" #ifndef _WIN32 -#include "jit/zend_jit_gdb.c" -#include "jit/zend_jit_perf_dump.c" +# include "jit/zend_jit_gdb.c" +# include "jit/zend_jit_perf_dump.c" #endif #ifdef HAVE_OPROFILE # include "jit/zend_jit_oprofile.c" #endif -#include "jit/zend_jit_vtune.c" +#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32) +#include "jit/zend_jit_vtune.c" #include "jit/zend_jit_x86.c" +#elif defined(__aarch64__) +#include "jit/zend_jit_arm64.c" +#endif #if _WIN32 # include @@ -298,15 +311,32 @@ static void handle_dasm_error(int ret) { case DASM_S_RANGE_PC: fprintf(stderr, "DASM_S_RANGE_PC %d\n", ret & 0xffffffu); break; +#ifdef DASM_S_RANGE_VREG case DASM_S_RANGE_VREG: fprintf(stderr, "DASM_S_RANGE_VREG\n"); break; +#endif +#ifdef DASM_S_UNDEF_L case DASM_S_UNDEF_L: fprintf(stderr, "DASM_S_UNDEF_L\n"); break; +#endif +#ifdef DASM_S_UNDEF_LG + case DASM_S_UNDEF_LG: + fprintf(stderr, "DASM_S_UNDEF_LG\n"); + break; +#endif +#ifdef DASM_S_RANGE_REL + case DASM_S_RANGE_REL: + fprintf(stderr, "DASM_S_RANGE_REL\n"); + break; +#endif case DASM_S_UNDEF_PC: fprintf(stderr, "DASM_S_UNDEF_PC\n"); break; + default: + fprintf(stderr, "DASM_S_%0x\n", ret & 0xff000000u); + break; } ZEND_UNREACHABLE(); } @@ -380,6 +410,10 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, return NULL; } +#ifdef __aarch64__ + dasm_venners_size = 0; +#endif + ret = dasm_encode(dasm_state, *dasm_ptr); if (ret != DASM_S_OK) { #if ZEND_DEBUG @@ -388,9 +422,16 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, return NULL; } +#ifdef __aarch64__ + size += dasm_venners_size; +#endif + entry = *dasm_ptr; *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT)); + /* flush the hardware I-cache */ + JIT_CACHE_FLUSH(entry, entry + size); + if (trace_num) { zend_jit_trace_add_code(entry, size); } @@ -4365,8 +4406,14 @@ ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached) return FAILURE; } - /* save JIT buffer pos */ zend_jit_unprotect(); +#ifdef __aarch64__ + /* reserve space for global labels veneers */ + dasm_labels_veneers = *dasm_ptr; + *dasm_ptr = (void**)*dasm_ptr + zend_lb_MAX; + memset(dasm_labels_veneers, 0, sizeof(void*) * zend_lb_MAX); +#endif + /* save JIT buffer pos */ dasm_ptr[1] = dasm_ptr[0]; zend_jit_protect(); @@ -4376,7 +4423,7 @@ ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached) ZEND_EXT_API void zend_jit_shutdown(void) { if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE) { - fprintf(stderr, "\nJIT memory usage: %td\n", (char*)*dasm_ptr - (char*)dasm_buf); + fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf)); } #ifdef HAVE_OPROFILE diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 6985ff141ea7c..753dec20ff732 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -53,6 +53,7 @@ #define ZEND_JIT_DEBUG_GDB (1<<8) #define ZEND_JIT_DEBUG_SIZE (1<<9) +#define ZEND_JIT_DEBUG_ASM_ADDR (1<<10) #define ZEND_JIT_DEBUG_TRACE_START (1<<12) #define ZEND_JIT_DEBUG_TRACE_STOP (1<<13) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc new file mode 100644 index 0000000000000..d1cd697ec55ee --- /dev/null +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -0,0 +1,15160 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | http://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Dmitry Stogov | + * | Xinchen Hui | + * | Hao Sun | + * +----------------------------------------------------------------------+ + */ + +|.arch arm64 + +|.define FP, x27 +|.define IP, x28 +|.define IPl, w28 +|.define RX, x28 // the same as VM IP reused as a general purpose reg +|.define LR, x30 +|.define CARG1, x0 +|.define CARG2, x1 +|.define CARG3, x2 +|.define CARG4, x3 +|.define CARG5, x4 +|.define CARG6, x5 +|.define CARG1w, w0 +|.define CARG2w, w1 +|.define CARG3w, w2 +|.define CARG4w, w3 +|.define CARG5w, w4 +|.define CARG6w, w5 +|.define RETVALx, x0 +|.define RETVALw, w0 +|.define FCARG1x, x0 +|.define FCARG1w, w0 +|.define FCARG2x, x1 +|.define FCARG2w, w1 +|.define SPAD, 0x20 // padding for CPU stack alignment +|.define NR_SPAD, 0x30 // padding for CPU stack alignment +|.define T3, [sp, #0x28] // Used to store old value of IP (CALL VM only) +|.define T2, [sp, #0x20] // Used to store old value of FP (CALL VM only) +|.define T1, [sp, #0x10] + +// We use REG0/1/2 and FPR0/1 to replace r0/1/2 and xmm0/1 in the x86 implementation. +// Scratch registers +|.define REG0, x8 +|.define REG0w, w8 +|.define REG1, x9 +|.define REG1w, w9 +|.define REG2, x10 +|.define REG2w, w10 +|.define FPR0, v0 +|.define FPR1, v1 +|.define FPR0d, d0 +|.define FPR1d, d1 + +|.define ZREG_REG0, ZREG_X8 +|.define ZREG_REG1, ZREG_X9 +|.define ZREG_REG2, ZREG_X10 +|.define ZREG_FPR0, ZREG_V0 +|.define ZREG_FPR1, ZREG_V1 + +// Temporaries, not preserved across calls +|.define TMP1, x15 +|.define TMP1w, w15 +|.define TMP2, x16 +|.define TMP2w, w16 +|.define TMP3, x17 // TODO: remember about hard-coded: mrs TMP3, tpidr_el0 +|.define TMP3w, w17 +|.define FPTMP, v16 +|.define FPTMPd, d16 + +|.define ZREG_TMP1, ZREG_X15 +|.define ZREG_TMP2, ZREG_X16 +|.define ZREG_TMP3, ZREG_X17 +|.define ZREG_FPTMP, ZREG_V16 + +|.define HYBRID_SPAD, #32 // padding for stack alignment + +#define TMP_ZVAL_OFFSET 16 +#define DASM_ALIGNMENT 16 + +/* Encoding of immediate. TODO: shift mode may be supported in the near future. */ +#define MAX_IMM12 0xfff // maximum value for imm12 +#define MAX_IMM16 0xffff // maximum value for imm16 +#define CMP_IMM MAX_IMM12 // cmp insn +#define MOVZ_IMM MAX_IMM16 // movz insn +#define ADD_SUB_IMM MAX_IMM12 // add/sub/adds/subs insn +#define LDR_STR_PIMM64 (MAX_IMM12*8) // ldr/str insn for 64-bit register: pimm is imm12 * 8 +#define LDR_STR_PIMM32 (MAX_IMM12*4) // ldr/str insn for 32-bit register: pimm is imm12 * 4 +#define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn + +#define B_IMM26 (1<<27) // signed imm26 * 4 + +static bool arm64_may_use_b(const void *addr) +{ + if (addr >= dasm_buf && addr < dasm_end) { + return (((char*)dasm_end - (char*)dasm_buf) < B_IMM26); + } else if (addr >= dasm_end) { + return (((char*)addr - (char*)dasm_buf) < B_IMM26); + } else if (addr < dasm_buf) { + return (((char*)dasm_end - (char*)addr) < B_IMM26); + } + return 0; +} + +#include "Zend/zend_cpuinfo.h" + +#ifdef HAVE_VALGRIND +# include +#endif + +/* The generated code may contain tautological comparisons, ignore them. */ +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wtautological-compare" +# pragma clang diagnostic ignored "-Wstring-compare" +#endif + +const char* zend_reg_name[] = { + "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", + "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", + "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", + "x24", "x25", "x26", "x27", "x28", "x29", "x30", "sp", + "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" +}; + +#ifdef HAVE_GCC_GLOBAL_REGS +# define GCC_GLOBAL_REGS 1 +#else +# define GCC_GLOBAL_REGS 0 +#endif + +# define ZREG_FCARG1x ZREG_X0 +# define ZREG_FCARG2x ZREG_X1 + +#if ZTS +static size_t tsrm_ls_cache_tcb_offset = 0; +#endif + +/* By default avoid JITing inline handlers if it does not seem profitable due to lack of + * type information. Disabling this option allows testing some JIT handlers in the + * presence of try/catch blocks, which prevent SSA construction. */ +#ifndef PROFITABILITY_CHECKS +# define PROFITABILITY_CHECKS 1 +#endif + +|.type EX, zend_execute_data, FP +|.type OP, zend_op +|.type ZVAL, zval + +|.actionlist dasm_actions + +|.globals zend_lb +static void* dasm_labels[zend_lb_MAX]; + +|.section code, cold_code, jmp_table + +#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) + +#define BP_JIT_IS 6 + +/* helper: determine whether an immediate value can be encoded as the immediate operand of logical instructions */ +static int logical_immediate_p (uint64_t value, uint32_t reg_size) +{ + /* fast path: power of two */ + if (value > 0 && !(value & (value - 1))) { + return 1; + } + + // TODO: slow path + return 0; +} + +/* Not Implemented Yet */ +|.macro NIY +|| //ZEND_ASSERT(0); +| brk #0 +|.endmacro + +|.macro NIY_STUB +|| //ZEND_ASSERT(0); +| brk #0 +|.endmacro + +|.macro ADD_HYBRID_SPAD +||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE +| add sp, sp, HYBRID_SPAD +||#endif +|.endmacro + +|.macro SUB_HYBRID_SPAD +||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE +| sub sp, sp, HYBRID_SPAD +||#endif +|.endmacro + +|.macro LOAD_ADDR, reg, addr +| // 48-bit virtual address +|| if (((uintptr_t)(addr)) == 0) { +| mov reg, xzr +|| } else if (((uintptr_t)(addr)) <= MOVZ_IMM) { +| movz reg, #((uint64_t)(addr)) +|| } else if ((uintptr_t)(addr) & 0xffff) { +| movz reg, #((uintptr_t)(addr) & 0xffff) +|| if (((uintptr_t)(addr) >> 16) & 0xffff) { +| movk reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 +|| } +|| if (((uintptr_t)(addr) >> 32) & 0xffff) { +| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 +|| } +|| } else if (((uintptr_t)(addr) >> 16) & 0xffff) { +| movz reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 +|| if (((uintptr_t)(addr) >> 32) & 0xffff) { +| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 +|| } +|| } else { +| movz reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 +|| } +|.endmacro + +// Type cast to unsigned is used to avoid undefined behavior. +|.macro LOAD_32BIT_VAL, reg, val +|| if (((uint32_t)(val)) <= MOVZ_IMM) { +| movz reg, #((uint32_t)(val)) +|| } else if (((uint32_t)(val) & 0xffff)) { +| movz reg, #((uint32_t)(val) & 0xffff) +|| if ((((uint32_t)(val) >> 16) & 0xffff)) { +| movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 +|| } +|| } else { +| movz reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 +|| } +|.endmacro + +|.macro LOAD_64BIT_VAL, reg, val +|| if (((uint64_t)(val)) == 0) { +| mov reg, xzr +|| } else if (((uint64_t)(val)) <= MOVZ_IMM) { +| movz reg, #((uint64_t)(val)) +|| } else if (~((uint64_t)(val)) <= MOVZ_IMM) { +| movn reg, #(~((uint64_t)(val))) +|| } else if ((uint64_t)(val) & 0xffff) { +| movz reg, #((uint64_t)(val) & 0xffff) +|| if (((uint64_t)(val) >> 16) & 0xffff) { +| movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 +|| } +|| if (((uint64_t)(val) >> 32) & 0xffff) { +| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 +|| } +|| if ((((uint64_t)(val) >> 48) & 0xffff)) { +| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 +|| } +|| } else if (((uint64_t)(val) >> 16) & 0xffff) { +| movz reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 +|| if (((uint64_t)(val) >> 32) & 0xffff) { +| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 +|| } +|| if ((((uint64_t)(val) >> 48) & 0xffff)) { +| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 +|| } +|| } else if (((uint64_t)(val) >> 32) & 0xffff) { +| movz reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 +|| if ((((uint64_t)(val) >> 48) & 0xffff)) { +| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 +|| } +|| } else { +| movz reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 +|| } +|.endmacro + +// Extract the low 8 bits from 'src_reg' into 'dst_reg'. 0xff can be encoded as imm for 'and' instruction. +|.macro GET_LOW_8BITS, dst_reg, src_reg +| and dst_reg, src_reg, #0xff +|.endmacro + +// Bitwise operation with constants. 'ins' can be and/orr/eor/ands. Operands are 32-bit. +|.macro BW_OP_32_WITH_CONST, ins, reg, op, val, tmp_reg +|| if (val == 0) { +| ins reg, op, wzr +|| } else if (logical_immediate_p((uint32_t)val, 32)) { +| ins reg, op, #val +|| } else { +| LOAD_32BIT_VAL tmp_reg, val +| ins reg, op, tmp_reg +|| } +|.endmacro + +// Bitwise operation with constants. 'ins' can be and/orr/eor/ands. Operands are 64-bit. +|.macro BW_OP_64_WITH_CONST, ins, reg, op, val, tmp_reg +|| if (val == 0) { +| ins reg, op, xzr +|| } else if (logical_immediate_p(val, 64)) { +| ins reg, op, #val +|| } else { +| LOAD_64BIT_VAL tmp_reg, val +| ins reg, op, tmp_reg +|| } +|.endmacro + +// Test operation 'tst' with constants. Operands are 32-bit. +|.macro TST_32_WITH_CONST, reg, val, tmp_reg +|| if (val == 0) { +| tst reg, wzr +|| } else if (logical_immediate_p((uint32_t)val, 32)) { +| tst reg, #val +|| } else { +| LOAD_32BIT_VAL tmp_reg, val +| tst reg, tmp_reg +|| } +|.endmacro + +// Test operation 'tst' with constants. Operands are 64-bit. +|.macro TST_64_WITH_CONST, reg, val, tmp_reg +|| if (val == 0) { +| tst reg, xzr +|| } else if (logical_immediate_p(val, 64)) { +| tst reg, #val +|| } else { +| LOAD_64BIT_VAL tmp_reg, val +| tst reg, tmp_reg +|| } +|.endmacro + +// Note: 1 is a valid immediate for logical instruction. +|.macro TST_64_WITH_ONE, reg +| tst reg, #1 +|.endmacro + +|.macro CMP_32_WITH_CONST, reg, val, tmp_reg +|| if (val == 0) { +| cmp reg, wzr +|| } else if (((int32_t)(val)) > 0 && ((int32_t)(val)) <= CMP_IMM) { +| cmp reg, #val +|| } else if (((int32_t)(val)) < 0 && ((int32_t)(val)) >= -CMP_IMM) { +| cmn reg, #-val +|| } else { +| LOAD_32BIT_VAL tmp_reg, val +| cmp reg, tmp_reg +|| } +|.endmacro + +|.macro CMP_64_WITH_CONST_32, reg, val, tmp_reg +|| if (val == 0) { +| cmp reg, xzr +|| } else if (((int32_t)(val)) > 0 && ((int32_t)(val)) <= CMP_IMM) { +| cmp reg, #val +|| } else if (((int32_t)(val)) < 0 && ((int32_t)(val)) >= -CMP_IMM) { +| cmn reg, #-val +|| } else { +| LOAD_32BIT_VAL tmp_reg, val +| cmp reg, tmp_reg +|| } +|.endmacro + +|.macro CMP_64_WITH_CONST, reg, val, tmp_reg +|| if (val == 0) { +| cmp reg, xzr +|| } else if (((int64_t)(val)) > 0 && ((int64_t)(val)) <= CMP_IMM) { +| cmp reg, #val +|| } else if (((int64_t)(val)) < 0 && ((int64_t)(val)) >= -CMP_IMM) { +| cmn reg, #-val +|| } else { +| LOAD_64BIT_VAL tmp_reg, val +| cmp reg, tmp_reg +|| } +|.endmacro + +|.macro ADD_SUB_32_WITH_CONST, ins, res_reg, op1_reg, val, tmp_reg +|| if (val == 0) { +| ins res_reg, op1_reg, wzr +|| } else if (((int32_t)(val)) > 0 && ((int32_t)(val)) <= ADD_SUB_IMM) { +| ins res_reg, op1_reg, #val +|| } else { +| LOAD_32BIT_VAL tmp_reg, val +| ins res_reg, op1_reg, tmp_reg +|| } +|.endmacro + +|.macro ADD_SUB_64_WITH_CONST_32, ins, res_reg, op1_reg, val, tmp_reg +|| if (val == 0) { +| ins res_reg, op1_reg, xzr +|| } else if (((int32_t)(val)) > 0 && ((int32_t)(val)) <= ADD_SUB_IMM) { +| ins res_reg, op1_reg, #val +|| } else { +| LOAD_32BIT_VAL tmp_reg, val +| ins res_reg, op1_reg, tmp_reg +|| } +|.endmacro + +|.macro ADD_SUB_64_WITH_CONST, ins, res_reg, op1_reg, val, tmp_reg +|| if (val == 0) { +| ins res_reg, op1_reg, xzr +|| } else if (((int64_t)(val)) > 0 && ((int64_t)(val)) <= ADD_SUB_IMM) { +| ins res_reg, op1_reg, #val +|| } else { +| LOAD_64BIT_VAL tmp_reg, val +| ins res_reg, op1_reg, tmp_reg +|| } +|.endmacro + +// Safe memory load/store with an unsigned 32-bit offset. +// 'op' can be used as 'tmp_reg' if 1) 'op' is one GPR, and 2) 'op' != 'base_reg', and 3) ins is 'ldr'. +|.macro SAFE_MEM_ACC_WITH_UOFFSET, ldr_str_ins, op, base_reg, offset, tmp_reg +|| if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { +| LOAD_32BIT_VAL tmp_reg, offset +| ldr_str_ins op, [base_reg, tmp_reg] +|| } else { +| ldr_str_ins op, [base_reg, #(offset)] +|| } +|.endmacro + +|.macro SAFE_MEM_ACC_WITH_UOFFSET_32, ldr_str_ins, op, base_reg, offset, tmp_reg +|| if (((uintptr_t)(offset)) > LDR_STR_PIMM32) { +| LOAD_32BIT_VAL tmp_reg, offset +| ldr_str_ins op, [base_reg, tmp_reg] +|| } else { +| ldr_str_ins op, [base_reg, #(offset)] +|| } +|.endmacro + +|.macro SAFE_MEM_ACC_WITH_UOFFSET_BYTE, ldrb_strb_ins, op, base_reg, offset, tmp_reg +|| if (((uintptr_t)(offset)) > LDRB_STRB_PIMM) { +| LOAD_32BIT_VAL tmp_reg, offset +| ldrb_strb_ins op, [base_reg, tmp_reg] +|| } else { +| ldrb_strb_ins op, [base_reg, #(offset)] +|| } +|.endmacro + +|.macro LOAD_TSRM_CACHE, reg +| .long 0xd53bd051 // TODO: hard-coded: mrs TMP3, tpidr_el0 +|| ZEND_ASSERT(tsrm_ls_cache_tcb_offset <= LDR_STR_PIMM64); +| ldr reg, [TMP3, #tsrm_ls_cache_tcb_offset] +|.endmacro + +|.macro LOAD_ADDR_ZTS, reg, struct, field +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| ADD_SUB_64_WITH_CONST_32 add, reg, TMP3, (struct.._offset + offsetof(zend_..struct, field)), reg +| .else +| LOAD_ADDR reg, &struct.field +| .endif +|.endmacro + +// Move the 48-bit address 'addr' into 'tmp_reg' and store it into the dest addr 'op1' +|.macro ADDR_STORE, op1, addr, tmp_reg +| LOAD_ADDR tmp_reg, addr +| str tmp_reg, op1 +|.endmacro + +// Move the 48-bit address 'addr' into 'tmp_reg1' and compare with the value inside address 'op1' +|.macro ADDR_CMP, op1, addr, tmp_reg1, tmp_reg2 +| LOAD_ADDR tmp_reg1, addr +| ldr tmp_reg2, op1 +| cmp tmp_reg2, tmp_reg1 +|.endmacro + +// Store the value from a register 'op' into memory 'addr' +|.macro MEM_STORE, str_ins, op, addr, tmp_reg +| LOAD_ADDR tmp_reg, addr +| str_ins op, [tmp_reg] +|.endmacro + +// 'op' is 64-bit register +|.macro MEM_STORE_ZTS, str_ins, op, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET str_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| .else +| MEM_STORE str_ins, op, &struct.field, tmp_reg +| .endif +|.endmacro + +// 'op' is 32-bit register +|.macro MEM_STORE_32_ZTS, str_ins, op, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET_32 str_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| .else +| MEM_STORE str_ins, op, &struct.field, tmp_reg +| .endif +|.endmacro + +|.macro MEM_STORE_BYTE_ZTS, strb_ins, op, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE strb_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| .else +| MEM_STORE strb_ins, op, &struct.field, tmp_reg +| .endif +|.endmacro + +// Load the value from memory 'addr' into a register 'op' +|.macro MEM_LOAD, ldr_ins, op, addr, tmp_reg +| LOAD_ADDR tmp_reg, addr +| ldr_ins op, [tmp_reg] +|.endmacro + +|.macro MEM_LOAD_ZTS, ldr_ins, op, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET ldr_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| .else +| MEM_LOAD ldr_ins, op, &struct.field, tmp_reg +| .endif +|.endmacro + +|.macro MEM_LOAD_32_ZTS, ldr_ins, op, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET_32 ldr_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| .else +| MEM_LOAD ldr_ins, op, &struct.field, tmp_reg +| .endif +|.endmacro + +|.macro MEM_LOAD_BYTE_ZTS, ldrb_ins, op, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| .else +| MEM_LOAD ldrb_ins, op, &struct.field, tmp_reg +| .endif +|.endmacro + +// Load the value from memory 'addr' into a tmp register 'tmp_reg1', +// and conduct arithmetic operations with 'op'. +// Operations can be add/sub/div/mul, and the computation result is stored into 'op'. +|.macro MEM_LOAD_OP, mem_ins, ldr_ins, op, addr, tmp_reg1, tmp_reg2 +| MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2 +| mem_ins op, op, tmp_reg1 +|.endmacro + +|.macro MEM_LOAD_OP_ZTS, mem_ins, ldr_ins, op, struct, field, tmp_reg1, tmp_reg2 +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET ldr_ins, tmp_reg2, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg1 +| mem_ins op, op, tmp_reg2 +| .else +| MEM_LOAD_OP mem_ins, ldr_ins, op, &struct.field, tmp_reg1, tmp_reg2 +| .endif +|.endmacro + +// Load the value from memory 'addr' into a tmp register 'tmp_reg1' and conduct arithmetic operations with 'op'. +// The computation result is stored back to memory 'addr'. 'op' can be either imm12 or register. +// For constant case, it should be guaranteed that 'op' can be represented by imm12 before using this macro. +|.macro MEM_LOAD_OP_STORE, mem_ins, ldr_ins, str_ins, op, addr, tmp_reg1, tmp_reg2 +| MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2 +| mem_ins tmp_reg1, tmp_reg1, op +| str_ins tmp_reg1, [tmp_reg2] +|.endmacro + +|.macro MEM_LOAD_OP_STORE_ZTS, mem_ins, ldr_ins, str_ins, op, struct, field, tmp_reg1, tmp_reg2 +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +|| if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > LDR_STR_PIMM64) { +| LOAD_32BIT_VAL tmp_reg1, (struct.._offset+offsetof(zend_..struct, field)) +| ldr_ins tmp_reg2, [TMP3, tmp_reg1] +| mem_ins tmp_reg2, tmp_reg2, op +| str_ins tmp_reg2, [TMP3, tmp_reg1] +|| } else { +| ldr_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] +| mem_ins tmp_reg2, tmp_reg2, op +| str_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] +|| } +| .else +| MEM_LOAD_OP_STORE mem_ins, ldr_ins, str_ins, op, &struct.field, tmp_reg1, tmp_reg2 +| .endif +|.endmacro + +|.macro LOAD_BASE_ADDR, reg, base, offset +|| if (offset) { +| ADD_SUB_64_WITH_CONST_32 add, reg, Rx(base), offset, reg +|| } else { +|| if (base == ZREG_RSP) { +| mov reg, sp +|| } else { +| mov reg, Rx(base) +|| } +|| } +|.endmacro + +|.macro EXT_CALL, func, tmp_reg +|| if (arm64_may_use_b(func)) { +| bl &func +|| } else { +| LOAD_ADDR tmp_reg, func +| blr tmp_reg +|| } +|.endmacro + +|.macro EXT_JMP, func, tmp_reg +|| if (arm64_may_use_b(func)) { +| b &func +|| } else { +| LOAD_ADDR tmp_reg, func +| br tmp_reg +|| } +|.endmacro + +|.macro SAVE_IP +|| if (GCC_GLOBAL_REGS) { +| str IP, EX->opline +|| } +|.endmacro + +|.macro LOAD_IP +|| if (GCC_GLOBAL_REGS) { +| ldr IP, EX->opline +|| } +|.endmacro + +|.macro LOAD_IP_ADDR, addr +|| if (GCC_GLOBAL_REGS) { +| LOAD_ADDR IP, addr +|| } else { +| ADDR_STORE EX->opline, addr, RX +|| } +|.endmacro + +|.macro LOAD_IP_ADDR_ZTS, struct, field, tmp_reg +| .if ZTS +|| if (GCC_GLOBAL_REGS) { +| LOAD_TSRM_CACHE IP +| SAFE_MEM_ACC_WITH_UOFFSET ldr, IP, IP, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +|| } else { +| LOAD_TSRM_CACHE RX +| ADD_SUB_64_WITH_CONST_32 add, RX, RX, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| str RX, EX->opline +|| } +| .else +| LOAD_IP_ADDR &struct.field +| .endif +|.endmacro + +|.macro GET_IP, reg +|| if (GCC_GLOBAL_REGS) { +| mov reg, IP +|| } else { +| ldr reg, EX->opline +|| } +|.endmacro + +// In x86 implementation, 'val' can be either a constant or a register. +// In AArch64, use ADD_IP for register case, +// and use ADD_IP_FROM_CST for constant case, where the value can be represented by ADD_SUB_IMM. +|.macro ADD_IP, val, tmp_reg +|| if (GCC_GLOBAL_REGS) { +| add IP, IP, val +|| } else { +| ldr tmp_reg, EX->opline +| add tmp_reg, tmp_reg, val +| str tmp_reg, EX->opline +|| } +|.endmacro + +|.macro ADD_IP_SHIFT, val, shift, tmp_reg +|| if (GCC_GLOBAL_REGS) { +| add IP, IP, val, shift +|| } else { +| ldr tmp_reg, EX->opline +| add tmp_reg, tmp_reg, val, shift +| str tmp_reg, EX->opline +|| } +|.endmacro + +|.macro ADD_IP_FROM_CST, val, tmp_reg +|| ZEND_ASSERT(val >=0 && val <= ADD_SUB_IMM); +|| if (GCC_GLOBAL_REGS) { +| add IP, IP, #val +|| } else { +| ldr tmp_reg, EX->opline +| add tmp_reg, tmp_reg, #val +| str tmp_reg, EX->opline +|| } +|.endmacro + +|.macro JMP_IP, tmp_reg +|| if (GCC_GLOBAL_REGS) { +| ldr tmp_reg, [IP] +| br tmp_reg +|| } else { +| ldr tmp_reg, EX:CARG1->opline +| br tmp_reg +|| } +|.endmacro + +|.macro CMP_IP, addr, tmp_reg1, tmp_reg2 +| LOAD_ADDR tmp_reg1, addr +|| if (GCC_GLOBAL_REGS) { +| cmp IP, tmp_reg1 +|| } else { +| ldr tmp_reg2, EX->opline +| cmp tmp_reg2, tmp_reg1 +|| } +|.endmacro + +|.macro LOAD_ZVAL_ADDR, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| LOAD_ADDR reg, Z_ZV(addr) +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| LOAD_BASE_ADDR reg, Z_REG(addr), Z_OFFSET(addr) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro GET_Z_TYPE_INFO, reg, zv +| ldr reg, [zv, #offsetof(zval,u1.type_info)] +|.endmacro + +|.macro SET_Z_TYPE_INFO, zv, type, tmp_reg +| LOAD_32BIT_VAL tmp_reg, type +| str tmp_reg, [zv, #offsetof(zval,u1.type_info)] +|.endmacro + +|.macro GET_ZVAL_TYPE, reg, addr, tmp_reg +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.v.type), tmp_reg +|.endmacro + +// 'reg' is 32-bit register +|.macro GET_ZVAL_TYPE_INFO, reg, addr, tmp_reg +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET_32 ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg +|.endmacro + +|.macro SET_ZVAL_TYPE_INFO, addr, type, tmp_reg1, tmp_reg2 +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| LOAD_32BIT_VAL tmp_reg1, type +| SAFE_MEM_ACC_WITH_UOFFSET_32 str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2 +|.endmacro + +// 'type' is 32-bit register +|.macro SET_ZVAL_TYPE_INFO_FROM_REG, addr, type, tmp_reg +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET_32 str, type, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg +|.endmacro + +|.macro GET_Z_PTR, reg, zv +| ldr reg, [zv] +|.endmacro + +|.macro SET_Z_PTR, zv, val +| mov aword [zv], val +|.endmacro + +|.macro GET_Z_W2, reg, zv +| mov reg, dword [zv+4] +|.endmacro + +|.macro SET_Z_W2, zv, reg +| mov dword [zv+4], reg +|.endmacro + +|.macro GET_ZVAL_PTR, reg, addr, tmp_reg +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg +|.endmacro + +|.macro SET_ZVAL_PTR, addr, val, tmp_reg +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET str, val, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg +|.endmacro + +|.macro UNDEF_OPLINE_RESULT, tmp_reg +| ldr REG0, EX->opline +| ldr REG0w, OP:REG0->result.var +| add REG0, FP, REG0 +| SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg +|.endmacro + +// Define DOUBLE_CMP to replace SSE_AVX_OP and SSE_OP for comparions in x86 implementation. +// Operand1 is from 'reg', and operand2 is from 'addr'. +|.macro DOUBLE_CMP, reg, addr, tmp_reg, fp_tmp_reg +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) +| ldr Rd(fp_tmp_reg-ZREG_V0), [Rx(tmp_reg)] +| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) +| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) +|| } else if (Z_MODE(addr) == IS_REG) { +| fcmp Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +// Define DOUBLE_GET_LONG to replace SSE_GET_LONG in x86 implementation. +// Convert the LONG value 'lval' into DOUBLE type, and move it into 'reg' +|.macro DOUBLE_GET_LONG, reg, lval, tmp_reg +|| if (lval == 0) { +| mov Rx(tmp_reg), xzr +| fmov Rd(reg-ZREG_V0), Rx(tmp_reg) +|| } else { +| LOAD_64BIT_VAL Rx(tmp_reg), lval +| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg) +|| } +|.endmacro + +// Define DOUBLE_GET_ZVAL_LVAL to replace SSE_GET_ZVAL_LVAL in x86 implementation. +// Convert the LONG value in 'addr' into DOUBLE type, and move it into 'reg' +|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg1, tmp_reg2 +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg1 +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rx(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg2) +| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg1) +|| } else if (Z_MODE(addr) == IS_REG) { +| scvtf Rd(reg-ZREG_V0), Rx(Z_REG(addr)) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +// Define DOUBLE_MATH_REG to replace AVX_MATH_REG in x86 implementation. +|.macro DOUBLE_MATH_REG, opcode, dst_reg, op1_reg, op2_reg +|| switch (opcode) { +|| case ZEND_ADD: +| fadd Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) +|| break; +|| case ZEND_SUB: +| fsub Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) +|| break; +|| case ZEND_MUL: +| fmul Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) +|| break; +|| case ZEND_DIV: +| fdiv Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) +|| break; +|| } +|.endmacro + +// Define LONG_ADD_SUB, LONG_BW_OP, LONG_MUL, LONG_CMP to replace the LONG_OP in x86 implementation. +// 'long_ins' should be addition or subtraction. +|.macro LONG_ADD_SUB, long_ins, reg, addr, tmp_reg1 +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| ADD_SUB_64_WITH_CONST long_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg1 +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg1 +| long_ins Rx(reg), Rx(reg), tmp_reg1 +|| } else if (Z_MODE(addr) == IS_REG) { +| long_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +// 'long_ins' should be 'and', 'orr' or 'eor' +|.macro LONG_BW_OP, long_ins, reg, addr, tmp_reg1 +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| BW_OP_64_WITH_CONST long_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg1 +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg1 +| long_ins Rx(reg), Rx(reg), tmp_reg1 +|| } else if (Z_MODE(addr) == IS_REG) { +| long_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro LONG_CMP, reg, addr, tmp_reg +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| CMP_64_WITH_CONST Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg +| cmp Rx(reg), tmp_reg +|| } else if (Z_MODE(addr) == IS_REG) { +| cmp Rx(reg), Rx(Z_REG(addr)) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +// long_ins should be addition or subtraction. +|.macro LONG_ADD_SUB_WITH_IMM, long_ins, op1_addr, lval, tmp_reg1, tmp_reg2 +|| ZEND_ASSERT(lval >=0 && lval <= ADD_SUB_IMM); +|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { +|| if (((uint32_t)(Z_OFFSET(op1_addr))) > LDR_STR_PIMM64) { +| LOAD_32BIT_VAL tmp_reg2, Z_OFFSET(op1_addr) +| ldr tmp_reg1, [Rx(Z_REG(op1_addr)), tmp_reg2] +| long_ins tmp_reg1, tmp_reg1, #lval +| str tmp_reg1, [Rx(Z_REG(op1_addr)), tmp_reg2] +|| } else { +| ldr tmp_reg1, [Rx(Z_REG(op1_addr)), #Z_OFFSET(op1_addr)] +| long_ins tmp_reg1, tmp_reg1, #lval +| str tmp_reg1, [Rx(Z_REG(op1_addr)), #Z_OFFSET(op1_addr)] +|| } +|| } else if (Z_MODE(op1_addr) == IS_REG) { +| long_ins Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)), #lval +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +// Define LONG_CMP_WITH_CONST to replace LONG_OP_WITH_CONST in the x86 implementation. +// Note that the 'long_ins' in all use sites of LONG_OP_WITH_CONST are always 'cmp'. +// Note that this macro is different from LONG_CMP. +|.macro LONG_CMP_WITH_CONST, op1_addr, lval, tmp_reg1, tmp_reg2 +|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(op1_addr)), Z_OFFSET(op1_addr), tmp_reg2 +| CMP_64_WITH_CONST tmp_reg1, lval, tmp_reg2 +|| } else if (Z_MODE(op1_addr) == IS_REG) { +| CMP_64_WITH_CONST Rx(Z_REG(op1_addr)), lval, tmp_reg1 +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro GET_ZVAL_LVAL, reg, addr, tmp_reg +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { +| mov Rx(reg), xzr +|| } else { +| LOAD_64BIT_VAL Rx(reg), Z_LVAL_P(Z_ZV(addr)) +|| } +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg +|| } else if (Z_MODE(addr) == IS_REG) { +|| if (reg != Z_REG(addr)) { +| mov Rx(reg), Rx(Z_REG(addr)) +|| } +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro LONG_MATH, opcode, reg, addr, tmp_reg1 +|| switch (opcode) { +|| case ZEND_ADD: +| LONG_ADD_SUB adds, reg, addr, tmp_reg1 +|| break; +|| case ZEND_SUB: +| LONG_ADD_SUB subs, reg, addr, tmp_reg1 +|| break; +|| case ZEND_BW_OR: +| LONG_BW_OP orr, reg, addr, tmp_reg1 +|| break; +|| case ZEND_BW_AND: +| LONG_BW_OP and, reg, addr, tmp_reg1 +|| break; +|| case ZEND_BW_XOR: +| LONG_BW_OP eor, reg, addr, tmp_reg1 +|| break; +|| default: +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro LONG_MATH_REG, opcode, dst_reg, src_reg1, src_reg2 +|| switch (opcode) { +|| case ZEND_ADD: +| adds dst_reg, src_reg1, src_reg2 +|| break; +|| case ZEND_SUB: +| subs dst_reg, src_reg1, src_reg2 +|| break; +|| case ZEND_BW_OR: +| orr dst_reg, src_reg1, src_reg2 +|| break; +|| case ZEND_BW_AND: +| and dst_reg, src_reg1, src_reg2 +|| break; +|| case ZEND_BW_XOR: +| eor dst_reg, src_reg1, src_reg2 +|| break; +|| default: +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +// In x86 implementation, argument 'lval' of SET_ZVAL_LVAL can be either a LONG constant +// or a register. Here, we separate it into two macros, SET_ZVAL_LVAL for the consant case, +// and SET_ZVAL_LVAL_FROM_REG for the register case. +|.macro SET_ZVAL_LVAL_FROM_REG, addr, reg, tmp_reg +|| if (Z_MODE(addr) == IS_REG) { +| mov Rx(Z_REG(addr)), reg +|| } else { +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg +|| } +|.endmacro + +|.macro SET_ZVAL_LVAL, addr, lval, tmp_reg1, tmp_reg2 +|| if (lval == 0) { +| SET_ZVAL_LVAL_FROM_REG addr, xzr, tmp_reg2 +|| } else { +| LOAD_64BIT_VAL tmp_reg1, lval +| SET_ZVAL_LVAL_FROM_REG addr, tmp_reg1, tmp_reg2 +|| } +|.endmacro + +// Define SET_ZVAL_DVAL to replace SSE_SET_ZVAL_DVAL in x86 implementation. +|.macro SET_ZVAL_DVAL, addr, reg, tmp_reg +|| if (Z_MODE(addr) == IS_REG) { +|| if (reg != Z_REG(addr)) { +| fmov Rd(Z_REG(addr)-ZREG_V0), Rd(reg-ZREG_V0) +|| } +|| } else { +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET str, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) +|| } +|.endmacro + +// Define GET_ZVAL_DVAL to replace SSE_GET_ZVAL_DVAL in x86 implementation. +|.macro GET_ZVAL_DVAL, reg, addr, tmp_reg +|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) +| ldr Rd(reg-ZREG_V0), [Rx(tmp_reg)] +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) +|| } else if (Z_MODE(addr) == IS_REG) { +| fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|| } +|.endmacro + +|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg +|| if (Z_TYPE_P(zv) > IS_TRUE) { +|| if (Z_TYPE_P(zv) == IS_DOUBLE) { +|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; +| LOAD_ADDR Rx(tmp_reg1), zv +| ldr Rd(dst_reg-ZREG_V0), [Rx(tmp_reg1)] +| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 +|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { +|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; +| DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), tmp_reg1 +| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 +|| } else { +| // In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr. +| // Note that imm32 is signed extended to 64 bits during mov. +| // In aarch64, we choose to handle both cases in the same way. Even though 4 mov's are used for 64-bit value and 2 mov's are +| // needed for 32-bit value, an extra ext insn is needed for 32-bit vlaue. +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) +|| } +|| } +|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { +|| if (dst_def_info == MAY_BE_DOUBLE) { +|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { +| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) +|| } +|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1< IS_TRUE) { +|| if (Z_TYPE_P(zv) == IS_DOUBLE) { +|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? +|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_MODE(res_addr) : fp_tmp_reg); +| LOAD_ADDR Rx(tmp_reg1), zv +| ldr Rd(dst_reg-ZREG_V0), [Rx(tmp_reg1)] +| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 +| SET_ZVAL_DVAL res_addr, dst_reg, tmp_reg2 +|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { +|| if (Z_MODE(dst_addr) == IS_REG) { +| DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), tmp_reg1 +| SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg2 +|| } else if (Z_MODE(res_addr) == IS_REG) { +| DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), tmp_reg1 +| SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg2 +|| } else { +| DOUBLE_GET_LONG fp_tmp_reg, Z_LVAL_P(zv), tmp_reg1 +| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg2 +| SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg2 +|| } +|| } else { +|| if (Z_MODE(dst_addr) == IS_REG) { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) +| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg1) +|| } else if (Z_MODE(res_addr) == IS_REG) { +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) +| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg1) +|| } else { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) +|| } +|| } +|| } +|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { +|| if (dst_def_info == MAY_BE_DOUBLE) { +|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { +| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) +|| } +|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<1, tmp_reg +|| } +| GC_ADDREF value_ptr_reg, tmp_reg +|1: +|| } +|.endmacro + +|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg, tmp_reg +|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { +|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg +|| } +| ldr tmp_reg, [value_ptr_reg] +| add tmp_reg, tmp_reg, #2 +| str tmp_reg, [value_ptr_reg] +|1: +|| } +|.endmacro + +|.macro ZVAL_DEREF, reg, info, tmp_reg +|| if (info & MAY_BE_REF) { +| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1, tmp_reg +| GET_Z_PTR reg, reg +| add reg, reg, #offsetof(zend_reference, val) +|1: +|| } +|.endmacro + +|.macro SET_EX_OPLINE, op, tmp_reg +|| if (op == last_valid_opline) { +|| zend_jit_use_last_valid_opline(); +| SAVE_IP +|| } else { +| ADDR_STORE EX->opline, op, tmp_reg +|| if (!GCC_GLOBAL_REGS) { +|| zend_jit_reset_last_valid_opline(); +|| } +|| } +|.endmacro + +// arg1 "zval" should be in FCARG1x +|.macro ZVAL_DTOR_FUNC, var_info, opline, tmp_reg +|| do { +|| if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_INDIRECT))) { +|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); +|| if (type == IS_STRING && !ZEND_DEBUG) { +| EXT_CALL _efree, tmp_reg +|| break; +|| } else if (type == IS_ARRAY) { +|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { +|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { +| SET_EX_OPLINE opline, tmp_reg +|| } +| EXT_CALL zend_array_destroy, tmp_reg +|| } else { +| EXT_CALL zend_jit_array_free, tmp_reg +|| } +|| break; +|| } else if (type == IS_OBJECT) { +|| if (opline) { +| SET_EX_OPLINE opline, REG0 +|| } +| EXT_CALL zend_objects_store_del, tmp_reg +|| break; +|| } +|| } +|| if (opline) { +| SET_EX_OPLINE opline, tmp_reg +|| } +| EXT_CALL rc_dtor_func, tmp_reg +|| } while(0); +|.endmacro + +|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline, tmp_reg1, tmp_reg2 +|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { +|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| // if (Z_REFCOUNTED_P(cv)) { +|| if (cold) { +| IF_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 +|.cold_code +|1: +|| } else { +| IF_NOT_ZVAL_REFCOUNTED addr, >4, tmp_reg1, tmp_reg2 +|| } +|| } +| // if (!Z_DELREF_P(cv)) { +| GET_ZVAL_PTR FCARG1x, addr, Rx(tmp_reg2) +| GC_DELREF FCARG1x, Rw(tmp_reg1) +|| if (RC_MAY_BE_1(op_info)) { +|| if (RC_MAY_BE_N(op_info)) { +|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { +| bne >3 +|| } else { +| bne >4 +|| } +|| } +| // zval_dtor_func(r); +| ZVAL_DTOR_FUNC op_info, opline, Rx(tmp_reg1) +|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { +| b >4 +|| } +|3: +|| } +|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { +|| if ((op_info) & MAY_BE_REF) { +|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, offsetof(zend_reference, val)); +| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1, tmp_reg1 +| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4, tmp_reg1, tmp_reg2 +| GET_ZVAL_PTR FCARG1x, ref_addr, Rx(tmp_reg2) +|1: +|| } +| IF_GC_MAY_NOT_LEAK FCARG1x, >4, Rw(tmp_reg1), Rw(tmp_reg2) +| // gc_possible_root(Z_COUNTED_P(z)) +| EXT_CALL gc_possible_root, Rx(tmp_reg1) +|| } +|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { +| b >4 +|.code +|| } +|4: +|| } +|.endmacro + +|.macro FREE_OP, op_type, op, op_info, cold, opline, tmp_reg1, tmp_reg2 +|| if (op_type & (IS_VAR|IS_TMP_VAR)) { +|| zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); +| ZVAL_PTR_DTOR addr, op_info, 0, cold, opline, tmp_reg1, tmp_reg2 +|| } +|.endmacro + +|.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2 +|| if (RC_MAY_BE_N(op_info)) { +|| if (Z_REG(addr) != ZREG_FP) { +| GET_ZVAL_LVAL ZREG_REG0, addr, Rx(tmp_reg1) +|| if (RC_MAY_BE_1(op_info)) { +| // if (GC_REFCOUNT() > 1) +| ldr Rw(tmp_reg1), [REG0] +| cmp Rw(tmp_reg1), #1 +| bls >2 +|| } +|| if (Z_REG(addr) != ZREG_FCARG1x || Z_OFFSET(addr) != 0) { +| LOAD_ZVAL_ADDR FCARG1x, addr +|| } +| EXT_CALL zend_jit_zval_array_dup, REG0 +| mov REG0, RETVALx +|2: +| mov FCARG1x, REG0 +|| } else { +| GET_ZVAL_LVAL ZREG_FCARG1x, addr, Rx(tmp_reg1) +|| if (RC_MAY_BE_1(op_info)) { +| // if (GC_REFCOUNT() > 1) +| ldr Rw(tmp_reg1), [FCARG1x] +| cmp Rw(tmp_reg1), #1 +|| if (cold) { +| bhi >1 +|.cold_code +|1: +|| } else { +| bls >2 +|| } +|| } +| IF_NOT_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 +| GC_DELREF FCARG1x, Rw(tmp_reg1) +|1: +| EXT_CALL zend_array_dup, REG0 +| mov REG0, RETVALx +| SET_ZVAL_PTR addr, REG0, Rx(tmp_reg1) +| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX, Rw(tmp_reg1), Rx(tmp_reg2) +| mov FCARG1x, REG0 +|| if (RC_MAY_BE_1(op_info)) { +|| if (cold) { +| b >2 +|.code +|| } +|| } +|2: +|| } +|| } else { +| GET_ZVAL_LVAL ZREG_FCARG1x, addr, Rx(tmp_reg1) +|| } +|.endmacro + +/* argument is passed in FCARG1x */ +|.macro EFREE_REFERENCE +||#if ZEND_DEBUG +| mov FCARG2x, xzr // filename +| mov CARG3w, wzr // lineno +| mov CARG4, xzr +| mov CARG5, xzr +| EXT_CALL _efree, REG0 +||#else +||#ifdef HAVE_BUILTIN_CONSTANT_P +| EXT_CALL _efree_32, REG0 +||#else +| EXT_CALL _efree, REG0 +||#endif +||#endif +|.endmacro + +|.macro EMALLOC, size, op_array, opline +||#if ZEND_DEBUG +|| const char *filename = op_array->filename ? op_array->filename->val : NULL; +| mov FCARG1x, #size +| LOAD_ADDR FCARG2x, filename +| LOAD_32BIT_VAL CARG3w, opline->lineno +| mov CARG4, xzr +| mov CARG5, xzr +| EXT_CALL _emalloc, REG0 +| mov REG0, RETVALx +||#else +||#ifdef HAVE_BUILTIN_CONSTANT_P +|| if (size > 24 && size <= 32) { +| EXT_CALL _emalloc_32, REG0 +| mov REG0, RETVALx +|| } else { +| mov FCARG1x, #size +| EXT_CALL _emalloc, REG0 +| mov REG0, RETVALx +|| } +||#else +| mov FCARG1x, #size +| EXT_CALL _emalloc, REG0 +| mov REG0, RETVALx +||#endif +||#endif +|.endmacro + +|.macro OBJ_RELEASE, reg, exit_label, tmp_reg1, tmp_reg2 +| GC_DELREF Rx(reg), Rw(tmp_reg1) +| bne >1 +| // zend_objects_store_del(obj); +|| if (reg != ZREG_FCARG1x) { +| mov FCARG1x, Rx(reg) +|| } +| EXT_CALL zend_objects_store_del, Rx(tmp_reg1) +| b exit_label +|1: +| IF_GC_MAY_NOT_LEAK Rx(reg), >1, Rw(tmp_reg1), Rw(tmp_reg2) +| // gc_possible_root(obj) +|| if (reg != ZREG_FCARG1x) { +| mov FCARG1x, Rx(reg) +|| } +| EXT_CALL gc_possible_root, Rx(tmp_reg1) +|1: +|.endmacro + +|.macro UNDEFINED_OFFSET, opline +|| if (opline == last_valid_opline) { +|| zend_jit_use_last_valid_opline(); +| bl ->undefined_offset_ex +|| } else { +| SET_EX_OPLINE opline, REG0 +| bl ->undefined_offset +|| } +|.endmacro + +|.macro UNDEFINED_INDEX, opline +|| if (opline == last_valid_opline) { +|| zend_jit_use_last_valid_opline(); +| bl ->undefined_index_ex +|| } else { +| SET_EX_OPLINE opline, REG0 +| bl ->undefined_index +|| } +|.endmacro + +|.macro CANNOT_ADD_ELEMENT, opline +|| if (opline == last_valid_opline) { +|| zend_jit_use_last_valid_opline(); +| bl ->cannot_add_element_ex +|| } else { +| SET_EX_OPLINE opline, REG0 +| bl ->cannot_add_element +|| } +|.endmacro + +static bool reuse_ip = 0; +static bool delayed_call_chain = 0; +static uint32_t delayed_call_level = 0; +static const zend_op *last_valid_opline = NULL; +static bool use_last_vald_opline = 0; +static bool track_last_valid_opline = 0; +static int jit_return_label = -1; +static uint32_t current_trace_num = 0; +static uint32_t allowed_opt_flags = 0; + +static void zend_jit_track_last_valid_opline(void) +{ + use_last_vald_opline = 0; + track_last_valid_opline = 1; +} + +static void zend_jit_use_last_valid_opline(void) +{ + if (track_last_valid_opline) { + use_last_vald_opline = 1; + track_last_valid_opline = 0; + } +} + +static bool zend_jit_trace_uses_initial_ip(void) +{ + return use_last_vald_opline; +} + +static void zend_jit_set_last_valid_opline(const zend_op *target_opline) +{ + if (!reuse_ip) { + track_last_valid_opline = 0; + last_valid_opline = target_opline; + } +} + +static void zend_jit_reset_last_valid_opline(void) +{ + track_last_valid_opline = 0; + last_valid_opline = NULL; +} + +static void zend_jit_start_reuse_ip(void) +{ + zend_jit_reset_last_valid_opline(); + reuse_ip = 1; +} + +static int zend_jit_reuse_ip(dasm_State **Dst) +{ + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | ldr RX, EX->call + } + return 1; +} + +static void zend_jit_stop_reuse_ip(void) +{ + reuse_ip = 0; +} + +static int zend_jit_interrupt_handler_stub(dasm_State **Dst) +{ + |->interrupt_handler: + | SAVE_IP + | //EG(vm_interrupt) = 0; + | MEM_STORE_BYTE_ZTS strb, wzr, executor_globals, vm_interrupt, TMP1 + | //if (EG(timed_out)) { + | MEM_LOAD_BYTE_ZTS ldrb, REG0w, executor_globals, timed_out, TMP1 + | cbz REG0w, >1 + | //zend_timeout(); + | EXT_CALL zend_timeout, TMP1 + |1: + | //} else if (zend_interrupt_function) { + if (zend_interrupt_function) { + | //zend_interrupt_function(execute_data); + | mov CARG1, FP + | EXT_CALL zend_interrupt_function, TMP1 + | //ZEND_VM_ENTER(); + | //execute_data = EG(current_execute_data); + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 + | LOAD_IP + } + | //ZEND_VM_CONTINUE() + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + | JMP_IP TMP1 + } else if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], # SPAD // stack alignment + | JMP_IP TMP1 + } else { + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | mov RETVALx, #1 // ZEND_VM_ENTER + | ret + } + + return 1; +} + +static int zend_jit_exception_handler_stub(dasm_State **Dst) +{ + |->exception_handler: + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + const void *handler = zend_get_opcode_handler_func(EG(exception_op)); + + | ADD_HYBRID_SPAD + | EXT_CALL handler, REG0 + | JMP_IP TMP1 + } else { + const void *handler = EG(exception_op)->handler; + + if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], # SPAD // stack alignment + | EXT_JMP handler, REG0 + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | mov FCARG1x, FP + | EXT_CALL handler, REG0 + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | tst RETVALw, RETVALw + | blt >1 + | mov RETVALw, #1 // ZEND_VM_ENTER + |1: + | ret + } else { + | mov FCARG1x, FP + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | EXT_JMP handler, REG0 + } + } + + return 1; +} + +static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) +{ + |->exception_handler_undef: + | MEM_LOAD_ZTS ldr, REG0, executor_globals, opline_before_exception, REG0 + | ldrb TMP1w, OP:REG0->result_type + | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w + | bne >1 + | ldr REG0w, OP:REG0->result.var + | add REG0, REG0, FP + | SET_Z_TYPE_INFO REG0, IS_UNDEF, TMP1w + |1: + | b ->exception_handler + + return 1; +} + +static int zend_jit_leave_function_stub(dasm_State **Dst) +{ + |->leave_function_handler: + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w + | bne >1 + | EXT_CALL zend_jit_leave_nested_func_helper, REG0 + | ADD_HYBRID_SPAD + | JMP_IP TMP1 + |1: + | EXT_CALL zend_jit_leave_top_func_helper, REG0 + | ADD_HYBRID_SPAD + | JMP_IP TMP1 + } else { + if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], # SPAD // stack alignment + } else { + | mov FCARG2x, FP + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + } + | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w + | bne >1 + | EXT_JMP zend_jit_leave_nested_func_helper, REG0 + |1: + | EXT_JMP zend_jit_leave_top_func_helper, REG0 + } + + return 1; +} + +static int zend_jit_leave_throw_stub(dasm_State **Dst) +{ + |->leave_throw_handler: + | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { + if (GCC_GLOBAL_REGS) { + | ldrb TMP1w, OP:IP->opcode + | cmp TMP1w, #ZEND_HANDLE_EXCEPTION + | beq >5 + | // EG(opline_before_exception) = opline; + | MEM_STORE_ZTS str, IP, executor_globals, opline_before_exception, TMP2 + |5: + | // opline = EG(exception_op); + | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 + | // HANDLE_EXCEPTION() + | b ->exception_handler + } else { + | GET_IP TMP1 + | ldrb TMP2w, OP:TMP1->opcode + | cmp TMP2w, #ZEND_HANDLE_EXCEPTION + | beq >5 + | // EG(opline_before_exception) = opline; + | MEM_STORE_ZTS str, TMP1, executor_globals, opline_before_exception, TMP2 + |5: + | // opline = EG(exception_op); + | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | mov RETVALx, #2 // ZEND_VM_LEAVE + | ret + } + + return 1; +} + +static int zend_jit_icall_throw_stub(dasm_State **Dst) +{ + |->icall_throw_handler: + | // zend_rethrow_exception(zend_execute_data *execute_data) + | ldr IP, EX->opline + | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { + | ldrb TMP1w, OP:IP->opcode + | cmp TMP1w,# ZEND_HANDLE_EXCEPTION + | beq >1 + | // EG(opline_before_exception) = opline; + | MEM_STORE_ZTS str, IP, executor_globals, opline_before_exception, TMP2 + |1: + | // opline = EG(exception_op); + | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 + || if (GCC_GLOBAL_REGS) { + | str IP, EX->opline + || } + | // HANDLE_EXCEPTION() + | b ->exception_handler + + return 1; +} + +static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) +{ + zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + |->throw_cannot_pass_by_ref: + | ldr REG0, EX->opline + | ldr REG1w, OP:REG0->result.var + | add REG1, REG1, RX + | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w + | // last EX(call) frame may be delayed + | ldr TMP1, EX->call + | cmp RX, TMP1 + | beq >1 + | ldr REG1, EX->call + | str REG1, EX:RX->prev_execute_data + | str RX, EX->call + |1: + | mov RX, REG0 + | ldr FCARG1w, OP:REG0->op2.num + | EXT_CALL zend_cannot_pass_by_reference, REG0 + | ldrb TMP1w, OP:RX->op1_type + | cmp TMP1w, #IS_TMP_VAR + | bne >9 + | ldr REG0w, OP:RX->op1.var + | add REG0, REG0, FP + | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 + |9: + | b ->exception_handler + + return 1; +} + +static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) +{ + |->undefined_offset_ex: + | SAVE_IP + | b ->undefined_offset + + return 1; +} + +static int zend_jit_undefined_offset_stub(dasm_State **Dst) +{ + |->undefined_offset: + | //sub r4, 8 + | ldr REG0, EX->opline + | ldr REG1w, OP:REG0->result.var + | add REG1, REG1, FP + | SET_Z_TYPE_INFO REG1, IS_NULL, TMP1w + | ldrb REG1w, OP:REG0->op2_type + | cmp REG1w, #IS_CONST + | bne >2 + | ldrsw REG1, OP:REG0->op2.constant + | add REG0, REG0, REG1 + | b >3 + |2: + | ldr REG0w, OP:REG0->op2.var + | add REG0, REG0, FP + |3: + | mov CARG1, #E_WARNING + | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT + | ldr CARG3, [REG0] + | EXT_JMP zend_error, REG0 // tail call + | //add r4, 8 // stack alignment + | //ret + + return 1; +} + +static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) +{ + |->undefined_index_ex: + | SAVE_IP + | b ->undefined_index + + return 1; +} + +static int zend_jit_undefined_index_stub(dasm_State **Dst) +{ + |->undefined_index: + | //sub r4, 8 + | ldr REG0, EX->opline + | ldr REG1w, OP:REG0->result.var + | add REG1, REG1, FP + | SET_Z_TYPE_INFO REG1, IS_NULL, TMP1w + | ldrb REG1w, OP:REG0->op2_type + | cmp REG1w, #IS_CONST + | bne >2 + | ldrsw REG1, OP:REG0->op2.constant + | add REG0, REG0, REG1 + | b >3 + |2: + | ldr REG0w, OP:REG0->op2.var + | add REG0, REG0, FP + |3: + | mov CARG1, #E_WARNING + | LOAD_ADDR CARG2, "Undefined array key \"%s\"" + | ldr CARG3, [REG0] + | add CARG3, CARG3, #offsetof(zend_string, val) + | EXT_JMP zend_error,REG0 // tail call + | //add r4, 8 + | //ret + + return 1; +} + +static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) +{ + |->cannot_add_element_ex: + | SAVE_IP + | b ->cannot_add_element + + return 1; +} + +static int zend_jit_cannot_add_element_stub(dasm_State **Dst) +{ + |->cannot_add_element: + | // sub r4, 8 + | ldr REG0, EX->opline + | ldrb TMP1w, OP:REG0->result_type + | cmp TMP1w, #IS_UNUSED + | beq >1 + | ldr REG0w, OP:REG0->result.var + | add REG0, REG0, FP + | SET_Z_TYPE_INFO REG0, IS_NULL, TMP1w + |1: + | mov CARG1, xzr + | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" + | EXT_JMP zend_throw_error, REG0 // tail call + | // add r4, 8 + | //ret + + return 1; +} + +static int zend_jit_undefined_function_stub(dasm_State **Dst) +{ + |->undefined_function: + | ldr REG0, EX->opline + | mov CARG1, xzr + | LOAD_ADDR CARG2, "Call to undefined function %s()" + | ldrsw CARG3, [REG0, #offsetof(zend_op, op2.constant)] + | ldr CARG3, [REG0, CARG3] + | add CARG3, CARG3, #offsetof(zend_string, val) + | EXT_CALL zend_throw_error, REG0 + | b ->exception_handler + return 1; +} + +static int zend_jit_negative_shift_stub(dasm_State **Dst) +{ + |->negative_shift: + | UNDEF_OPLINE_RESULT TMP1w + | LOAD_ADDR CARG1, zend_ce_arithmetic_error + | LOAD_ADDR CARG2, "Bit shift by negative number" + | EXT_CALL zend_throw_error, REG0 + | b ->exception_handler + return 1; +} + +static int zend_jit_mod_by_zero_stub(dasm_State **Dst) +{ + |->mod_by_zero: + | UNDEF_OPLINE_RESULT TMP1w + | LOAD_ADDR CARG1, zend_ce_division_by_zero_error + | LOAD_ADDR CARG2, "Modulo by zero" + | EXT_CALL zend_throw_error, REG0 + | b ->exception_handler + return 1; +} + +static int zend_jit_invalid_this_stub(dasm_State **Dst) +{ + |->invalid_this: + | UNDEF_OPLINE_RESULT TMP1w + | mov CARG1, xzr + | LOAD_ADDR CARG2, "Using $this when not in object context" + | EXT_CALL zend_throw_error, REG0 + | b ->exception_handler + + return 1; +} + +static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + + |->hybrid_runtime_jit: + | EXT_CALL zend_runtime_jit, REG0 + | JMP_IP TMP1 + return 1; +} + +static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + + |->hybrid_profile_jit: + | // ++zend_jit_profile_counter; + | LOAD_ADDR REG0, &zend_jit_profile_counter + | ldr TMP1, [REG0] + | add TMP1, TMP1, #1 + | str TMP1, [REG0] + | // op_array = (zend_op_array*)EX(func); + | ldr REG0, EX->func + | // run_time_cache = EX(run_time_cache); + | ldr REG2, EX->run_time_cache + | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | // ++ZEND_COUNTER_INFO(op_array) + || if ((zend_jit_profile_counter_rid * sizeof(void*)) > LDR_STR_PIMM64) { + | LOAD_32BIT_VAL TMP1, (zend_jit_profile_counter_rid * sizeof(void*)) + | ldr TMP2, [REG2, TMP1] + | add TMP2, TMP2, #1 + | str TMP2, [REG2, TMP1] + || } else { + | ldr TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] + | add TMP2, TMP2, #1 + | str TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] + || } + | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() + | ldr TMP1, [REG0, #offsetof(zend_jit_op_array_extension, orig_handler)] + | br TMP1 + return 1; +} + +static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + + |->hybrid_hot_code: + || ZEND_ASSERT(ZEND_JIT_COUNTER_INIT <= MOVZ_IMM); + | movz TMP1w, #ZEND_JIT_COUNTER_INIT + | strh TMP1w, [REG2] + | mov FCARG1x, FP + | GET_IP FCARG2x + | EXT_CALL zend_jit_hot_func, REG0 + | JMP_IP TMP1 + return 1; +} + +/* + * This code is based Mike Pall's "Hashed profile counters" idea, implemented + * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual + * property disclosure and research opportunities" email + * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html + * + * In addition we use a variation of Knuth's multiplicative hash function + * described at https://code.i-harness.com/en/q/a21ce + * + * uint64_t hash(uint64_t x) { + * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; + * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; + * x = x ^ (x >> 31); + * return x; + * } + * + * uint_32_t hash(uint32_t x) { + * x = ((x >> 16) ^ x) * 0x45d9f3b; + * x = ((x >> 16) ^ x) * 0x45d9f3b; + * x = (x >> 16) ^ x; + * return x; + * } + * + */ +static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) +{ + | ldr REG0, EX->func + | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG2, [REG1, #offsetof(zend_jit_op_array_hot_extension, counter)] + | ldrh TMP2w, [REG2] + | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w + | strh TMP2w, [REG2] + | ble ->hybrid_hot_code + | GET_IP REG2 + | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] + | sub REG2, REG2, TMP1 + | // divide by sizeof(zend_op) + || ZEND_ASSERT(sizeof(zend_op) == 32); + | asr REG2, REG2, #2 + | add TMP1, REG1, REG2 + | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_hot_extension, orig_handlers)] + | br TMP1 + return 1; +} + +static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { + return 1; + } + + |->hybrid_func_hot_counter: + + return zend_jit_hybrid_hot_counter_stub(Dst, + ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); +} + +static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { + return 1; + } + + |->hybrid_loop_hot_counter: + + return zend_jit_hybrid_hot_counter_stub(Dst, + ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); +} + +static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + + // On entry from counter stub: + // REG2 -> zend_op_trace_info.counter + + |->hybrid_hot_trace: + | mov TMP1w, #ZEND_JIT_COUNTER_INIT + | strh TMP1w, [REG2] + | mov FCARG1x, FP + | GET_IP FCARG2x + | EXT_CALL zend_jit_trace_hot_root, REG0 + | cmp RETVALw, wzr // Result is < 0 on failure. + | blt >1 + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, REG0 + | LOAD_IP + | JMP_IP TMP1 + |1: + | EXT_JMP zend_jit_halt_op->handler, REG0 + + return 1; +} + +static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) +{ + | ldr REG0, EX->func + | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG1, [REG1, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add TMP1, REG1, IP + | ldr REG2, [TMP1, #offsetof(zend_op_trace_info, counter)] + | ldrh TMP2w, [REG2] + | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w + | strh TMP2w, [REG2] + | ble ->hybrid_hot_trace + // Note: "REG1 + IP" is re-calculated as TMP1 is used as temporary register by the prior + // ADD_SUB_32_WITH_CONST. Will optimize in the future if more temporary registers are available. + | add TMP1, REG1, IP + | ldr TMP2, [TMP1, #offsetof(zend_op_trace_info, orig_handler)] + | br TMP2 + + return 1; +} + +static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { + return 1; + } + + |->hybrid_func_trace_counter: + + return zend_jit_hybrid_trace_counter_stub(Dst, + ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); +} + +static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { + return 1; + } + + |->hybrid_ret_trace_counter: + + return zend_jit_hybrid_trace_counter_stub(Dst, + ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); +} + +static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { + return 1; + } + + |->hybrid_loop_trace_counter: + + return zend_jit_hybrid_trace_counter_stub(Dst, + ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); +} + +static int zend_jit_trace_halt_stub(dasm_State **Dst) +{ + |->trace_halt: + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + | EXT_JMP zend_jit_halt_op->handler, REG0 + } else if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], # SPAD // stack alignment + | ret // PC must be zero + } else { + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | movn RETVALx, #0 // ZEND_VM_RETURN (-1) + | ret + } + return 1; +} + +static int zend_jit_trace_exit_stub(dasm_State **Dst) +{ + |->trace_exit: + | + | // Save CPU registers(32 GP regs + 32 FP regs) on stack in the order of d31 to x0 + | + | stp d30, d31, [sp, #-16]! + | stp d28, d29, [sp, #-16]! + | stp d26, d27, [sp, #-16]! + | stp d24, d25, [sp, #-16]! + | stp d22, d23, [sp, #-16]! + | stp d20, d21, [sp, #-16]! + | stp d18, d19, [sp, #-16]! + | stp d16, d17, [sp, #-16]! + | //stp d14, d15, [sp, #-16]! // we don't use preserved registers yet + | //stp d12, d13, [sp, #-16]! + | //stp d10, d11, [sp, #-16]! + | //stp d8, d9, [sp, #-16]! + | stp d6, d7, [sp, #(-16*5)]! + | stp d4, d5, [sp, #-16]! + | stp d2, d3, [sp, #-16]! + | stp d0, d1, [sp, #-16]! + | + | //str x30, [sp, #-16]! // we don't use callee-saved registers yet (x31 can be omitted) + | stp x28, x29, [sp, #(-16*2)]! // we have to store RX (x28) + | //stp x26, x27, [sp, #-16]! // we don't use callee-saved registers yet + | //stp x24, x25, [sp, #-16]! + | //stp x22, x23, [sp, #-16]! + | //stp x20, x21, [sp, #-16]! + | //stp x18, x19, [sp, #-16]! + | //stp x16, x17, [sp, #-16]! // we don't need temporary registers + | stp x14, x15, [sp, #-(16*7)]! + | stp x12, x13, [sp, #-16]! + | stp x10, x11, [sp, #-16]! + | stp x8, x9, [sp, #-16]! + | stp x6, x7, [sp, #-16]! + | stp x4, x5, [sp, #-16]! + | stp x2, x3, [sp, #-16]! + | stp x0, x1, [sp, #-16]! + | + | mov FCARG1w, TMP1w // exit_num + | mov FCARG2x, sp + | + | // EX(opline) = opline + | SAVE_IP + | // zend_jit_trace_exit(trace_num, exit_num) + | EXT_CALL zend_jit_trace_exit, REG0 + | + | add sp, sp, #(32 * 16) // restore sp + | + + | tst RETVALw, RETVALw + | bne >1 // not zero + + | // execute_data = EG(current_execute_data) + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, REG0 + | // opline = EX(opline) + | LOAD_IP + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + | JMP_IP TMP1 + } else if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], # SPAD // stack alignment + | JMP_IP TMP1 + } else { + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | mov RETVALx, #1 // ZEND_VM_ENTER + | ret + } + + |1: + | blt ->trace_halt + + | // execute_data = EG(current_execute_data) + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, REG0 + | // opline = EX(opline) + | LOAD_IP + + | // check for interrupt (try to avoid this ???) + | MEM_LOAD_BYTE_ZTS ldrb, REG0w, executor_globals, vm_interrupt, TMP1 + | cbnz REG0w, ->interrupt_handler + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + | ldr REG0, EX->func + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add REG0, IP, REG0 + | ldr REG0, [REG0] + | br REG0 + } else if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], # SPAD // stack alignment + | ldr REG0, EX->func + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add REG0, IP, REG0 + | ldr REG0, [REG0] + | br REG0 + } else { + | ldr IP, EX->opline + | mov FCARG1x, FP + | ldr REG0, EX->func + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add REG0, IP, REG0 + | ldr REG0, [REG0] + | blr REG0 + | + | tst RETVALw, RETVALw + | blt ->trace_halt + | + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | mov RETVALx, #1 // ZEND_VM_ENTER + | ret + } + + return 1; +} + +static int zend_jit_trace_escape_stub(dasm_State **Dst) +{ + |->trace_escape: + | + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + | JMP_IP, TMP1 + } else if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], # SPAD // stack alignment + | JMP_IP, TMP1 + } else { + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | mov RETVALx, #1 // ZEND_VM_ENTER + | ret + } + + return 1; +} + +/* Keep 32 exit points in a single code block */ +#define ZEND_JIT_EXIT_POINTS_SPACING 4 // bl = bytes +#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points + +static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) +{ + uint32_t i; + + | bl >2 + |1: + for (i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { + | bl >2 + } + |2: + | adr TMP1, <1 + | sub TMP1, lr, TMP1 + | lsr TMP1, TMP1, #2 + if (n) { + | ADD_SUB_32_WITH_CONST add, TMP1w, TMP1w, n, TMP2w + } + | b ->trace_exit // pass exit_num in TMP1w + + return 1; +} + +#ifdef CONTEXT_THREADED_JIT +static int zend_jit_context_threaded_call_stub(dasm_State **Dst) +{ + |->context_threaded_call: + | NIY_STUB // TODO + return 1; +} +#endif + +static int zend_jit_assign_to_variable(dasm_State **Dst, + const zend_op *opline, + zend_jit_addr var_use_addr, + zend_jit_addr var_addr, + uint32_t var_info, + uint32_t var_def_info, + zend_uchar val_type, + zend_jit_addr val_addr, + uint32_t val_info, + zend_jit_addr res_addr, + bool check_exception); + +static int zend_jit_assign_const_stub(dasm_State **Dst) +{ + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; + + |->assign_const: + | stp x29, x30, [sp,#-32]! + | mov x29, sp + if (!zend_jit_assign_to_variable( + Dst, NULL, + var_addr, var_addr, -1, -1, + IS_CONST, val_addr, val_info, + 0, 0)) { + return 0; + } + | ldp x29, x30, [sp],#32 + | ret + return 1; +} + +static int zend_jit_assign_tmp_stub(dasm_State **Dst) +{ + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; + + |->assign_tmp: + | stp x29, x30, [sp,#-32]! + | mov x29, sp + if (!zend_jit_assign_to_variable( + Dst, NULL, + var_addr, var_addr, -1, -1, + IS_TMP_VAR, val_addr, val_info, + 0, 0)) { + return 0; + } + | ldp x29, x30, [sp],#32 + | ret + return 1; +} + +static int zend_jit_assign_var_stub(dasm_State **Dst) +{ + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; + + |->assign_var: + | stp x29, x30, [sp,#-32]! + | mov x29, sp + if (!zend_jit_assign_to_variable( + Dst, NULL, + var_addr, var_addr, -1, -1, + IS_VAR, val_addr, val_info, + 0, 0)) { + return 0; + } + | ldp x29, x30, [sp],#32 + | ret + return 1; +} + +static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) +{ + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; + + |->assign_cv_noref: + | stp x29, x30, [sp,#-32]! + | mov x29, sp + if (!zend_jit_assign_to_variable( + Dst, NULL, + var_addr, var_addr, -1, -1, + IS_CV, val_addr, val_info, + 0, 0)) { + return 0; + } + | ldp x29, x30, [sp],#32 + | ret + return 1; +} + +static int zend_jit_assign_cv_stub(dasm_State **Dst) +{ + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; + + |->assign_cv: + | stp x29, x30, [sp,#-32]! + | mov x29, sp + if (!zend_jit_assign_to_variable( + Dst, NULL, + var_addr, var_addr, -1, -1, + IS_CV, val_addr, val_info, + 0, 0)) { + return 0; + } + | ldp x29, x30, [sp],#32 + | ret + return 1; +} + +static const zend_jit_stub zend_jit_stubs[] = { + JIT_STUB(interrupt_handler), + JIT_STUB(exception_handler), + JIT_STUB(exception_handler_undef), + JIT_STUB(leave_function), + JIT_STUB(leave_throw), + JIT_STUB(icall_throw), + JIT_STUB(throw_cannot_pass_by_ref), + JIT_STUB(undefined_offset), + JIT_STUB(undefined_index), + JIT_STUB(cannot_add_element), + JIT_STUB(undefined_offset_ex), + JIT_STUB(undefined_index_ex), + JIT_STUB(cannot_add_element_ex), + JIT_STUB(undefined_function), + JIT_STUB(negative_shift), + JIT_STUB(mod_by_zero), + JIT_STUB(invalid_this), + JIT_STUB(trace_halt), + JIT_STUB(trace_exit), + JIT_STUB(trace_escape), + JIT_STUB(hybrid_runtime_jit), + JIT_STUB(hybrid_profile_jit), + JIT_STUB(hybrid_hot_code), + JIT_STUB(hybrid_func_hot_counter), + JIT_STUB(hybrid_loop_hot_counter), + JIT_STUB(hybrid_hot_trace), + JIT_STUB(hybrid_func_trace_counter), + JIT_STUB(hybrid_ret_trace_counter), + JIT_STUB(hybrid_loop_trace_counter), + JIT_STUB(assign_const), + JIT_STUB(assign_tmp), + JIT_STUB(assign_var), + JIT_STUB(assign_cv_noref), + JIT_STUB(assign_cv), +#ifdef CONTEXT_THREADED_JIT + JIT_STUB(context_threaded_call), +#endif +}; + +#if ZTS && defined(ZEND_WIN32) +extern uint32_t _tls_index; +extern char *_tls_start; +extern char *_tls_end; +#endif + +static int zend_jit_setup(void) +{ + allowed_opt_flags = 0; + +#if ZTS +# ifdef _WIN64 + tsrm_tls_index = _tls_index * sizeof(void*); + + /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ + /* Probably, it might be better solution */ + do { + void ***tls_mem = ((void**)__readgsqword(0x58))[_tls_index]; + void *val = _tsrm_ls_cache; + size_t offset = 0; + size_t size = (char*)&_tls_end - (char*)&_tls_start; + + while (offset < size) { + if (*tls_mem == val) { + tsrm_tls_offset = offset; + break; + } + tls_mem++; + offset += sizeof(void*); + } + if (offset >= size) { + // TODO: error message ??? + return FAILURE; + } + } while(0); +# elif ZEND_WIN32 + tsrm_tls_index = _tls_index * sizeof(void*); + + /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ + /* Probably, it might be better solution */ + do { + void ***tls_mem = ((void***)__readfsdword(0x2c))[_tls_index]; + void *val = _tsrm_ls_cache; + size_t offset = 0; + size_t size = (char*)&_tls_end - (char*)&_tls_start; + + while (offset < size) { + if (*tls_mem == val) { + tsrm_tls_offset = offset; + break; + } + tls_mem++; + offset += sizeof(void*); + } + if (offset >= size) { + // TODO: error message ??? + return FAILURE; + } + } while(0); +# elif defined(__APPLE__) && defined(__x86_64__) + tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (tsrm_ls_cache_tcb_offset == 0) { + size_t *ti; + __asm__( + "leaq __tsrm_ls_cache(%%rip),%0" + : "=r" (ti)); + tsrm_tls_offset = ti[2]; + tsrm_tls_index = ti[1] * 8; + } +# elif defined(__GNUC__) && defined(__x86_64__) + tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (tsrm_ls_cache_tcb_offset == 0) { +#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) + size_t ret; + + asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" + : "=r" (ret)); + tsrm_ls_cache_tcb_offset = ret; +#else + size_t *ti; + + __asm__( + "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" + : "=a" (ti)); + tsrm_tls_offset = ti[1]; + tsrm_tls_index = ti[0] * 16; +#endif + } +# elif defined(__GNUC__) && defined(__i386__) + tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (tsrm_ls_cache_tcb_offset == 0) { +#if !defined(__FreeBSD__) + size_t ret; + + asm ("leal _tsrm_ls_cache@ntpoff,%0\n" + : "=a" (ret)); + tsrm_ls_cache_tcb_offset = ret; +#else + size_t *ti, _ebx, _ecx, _edx; + + __asm__( + "call 1f\n" + ".subsection 1\n" + "1:\tmovl (%%esp), %%ebx\n\t" + "ret\n" + ".previous\n\t" + "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" + "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" + "call ___tls_get_addr@plt\n\t" + "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" + : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); + tsrm_tls_offset = ti[1]; + tsrm_tls_index = ti[0] * 8; +#endif + } +# elif defined(__aarch64__) + tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); + ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0); +# elif +# endif +#endif + + return SUCCESS; +} + +static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) +{ + | brk #0 + return 1; +} + +static int zend_jit_align_func(dasm_State **Dst) +{ + reuse_ip = 0; + delayed_call_chain = 0; + last_valid_opline = NULL; + use_last_vald_opline = 0; + track_last_valid_opline = 0; + jit_return_label = -1; + |.align 16 + return 1; +} + +static int zend_jit_prologue(dasm_State **Dst) +{ + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | SUB_HYBRID_SPAD + } else if (GCC_GLOBAL_REGS) { + | stp x29, x30, [sp, # -SPAD]! // stack alignment + } else { + | stp x29, x30, [sp, # -NR_SPAD]! // stack alignment + | stp FP, RX, T2 // save FP and IP + | mov FP, FCARG1x + } + return 1; +} + +static int zend_jit_label(dasm_State **Dst, unsigned int label) +{ + |=>label: + return 1; +} + +static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) +{ + | // call->prev_execute_data = EX(call); + if (call_level == 1) { + | str xzr, EX:RX->prev_execute_data + } else { + | ldr REG0, EX->call + | str REG0, EX:RX->prev_execute_data + } + | // EX(call) = call; + | str RX, EX->call + + delayed_call_chain = 0; + + return 1; +} + +static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) +{ + if (last_valid_opline == opline) { + zend_jit_use_last_valid_opline(); + } else if (GCC_GLOBAL_REGS && last_valid_opline) { + zend_jit_use_last_valid_opline(); + | LOAD_64BIT_VAL TMP1, (opline - last_valid_opline) * sizeof(zend_op) + | ADD_IP TMP1, TMP2 + } else { + | LOAD_IP_ADDR opline + } + zend_jit_set_last_valid_opline(opline); + + return 1; +} + +static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) +{ + if (delayed_call_chain) { + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } + } + if (!zend_jit_set_ip(Dst, opline)) { + return 0; + } + reuse_ip = 0; + return 1; +} + +static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) +{ + | MEM_LOAD_BYTE_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 + if (exit_addr) { + | cbnz TMP1w, &exit_addr + } else if (last_valid_opline == opline) { + || zend_jit_use_last_valid_opline(); + | cbnz TMP1w, ->interrupt_handler + } else { + | cbnz TMP1w, >1 + |.cold_code + |1: + | LOAD_IP_ADDR opline + | b ->interrupt_handler + |.code + } + return 1; +} + +static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) +{ + if (timeout_exit_addr) { + | MEM_LOAD_BYTE_ZTS ldrb, REG0w, executor_globals, vm_interrupt, TMP1 + | cbz REG0w, =>loop_label + | b &timeout_exit_addr + } else { + | b =>loop_label + } + return 1; +} + +static int zend_jit_check_exception(dasm_State **Dst) +{ + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbnz REG0, ->exception_handler + return 1; +} + +static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) +{ + if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbnz REG0, ->exception_handler_undef + return 1; + } + return zend_jit_check_exception(Dst); +} + +static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) +{ + zend_regset regset = ZEND_REGSET_SCRATCH; + + // In the x86 implementation, this clause would be conducted if ZTS is enabled or the addressing mode is 64-bit. + { + /* assignment to EG(jit_trace_num) shouldn't clober CPU register used by deoptimizer */ + if (parent) { + int i; + int parent_vars_count = parent->exit_info[exit_num].stack_size; + zend_jit_trace_stack *parent_stack = + parent->stack_map + + parent->exit_info[exit_num].stack_offset; + + for (i = 0; i < parent_vars_count; i++) { + if (STACK_REG(parent_stack, i) != ZREG_NONE) { + if (STACK_REG(parent_stack, i) < ZREG_NUM) { + ZEND_REGSET_EXCL(regset, STACK_REG(parent_stack, i)); + } else if (STACK_REG(parent_stack, i) == ZREG_ZVAL_COPY_GPR0) { + ZEND_REGSET_EXCL(regset, ZREG_REG0); + } + } + } + } + } + + if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { + ZEND_REGSET_EXCL(regset, ZREG_REG0); + } + + current_trace_num = trace_num; + + | // EG(jit_trace_num) = trace_num; + if (regset == ZEND_REGSET_EMPTY || ZEND_REGSET_IS_SINGLETON(regset)) { + | sub sp, sp, #16 + | stp TMP1, TMP2, [sp] // save TMP1 and TMP2 + | LOAD_32BIT_VAL TMP1w, trace_num + | MEM_STORE_32_ZTS str, TMP1w, executor_globals, jit_trace_num, TMP2 + | ldp TMP1, TMP2, [sp] // retore TMP1 and TMP2 + | add sp, sp, #16 + } else { + zend_reg tmp1 = ZEND_REGSET_FIRST(regset); + zend_reg tmp2 = ZEND_REGSET_FIRST(ZEND_REGSET_EXCL(regset, tmp1)); + + | LOAD_32BIT_VAL Rw(tmp1), trace_num + | MEM_STORE_32_ZTS str, Rw(tmp1), executor_globals, jit_trace_num, Rx(tmp2) + (void)tmp1; + (void)tmp2; + } + + return 1; +} + +static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) +{ + int ret = 0; + uint8_t *p, *end; + size_t code_size = size; + ptrdiff_t delta; + + if (jmp_table_size) { + const void **jmp_slot = (const void **)((char*)code + size); + + code_size -= jmp_table_size * sizeof(void*); + do { + jmp_slot--; + if (*jmp_slot == from_addr) { + *jmp_slot = to_addr; + ret++; + } + } while (--jmp_table_size); + } + + p = (uint8_t*)code; + end = p + code_size; + while (p < end) { + uint32_t *ins_ptr = (uint32_t*)p; + uint32_t ins = *ins_ptr; + if ((ins & 0xfc000000u) == 0x14000000u) { + // B (imm26:0..25) + delta = (uint32_t*)from_addr - ins_ptr; + if (((ins ^ (uint32_t)delta) & 0x01ffffffu) == 0) { + delta = (uint32_t*)to_addr - ins_ptr; + if (((delta + 0x02000000) >> 26) != 0) { + abort(); // brnach target out of range + } + *ins_ptr = (ins & 0xfc000000u) | ((uint32_t)delta & 0x03ffffffu); + ret++; + } + } else if ((ins & 0xff000000u) == 0x54000000u || + (ins & 0x7e000000u) == 0x34000000u) { + // B.cond, CBZ, CBNZ (imm19:5..23) + delta = (uint32_t*)from_addr - ins_ptr; + if (((ins ^ ((uint32_t)delta << 5)) & 0x00ffffe0u) == 0) { + delta = (uint32_t*)to_addr - ins_ptr; + if (((delta + 0x40000) >> 19) != 0) { + abort(); // brnach target out of range + } + *ins_ptr = (ins & 0xff00001fu) | (((uint32_t)delta & 0x7ffffu) << 5); + ret++; + } + } else if ((ins & 0x7e000000u) == 0x36000000u) { + // TBZ, TBNZ (imm14:5..18) + delta = (uint32_t*)from_addr - ins_ptr; + if (((ins ^ ((uint32_t)delta << 5)) & 0x0007ffe0u) == 0) { + delta = (uint32_t*)to_addr - ins_ptr; + if (((delta + 0x2000) >> 14) != 0) { + abort(); // brnach target out of range + } + *ins_ptr = (ins & 0xfff8001fu) | (((uint32_t)delta & 0x3fffu) << 5); + ret++; + } + } + p += 4; + } + + JIT_CACHE_FLUSH(code, (char*)code + size); + +#ifdef HAVE_VALGRIND + VALGRIND_DISCARD_TRANSLATIONS(code, size); +#endif + + return ret; +} + +static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) +{ + return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); +} + +static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) +{ + const void *link_addr; + size_t prologue_size; + + /* Skip prologue. */ + // TODO: don't hardcode this ??? + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { +#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE + prologue_size = 0; +#else + // sub sp, sp, #0x20 + prologue_size = 4; +#endif + } else if (GCC_GLOBAL_REGS) { + // stp x29, x30, [sp, # -SPAD]! + prologue_size = 4; + } else { + // stp x29, x30, [sp, # -NR_SPAD]! // stack alignment + // stp FP, RX, T2 + // mov FP, FCARG1x + prologue_size = 12; + } + link_addr = (const void*)((const char*)t->code_start + prologue_size); + + if (timeout_exit_addr) { + /* Check timeout for links to LOOP */ + | MEM_LOAD_BYTE_ZTS ldrb, REG0w, executor_globals, vm_interrupt, TMP1 + | cbz REG0w, &link_addr + | b &timeout_exit_addr + } else { + | b &link_addr + } + return 1; +} + +static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) +{ + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + if (!original_handler) { + | JMP_IP TMP1 + } else { + | ldr REG0, EX->func + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add REG0, IP, REG0 + | ldr REG0, [REG0] + | br REG0 + } + } else if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], # SPAD // stack alignment + if (!original_handler) { + | JMP_IP TMP1 + } else { + | ldr REG0, EX->func + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add REG0, IP, REG0 + | ldr REG0, [REG0] + | br REG0 + } + } else { + if (original_handler) { + | mov FCARG1x, FP + | ldr REG0, EX->func + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add REG0, IP, REG0 + | ldr REG0, [REG0] + | blr REG0 + } + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | mov RETVALx, #2 // ZEND_VM_LEAVE + | ret + } + return 1; +} + +static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) +{ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); + + if (!exit_addr) { + return 0; + } + + | IF_NOT_ZVAL_TYPE var_addr, type, &exit_addr, ZREG_TMP1 + + return 1; +} + +static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) +{ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); + + if (!exit_addr) { + return 0; + } + + | GET_ZVAL_LVAL ZREG_FCARG1x, var_addr, TMP1 + if (op_info & MAY_BE_ARRAY_PACKED) { + | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] + | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w + | beq &exit_addr + } else { + | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] + | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w + | bne &exit_addr + } + + return 1; +} + +static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) +{ + zend_jit_op_array_trace_extension *jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + size_t offset = jit_extension->offset; + const void *handler = + (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; + + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, FP + } + | EXT_CALL handler, REG0 + if (may_throw + && opline->opcode != ZEND_RETURN + && opline->opcode != ZEND_RETURN_BY_REF) { + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbnz REG0, ->exception_handler + } + + while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { + trace++; + } + + if (!GCC_GLOBAL_REGS + && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { + if (opline->opcode == ZEND_RETURN || + opline->opcode == ZEND_RETURN_BY_REF || + opline->opcode == ZEND_DO_UCALL || + opline->opcode == ZEND_DO_FCALL_BY_NAME || + opline->opcode == ZEND_DO_FCALL || + opline->opcode == ZEND_GENERATOR_CREATE) { + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 + } + } + + if (zend_jit_trace_may_exit(op_array, opline)) { + if (opline->opcode == ZEND_RETURN || + opline->opcode == ZEND_RETURN_BY_REF || + opline->opcode == ZEND_GENERATOR_CREATE) { + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { +#if 0 + /* this check should be handled by the following OPLINE guard or jmp [IP] */ + | LOAD_ADDR TMP1, zend_jit_halt_op + | cmp IP, TMP1 + | beq ->trace_halt +#endif + } else if (GCC_GLOBAL_REGS) { + | cbz IP, ->trace_halt + } else { + | tst RETVALw, RETVALw + | blt ->trace_halt + } + } else if (opline->opcode == ZEND_EXIT || + opline->opcode == ZEND_GENERATOR_RETURN || + opline->opcode == ZEND_YIELD || + opline->opcode == ZEND_YIELD_FROM) { + | b ->trace_halt + } + if (trace->op != ZEND_JIT_TRACE_END || + (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && + trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { + + const zend_op *next_opline = trace->opline; + const zend_op *exit_opline = NULL; + uint32_t exit_point; + const void *exit_addr; + uint32_t old_info = 0; + uint32_t old_res_info = 0; + zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; + + if (zend_is_smart_branch(opline)) { + bool exit_if_true = 0; + exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); + } else { + switch (opline->opcode) { + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_JMP_NULL: + case ZEND_FE_RESET_R: + case ZEND_FE_RESET_RW: + exit_opline = (trace->opline == opline + 1) ? + OP_JMP_ADDR(opline, opline->op2) : + opline + 1; + break; + case ZEND_JMPZNZ: + exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ? + ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : + OP_JMP_ADDR(opline, opline->op2); + break; + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + if (opline->op2_type == IS_CV) { + old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); + } + exit_opline = (trace->opline == opline + 1) ? + ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : + opline + 1; + break; + + } + } + + if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { + old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); + } + exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); + } + switch (opline->opcode) { + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + if (opline->op2_type == IS_CV) { + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); + } + break; + } + + if (!exit_addr) { + return 0; + } + | CMP_IP next_opline, TMP1, TMP2 + | bne &exit_addr + } + } + + zend_jit_set_last_valid_opline(trace->opline); + + return 1; +} + +static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) +{ + const void *handler; + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + handler = zend_get_opcode_handler_func(opline); + } else { + handler = opline->handler; + } + + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, FP + } + | EXT_CALL handler, REG0 + if (may_throw) { + zend_jit_check_exception(Dst); + } + + /* Skip the following OP_DATA */ + switch (opline->opcode) { + case ZEND_ASSIGN_DIM: + case ZEND_ASSIGN_OBJ: + case ZEND_ASSIGN_STATIC_PROP: + case ZEND_ASSIGN_DIM_OP: + case ZEND_ASSIGN_OBJ_OP: + case ZEND_ASSIGN_STATIC_PROP_OP: + case ZEND_ASSIGN_STATIC_PROP_REF: + case ZEND_ASSIGN_OBJ_REF: + zend_jit_set_last_valid_opline(opline + 2); + break; + default: + zend_jit_set_last_valid_opline(opline + 1); + break; + } + + return 1; +} + +static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) +{ + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + if (opline->opcode == ZEND_DO_UCALL || + opline->opcode == ZEND_DO_FCALL_BY_NAME || + opline->opcode == ZEND_DO_FCALL || + opline->opcode == ZEND_RETURN) { + + /* Use inlined HYBRID VM handler */ + const void *handler = opline->handler; + + | ADD_HYBRID_SPAD + | EXT_JMP handler, REG0 + } else { + const void *handler = zend_get_opcode_handler_func(opline); + + | EXT_CALL handler, REG0 + | ADD_HYBRID_SPAD + | JMP_IP TMP1 + } + } else { + const void *handler = opline->handler; + + if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], # SPAD // stack alignment + } else { + | mov FCARG1x, FP + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + } + | EXT_JMP handler, REG0 + } + zend_jit_reset_last_valid_opline(); + return 1; +} + +static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) +{ + uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | CMP_IP opline, TMP1, TMP2 + | bne &exit_addr + + zend_jit_set_last_valid_opline(opline); + + return 1; +} + +static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) +{ + | b =>target_label + return 1; +} + +static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) +{ + | CMP_IP next_opline, TMP1, TMP2 + | bne =>target_label + + zend_jit_set_last_valid_opline(next_opline); + + return 1; +} + +#ifdef CONTEXT_THREADED_JIT +static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) +{ + | NIY // TODO + return 1; +} +#endif + +static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) +{ +#ifdef CONTEXT_THREADED_JIT + return zend_jit_context_threaded_call(Dst, opline, next_block); +#else + return zend_jit_tail_handler(Dst, opline); +#endif +} + +static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) +{ + ZEND_ASSERT(Z_MODE(src) == IS_REG); + ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); + + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | SET_ZVAL_LVAL_FROM_REG dst, Rx(Z_REG(src)), TMP1 + if (set_type) { + | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 + } + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | SET_ZVAL_DVAL dst, Z_REG(src), ZREG_TMP1 + if (set_type) { + | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 + } + } else { + ZEND_UNREACHABLE(); + } + return 1; +} + +static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) +{ + ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); + ZEND_ASSERT(Z_MODE(dst) == IS_REG); + + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | GET_ZVAL_LVAL Z_REG(dst), src, TMP1 + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | GET_ZVAL_DVAL Z_REG(dst), src, ZREG_TMP1 + } else { + ZEND_UNREACHABLE(); + } + return 1; +} + +static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type) +{ + zend_jit_addr src = ZEND_ADDR_REG(reg); + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); + + return zend_jit_spill_store(Dst, src, dst, info, set_type); +} + +static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) +{ + if (Z_MODE(src) == IS_REG && Z_STORE(src)) { + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); + return zend_jit_spill_store(Dst, src, dst, info, 1); + } + return 1; +} + +static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) +{ + if (Z_MODE(src) == IS_REG && Z_STORE(src)) { + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); + bool set_type = 1; + + if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == + (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { + if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { + set_type = 0; + } + } + return zend_jit_spill_store(Dst, src, dst, info, set_type); + } + return 1; +} + +static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) +{ + zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); + zend_jit_addr dst = ZEND_ADDR_REG(reg); + + return zend_jit_load_reg(Dst, src, dst, info); +} + +static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) +{ + if (!zend_jit_same_addr(src, dst)) { + if (Z_MODE(src) == IS_REG) { + if (Z_MODE(dst) == IS_REG) { + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | mov Rx(Z_REG(dst)), Rx(Z_REG(src)) + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | fmov Rd(Z_REG(dst)-ZREG_V0), Rd(Z_REG(src)-ZREG_V0) + } else { + ZEND_UNREACHABLE(); + } + } else if (Z_MODE(dst) == IS_MEM_ZVAL) { + if (!Z_LOAD(src) && !Z_STORE(src)) { + if (!zend_jit_spill_store(Dst, src, dst, info, + JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + JIT_G(current_frame) == NULL || + STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || + (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) + )) { + return 0; + } + } + } else { + ZEND_UNREACHABLE(); + } + } else if (Z_MODE(src) == IS_MEM_ZVAL) { + if (Z_MODE(dst) == IS_REG) { + if (!zend_jit_load_reg(Dst, src, dst, info)) { + return 0; + } + } else { + ZEND_UNREACHABLE(); + } + } else { + ZEND_UNREACHABLE(); + } + } + return 1; +} + +static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) +{ + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 + + if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { + if (!zend_jit_save_call_chain(Dst, -1)) { + return 0; + } + } + + ZEND_ASSERT(opline); + + | LOAD_IP_ADDR (opline - 1) + | b ->trace_escape + |1: + + return 1; +} + +static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) +{ + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); + + if (reg == ZREG_LONG_MIN_MINUS_1) { + uint64_t val = 0xc3e0000000000000; + | SET_ZVAL_LVAL dst, val, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 + } else if (reg == ZREG_LONG_MIN) { + uint64_t val = 0x8000000000000000; + | SET_ZVAL_LVAL dst, val, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 + } else if (reg == ZREG_LONG_MAX) { + uint64_t val = 0x7fffffffffffffff; + | SET_ZVAL_LVAL dst, val, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 + } else if (reg == ZREG_LONG_MAX_PLUS_1) { + uint64_t val = 0x43e0000000000000; + | SET_ZVAL_LVAL dst, val, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 + } else if (reg == ZREG_NULL) { + | SET_ZVAL_TYPE_INFO dst, IS_NULL, TMP1w, TMP2 + } else if (reg == ZREG_ZVAL_TRY_ADDREF) { + | IF_NOT_ZVAL_REFCOUNTED dst, >1, ZREG_TMP1, ZREG_TMP2 + | GET_ZVAL_PTR TMP1, dst, TMP2 + | GC_ADDREF TMP1, TMP2w + |1: + } else if (reg == ZREG_ZVAL_COPY_GPR0) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF -1, REG1w, REG2, TMP1w + } else { + ZEND_UNREACHABLE(); + } + return 1; +} + +static int zend_jit_free_trampoline(dasm_State **Dst) +{ + | // if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) + | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] + | TST_32_WITH_CONST TMP1w, ZEND_ACC_CALL_VIA_TRAMPOLINE, TMP2w + | beq >1 + | mov FCARG1x, REG0 + | EXT_CALL zend_jit_free_trampoline_helper, REG0 + |1: + return 1; +} + +static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) +{ + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 + } + if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { + return 0; + } + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | LONG_ADD_SUB_WITH_IMM adds, op1_def_addr, Z_L(1), TMP1, TMP2 + } else { + | LONG_ADD_SUB_WITH_IMM subs, op1_def_addr, Z_L(1), TMP1, TMP2 + } + + if (may_overflow && + (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || + ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { + int32_t exit_point; + const void *exit_addr; + zend_jit_trace_stack *stack; + uint32_t old_op1_info, old_res_info = 0; + + stack = JIT_G(current_frame)->stack; + old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); + } else { + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); + } + if (opline->result_type != IS_UNUSED) { + old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + if (opline->opcode == ZEND_PRE_INC) { + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); + } else if (opline->opcode == ZEND_PRE_DEC) { + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); + } else if (opline->opcode == ZEND_POST_INC) { + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); + } else if (opline->opcode == ZEND_POST_DEC) { + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); + } + } + + exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + | bvs &exit_addr + + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); + if (opline->result_type != IS_UNUSED) { + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); + } + } else if (may_overflow) { + | bvs >1 + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + |.cold_code + |1: + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + uint64_t val = 0x43e0000000000000; + if (Z_MODE(op1_def_addr) == IS_REG) { + | LOAD_64BIT_VAL TMP1, val + | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 + } else { + | SET_ZVAL_LVAL op1_def_addr, val, REG0, TMP1 + } + } else { + uint64_t val = 0xc3e0000000000000; + if (Z_MODE(op1_def_addr) == IS_REG) { + | LOAD_64BIT_VAL TMP1, val + | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 + } else { + | SET_ZVAL_LVAL op1_def_addr, val, REG0, TMP1 + } + } + if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { + | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE, TMP1w, TMP2 + } + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + | b >3 + |.code + } else { + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + } + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { + |.cold_code + |2: + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | SET_EX_OPLINE opline, REG0 + if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2, ZREG_TMP1 + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 + | EXT_CALL zend_jit_undefined_op_helper, REG0 + op1_info |= MAY_BE_NULL; + } + |2: + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + + | // ZVAL_DEREF(var_ptr); + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >2, TMP1w + | GET_Z_PTR FCARG1x, FCARG1x + | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] + | cbz TMP1, >1 + if (RETURN_VALUE_USED(opline)) { + | LOAD_ZVAL_ADDR FCARG2x, res_addr + } else { + | mov FCARG2x, xzr + } + if (opline->opcode == ZEND_PRE_INC) { + | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 + } else if (opline->opcode == ZEND_PRE_DEC) { + | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 + } else if (opline->opcode == ZEND_POST_INC) { + | EXT_CALL zend_jit_post_inc_typed_ref, REG0 + } else if (opline->opcode == ZEND_POST_DEC) { + | EXT_CALL zend_jit_post_dec_typed_ref, REG0 + } else { + ZEND_UNREACHABLE(); + } + zend_jit_check_exception(Dst); + | b >3 + |1: + | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) + |2: + } + + if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + + | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF op1_info, REG0w, REG2, TMP1w + } + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { + | LOAD_ZVAL_ADDR FCARG2x, res_addr + | EXT_CALL zend_jit_pre_inc, REG0 + } else { + | EXT_CALL increment_function, REG0 + } + } else { + if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { + | LOAD_ZVAL_ADDR FCARG2x, res_addr + | EXT_CALL zend_jit_pre_dec, REG0 + } else { + | EXT_CALL decrement_function, REG0 + } + } + if (may_throw) { + zend_jit_check_exception(Dst); + } + } else { + zend_reg tmp_reg; + + if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + if (Z_MODE(op1_def_addr) == IS_REG) { + tmp_reg = Z_REG(op1_def_addr); + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { + tmp_reg = Z_REG(op1_addr); + } else { + tmp_reg = ZREG_FPR0; + } + | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + uint64_t val = 0x3ff0000000000000; // 1.0 + | LOAD_64BIT_VAL TMP1, val + | fmov FPTMPd, TMP1 + | fadd Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMPd + } else { + uint64_t val = 0x3ff0000000000000; // 1.0 + | LOAD_64BIT_VAL TMP1, val + | fmov FPTMPd, TMP1 + | fsub Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMPd + } + | SET_ZVAL_DVAL op1_def_addr, tmp_reg, ZREG_TMP1 + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF op1_def_info, REG0w, REG1, TMP1w + } + } + | b >3 + |.code + } + |3: + if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { + return 0; + } + if (opline->result_type != IS_UNUSED) { + if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { + return 0; + } + } + return 1; +} + +static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) +{ + if ((opline+1)->opcode == ZEND_OP_DATA + && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) + && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { + return 1; + } + return + ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && + JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || + ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && + JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || + ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && + JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); +} + +static int zend_jit_math_long_long(dasm_State **Dst, + const zend_op *opline, + zend_uchar opcode, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_info, + uint32_t res_use_info, + int may_overflow) +{ + bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + zend_reg result_reg; + zend_reg tmp_reg = ZREG_REG0; + bool use_ovf_flag = 1; + + if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { + if (may_overflow && (res_info & MAY_BE_GUARD) + && JIT_G(current_frame) + && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { + result_reg = ZREG_REG0; + } else { + result_reg = Z_REG(res_addr); + } + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { + result_reg = Z_REG(op1_addr); + } else if (Z_REG(res_addr) != ZREG_REG0) { + result_reg = ZREG_REG0; + } else { + /* ASSIGN_DIM_OP */ + result_reg = ZREG_FCARG1x; + tmp_reg = ZREG_FCARG1x; + } + + if (opcode == ZEND_MUL && + Z_MODE(op2_addr) == IS_CONST_ZVAL && + Z_LVAL_P(Z_ZV(op2_addr)) == 2) { + if (Z_MODE(op1_addr) == IS_REG) { + | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) + } else { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) + } + } else if (opcode == ZEND_MUL && + Z_MODE(op2_addr) == IS_CONST_ZVAL && + !may_overflow && + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | lsl Rx(result_reg), Rx(result_reg), TMP1 + } else if (opcode == ZEND_MUL && + Z_MODE(op1_addr) == IS_CONST_ZVAL && + Z_LVAL_P(Z_ZV(op1_addr)) == 2) { + if (Z_MODE(op2_addr) == IS_REG) { + | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) + } else { + | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 + | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) + } + } else if (opcode == ZEND_MUL && + Z_MODE(op1_addr) == IS_CONST_ZVAL && + !may_overflow && + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { + | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 + | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) + | lsl Rx(result_reg), Rx(result_reg), TMP1 + } else if (opcode == ZEND_DIV && + (Z_MODE(op2_addr) == IS_CONST_ZVAL && + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | asr Rx(result_reg), Rx(result_reg), #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) +#if 0 + /* x86 specific optimizations through LEA instraction are not supported on ARM */ + } else if (opcode == ZEND_ADD && + !may_overflow && + Z_MODE(op1_addr) == IS_REG && + Z_MODE(op2_addr) == IS_CONST_ZVAL) { + | NIY // TODO: test + } else if (opcode == ZEND_ADD && + !may_overflow && + Z_MODE(op2_addr) == IS_REG && + Z_MODE(op1_addr) == IS_CONST_ZVAL) { + | NIY // TODO: test + } else if (opcode == ZEND_SUB && + !may_overflow && + Z_MODE(op1_addr) == IS_REG && + Z_MODE(op2_addr) == IS_CONST_ZVAL) { + | NIY // TODO: test +#endif + } else if (opcode == ZEND_MUL) { + | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 + | mul Rx(result_reg), TMP1, TMP2 + if(may_overflow) { + /* Use 'smulh' to get the upper 64 bits fo the 128-bit result. + * For signed multiplication, the top 65 bits of the result will contain + * either all zeros or all ones if no overflow occurred. + * Note that 'cmp, TMP1, Rx(result_reg), asr, #63' is not supported by DynASM/arm64 + * currently, and we put 'asr' and 'cmp' separately. + * Flag: bne -> overflow. beq -> no overflow. + */ + use_ovf_flag = 0; + | smulh TMP1, TMP1, TMP2 + | cmp TMP1, Rx(result_reg), asr #63 + } + } else { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + if ((opcode == ZEND_ADD || opcode == ZEND_SUB) + && Z_MODE(op2_addr) == IS_CONST_ZVAL + && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { + /* +/- 0 */ + may_overflow = 0; + } else if (same_ops && opcode != ZEND_DIV) { + | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) + } else { + | LONG_MATH opcode, result_reg, op2_addr, TMP1 + } + } + if (may_overflow) { + if (res_info & MAY_BE_GUARD) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { + if (use_ovf_flag) { + | bvs &exit_addr + } else { + | bne &exit_addr + } + if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { + | mov Rx(Z_REG(res_addr)), Rx(result_reg) + } + } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + if (use_ovf_flag) { + | bvc &exit_addr + } else { + | beq &exit_addr + } + } else { + ZEND_UNREACHABLE(); + } + } else { + if (res_info & MAY_BE_LONG) { + if (use_ovf_flag) { + | bvs >1 + } else { + | bne >1 + } + } else { + if (use_ovf_flag) { + | bvc >1 + } else { + | beq >1 + } + } + } + } + + if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { + | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + } + } + } + + if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { + zend_reg tmp_reg1 = ZREG_FPR0; + zend_reg tmp_reg2 = ZREG_FPR1; + + if (res_info & MAY_BE_LONG) { + |.cold_code + |1: + } + + do { + if ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || + (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)) { + if (opcode == ZEND_ADD) { + uint64_t val = 0x43e0000000000000; + if (Z_MODE(res_addr) == IS_REG) { + | LOAD_64BIT_VAL TMP1, val + | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 + } else { + | SET_ZVAL_LVAL res_addr, val, REG0, TMP1 + } + break; + } else if (opcode == ZEND_SUB) { + uint64_t val = 0xc3e0000000000000; + if (Z_MODE(res_addr) == IS_REG) { + | LOAD_64BIT_VAL TMP1, val + | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 + } else { + | SET_ZVAL_LVAL res_addr, val, REG0, TMP1 + } + break; + } + } + + | DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg, ZREG_TMP1 + | DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg, ZREG_TMP1 + | DOUBLE_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 + | SET_ZVAL_DVAL res_addr, tmp_reg1, ZREG_TMP1 + } while (0); + + if (Z_MODE(res_addr) == IS_MEM_ZVAL + && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 + } + if (res_info & MAY_BE_LONG) { + | b >2 + |.code + } + |2: + } + + return 1; +} + +static int zend_jit_math_long_double(dasm_State **Dst, + zend_uchar opcode, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_use_info) +{ + zend_reg result_reg = + (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_FPR0; + zend_reg op2_reg; + + | DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, ZREG_TMP1, ZREG_TMP2 + + if (Z_MODE(op2_addr) == IS_REG) { + op2_reg = Z_REG(op2_addr); + } else { + op2_reg = ZREG_FPTMP; + | GET_ZVAL_DVAL op2_reg, op2_addr, ZREG_TMP1 + } + + | DOUBLE_MATH_REG opcode, result_reg, result_reg, op2_reg + + | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 + } + } + + return 1; +} + +static int zend_jit_math_double_long(dasm_State **Dst, + zend_uchar opcode, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_use_info) +{ + zend_reg result_reg, op1_reg, op2_reg; + + if (zend_is_commutative(opcode) + && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else { + result_reg = ZREG_FPR0; + } + | DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 + if (Z_MODE(op1_addr) == IS_REG) { + op1_reg = Z_REG(op1_addr); + } else { + op1_reg = ZREG_FPTMP; + | GET_ZVAL_DVAL op1_reg, op1_addr, ZREG_TMP1 + } + | DOUBLE_MATH_REG opcode, result_reg, result_reg, op1_reg + } else { + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { + result_reg = Z_REG(op1_addr); + } else { + result_reg = ZREG_FPR0; + } + + if (Z_MODE(op1_addr) == IS_REG) { + op1_reg = Z_REG(op1_addr); + } else { + | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 + op1_reg = result_reg; + } + if ((opcode == ZEND_ADD || opcode == ZEND_SUB) + && Z_MODE(op2_addr) == IS_CONST_ZVAL + && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { + /* +/- 0 */ + } else { + op2_reg = ZREG_FPTMP; + | DOUBLE_GET_ZVAL_LVAL op2_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 + | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg + } + } + + | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 + } + } + } + + return 1; +} + +static int zend_jit_math_double_double(dasm_State **Dst, + zend_uchar opcode, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_use_info) +{ + bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + zend_reg result_reg, op1_reg, op2_reg; + zend_jit_addr val_addr; + + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { + result_reg = Z_REG(op1_addr); + } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { + result_reg = Z_REG(op2_addr); + } else { + result_reg = ZREG_FPR0; + } + + if (Z_MODE(op1_addr) == IS_REG) { + op1_reg = Z_REG(op1_addr); + val_addr = op2_addr; + } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { + op1_reg = Z_REG(op2_addr); + val_addr = op1_addr; + } else { + | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 + op1_reg = result_reg; + val_addr = op2_addr; + } + + if ((opcode == ZEND_MUL) && + Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { + | DOUBLE_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg + } else { + if (same_ops) { + op2_reg = op1_reg; + } else if (Z_MODE(val_addr) == IS_REG) { + op2_reg = Z_REG(val_addr); + } else { + op2_reg = ZREG_FPTMP; + | GET_ZVAL_DVAL op2_reg, val_addr, ZREG_TMP1 + } + | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg + } + + | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 + } + } + } + return 1; +} + +static int zend_jit_math_helper(dasm_State **Dst, + const zend_op *opline, + zend_uchar opcode, + zend_uchar op1_type, + znode_op op1, + zend_jit_addr op1_addr, + uint32_t op1_info, + zend_uchar op2_type, + znode_op op2, + zend_jit_addr op2_addr, + uint32_t op2_info, + uint32_t res_var, + zend_jit_addr res_addr, + uint32_t res_info, + uint32_t res_use_info, + int may_overflow, + int may_throw) +/* Labels: 1,2,3,4,5,6 */ +{ + bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { + if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 + } + } + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { + if (op2_info & MAY_BE_DOUBLE) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, ZREG_TMP1 + |.cold_code + |1: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 + } + if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + | b >5 + |.code + } else { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 + } + } + if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + |.cold_code + |3: + if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops) { + | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6, ZREG_TMP1 + } + } + if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + | b >5 + } + if (!same_ops) { + |1: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 + } + if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + | b >5 + } + |.code + } + } else if ((op1_info & MAY_BE_DOUBLE) && + !(op1_info & MAY_BE_LONG) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops && (op2_info & MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 + } + } + if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + } + if (!same_ops && (op2_info & MAY_BE_LONG)) { + if (op2_info & MAY_BE_DOUBLE) { + |.cold_code + } + |1: + if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 + } + if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + if (op2_info & MAY_BE_DOUBLE) { + | b >5 + |.code + } + } + } else if ((op2_info & MAY_BE_DOUBLE) && + !(op2_info & MAY_BE_LONG) && + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 + } + if (op1_info & MAY_BE_DOUBLE) { + if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops && (op1_info & MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 + } + } + if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + } + if (!same_ops && (op1_info & MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + |.cold_code + } + |1: + if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 + } + if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + | b >5 + |.code + } + } + } + + |5: + + if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + |.cold_code + } + |6: + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + | LOAD_ZVAL_ADDR FCARG1x, real_addr + } else if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, res_addr + } + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } + | LOAD_ZVAL_ADDR CARG3, op2_addr + | SET_EX_OPLINE opline, REG0 + if (opcode == ZEND_ADD) { + | EXT_CALL add_function, REG0 + } else if (opcode == ZEND_SUB) { + | EXT_CALL sub_function, REG0 + } else if (opcode == ZEND_MUL) { + | EXT_CALL mul_function, REG0 + } else if (opcode == ZEND_DIV) { + | EXT_CALL div_function, REG0 + } else { + ZEND_UNREACHABLE(); + } + | FREE_OP op1_type, op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception(Dst); + } + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { + return 0; + } + } + if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + | b <5 + |.code + } + } + + return 1; +} + +static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) +{ + ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); + ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); + + if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { + return 0; + } + if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { + return 0; + } + return 1; +} + +static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr) +{ + zend_jit_addr op1_addr = OP1_ADDR(); + zend_jit_addr op2_addr = OP2_ADDR(); + + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | EXT_CALL zend_jit_add_arrays_helper, REG0 + | SET_ZVAL_PTR res_addr, RETVALx, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX, TMP1w, TMP2 + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + return 1; +} + +static int zend_jit_long_math_helper(dasm_State **Dst, + const zend_op *opline, + zend_uchar opcode, + zend_uchar op1_type, + znode_op op1, + zend_jit_addr op1_addr, + uint32_t op1_info, + zend_ssa_range *op1_range, + zend_uchar op2_type, + znode_op op2, + zend_jit_addr op2_addr, + uint32_t op2_info, + zend_ssa_range *op2_range, + uint32_t res_var, + zend_jit_addr res_addr, + uint32_t res_info, + uint32_t res_use_info, + int may_throw) +/* Labels: 6 */ +{ + bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + zend_reg result_reg; + zval tmp; + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 + } + if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 + } + + if (opcode == ZEND_MOD && Z_MODE(op2_addr) == IS_CONST_ZVAL && + op1_range && + op1_range->min >= 0) { + zend_long l = Z_LVAL_P(Z_ZV(op2_addr)); + + if (zend_long_is_power_of_two(l)) { + /* Optimisation for mod of power of 2 */ + opcode = ZEND_BW_AND; + ZVAL_LONG(&tmp, l - 1); + op2_addr = ZEND_ADDR_CONST_ZVAL(&tmp); + } + } + + if (opcode == ZEND_MOD) { + result_reg = ZREG_REG0; + } else if (Z_MODE(res_addr) == IS_REG) { + if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) + && opline->op2_type != IS_CONST) { + result_reg = ZREG_REG0; + } else { + result_reg = Z_REG(res_addr); + } + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { + result_reg = Z_REG(op1_addr); + } else if (Z_REG(res_addr) != ZREG_REG0) { + result_reg = ZREG_REG0; + } else { + /* ASSIGN_DIM_OP */ + result_reg = ZREG_FCARG1x; + } + + if (opcode == ZEND_SL) { + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); + + if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { + if (EXPECTED(op2_lval > 0)) { + | mov Rx(result_reg), xzr + } else { + | SET_EX_OPLINE opline, REG0 + | b ->negative_shift + } + } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { + | add Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) + } else { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | mov TMP1w, #op2_lval + | lsl Rx(result_reg), Rx(result_reg), TMP1 + } + } else { + if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_REG1) { + | GET_ZVAL_LVAL ZREG_REG1, op2_addr, TMP1 + } + if (!op2_range || + op2_range->min < 0 || + op2_range->max >= SIZEOF_ZEND_LONG * 8) { + + | cmp REG1, #(SIZEOF_ZEND_LONG*8) + | bhs >1 + |.cold_code + |1: + | mov Rx(result_reg), xzr + | cmp REG1, xzr + | bgt >1 + | SET_EX_OPLINE opline, REG0 + | b ->negative_shift + |.code + } + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | lsl Rx(result_reg), Rx(result_reg), REG1 + |1: + } + } else if (opcode == ZEND_SR) { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); + + if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { + if (EXPECTED(op2_lval > 0)) { + | asr Rx(result_reg), Rx(result_reg), #((SIZEOF_ZEND_LONG * 8) - 1) + } else { + | SET_EX_OPLINE opline, REG0 + | b ->negative_shift + } + } else { + | mov TMP1w, #op2_lval + | asr Rx(result_reg), Rx(result_reg), TMP1 + } + } else { + if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_REG1) { + | GET_ZVAL_LVAL ZREG_REG1, op2_addr, TMP1 + } + if (!op2_range || + op2_range->min < 0 || + op2_range->max >= SIZEOF_ZEND_LONG * 8) { + | cmp REG1, #(SIZEOF_ZEND_LONG*8) + | bhs >1 + |.cold_code + |1: + | cmp REG1, xzr + | mov REG1w, #((SIZEOF_ZEND_LONG * 8) - 1) + | bgt >1 + | SET_EX_OPLINE opline, REG0 + | b ->negative_shift + |.code + } + |1: + | asr Rx(result_reg), Rx(result_reg), REG1 + } + } else if (opcode == ZEND_MOD) { + // REG0 -> result_reg + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); + + if (op2_lval == 0) { + | SET_EX_OPLINE opline, REG0 + | b ->mod_by_zero + } else if (op2_lval == -1) { + | mov REG0, xzr + } else { + | GET_ZVAL_LVAL ZREG_REG1, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_REG2, op2_addr, TMP1 + | sdiv REG0, REG1, REG2 + | msub REG0, REG0, REG2, REG1 + } + } else { + if (!op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { + if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { + | SAFE_MEM_ACC_WITH_UOFFSET ldr, TMP1, Rx(Z_REG(op2_addr)), Z_OFFSET(op2_addr), TMP2 + | cbz TMP1, >1 + } else if (Z_MODE(op2_addr) == IS_REG) { + | cbz Rx(Z_REG(op2_addr)), >1 + } + |.cold_code + |1: + | SET_EX_OPLINE opline, REG0 + | b ->mod_by_zero + |.code + } + + /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ + if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { + if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { + | SAFE_MEM_ACC_WITH_UOFFSET ldr, TMP1, Rx(Z_REG(op2_addr)), Z_OFFSET(op2_addr), TMP2 + | cmn TMP1, #1 + } else if (Z_MODE(op2_addr) == IS_REG) { + | cmn Rx(Z_REG(op2_addr)), #1 + } + | beq >1 + |.cold_code + |1: + | SET_ZVAL_LVAL_FROM_REG res_addr, xzr, TMP1 + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + } + } + } + | b >5 + |.code + } + + | GET_ZVAL_LVAL ZREG_REG1, op1_addr, TMP1 + if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { + | ldr TMP1, [Rx(Z_REG(op2_addr)), #Z_OFFSET(op2_addr)] + | sdiv REG0, REG1, TMP1 + | msub REG0, REG0, TMP1, REG1 + } else if (Z_MODE(op2_addr) == IS_REG) { + | sdiv REG0, REG1, Rx(Z_REG(op2_addr)) + | msub REG0, REG0, Rx(Z_REG(op2_addr)), REG1 + } + } + } else if (same_ops) { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) + } else { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | LONG_MATH opcode, result_reg, op2_addr, TMP1 + } + + if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { + | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 + } + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + } + } + } + + if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { + if ((op1_info & MAY_BE_LONG) && + (op2_info & MAY_BE_LONG)) { + |.cold_code + } + |6: + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + | LOAD_ZVAL_ADDR FCARG1x, real_addr + } else if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, res_addr + } + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } + | LOAD_ZVAL_ADDR CARG3, op2_addr + | SET_EX_OPLINE opline, REG0 + if (opcode == ZEND_BW_OR) { + | EXT_CALL bitwise_or_function, REG0 + } else if (opcode == ZEND_BW_AND) { + | EXT_CALL bitwise_and_function, REG0 + } else if (opcode == ZEND_BW_XOR) { + | EXT_CALL bitwise_xor_function, REG0 + } else if (opcode == ZEND_SL) { + | EXT_CALL shift_left_function, REG0 + } else if (opcode == ZEND_SR) { + | EXT_CALL shift_right_function, REG0 + } else if (opcode == ZEND_MOD) { + | EXT_CALL mod_function, REG0 + } else { + ZEND_UNREACHABLE(); + } + | FREE_OP op1_type, op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception(Dst); + } + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { + return 0; + } + } + if ((op1_info & MAY_BE_LONG) && + (op2_info & MAY_BE_LONG)) { + | b >5 + |.code + } + } + |5: + + return 1; +} + +static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) +{ + ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); + ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); + + if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, + opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, + opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, + opline->result.var, res_addr, res_info, res_use_info, may_throw)) { + return 0; + } + if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { + return 0; + } + return 1; +} + +static int zend_jit_concat_helper(dasm_State **Dst, + const zend_op *opline, + zend_uchar op1_type, + znode_op op1, + zend_jit_addr op1_addr, + uint32_t op1_info, + zend_uchar op2_type, + znode_op op2, + zend_jit_addr op2_addr, + uint32_t op2_info, + zend_jit_addr res_addr, + int may_throw) +{ + if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 + } + if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6, ZREG_TMP1 + } + if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { + if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, res_addr + } + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + | EXT_CALL zend_jit_fast_assign_concat_helper, REG0 + } else { + if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, res_addr + } + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + | LOAD_ZVAL_ADDR CARG3, op2_addr + | EXT_CALL zend_jit_fast_concat_helper, REG0 + } + /* concatination with empty string may increase refcount */ + op1_info |= MAY_BE_RCN; + op2_info |= MAY_BE_RCN; + | FREE_OP op1_type, op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + |5: + } + if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || + (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { + if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { + |.cold_code + |6: + } + if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, res_addr + } + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + | LOAD_ZVAL_ADDR CARG3, op2_addr + | SET_EX_OPLINE opline, REG0 + | EXT_CALL concat_function, REG0 + /* concatination with empty string may increase refcount */ + op1_info |= MAY_BE_RCN; + op2_info |= MAY_BE_RCN; + | FREE_OP op1_type, op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception(Dst); + } + if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { + | b <5 + |.code + } + } + + return 1; +} + +static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) +{ + zend_jit_addr op1_addr, op2_addr; + + ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); + ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); + + op1_addr = OP1_ADDR(); + op2_addr = OP2_ADDR(); + + return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); +} + +static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) +/* Labels: 1,2,3,4,5 */ +{ + zend_jit_addr op2_addr = OP2_ADDR(); + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && type == BP_VAR_R + && !exit_addr) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + } + + if (op2_info & MAY_BE_LONG) { + bool op2_loaded = 0; + bool packed_loaded = 0; + + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { + | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 + } + if (op1_info & MAY_BE_PACKED_GUARD) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + if (op1_info & MAY_BE_ARRAY_PACKED) { + | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] + | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w + | beq &exit_addr + } else { + | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] + | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w + | bne &exit_addr + } + } + if (type == BP_VAR_W) { + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + op2_loaded = 1; + } + if (op1_info & MAY_BE_ARRAY_PACKED) { + zend_long val = -1; + + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + val = Z_LVAL_P(Z_ZV(op2_addr)); + if (val >= 0 && val < HT_MAX_SIZE) { + packed_loaded = 1; + } + } else { + if (!op2_loaded) { + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + op2_loaded = 1; + } + packed_loaded = 1; + } + if (packed_loaded) { + | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + if (op1_info & MAY_BE_ARRAY_HASH) { + | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] + | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w + | beq >4 // HASH_FIND + } + | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) + + | ldr REG0w, [FCARG1x, #offsetof(zend_array, nNumUsed)] + if (val == 0) { + | cmp REG0, xzr + } else if (val > 0 && !op2_loaded) { + | CMP_64_WITH_CONST REG0, val, TMP1 + } else { + | cmp REG0, FCARG2x + } + + if (type == BP_JIT_IS) { + if (not_found_exit_addr) { + | bls ¬_found_exit_addr + } else { + | bls >9 // NOT_FOUND + } + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + | bls &exit_addr + } else if (type == BP_VAR_IS && not_found_exit_addr) { + | bls ¬_found_exit_addr + } else if (type == BP_VAR_IS && found_exit_addr) { + | bls >7 // NOT_FOUND + } else { + | bls >2 // NOT_FOUND + } + | // _ret = &_ht->arData[_h].val; + if (val >= 0) { + | ldr REG0, [FCARG1x, #offsetof(zend_array, arData)] + if (val != 0) { + | ADD_SUB_64_WITH_CONST add, REG0, REG0, (val * sizeof(Bucket)), TMP1 + } + } else { + | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] + | add REG0, TMP1, FCARG2x, lsl #5 + } + } + } + switch (type) { + case BP_JIT_IS: + if (op1_info & MAY_BE_ARRAY_HASH) { + if (packed_loaded) { + | b >5 + } + |4: + if (!op2_loaded) { + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + } + | EXT_CALL _zend_hash_index_find, REG0 + | mov REG0, RETVALx + if (not_found_exit_addr) { + | cbz REG0, ¬_found_exit_addr + } else { + | cbz REG0, >9 // NOT_FOUND + } + if (op2_info & MAY_BE_STRING) { + | b >5 + } + } else if (packed_loaded) { + if (op2_info & MAY_BE_STRING) { + | b >5 + } + } else if (not_found_exit_addr) { + | b ¬_found_exit_addr + } else { + | b >9 // NOT_FOUND + } + break; + case BP_VAR_R: + case BP_VAR_IS: + case BP_VAR_UNSET: + if (packed_loaded) { + if (op1_info & MAY_BE_ARRAY_HASH) { + | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + /* perform IS_UNDEF check only after result type guard (during deoptimization) */ + if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_HASH)) { + | IF_Z_TYPE REG0, IS_UNDEF, &exit_addr, TMP1w + } + } else if (type == BP_VAR_IS && not_found_exit_addr) { + | IF_Z_TYPE REG0, IS_UNDEF, ¬_found_exit_addr, TMP1w + } else if (type == BP_VAR_IS && found_exit_addr) { + | IF_Z_TYPE REG0, IS_UNDEF, >7, TMP1w // NOT_FOUND + } else { + | IF_Z_TYPE REG0, IS_UNDEF, >2, TMP1w // NOT_FOUND + } + } + if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_HASH))) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + | b &exit_addr + } else if (type == BP_VAR_IS && not_found_exit_addr) { + | b ¬_found_exit_addr + } else if (type == BP_VAR_IS && found_exit_addr) { + | b >7 // NOT_FOUND + } else { + | b >2 // NOT_FOUND + } + } + if (op1_info & MAY_BE_ARRAY_HASH) { + |4: + if (!op2_loaded) { + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + } + | EXT_CALL _zend_hash_index_find, REG0 + | mov REG0, RETVALx + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + | cbz REG0, &exit_addr + } else if (type == BP_VAR_IS && not_found_exit_addr) { + | cbz REG0, ¬_found_exit_addr + } else if (type == BP_VAR_IS && found_exit_addr) { + | cbz REG0, >7 // NOT_FOUND + } else { + | cbz REG0, >2 // NOT_FOUND + } + } + |.cold_code + |2: + switch (type) { + case BP_VAR_R: + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { + | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); + | // retval = &EG(uninitialized_zval); + | UNDEFINED_OFFSET opline + | b >9 + } + break; + case BP_VAR_IS: + case BP_VAR_UNSET: + if (!not_found_exit_addr && !found_exit_addr) { + | // retval = &EG(uninitialized_zval); + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + | b >9 + } + break; + default: + ZEND_UNREACHABLE(); + } + |.code + break; + case BP_VAR_RW: + if (packed_loaded) { + | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w + } + |2: + |4: + if (!op2_loaded) { + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + } + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_hash_index_lookup_rw, REG0 + | mov REG0, RETVALx + | cbz REG0, >9 + break; + case BP_VAR_W: + if (packed_loaded) { + | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w + } + if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || packed_loaded) { + |2: + | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); + if (!op2_loaded) { + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + } + | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval + | EXT_CALL zend_hash_index_add_new, REG0 + | mov REG0, RETVALx + if (op1_info & MAY_BE_ARRAY_HASH) { + | b >8 + } + } + if (op1_info & MAY_BE_ARRAY_HASH) { + |4: + if (!op2_loaded) { + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + } + | EXT_CALL zend_hash_index_lookup, REG0 + | mov REG0, RETVALx + } + break; + default: + ZEND_UNREACHABLE(); + } + + if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { + | b >8 + } + } + + if (op2_info & MAY_BE_STRING) { + |3: + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { + | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) + | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, ZREG_TMP1 + } + | // offset_key = Z_STR_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | // retval = zend_hash_find(ht, offset_key); + switch (type) { + case BP_JIT_IS: + if (opline->op2_type != IS_CONST) { + | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] + | cmp TMP1w, #((uint8_t) ('9')) + | ble >1 + |.cold_code + |1: + | EXT_CALL zend_jit_symtable_find, REG0 + | b >1 + |.code + | EXT_CALL zend_hash_find, REG0 + |1: + } else { + | EXT_CALL _zend_hash_find_known_hash, REG0 + } + | mov REG0, RETVALx + if (not_found_exit_addr) { + | cbz REG0, ¬_found_exit_addr + } else { + | cbz REG0, >9 // NOT_FOUND + } + break; + case BP_VAR_R: + case BP_VAR_IS: + case BP_VAR_UNSET: + if (opline->op2_type != IS_CONST) { + | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] + | cmp TMP1w, #((uint8_t) ('9')) + | ble >1 + |.cold_code + |1: + | EXT_CALL zend_jit_symtable_find, REG0 + | b >1 + |.code + | EXT_CALL zend_hash_find, REG0 + |1: + } else { + | EXT_CALL _zend_hash_find_known_hash, REG0 + } + | mov REG0, RETVALx + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + | cbz REG0, &exit_addr + } else if (type == BP_VAR_IS && not_found_exit_addr) { + | cbz REG0, ¬_found_exit_addr + } else if (type == BP_VAR_IS && found_exit_addr) { + | cbz REG0, >7 + } else { + | cbz REG0, >2 // NOT_FOUND + |.cold_code + |2: + switch (type) { + case BP_VAR_R: + // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); + | UNDEFINED_INDEX opline + | b >9 + break; + case BP_VAR_IS: + case BP_VAR_UNSET: + | // retval = &EG(uninitialized_zval); + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + | b >9 + break; + default: + ZEND_UNREACHABLE(); + } + |.code + } + break; + case BP_VAR_RW: + | SET_EX_OPLINE opline, REG0 + if (opline->op2_type != IS_CONST) { + | EXT_CALL zend_jit_symtable_lookup_rw, REG0 + } else { + | EXT_CALL zend_jit_hash_lookup_rw, REG0 + } + | mov REG0, RETVALx + | cbz REG0, >9 + break; + case BP_VAR_W: + if (opline->op2_type != IS_CONST) { + | EXT_CALL zend_jit_symtable_lookup_w, REG0 + } else { + | EXT_CALL zend_hash_lookup, REG0 + } + | mov REG0, RETVALx + break; + default: + ZEND_UNREACHABLE(); + } + } + + if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { + |5: + if (op1_info & MAY_BE_ARRAY_OF_REF) { + | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w + } + | ldrb TMP1w, [REG0,#offsetof(zval, u1.v.type)] + | cmp TMP1w, #IS_NULL + if (not_found_exit_addr) { + | ble ¬_found_exit_addr + } else if (found_exit_addr) { + | bgt &found_exit_addr + } else { + | ble >9 // NOT FOUND + } + } + + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + |.cold_code + |3: + } + | SET_EX_OPLINE opline, REG0 + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + switch (type) { + case BP_VAR_R: + | LOAD_ZVAL_ADDR CARG3, res_addr + | EXT_CALL zend_jit_fetch_dim_r_helper, REG0 + | mov REG0, RETVALx + | b >9 + break; + case BP_JIT_IS: + | EXT_CALL zend_jit_fetch_dim_isset_helper, REG0 + | mov REG0, RETVALx + if (not_found_exit_addr) { + | cbz REG0, ¬_found_exit_addr + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + | b >8 + } + } else if (found_exit_addr) { + | cbnz REG0, &found_exit_addr + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + | b >9 + } + } else { + | cbnz REG0, >8 + | b >9 + } + break; + case BP_VAR_IS: + case BP_VAR_UNSET: + | LOAD_ZVAL_ADDR CARG3, res_addr + | EXT_CALL zend_jit_fetch_dim_is_helper, REG0 + | mov REG0, RETVALx + | b >9 + break; + case BP_VAR_RW: + | EXT_CALL zend_jit_fetch_dim_rw_helper, REG0 + | mov REG0, RETVALx + | cbnz REG0, >8 + | b >9 + break; + case BP_VAR_W: + | EXT_CALL zend_jit_fetch_dim_w_helper, REG0 + | mov REG0, RETVALx + | cbnz REG0, >8 + | b >9 + break; + default: + ZEND_UNREACHABLE(); + } + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + |.code + } + } + + return 1; +} + +static int zend_jit_simple_assign(dasm_State **Dst, + const zend_op *opline, + zend_jit_addr var_addr, + uint32_t var_info, + uint32_t var_def_info, + zend_uchar val_type, + zend_jit_addr val_addr, + uint32_t val_info, + zend_jit_addr res_addr, + int in_cold, + int save_r1) +/* Labels: 1,2,3 */ +{ + zend_reg tmp_reg; + + if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_REG0) { + tmp_reg = ZREG_REG0; + } else { + /* ASSIGN_DIM */ + tmp_reg = ZREG_FCARG1x; + } + + if (Z_MODE(val_addr) == IS_CONST_ZVAL) { + zval *zv = Z_ZV(val_addr); + + if (!res_addr) { + | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 + } else { + | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 + } + if (Z_REFCOUNTED_P(zv)) { + if (!res_addr) { + | ADDREF_CONST zv, TMP1, TMP2 + } else { + | ADDREF_CONST_2 zv, TMP1, TMP2 + } + } + } else { + if (val_info & MAY_BE_UNDEF) { + if (in_cold) { + | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, ZREG_TMP1 + } else { + | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 + |.cold_code + |1: + } + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + if (save_r1) { + | str FCARG1x, T1 // save + } + | SET_ZVAL_TYPE_INFO var_addr, IS_NULL, TMP1w, TMP2 + if (res_addr) { + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + } + if (opline) { + | SET_EX_OPLINE opline, Rx(tmp_reg) + } + ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); + | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) + | EXT_CALL zend_jit_undefined_op_helper, REG0 + if (save_r1) { + | ldr FCARG1x, T1 // restore + } + | b >3 + if (in_cold) { + |2: + } else { + |.code + } + } + if (val_info & MAY_BE_REF) { + if (val_type == IS_CV) { + ZEND_ASSERT(Z_REG(var_addr) != ZREG_REG2); + if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_REG2 || Z_OFFSET(val_addr) != 0) { + | LOAD_ZVAL_ADDR REG2, val_addr + } + | ZVAL_DEREF REG2, val_info, TMP1w + val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); + } else { + zend_jit_addr ref_addr; + + if (in_cold) { + | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 + } else { + | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 + |.cold_code + |1: + } + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | GET_ZVAL_PTR REG2, val_addr, TMP1 + | GC_DELREF REG2, TMP1w + | // ZVAL_COPY_VALUE(return_value, &ref->val); + ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, offsetof(zend_reference, val)); + if (!res_addr) { + | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } else { + | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + | beq >2 // GC_DELREF() reached zero + | IF_NOT_REFCOUNTED REG2w, >3, TMP1w + if (!res_addr) { + | GC_ADDREF Rx(tmp_reg), TMP1w + } else { + | GC_ADDREF_2 Rx(tmp_reg), TMP1w + } + | b >3 + |2: + if (res_addr) { + | IF_NOT_REFCOUNTED REG2w, >2, TMP1w + | GC_ADDREF Rx(tmp_reg), TMP1w + |2: + } + if (save_r1) { + | str FCARG1x, T1 // save + } + | GET_ZVAL_PTR FCARG1x, val_addr, TMP1 + | EFREE_REFERENCE + if (save_r1) { + | ldr FCARG1x, T1 // restore + } + | b >3 + if (in_cold) { + |1: + } else { + |.code + } + } + } + + if (!res_addr) { + | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } else { + | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + + if (val_type == IS_CV) { + if (!res_addr) { + | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w + } else { + | TRY_ADDREF_2 val_info, REG2w, Rx(tmp_reg), TMP1w + } + } else { + if (res_addr) { + | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w + } + } + |3: + } + return 1; +} + +static int zend_jit_assign_to_typed_ref(dasm_State **Dst, + const zend_op *opline, + zend_uchar val_type, + zend_jit_addr val_addr, + bool check_exception) +{ + | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { + | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] + | cbnz TMP1, >2 + |.cold_code + |2: + if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2x || Z_OFFSET(val_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG2x, val_addr + } + if (opline) { + | SET_EX_OPLINE opline, REG0 + } + if (val_type == IS_CONST) { + | EXT_CALL zend_jit_assign_const_to_typed_ref, REG0 + } else if (val_type == IS_TMP_VAR) { + | EXT_CALL zend_jit_assign_tmp_to_typed_ref, REG0 + } else if (val_type == IS_VAR) { + | EXT_CALL zend_jit_assign_var_to_typed_ref, REG0 + } else if (val_type == IS_CV) { + | EXT_CALL zend_jit_assign_cv_to_typed_ref, REG0 + } else { + ZEND_UNREACHABLE(); + } + if (check_exception) { + | // if (UNEXPECTED(EG(exception) != NULL)) { + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbz REG0, >8 // END OF zend_jit_assign_to_variable() + | b ->exception_handler + } else { + | b >8 + } + |.code + + return 1; +} + +static int zend_jit_assign_to_variable_call(dasm_State **Dst, + const zend_op *opline, + zend_jit_addr __var_use_addr, + zend_jit_addr var_addr, + uint32_t __var_info, + uint32_t __var_def_info, + zend_uchar val_type, + zend_jit_addr val_addr, + uint32_t val_info, + zend_jit_addr __res_addr, + bool __check_exception) +{ + if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1x || Z_OFFSET(var_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, var_addr + } + if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2x || Z_OFFSET(val_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG2x, val_addr + } + if (opline) { + | SET_EX_OPLINE opline, REG0 + } + if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { + | bl ->assign_tmp + } else if (val_type == IS_CONST) { + | bl ->assign_const + } else if (val_type == IS_TMP_VAR) { + | bl ->assign_tmp + } else if (val_type == IS_VAR) { + if (!(val_info & MAY_BE_REF)) { + | bl ->assign_tmp + } else { + | bl ->assign_var + } + } else if (val_type == IS_CV) { + if (!(val_info & MAY_BE_REF)) { + | bl ->assign_cv_noref + } else { + | bl ->assign_cv + } + } else { + ZEND_UNREACHABLE(); + } + + return 1; +} + +static int zend_jit_assign_to_variable(dasm_State **Dst, + const zend_op *opline, + zend_jit_addr var_use_addr, + zend_jit_addr var_addr, + uint32_t var_info, + uint32_t var_def_info, + zend_uchar val_type, + zend_jit_addr val_addr, + uint32_t val_info, + zend_jit_addr res_addr, + bool check_exception) +/* Labels: 1,2,3,4,5,8 */ +{ + int done = 0; + zend_reg ref_reg, tmp_reg; + + if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_REG0) { + ref_reg = ZREG_FCARG1x; + tmp_reg = ZREG_REG0; + } else { + /* ASSIGN_DIM */ + ref_reg = ZREG_REG0; + tmp_reg = ZREG_FCARG1x; + } + + if (var_info & MAY_BE_REF) { + if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { + | LOAD_ZVAL_ADDR Rx(ref_reg), var_use_addr + var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); + } + | // if (Z_ISREF_P(variable_ptr)) { + | IF_NOT_Z_TYPE Rx(ref_reg), IS_REFERENCE, >1, TMP1w + | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { + | GET_Z_PTR FCARG1x, Rx(ref_reg) + if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, check_exception)) { + return 0; + } + | add Rx(ref_reg), FCARG1x, #offsetof(zend_reference, val) + |1: + } + if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + if (RC_MAY_BE_1(var_info)) { + int in_cold = 0; + + if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_ZVAL_REFCOUNTED var_use_addr, >1, ZREG_TMP1, ZREG_TMP2 + |.cold_code + |1: + in_cold = 1; + } + if (Z_REG(var_use_addr) == ZREG_FCARG1x || Z_REG(var_use_addr) == ZREG_REG0) { + bool keep_gc = 0; + + | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 +#if 0 + // TODO: This optiization doesn't work on ARM + if (tmp_reg == ZREG_FCARG1x) { + if (Z_MODE(val_addr) == IS_REG) { + keep_gc = 1; + } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { + keep_gc = 1; + } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { + zval *zv = Z_ZV(val_addr); + if (Z_TYPE_P(zv) == IS_DOUBLE) { + if (Z_DVAL_P(zv) == 0) { + keep_gc = 1; + } + } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { + keep_gc = 1; + } + } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { + if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { + keep_gc = 1; + } + } + } +#endif + if (!keep_gc) { + | str Rx(tmp_reg), T1 // save + } + if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0)) { + return 0; + } + if (!keep_gc) { + | ldr FCARG1x, T1 // restore + } + } else { + | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 + if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1)) { + return 0; + } + } + | GC_DELREF FCARG1x, TMP1w + if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { + | bne >4 + } else { + | bne >8 + } + | ZVAL_DTOR_FUNC var_info, opline, TMP1 + if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { + if (check_exception) { + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbz REG0, >8 + | b ->exception_handler + } else { + | b >8 + } + } + if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { + |4: + | IF_GC_MAY_NOT_LEAK FCARG1x, >8, TMP1w, TMP2w + | EXT_CALL gc_possible_root, REG0 + if (in_cold) { + | b >8 + } + } + if (in_cold) { + |.code + } else { + done = 1; + } + } else /* if (RC_MAY_BE_N(var_info)) */ { + if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5, ZREG_TMP1, ZREG_TMP2 + } + if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { + if (Z_REG(var_use_addr) == ZREG_FP) { + | str Rx(Z_REG(var_use_addr)), T1 // save + } + | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 + | GC_DELREF FCARG1x, TMP1w + | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w + | EXT_CALL gc_possible_root, TMP1 + if (Z_REG(var_use_addr) != ZREG_FP) { + | ldr Rx(Z_REG(var_use_addr)), T1 // restore + } + } else { + | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 + | GC_DELREF Rx(tmp_reg), TMP1w + } + |5: + } + } + + if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0)) { + return 0; + } + + |8: + + return 1; +} + +static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, int may_throw) +{ + zend_jit_addr op2_addr, op3_addr, res_addr; + + op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; + op3_addr = OP1_DATA_ADDR(); + if (opline->result_type == IS_UNUSED) { + res_addr = 0; + } else { + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + } + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr, ZREG_TMP1 + + val_info &= ~MAY_BE_UNDEF; + } + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w + | GET_Z_PTR FCARG2x, FCARG1x + | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] + | cmp TMP1w, #IS_ARRAY + | bne >2 + | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) + | b >3 + |.cold_code + |2: + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 + | mov FCARG1x, RETVALx + | cbnz FCARG1x, >1 + | b ->exception_handler_undef + |.code + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 + } + |3: + | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | CMP_ZVAL_TYPE op1_addr, IS_FALSE, ZREG_TMP1 + | bgt >7 + } + | // ZVAL_ARR(container, zend_new_array(8)); + if (Z_REG(op1_addr) != ZREG_FP) { + | str Rx(Z_REG(op1_addr)), T1 // save + } + | EXT_CALL _zend_new_array_0, REG0 + | mov REG0, RETVALx + if (Z_REG(op1_addr) != ZREG_FP) { + | ldr Rx(Z_REG(op1_addr)), T1 // restore + } + | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 + | mov FCARG1x, REG0 + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |6: + if (opline->op2_type == IS_UNUSED) { + uint32_t var_info = MAY_BE_NULL; + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); + | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval + | EXT_CALL zend_hash_next_index_insert, REG0 + | // if (UNEXPECTED(!var_ptr)) { + | mov REG0, RETVALx + | cbz REG0, >1 + |.cold_code + |1: + | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); + | CANNOT_ADD_ELEMENT opline + | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); + | b >9 + |.code + + if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) { + return 0; + } + } else { + uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, NULL, NULL, NULL)) { + return 0; + } + + if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { + var_info |= MAY_BE_REF; + } + if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + var_info |= MAY_BE_RC1; + } + + |8: + | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); + if (opline->op1_type == IS_VAR) { + ZEND_ASSERT(opline->result_type == IS_UNUSED); + if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { + return 0; + } + } else { + if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { + return 0; + } + } + } + } + + if (((op1_info & MAY_BE_ARRAY) && + (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE))) || + (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) { + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |.cold_code + |7: + } + + if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) && + (op1_info & MAY_BE_ARRAY)) { + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | CMP_ZVAL_TYPE op1_addr, IS_FALSE, ZREG_TMP1 + | bgt >2 + } + | // ZVAL_ARR(container, zend_new_array(8)); + if (Z_REG(op1_addr) != ZREG_FP) { + | str Rx(Z_REG(op1_addr)), T1 // save + } + | EXT_CALL _zend_new_array_0, REG0 + | mov REG0, RETVALx + if (Z_REG(op1_addr) != ZREG_FP) { + | ldr Rx(Z_REG(op1_addr)), T1 // restore + } + | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 + | mov FCARG1x, REG0 + | // ZEND_VM_C_GOTO(assign_dim_op_new_array); + | b <6 + |2: + } + + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type == IS_UNUSED) { + | mov FCARG2x, xzr + } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + if (opline->result_type == IS_UNUSED) { + | mov CARG4, xzr + } else { + | LOAD_ZVAL_ADDR CARG4, res_addr + } + | LOAD_ZVAL_ADDR CARG3, op3_addr + | EXT_CALL zend_jit_assign_dim_helper, REG0 + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { + /* ASSIGN_DIM may increase refcount of the value */ + val_info |= MAY_BE_RCN; + } +#endif + + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | b >9 // END + } + |.code + } + } + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { + /* ASSIGN_DIM may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + + |9: + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + + if (may_throw) { + zend_jit_check_exception(Dst); + } + + return 1; +} + +static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, int may_throw) +{ + zend_jit_addr op2_addr, op3_addr, var_addr; + + ZEND_ASSERT(opline->result_type == IS_UNUSED); + + op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; + op3_addr = OP1_DATA_ADDR(); + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w + | GET_Z_PTR FCARG2x, FCARG1x + | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] + | cmp TMP1w, #IS_ARRAY + | bne >2 + | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) + | b >3 + |.cold_code + |2: + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 + | mov FCARG1x, RETVALx + | cbnz RETVALx, >1 + | b ->exception_handler_undef + |.code + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 + } + |3: + | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 + } + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | CMP_ZVAL_TYPE op1_addr, IS_FALSE, ZREG_TMP1 + | bgt >7 + } + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 + } + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1x, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + |1: + } + | // ZVAL_ARR(container, zend_new_array(8)); + if (Z_REG(op1_addr) != ZREG_FP) { + | str Rx(Z_REG(op1_addr)), T1 // save + } + | EXT_CALL _zend_new_array_0, REG0 + | mov REG0, RETVALx + if (Z_REG(op1_addr) != ZREG_FP) { + | ldr Rx(Z_REG(op1_addr)), T1 // restore + } + | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 + | mov FCARG1x, REG0 + if (op1_info & MAY_BE_ARRAY) { + | b >1 + |.code + |1: + } + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + uint32_t var_info; + uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); + + |6: + if (opline->op2_type == IS_UNUSED) { + var_info = MAY_BE_NULL; + + | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); + | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval + | EXT_CALL zend_hash_next_index_insert, REG0 + | mov REG0, RETVALx + | // if (UNEXPECTED(!var_ptr)) { + | cbz REG0, >1 + |.cold_code + |1: + | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); + | CANNOT_ADD_ELEMENT opline + | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); + | b >9 + |.code + } else { + var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); + if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { + var_info |= MAY_BE_REF; + } + if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + var_info |= MAY_BE_RC1; + } + + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, NULL, NULL, NULL)) { + return 0; + } + + |8: + if (op1_info & (MAY_BE_ARRAY_OF_REF)) { + binary_op_type binary_op = get_binary_op(opline->extended_value); + | IF_NOT_Z_TYPE, REG0, IS_REFERENCE, >1, TMP1w + | GET_Z_PTR FCARG1x, REG0 + | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] + | cbnz TMP1, >2 + | add REG0, FCARG1x, #offsetof(zend_reference, val) + |.cold_code + |2: + | LOAD_ZVAL_ADDR FCARG2x, op3_addr + | LOAD_ADDR CARG3, binary_op + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 + zend_jit_check_exception(Dst); + | b >9 + |.code + |1: + } + } + + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + switch (opline->extended_value) { + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + case ZEND_DIV: + if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, + 1 /* may overflow */, may_throw)) { + return 0; + } + break; + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + case ZEND_SL: + case ZEND_SR: + case ZEND_MOD: + if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, + IS_CV, opline->op1, var_addr, var_info, NULL, + (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, + op1_data_range, + 0, var_addr, var_def_info, var_info, may_throw)) { + return 0; + } + break; + case ZEND_CONCAT: + if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, + may_throw)) { + return 0; + } + break; + default: + ZEND_UNREACHABLE(); + } + } + + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + binary_op_type binary_op; + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |.cold_code + |7: + } + + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type == IS_UNUSED) { + | mov FCARG2x, xzr + } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + binary_op = get_binary_op(opline->extended_value); + | LOAD_ZVAL_ADDR CARG3, op3_addr + | LOAD_ADDR CARG4, binary_op + | EXT_CALL zend_jit_assign_dim_op_helper, REG0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + | b >9 // END + |.code + } + } + + |9: + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + + return 1; +} + +static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) +{ + zend_jit_addr op1_addr, op2_addr; + + ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); + ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); + + op1_addr = OP1_ADDR(); + op2_addr = OP2_ADDR(); + + if (op1_info & MAY_BE_REF) { + binary_op_type binary_op = get_binary_op(opline->extended_value); + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >1, TMP1w + | GET_Z_PTR FCARG1x, FCARG1x + | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] + | cbnz TMP1, >2 + | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) + |.cold_code + |2: + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + | LOAD_ADDR CARG3, binary_op + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 + zend_jit_check_exception(Dst); + | b >9 + |.code + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + int result; + switch (opline->extended_value) { + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + case ZEND_DIV: + result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); + break; + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + case ZEND_SL: + case ZEND_SR: + case ZEND_MOD: + result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, + opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, + opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, + opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); + break; + case ZEND_CONCAT: + result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); + break; + default: + ZEND_UNREACHABLE(); + } + |9: + return result; +} + +static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, + zend_ssa_range *op1_range, + zend_jit_addr op1_addr, + zend_ssa_range *op2_range, + zend_jit_addr op2_addr, + bool *result) +{ + zend_long op1_min; + zend_long op1_max; + zend_long op2_min; + zend_long op2_max; + + if (op1_range) { + op1_min = op1_range->min; + op1_max = op1_range->max; + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { + ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG); + op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr)); + } else { + return 0; + } + + if (op2_range) { + op2_min = op2_range->min; + op2_max = op2_range->max; + } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG); + op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr)); + } else { + return 0; + } + + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { + *result = 1; + return 1; + } else if (op1_max < op2_min || op1_min > op2_max) { + *result = 0; + return 1; + } + return 0; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { + *result = 0; + return 1; + } else if (op1_max < op2_min || op1_min > op2_max) { + *result = 1; + return 1; + } + return 0; + case ZEND_IS_SMALLER: + if (op1_max < op2_min) { + *result = 1; + return 1; + } else if (op1_min >= op2_max) { + *result = 0; + return 1; + } + return 0; + case ZEND_IS_SMALLER_OR_EQUAL: + if (op1_max <= op2_min) { + *result = 1; + return 1; + } else if (op1_min > op2_max) { + *result = 0; + return 1; + } + return 0; + default: + ZEND_UNREACHABLE(); + } + return 0; +} + +static int zend_jit_cmp_long_long(dasm_State **Dst, + const zend_op *opline, + zend_ssa_range *op1_range, + zend_jit_addr op1_addr, + zend_ssa_range *op2_range, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + zend_uchar smart_branch_opcode, + uint32_t target_label, + uint32_t target_label2, + const void *exit_addr, + bool skip_comparison) +{ + bool swap = 0; + bool result; + + if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { + if (!smart_branch_opcode || + smart_branch_opcode == ZEND_JMPZ_EX || + smart_branch_opcode == ZEND_JMPNZ_EX) { + | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE), TMP1w, TMP2 + } + if (smart_branch_opcode && !exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ || + smart_branch_opcode == ZEND_JMPZ_EX) { + if (!result) { + | b => target_label + } + } else if (smart_branch_opcode == ZEND_JMPNZ || + smart_branch_opcode == ZEND_JMPNZ_EX) { + if (result) { + | b => target_label + } + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + if (!result) { + | b => target_label + } else { + | b => target_label2 + } + } else { + ZEND_UNREACHABLE(); + } + } + return 1; + } + + if (skip_comparison) { + if (Z_MODE(op1_addr) != IS_REG && + (Z_MODE(op2_addr) == IS_REG || + (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { + swap = 1; + } + } else if (Z_MODE(op1_addr) == IS_REG) { + if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { + | cmp Rx(Z_REG(op1_addr)), xzr + } else { + | LONG_CMP Z_REG(op1_addr), op2_addr, TMP1 + } + } else if (Z_MODE(op2_addr) == IS_REG) { + if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { + | cmp Rx(Z_REG(op2_addr)), xzr + } else { + | LONG_CMP Z_REG(op2_addr), op1_addr, TMP1 + } + swap = 1; + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { + | LONG_CMP_WITH_CONST op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2 + swap = 1; + } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { + | LONG_CMP_WITH_CONST op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2 + } else { + | GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1 + if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { + | cmp Rx(ZREG_REG0), xzr + } else { + | LONG_CMP ZREG_REG0, op2_addr, TMP1 + } + } + + if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ_EX || + smart_branch_opcode == ZEND_JMPNZ_EX) { + + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | cset REG0w, eq + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | cset REG0w, ne + break; + case ZEND_IS_SMALLER: + if (swap) { + | cset REG0w, gt + } else { + | cset REG0w, lt + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | cset REG0w, ge + } else { + | cset REG0w, le + } + break; + default: + ZEND_UNREACHABLE(); + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + if (smart_branch_opcode == ZEND_JMPZ || + smart_branch_opcode == ZEND_JMPZ_EX) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + if (exit_addr) { + | bne &exit_addr + } else { + | bne => target_label + } + break; + case ZEND_IS_NOT_EQUAL: + if (exit_addr) { + | beq &exit_addr + } else { + | beq => target_label + } + break; + case ZEND_IS_NOT_IDENTICAL: + if (exit_addr) { + | bne &exit_addr + } else { + | beq => target_label + } + break; + case ZEND_IS_SMALLER: + if (swap) { + if (exit_addr) { + | ble &exit_addr + } else { + | ble => target_label + } + } else { + if (exit_addr) { + | bge &exit_addr + } else { + | bge => target_label + } + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + if (exit_addr) { + | blt &exit_addr + } else { + | blt => target_label + } + } else { + if (exit_addr) { + | bgt &exit_addr + } else { + | bgt => target_label + } + } + break; + default: + ZEND_UNREACHABLE(); + } + } else if (smart_branch_opcode == ZEND_JMPNZ || + smart_branch_opcode == ZEND_JMPNZ_EX) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + if (exit_addr) { + | beq &exit_addr + } else { + | beq => target_label + } + break; + case ZEND_IS_NOT_EQUAL: + if (exit_addr) { + | bne &exit_addr + } else { + | bne => target_label + } + break; + case ZEND_IS_NOT_IDENTICAL: + if (exit_addr) { + | beq &exit_addr + } else { + | bne => target_label + } + break; + case ZEND_IS_SMALLER: + if (swap) { + if (exit_addr) { + | bgt &exit_addr + } else { + | bgt => target_label + } + } else { + if (exit_addr) { + | blt &exit_addr + } else { + | blt => target_label + } + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + if (exit_addr) { + | bge &exit_addr + } else { + | bge => target_label + } + } else { + if (exit_addr) { + | ble &exit_addr + } else { + | ble => target_label + } + } + break; + default: + ZEND_UNREACHABLE(); + } + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | bne => target_label + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | beq => target_label + break; + case ZEND_IS_SMALLER: + if (swap) { + | ble => target_label + } else { + | bge => target_label + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | blt => target_label + } else { + | bgt => target_label + } + break; + default: + ZEND_UNREACHABLE(); + } + | b => target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | cset REG0w, eq + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | cset REG0w, ne + break; + case ZEND_IS_SMALLER: + if (swap) { + | cset REG0w, gt + } else { + | cset REG0w, lt + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | cset REG0w, ge + } else { + | cset REG0w, le + } + break; + default: + ZEND_UNREACHABLE(); + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + + return 1; +} + +static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + if (exit_addr) { + | bne &exit_addr + } else { + | bne => target_label + } + break; + case ZEND_IS_NOT_EQUAL: + | bvs >1 + if (exit_addr) { + | beq &exit_addr + } else { + | beq => target_label + } + |1: + break; + case ZEND_IS_NOT_IDENTICAL: + if (exit_addr) { + | bvs &exit_addr + | bne &exit_addr + } else { + | bvs >1 + | beq => target_label + |1: + } + break; + case ZEND_IS_SMALLER: + if (swap) { + if (exit_addr) { + | bvs &exit_addr + | bls &exit_addr + } else { + | bvs => target_label + | bls => target_label + } + } else { + if (exit_addr) { + | bhs &exit_addr + } else { + | bhs => target_label + } + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + if (exit_addr) { + | bvs &exit_addr + | blo &exit_addr + } else { + | bvs => target_label + | blo => target_label + } + } else { + if (exit_addr) { + | bhi &exit_addr + } else { + | bhi => target_label + } + } + break; + default: + ZEND_UNREACHABLE(); + } + } else if (smart_branch_opcode == ZEND_JMPNZ) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | bvs >1 + if (exit_addr) { + | beq &exit_addr + } else { + | beq => target_label + } + |1: + break; + case ZEND_IS_NOT_EQUAL: + if (exit_addr) { + | bne &exit_addr + } else { + | bne => target_label + } + break; + case ZEND_IS_NOT_IDENTICAL: + if (exit_addr) { + | bvs >1 + | beq &exit_addr + |1: + } else { + | bne => target_label + } + break; + case ZEND_IS_SMALLER: + if (swap) { + | bvs >1 // Always False if involving NaN + if (exit_addr) { + | bhi &exit_addr + } else { + | bhi => target_label + } + |1: + } else { + | bvs >1 + if (exit_addr) { + | blo &exit_addr + } else { + | blo => target_label + } + |1: + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | bvs >1 // Always False if involving NaN + if (exit_addr) { + | bhs &exit_addr + } else { + | bhs => target_label + } + |1: + } else { + | bvs >1 + if (exit_addr) { + | bls &exit_addr + } else { + | bls => target_label + } + |1: + } + break; + default: + ZEND_UNREACHABLE(); + } + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | bne => target_label + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | bvs => target_label2 + | beq => target_label + break; + case ZEND_IS_SMALLER: + if (swap) { + | bvs => target_label + | bls => target_label + } else { + | bhs => target_label + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | bvs => target_label + | blo => target_label + } else { + | bhi => target_label + } + break; + default: + ZEND_UNREACHABLE(); + } + | b => target_label2 + } else if (smart_branch_opcode == ZEND_JMPZ_EX) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bne => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | bvs >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | beq => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + break; + case ZEND_IS_SMALLER: + if (swap) { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bvs => target_label + | bls => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bhs => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bvs => target_label + | blo => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bhi => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + break; + default: + ZEND_UNREACHABLE(); + } + } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | bvs >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | beq => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | bvs => target_label + | bne => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + break; + case ZEND_IS_SMALLER: + if (swap) { + | cset REG0w, hi + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + | bvs >1 // Always False if involving NaN + | bhi => target_label + |1: + } else { + | bvs >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | blo => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | cset REG0w, hs + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + | bvs >1 // Always False if involving NaN + | bhs => target_label + |1: + } else { + | bvs >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | bls => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + break; + default: + ZEND_UNREACHABLE(); + } + } else { + ZEND_UNREACHABLE(); + } + } else { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | bvs >1 + | mov REG0, #IS_TRUE + | beq >2 + |1: + | mov REG0, #IS_FALSE + |2: + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | bvs >1 + | mov REG0, #IS_FALSE + | beq >2 + |1: + | mov REG0, #IS_TRUE + |2: + break; + case ZEND_IS_SMALLER: + | bvs >1 + | mov REG0, #IS_TRUE + || if (swap) { + | bhi >2 + || } else { + | blo >2 + || } + |1: + | mov REG0, #IS_FALSE + |2: + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | bvs >1 + | mov REG0, #IS_TRUE + || if (swap) { + | bhs >2 + || } else { + | bls >2 + || } + |1: + | mov REG0, #IS_FALSE + |2: + break; + default: + ZEND_UNREACHABLE(); + } + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + + return 1; +} + +static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + zend_reg tmp_reg = ZREG_FPR0; + + | DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_REG0, ZREG_TMP1 + | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP + + return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); +} + +static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + zend_reg tmp_reg = ZREG_FPR0; + + | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_REG0, ZREG_TMP1 + | DOUBLE_CMP tmp_reg, op1_addr, ZREG_TMP1, ZREG_FPTMP + + return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); +} + +static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + bool swap = 0; + + if (Z_MODE(op1_addr) == IS_REG) { + | DOUBLE_CMP Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP + } else if (Z_MODE(op2_addr) == IS_REG) { + | DOUBLE_CMP Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP + swap = 1; + } else { + zend_reg tmp_reg = ZREG_FPR0; + + | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 + | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP + } + + return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); +} + +static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + | LONG_CMP_WITH_CONST res_addr, Z_L(0), TMP1, TMP2 + if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ_EX || + smart_branch_opcode == ZEND_JMPNZ_EX) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | cset REG0w, eq + break; + case ZEND_IS_NOT_EQUAL: + | cset REG0w, ne + break; + case ZEND_IS_SMALLER: + | cset REG0w, lt + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | cset REG0w, le + break; + default: + ZEND_UNREACHABLE(); + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + if (smart_branch_opcode == ZEND_JMPZ || + smart_branch_opcode == ZEND_JMPZ_EX) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + if (exit_addr) { + | bne &exit_addr + } else { + | bne => target_label + } + break; + case ZEND_IS_NOT_EQUAL: + if (exit_addr) { + | beq &exit_addr + } else { + | beq => target_label + } + break; + case ZEND_IS_SMALLER: + if (exit_addr) { + | bge &exit_addr + } else { + | bge => target_label + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (exit_addr) { + | bgt &exit_addr + } else { + | bgt => target_label + } + break; + default: + ZEND_UNREACHABLE(); + } + } else if (smart_branch_opcode == ZEND_JMPNZ || + smart_branch_opcode == ZEND_JMPNZ_EX) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + if (exit_addr) { + | beq &exit_addr + } else { + | beq => target_label + } + break; + case ZEND_IS_NOT_EQUAL: + if (exit_addr) { + | bne &exit_addr + } else { + | bne => target_label + } + break; + case ZEND_IS_SMALLER: + if (exit_addr) { + | blt &exit_addr + } else { + | blt => target_label + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (exit_addr) { + | ble &exit_addr + } else { + | ble => target_label + } + break; + default: + ZEND_UNREACHABLE(); + } + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | bne => target_label + break; + case ZEND_IS_NOT_EQUAL: + | beq => target_label + break; + case ZEND_IS_SMALLER: + | bge => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | bgt => target_label + break; + default: + ZEND_UNREACHABLE(); + } + | b => target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | cset REG0w, eq + break; + case ZEND_IS_NOT_EQUAL: + | cset REG0w, ne + break; + case ZEND_IS_SMALLER: + | cset REG0w, lt + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | cset REG0w, le + break; + default: + ZEND_UNREACHABLE(); + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + + return 1; +} + +static int zend_jit_cmp(dasm_State **Dst, + const zend_op *opline, + uint32_t op1_info, + zend_ssa_range *op1_range, + zend_jit_addr op1_addr, + uint32_t op2_info, + zend_ssa_range *op2_range, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + int may_throw, + zend_uchar smart_branch_opcode, + uint32_t target_label, + uint32_t target_label2, + const void *exit_addr, + bool skip_comparison) +{ + bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); + bool has_slow; + + has_slow = + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); + + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { + if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 + } + } + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { + if (op2_info & MAY_BE_DOUBLE) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 + |.cold_code + |3: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 + } + if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + | b >6 + |.code + } else { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 + } + } + if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + |.cold_code + |4: + if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 + } + } + if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + | b >6 + } + if (!same_ops) { + |5: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 + } + if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + | b >6 + } + |.code + } + } else if ((op1_info & MAY_BE_DOUBLE) && + !(op1_info & MAY_BE_LONG) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops && (op2_info & MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 + } + } + if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + } + if (!same_ops && (op2_info & MAY_BE_LONG)) { + if (op2_info & MAY_BE_DOUBLE) { + |.cold_code + } + |3: + if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 + } + if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + if (op2_info & MAY_BE_DOUBLE) { + | b >6 + |.code + } + } + } else if ((op2_info & MAY_BE_DOUBLE) && + !(op2_info & MAY_BE_LONG) && + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 + } + if (op1_info & MAY_BE_DOUBLE) { + if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops && (op1_info & MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 + } + } + if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + } + if (!same_ops && (op1_info & MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + |.cold_code + } + |3: + if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 + } + if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + | b >6 + |.code + } + } + } + + if (has_slow || + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + if (has_slow) { + |.cold_code + |9: + } + | SET_EX_OPLINE opline, REG0 + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { + | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w + | LOAD_32BIT_VAL FCARG1x, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval + |1: + } + if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 + | str FCARG2x, T1 // save + | LOAD_32BIT_VAL FCARG1x, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + | ldr FCARG2x, T1 // restore + | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval + | b >2 + |1: + | LOAD_ZVAL_ADDR CARG3, op2_addr + |2: + } else { + | LOAD_ZVAL_ADDR CARG3, op2_addr + } + | LOAD_ZVAL_ADDR FCARG1x, res_addr + | EXT_CALL compare_function, REG0 + if (opline->opcode != ZEND_CASE) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + if (has_slow) { + | b >6 + |.code + } + } + + |6: + + return 1; +} + +static int zend_jit_identical(dasm_State **Dst, + const zend_op *opline, + uint32_t op1_info, + zend_ssa_range *op1_range, + zend_jit_addr op1_addr, + uint32_t op2_info, + zend_ssa_range *op2_range, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + int may_throw, + zend_uchar smart_branch_opcode, + uint32_t target_label, + uint32_t target_label2, + const void *exit_addr, + bool skip_comparison) +{ + uint32_t identical_label = (uint32_t)-1; + uint32_t not_identical_label = (uint32_t)-1; + + if (smart_branch_opcode && !exit_addr) { + if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { + if (smart_branch_opcode == ZEND_JMPZ) { + not_identical_label = target_label; + } else if (smart_branch_opcode == ZEND_JMPNZ) { + identical_label = target_label; + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + not_identical_label = target_label; + identical_label = target_label2; + } else { + ZEND_UNREACHABLE(); + } + } else { + if (smart_branch_opcode == ZEND_JMPZ) { + identical_label = target_label; + } else if (smart_branch_opcode == ZEND_JMPNZ) { + not_identical_label = target_label; + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + identical_label = target_label; + not_identical_label = target_label2; + } else { + ZEND_UNREACHABLE(); + } + } + } + + if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && + (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { + if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { + return 0; + } + return 1; + } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && + (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { + if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + return 1; + } + + if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { + op1_info |= MAY_BE_NULL; + op2_info |= MAY_BE_NULL; + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w + |.cold_code + |1: + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval + | b >1 + |.code + |1: + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w + |.cold_code + |1: + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SET_EX_OPLINE opline, REG0 + | str FCARG1x, T1 // save + | LOAD_32BIT_VAL FCARG1w, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | ldr FCARG1x, T1 // restore + | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval + | b >1 + |.code + |1: + } else if (op1_info & MAY_BE_UNDEF) { + op1_info |= MAY_BE_NULL; + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w + |.cold_code + |1: + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval + | b >1 + |.code + |1: + if (opline->op2_type != IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + } else if (op2_info & MAY_BE_UNDEF) { + op2_info |= MAY_BE_NULL; + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w + |.cold_code + |1: + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval + | b >1 + |.code + |1: + if (opline->op1_type != IS_CONST) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + } else if ((op1_info & op2_info & MAY_BE_ANY) != 0) { + if (opline->op1_type != IS_CONST) { + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type != IS_CONST) { + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + } + + if ((op1_info & op2_info & MAY_BE_ANY) == 0) { + if ((opline->opcode != ZEND_CASE_STRICT && + (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || + ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { + | SET_EX_OPLINE opline, REG0 + if (opline->opcode != ZEND_CASE_STRICT) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + if (smart_branch_opcode) { + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b &exit_addr + } + } else if (not_identical_label != (uint32_t)-1) { + | b =>not_identical_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 + if (may_throw) { + zend_jit_check_exception(Dst); + } + } + return 1; + } + + if (opline->op1_type & (IS_CV|IS_VAR)) { + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + } + if (opline->op2_type & (IS_CV|IS_VAR)) { + | ZVAL_DEREF FCARG2x, op2_info, TMP1w + } + + if (has_concrete_type(op1_info) + && has_concrete_type(op2_info) + && concrete_type(op1_info) == concrete_type(op2_info) + && concrete_type(op1_info) <= IS_TRUE) { + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | b &exit_addr + } + } else if (identical_label != (uint32_t)-1) { + | b =>identical_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 + } + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { + if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | b &exit_addr + } + } else if (identical_label != (uint32_t)-1) { + | b =>identical_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 + } + } else { + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b &exit_addr + } + } else if (not_identical_label != (uint32_t)-1) { + | b =>not_identical_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 + } + } + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { + zval *val = Z_ZV(op1_addr); + + | ldrb TMP1w, [FCARG2x, #offsetof(zval, u1.v.type)] + | cmp TMP1w, #Z_TYPE_P(val) + if (smart_branch_opcode) { + if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { + | bne >8 + | SET_EX_OPLINE opline, REG0 + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { + | b &exit_addr + } else if (identical_label != (uint32_t)-1) { + | b =>identical_label + } else { + | b >9 + } + |8: + } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { + | beq &exit_addr + } else if (identical_label != (uint32_t)-1) { + | beq =>identical_label + } else { + | beq >9 + } + } else { + if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { + | cset REG0w, eq + } else { + | cset REG0w, ne + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { + | SET_EX_OPLINE opline, REG0 + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + } + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b &exit_addr + } + } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { + | b =>not_identical_label + } + } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { + zval *val = Z_ZV(op2_addr); + + | ldrb TMP1w, [FCARG1x, #offsetof(zval, u1.v.type)] + | cmp TMP1w, #Z_TYPE_P(val) + if (smart_branch_opcode) { + if (opline->opcode != ZEND_CASE_STRICT + && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { + | bne >8 + | SET_EX_OPLINE opline, REG0 + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { + | b &exit_addr + } else if (identical_label != (uint32_t)-1) { + | b =>identical_label + } else { + | b >9 + } + |8: + } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { + | beq &exit_addr + } else if (identical_label != (uint32_t)-1) { + | beq =>identical_label + } else { + | beq >9 + } + } else { + if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { + | cset REG0w, eq + } else { + | cset REG0w, ne + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + if (opline->opcode != ZEND_CASE_STRICT + && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { + | SET_EX_OPLINE opline, REG0 + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + } + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b &exit_addr + } + } else if (not_identical_label != (uint32_t)-1) { + | b =>not_identical_label + } + } + } else { + if (opline->op1_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + | EXT_CALL zend_is_identical, REG0 + if ((opline->opcode != ZEND_CASE_STRICT && + (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || + ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { + | str RETVALw, T1 // save + | SET_EX_OPLINE opline, REG0 + if (opline->opcode != ZEND_CASE_STRICT) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | ldr RETVALw, T1 // restore + } + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | cbnz RETVALw, &exit_addr + } else { + | cbz RETVALw, &exit_addr + } + } else if (not_identical_label != (uint32_t)-1) { + | cbz RETVALw, =>not_identical_label + if (identical_label != (uint32_t)-1) { + | b =>identical_label + } + } else if (identical_label != (uint32_t)-1) { + | cbnz RETVALw, =>identical_label + } + } else { + if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { + | add RETVALw, RETVALw, #2 + } else { + | neg RETVALw, RETVALw + | add RETVALw, RETVALw, #3 + } + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, RETVALw, TMP1 + } + } + + |9: + if (may_throw) { + zend_jit_check_exception(Dst); + } + return 1; +} + +static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) +{ + uint32_t true_label = -1; + uint32_t false_label = -1; + bool set_bool = 0; + bool set_bool_not = 0; + bool set_delayed = 0; + bool jmp_done = 0; + + if (branch_opcode == ZEND_BOOL) { + set_bool = 1; + } else if (branch_opcode == ZEND_BOOL_NOT) { + set_bool = 1; + set_bool_not = 1; + } else if (branch_opcode == ZEND_JMPZ) { + false_label = target_label; + } else if (branch_opcode == ZEND_JMPNZ) { + true_label = target_label; + } else if (branch_opcode == ZEND_JMPZNZ) { + true_label = target_label2; + false_label = target_label; + } else if (branch_opcode == ZEND_JMPZ_EX) { + set_bool = 1; + false_label = target_label; + } else if (branch_opcode == ZEND_JMPNZ_EX) { + set_bool = 1; + true_label = target_label; + } else { + ZEND_UNREACHABLE(); + } + + if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { + if (zend_is_true(Z_ZV(op1_addr))) { + /* Always TRUE */ + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + } + if (true_label != (uint32_t)-1) { + | b =>true_label + } + } else { + /* Always FALSE */ + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + } + if (false_label != (uint32_t)-1) { + | b =>false_label + } + } + return 1; + } + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { + if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { + /* Always TRUE */ + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + } + if (true_label != (uint32_t)-1) { + | b =>true_label + } + } else { + if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { + /* Always FALSE */ + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + } + } else { + | CMP_ZVAL_TYPE op1_addr, IS_TRUE, ZREG_TMP1 + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { + if ((op1_info & MAY_BE_LONG) && + !(op1_info & MAY_BE_UNDEF) && + !set_bool) { + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ) { + | blt >9 + } else { + | blt &exit_addr + } + } else if (false_label != (uint32_t)-1) { + | blt =>false_label + } else { + | blt >9 + } + jmp_done = 1; + } else { + | bgt >2 + } + } + if (!(op1_info & MAY_BE_TRUE)) { + /* It's FALSE */ + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + } + } else { + if (exit_addr) { + if (set_bool) { + | bne >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | b &exit_addr + } else { + | b >9 + } + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { + if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { + | bne &exit_addr + } + } + } else { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | beq &exit_addr + } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { + | bne &exit_addr + } else { + | beq >9 + } + } + } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + if (set_bool) { + | bne >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + if (true_label != (uint32_t)-1) { + | b =>true_label + } else { + | b >9 + } + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } else { + if (true_label != (uint32_t)-1) { + | beq =>true_label + } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { + | bne =>false_label + jmp_done = 1; + } else { + | beq >9 + } + } + } else if (set_bool) { + | cset REG0w, eq + if (set_bool_not) { + | neg REG0w, REG0w + | add REG0w, REG0w, #3 + } else { + | add REG0w, REG0w, #2 + } + if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { + set_delayed = 1; + } else { + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + } + } + } + + /* It's FALSE, but may be UNDEF */ + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & MAY_BE_ANY) { + if (set_delayed) { + | CMP_ZVAL_TYPE op1_addr, IS_UNDEF, ZREG_TMP1 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + | beq >1 + } else { + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 + } + |.cold_code + |1: + } + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_undefined_op_helper, REG0 + + if (may_throw) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { + return 0; + } + } + + if (exit_addr) { + if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { + | b &exit_addr + } + } else if (false_label != (uint32_t)-1) { + | b =>false_label + } + if (op1_info & MAY_BE_ANY) { + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | b >9 + } + } else if (false_label == (uint32_t)-1) { + | b >9 + } + |.code + } + } + + if (!jmp_done) { + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + if (op1_info & MAY_BE_LONG) { + | b >9 + } + } else if (op1_info & MAY_BE_LONG) { + | b &exit_addr + } + } else if (false_label != (uint32_t)-1) { + | b =>false_label + } else if (op1_info & MAY_BE_LONG) { + | b >9 + } + } + } + } + + if (op1_info & MAY_BE_LONG) { + |2: + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 + } + if (Z_MODE(op1_addr) == IS_REG) { + | tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) + } else { + | LONG_CMP_WITH_CONST op1_addr, Z_L(0), TMP1, TMP2 + } + if (set_bool) { + | cset REG0w, ne + if (set_bool_not) { + | neg REG0w, REG0w + | add REG0w, REG0w, #3 + } else { + | add REG0w, REG0w, #2 + } + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | bne &exit_addr + } else { + | beq &exit_addr + } + } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + if (true_label != (uint32_t)-1) { + | bne =>true_label + if (false_label != (uint32_t)-1) { + | b =>false_label + } + } else { + | beq =>false_label + } + } + } + + if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | mov TMP1, xzr + | fmov FPR0d, TMP1 + | DOUBLE_CMP ZREG_FPR0, op1_addr, ZREG_TMP1, ZREG_FPTMP + + if (set_bool) { + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | bvs >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | bne &exit_addr + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bvs &exit_addr + | beq &exit_addr + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + } else if (false_label != (uint32_t)-1) { // JMPZ_EX + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bvs >1 + | beq => false_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + |1: + } else if (true_label != (uint32_t)-1) { // JMPNZ_EX + | bvs >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | bne => true_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } else if (set_bool_not) { // BOOL_NOT + | bvs >1 + | mov REG0w, #IS_TRUE + | beq >2 + |1: + | mov REG0w, #IS_FALSE + |2: + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } else { // BOOL + | bvs >1 + | mov REG0w, #IS_TRUE + | bne >2 + |1: + | mov REG0w, #IS_FALSE + |2: + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + } else { + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | bvs >1 + | bne &exit_addr + |1: + } else { + | bvs &exit_addr + | beq &exit_addr + } + } else { + ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); + if (false_label != (uint32_t)-1) { + | bvs =>false_label + } else { + | bvs >1 + } + if (true_label != (uint32_t)-1) { + | bne =>true_label + if (false_label != (uint32_t)-1) { + | b =>false_label + } + } else { + | beq =>false_label + } + |1: + } + } + } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + |.cold_code + |2: + } + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_is_true, REG0 + | mov REG0, RETVALx + + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3, ZREG_TMP1, ZREG_TMP2 + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | GC_DELREF FCARG1x, TMP1w + | bne >3 + // In x86, r0 is used in macro ZVAL_DTOR_FUNC as temporary register, hence, r0 should be saved/restored + // before/after this macro. In AArch64, TMP1 is used, but we still have to store REG0, + // because it's clobbered by function call. + | str REG0, T1 // save + | ZVAL_DTOR_FUNC op1_info, opline, TMP1 + | ldr REG0, T1 // restore + |3: + } + if (may_throw) { + | MEM_LOAD_ZTS ldr, REG1, executor_globals, exception, TMP1 + | cbnz REG1, ->exception_handler_undef + } + + if (set_bool) { + if (set_bool_not) { + | neg REG0w, REG0w + | add REG0w, REG0w, #3 + } else { + | add REG0w, REG0w, #2 + } + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + if (exit_addr) { + | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | bne &exit_addr + } else { + | beq &exit_addr + } + } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 + if (true_label != (uint32_t)-1) { + | bne =>true_label + if (false_label != (uint32_t)-1) { + | b =>false_label + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | b >9 + } + } else { + | beq =>false_label + } + } + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | b >9 + |.code + } + } else { + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | cbnz REG0w, &exit_addr + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | b >9 + } + } else { + | cbz REG0w, &exit_addr + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | b >9 + } + } + } else if (true_label != (uint32_t)-1) { + | cbnz REG0w, =>true_label + if (false_label != (uint32_t)-1) { + | b =>false_label + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | b >9 + } + } else { + | cbz REG0w, =>false_label + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | b >9 + } + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + |.code + } + } + } + + |9: + + return 1; +} + +static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) +{ + if (op1_addr != op1_def_addr) { + if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { + return 0; + } + if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { + op1_addr = op1_def_addr; + } + } + + if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0)) { + return 0; + } + if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { + return 0; + } + return 1; +} + +static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) +{ + ZEND_ASSERT(opline->op1_type == IS_CV); + + if (op2_addr != op2_def_addr) { + if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { + return 0; + } + if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { + op2_addr = op2_def_addr; + } + } + + if (Z_MODE(op1_addr) != IS_REG + && Z_MODE(op1_use_addr) == IS_REG + && !Z_LOAD(op1_use_addr) + && !Z_STORE(op1_use_addr)) { + /* Force type update */ + op1_info |= MAY_BE_UNDEF; + } + if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, + may_throw)) { + return 0; + } + if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { + return 0; + } + if (opline->result_type != IS_UNUSED) { + if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { + return 0; + } + } + + return 1; +} + +/* copy of hidden zend_closure */ +typedef struct _zend_closure { + zend_object std; + zend_function func; + zval this_ptr; + zend_class_entry *called_scope; + zif_handler orig_internal_handler; +} zend_closure; + +static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) +{ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | // Check Stack Overflow + | MEM_LOAD_ZTS ldr, REG1, executor_globals, vm_stack_end, TMP1 + | MEM_LOAD_OP_ZTS sub, ldr, REG1, executor_globals, vm_stack_top, TMP1, TMP2 + | CMP_64_WITH_CONST_32 REG1, used_stack, TMP1 + | blo &exit_addr + + return 1; +} + +static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool use_this, bool stack_check) +{ + uint32_t used_stack; + + // REG0 -> zend_function + // FCARG1 -> used_stack + + if (func) { + used_stack = zend_vm_calc_used_stack(opline->extended_value, func); + } else { + used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); + + | // if (EXPECTED(ZEND_USER_CODE(func->type))) { + if (!is_closure) { + | LOAD_32BIT_VAL FCARG1w, used_stack + | // Check whether REG0 is an internal function. + | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] + | TST_32_WITH_CONST TMP1w, 1, TMP2w + | bne >1 + } else { + | LOAD_32BIT_VAL FCARG1w, used_stack + } + | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); + | LOAD_32BIT_VAL REG2w, opline->extended_value + if (!is_closure) { + | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.num_args)] + | cmp REG2w, TMP1w + | csel REG2w, REG2w, TMP1w, le + | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.last_var)] + | sub REG2w, REG2w, TMP1w + | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.T)] + | sub REG2w, REG2w, TMP1w + } else { + | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.num_args)] + | cmp REG2w, TMP1w + | csel REG2w, REG2w, TMP1w, le + | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.last_var)] + | sub REG2w, REG2w, TMP1w + | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.T)] + | sub REG2w, REG2w, TMP1w + } + | sxtw REG2, REG2w + | sub FCARG1x, FCARG1x, REG2, lsl #4 + |1: + } + + zend_jit_start_reuse_ip(); + + | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { + | MEM_LOAD_ZTS ldr, RX, executor_globals, vm_stack_top, TMP1 + + if (stack_check) { + | // Check Stack Overflow + | MEM_LOAD_ZTS ldr, REG2, executor_globals, vm_stack_end, TMP1 + | sub REG2, REG2, RX + if (func) { + | CMP_64_WITH_CONST_32 REG2, used_stack, TMP1 + } else { + | cmp REG2, FCARG1x + } + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | blo &exit_addr + } else { + | blo >1 + | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); + |.cold_code + |1: + if (func) { + | LOAD_32BIT_VAL FCARG1w, used_stack + } + if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_int_extend_stack_helper, REG0 + } else { + if (!is_closure) { + | mov FCARG2x, REG0 + } else { + | add FCARG2x, REG0, #offsetof(zend_closure, func) + } + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_extend_stack_helper, REG0 + } + | mov RX, RETVALx + | b >1 + |.code + } + } + + if (func) { + || if (used_stack <= ADD_SUB_IMM) { + | MEM_LOAD_OP_STORE_ZTS add, ldr, str, #used_stack, executor_globals, vm_stack_top, REG2, TMP1 + || } else { + | LOAD_32BIT_VAL TMP1w, used_stack + | MEM_LOAD_OP_STORE_ZTS add, ldr, str, TMP1, executor_globals, vm_stack_top, REG2, TMP2 + || } + } else { + | MEM_LOAD_OP_STORE_ZTS add, ldr, str, FCARG1x, executor_globals, vm_stack_top, REG2, TMP1 + } + | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { + | // ZEND_SET_CALL_INFO(call, 0, call_info); + | LOAD_32BIT_VAL TMP1w, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) + | str TMP1w, EX:RX->This.u1.type_info + } + if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { + | // call->func = func; + |1: + | ADDR_STORE EX:RX->func, func, REG1 + } else { + if (!is_closure) { + | // call->func = func; + if (func + && op_array == &func->op_array + && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) + && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { + | LOAD_ADDR TMP1, func + | str TMP1, EX:RX->func + } else { + | str REG0, EX:RX->func + } + } else { + | // call->func = &closure->func; + | add REG1, REG0, #offsetof(zend_closure, func) + | str REG1, EX:RX->func + } + |1: + } + if (opline->opcode == ZEND_INIT_METHOD_CALL) { + | // Z_PTR(call->This) = obj; + | ldr REG1, T1 + | str REG1, EX:RX->This.value.ptr + if (opline->op1_type == IS_UNUSED || use_this) { + | // call->call_info |= ZEND_CALL_HAS_THIS; + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS + | str TMP1w, EX:RX->This.u1.type_info + } else { + | LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS + | ldr TMP2w, EX:RX->This.u1.type_info + | orr TMP2w, TMP2w, TMP1w + | str TMP2w, EX:RX->This.u1.type_info + } + } else { + if (opline->op1_type == IS_CV) { + | // GC_ADDREF(obj); + | GC_ADDREF REG1, TMP1w + } + | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) + | str TMP1w, EX:RX->This.u1.type_info + } else { + | ldr TMP1w, EX:RX->This.u1.type_info + | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS), TMP2w + | str TMP1w, EX:RX->This.u1.type_info + } + } + } else if (!is_closure) { + | // Z_CE(call->This) = called_scope; + | str xzr, EX:RX->This.value.ptr + } else { + if (opline->op2_type == IS_CV) { + | // GC_ADDREF(closure); + | ldr TMP1w, [REG0] + | add TMP1w, TMP1w, #1 + | str TMP1w, [REG0] + } + | // object_or_called_scope = closure->called_scope; + | ldr REG1, [REG0, #offsetof(zend_closure, called_scope)] + | str REG1, EX:RX->This.value.ptr + | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | + | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); + | ldr REG2w, [REG0, #offsetof(zend_closure, func.common.fn_flags)] + | BW_OP_32_WITH_CONST and, REG2w, REG2w, ZEND_ACC_FAKE_CLOSURE, TMP1w + | BW_OP_32_WITH_CONST orr, REG2w, REG2w, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE), TMP1w + | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { + | ldrb TMP1w, [REG0, #offsetof(zend_closure, this_ptr.u1.v.type)] + | cmp TMP1w, #IS_UNDEF + | beq >1 + | // call_info |= ZEND_CALL_HAS_THIS; + | BW_OP_32_WITH_CONST orr, REG2w, REG2w, ZEND_CALL_HAS_THIS, TMP1w + | // object_or_called_scope = Z_OBJ(closure->this_ptr); + | ldr REG1, [REG0, #offsetof(zend_closure, this_ptr.value.ptr)] + |1: + | // ZEND_SET_CALL_INFO(call, 0, call_info); + | ldr TMP1w, EX:RX->This.u1.type_info + | orr TMP1w, TMP1w, REG2w + | str TMP1w, EX:RX->This.u1.type_info + | // Z_PTR(call->This) = object_or_called_scope; + | str REG1, EX:RX->This.value.ptr + | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.run_time_cache__ptr)] + | cbnz TMP1, >1 + | add FCARG1x, REG0, #offsetof(zend_closure, func) + | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 + |1: + } + | // ZEND_CALL_NUM_ARGS(call) = num_args; + | LOAD_32BIT_VAL TMP1w, opline->extended_value + | str TMP1w, EX:RX->This.u2.num_args + + return 1; +} + +static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, zend_jit_trace_rec *trace) +{ + int skip; + + if (trace) { + zend_jit_trace_rec *p = trace; + + ssa_op++; + while (1) { + if (p->op == ZEND_JIT_TRACE_VM) { + switch (p->opline->opcode) { + case ZEND_SEND_ARRAY: + case ZEND_SEND_USER: + case ZEND_SEND_UNPACK: + case ZEND_INIT_FCALL: + case ZEND_INIT_METHOD_CALL: + case ZEND_INIT_STATIC_METHOD_CALL: + case ZEND_INIT_FCALL_BY_NAME: + case ZEND_INIT_NS_FCALL_BY_NAME: + case ZEND_INIT_DYNAMIC_CALL: + case ZEND_NEW: + case ZEND_INIT_USER_CALL: + case ZEND_FAST_CALL: + case ZEND_JMP: + case ZEND_JMPZNZ: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_FE_RESET_R: + case ZEND_FE_RESET_RW: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_JMP_NULL: + case ZEND_ASSERT_CHECK: + case ZEND_CATCH: + case ZEND_DECLARE_ANON_CLASS: + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + return 1; + case ZEND_DO_ICALL: + case ZEND_DO_UCALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_DO_FCALL: + return 0; + case ZEND_SEND_VAL: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: + case ZEND_SEND_REF: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + /* skip */ + break; + default: + if (zend_may_throw(opline, ssa_op, op_array, ssa)) { + return 1; + } + } + ssa_op += zend_jit_trace_op_len(opline); + } else if (p->op == ZEND_JIT_TRACE_ENTER || + p->op == ZEND_JIT_TRACE_BACK || + p->op == ZEND_JIT_TRACE_END) { + return 1; + } + p++; + } + } + + if (!call_info) { + const zend_op *end = op_array->opcodes + op_array->last; + + opline++; + ssa_op++; + skip = 1; + while (opline != end) { + if (!skip) { + if (zend_may_throw(opline, ssa_op, op_array, ssa)) { + return 1; + } + } + switch (opline->opcode) { + case ZEND_SEND_VAL: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: + case ZEND_SEND_REF: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + skip = 0; + break; + case ZEND_SEND_ARRAY: + case ZEND_SEND_USER: + case ZEND_SEND_UNPACK: + case ZEND_INIT_FCALL: + case ZEND_INIT_METHOD_CALL: + case ZEND_INIT_STATIC_METHOD_CALL: + case ZEND_INIT_FCALL_BY_NAME: + case ZEND_INIT_NS_FCALL_BY_NAME: + case ZEND_INIT_DYNAMIC_CALL: + case ZEND_NEW: + case ZEND_INIT_USER_CALL: + case ZEND_FAST_CALL: + case ZEND_JMP: + case ZEND_JMPZNZ: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_FE_RESET_R: + case ZEND_FE_RESET_RW: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_JMP_NULL: + case ZEND_ASSERT_CHECK: + case ZEND_CATCH: + case ZEND_DECLARE_ANON_CLASS: + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + return 1; + case ZEND_DO_ICALL: + case ZEND_DO_UCALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_DO_FCALL: + end = opline; + if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { + /* INIT_FCALL and DO_FCALL in different BasicBlocks */ + return 1; + } + return 0; + } + opline++; + ssa_op++; + } + + return 1; + } else { + const zend_op *end = call_info->caller_call_opline; + + if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { + /* INIT_FCALL and DO_FCALL in different BasicBlocks */ + return 1; + } + + opline++; + ssa_op++; + skip = 1; + while (opline != end) { + if (skip) { + switch (opline->opcode) { + case ZEND_SEND_VAL: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: + case ZEND_SEND_REF: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + skip = 0; + break; + case ZEND_SEND_ARRAY: + case ZEND_SEND_USER: + case ZEND_SEND_UNPACK: + return 1; + } + } else { + if (zend_may_throw(opline, ssa_op, op_array, ssa)) { + return 1; + } + } + opline++; + ssa_op++; + } + + return 0; + } +} + +static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) +{ + int32_t exit_point; + const void *exit_addr; + + if (func->type == ZEND_INTERNAL_FUNCTION) { +#ifdef ZEND_WIN32 + // TODO: ASLR may cause different addresses in different workers ??? + return 0; +#endif + } else if (func->type == ZEND_USER_FUNCTION) { + if (!zend_accel_in_shm(func->op_array.opcodes)) { + /* op_array and op_array->opcodes are not persistent. We can't link. */ + return 0; + } + } else { + ZEND_UNREACHABLE(); + return 0; + } + + exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + + | // call = EX(call); + | ldr REG1, EX->call + while (level > 0) { + | ldr REG1, EX:REG1->prev_execute_data + level--; + } + + if (func->type == ZEND_USER_FUNCTION && + (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || + (func->common.fn_flags & ZEND_ACC_CLOSURE) || + !func->common.function_name)) { + const zend_op *opcodes = func->op_array.opcodes; + + | ldr REG1, EX:REG1->func + | LOAD_ADDR REG2, ((ptrdiff_t)opcodes) + | ldr TMP1, [REG1, #offsetof(zend_op_array, opcodes)] + | cmp TMP1, REG2 + | bne &exit_addr + } else { + | LOAD_ADDR REG2, ((ptrdiff_t)func) + | ldr TMP1, EX:REG1->func + | cmp TMP1, REG2 + | bne &exit_addr + } + + return 1; +} + +static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, bool stack_check) +{ + zend_func_info *info = ZEND_FUNC_INFO(op_array); + zend_call_info *call_info = NULL; + zend_function *func = NULL; + + if (delayed_call_chain) { + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } + } + + if (info) { + call_info = info->callee_info; + while (call_info && call_info->caller_init_opline != opline) { + call_info = call_info->next_callee; + } + if (call_info && call_info->callee_func && !call_info->is_prototype) { + func = call_info->callee_func; + } + } + + if (!func + && trace + && trace->op == ZEND_JIT_TRACE_INIT_CALL) { + func = (zend_function*)trace->func; + } + + if (opline->opcode == ZEND_INIT_FCALL + && func + && func->type == ZEND_INTERNAL_FUNCTION) { + /* load constant address later */ + } else if (func && op_array == &func->op_array) { + /* recursive call */ + | ldr REG0, EX->func + } else { + | // if (CACHED_PTR(opline->result.num)) + | ldr REG0, EX->run_time_cache + | ldr REG0, [REG0, #opline->result.num] + | cbz REG0, >1 + |.cold_code + |1: + if (opline->opcode == ZEND_INIT_FCALL + && func + && func->type == ZEND_USER_FUNCTION + && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { + | LOAD_ADDR FCARG1x, func + | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 + | ldr REG1, EX->run_time_cache + | mov REG0, RETVALx + || ZEND_ASSERT(opline->result.num <= LDR_STR_PIMM64); + | str REG0, [REG1, #opline->result.num] + | b >3 + } else { + zval *zv = RT_CONSTANT(opline, opline->op2); + + if (opline->opcode == ZEND_INIT_FCALL) { + | LOAD_ADDR FCARG1x, Z_STR_P(zv); + | EXT_CALL zend_jit_find_func_helper, REG0 + } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { + | LOAD_ADDR FCARG1x, Z_STR_P(zv + 1); + | EXT_CALL zend_jit_find_func_helper, REG0 + } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { + | LOAD_ADDR FCARG1x, zv; + | EXT_CALL zend_jit_find_ns_func_helper, REG0 + } else { + ZEND_UNREACHABLE(); + } + | // CACHE_PTR(opline->result.num, fbc); + | ldr REG1, EX->run_time_cache + | // Get the return value of function zend_jit_find_func_helper/zend_jit_find_ns_func_helper + | mov REG0, RETVALx + | str REG0, [REG1, #opline->result.num] + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + if (!func || opline->opcode == ZEND_INIT_FCALL) { + | cbnz REG0, >3 + } else if (func->type == ZEND_USER_FUNCTION + && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { + const zend_op *opcodes = func->op_array.opcodes; + + | LOAD_ADDR REG1, ((ptrdiff_t)opcodes) + | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] + | cmp TMP1, REG1 + | beq >3 + } else { + | LOAD_ADDR REG1, ((ptrdiff_t)func) + | cmp REG0, REG1 + | beq >3 + } + | b &exit_addr + } else { + | cbnz REG0, >3 + | // SAVE_OPLINE(); + | SET_EX_OPLINE opline, REG0 + | b ->undefined_function + } + } + |.code + |3: + } + + if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, stack_check)) { + return 0; + } + + if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, trace)) { + if (!zend_jit_save_call_chain(Dst, call_level)) { + return 0; + } + } else { + delayed_call_chain = 1; + delayed_call_level = call_level; + } + + return 1; +} + +static int zend_jit_init_method_call(dasm_State **Dst, + const zend_op *opline, + uint32_t b, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + int call_level, + uint32_t op1_info, + zend_jit_addr op1_addr, + zend_class_entry *ce, + bool ce_is_instanceof, + bool use_this, + zend_class_entry *trace_ce, + zend_jit_trace_rec *trace, + bool stack_check, + bool polymorphic_side_trace) +{ + zend_func_info *info = ZEND_FUNC_INFO(op_array); + zend_call_info *call_info = NULL; + zend_function *func = NULL; + zval *function_name; + + ZEND_ASSERT(opline->op2_type == IS_CONST); + ZEND_ASSERT(op1_info & MAY_BE_OBJECT); + + function_name = RT_CONSTANT(opline, opline->op2); + + if (info) { + call_info = info->callee_info; + while (call_info && call_info->caller_init_opline != opline) { + call_info = call_info->next_callee; + } + if (call_info && call_info->callee_func && !call_info->is_prototype) { + func = call_info->callee_func; + } + } + + if (polymorphic_side_trace) { + /* function is passed in r0 from parent_trace */ + } else { + if (opline->op1_type == IS_UNUSED || use_this) { + zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); + + | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 + } else { + if (op1_info & MAY_BE_REF) { + if (opline->op1_type == IS_CV) { + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } else { + /* Hack: Convert reference to regular value to simplify JIT code */ + ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | EXT_CALL zend_jit_unref_helper, REG0 + |1: + } + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 + |.cold_code + |1: + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | SET_EX_OPLINE opline, REG0 + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { + | EXT_CALL zend_jit_invalid_method_call_tmp, REG0 + } else { + | EXT_CALL zend_jit_invalid_method_call, REG0 + } + | b ->exception_handler + |.code + } + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + } + + if (delayed_call_chain) { + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } + } + + | str FCARG1x, T1 // save + + if (func) { + | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); + | ldr REG0, EX->run_time_cache + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 + | cbz REG0, >1 + } else { + | // if (CACHED_PTR(opline->result.num) == obj->ce)) { + | ldr REG0, EX->run_time_cache + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG2, REG0, opline->result.num, TMP1 + | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] + | cmp REG2, TMP1 + | bne >1 + | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 + } + + |.cold_code + |1: + | LOAD_ADDR FCARG2x, function_name + if (TMP_ZVAL_OFFSET == 0) { + | mov CARG3, sp + } else { + | add CARG3, sp, #TMP_ZVAL_OFFSET + } + | SET_EX_OPLINE opline, REG0 + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { + | EXT_CALL zend_jit_find_method_tmp_helper, REG0 + } else { + | EXT_CALL zend_jit_find_method_helper, REG0 + } + | mov REG0, RETVALx + | cbnz REG0, >2 + | b ->exception_handler + |.code + |2: + } + + if (!func + && trace + && trace->op == ZEND_JIT_TRACE_INIT_CALL + && trace->func + ) { + int32_t exit_point; + const void *exit_addr; + + exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_METHOD_CALL); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + + func = (zend_function*)trace->func; + + if (func->type == ZEND_USER_FUNCTION && + (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || + (func->common.fn_flags & ZEND_ACC_CLOSURE) || + !func->common.function_name)) { + const zend_op *opcodes = func->op_array.opcodes; + + | LOAD_ADDR TMP1, opcodes + | ldr TMP2, [REG0, #offsetof(zend_op_array, opcodes)] + | cmp TMP2, TMP1 + | bne &exit_addr + } else { + | LOAD_ADDR TMP1, func + | cmp REG0, TMP1 + | bne &exit_addr + } + } + + if (!func) { + | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { + | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] + | TST_32_WITH_CONST TMP1w, ZEND_ACC_STATIC, TMP2w + | bne >1 + |.cold_code + |1: + } + + if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { + | ldr FCARG1x, T1 // restore + | mov FCARG2x, REG0 + | LOAD_32BIT_VAL CARG3w, opline->extended_value + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { + | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, REG0 + } else { + | EXT_CALL zend_jit_push_static_metod_call_frame, REG0 + } + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !use_this)) { + | cbz RETVALx, ->exception_handler + } + | mov RX, RETVALx + } + + if (!func) { + | b >9 + |.code + } + + if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { + if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, use_this, stack_check)) { + return 0; + } + } + + if (!func) { + |9: + } + zend_jit_start_reuse_ip(); + + if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, trace)) { + if (!zend_jit_save_call_chain(Dst, call_level)) { + return 0; + } + } else { + delayed_call_chain = 1; + delayed_call_level = call_level; + } + + return 1; +} + +static int zend_jit_init_closure_call(dasm_State **Dst, + const zend_op *opline, + uint32_t b, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + int call_level, + zend_jit_trace_rec *trace, + bool stack_check) +{ + zend_function *func = NULL; + zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + + | GET_ZVAL_PTR REG0, op2_addr, TMP1 + + if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure + && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | LOAD_ADDR FCARG1x, ((ptrdiff_t)zend_ce_closure) + | ldr, TMP1, [REG0, #offsetof(zend_object, ce)] + | cmp TMP1, FCARG1x + | bne &exit_addr + if (ssa->var_info && ssa_op->op2_use >= 0) { + ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; + ssa->var_info[ssa_op->op2_use].is_instanceof = 0; + } + } + + if (trace + && trace->op == ZEND_JIT_TRACE_INIT_CALL + && trace->func + && trace->func->type == ZEND_USER_FUNCTION) { + const zend_op *opcodes; + int32_t exit_point; + const void *exit_addr; + + func = (zend_function*)trace->func; + opcodes = func->op_array.opcodes; + exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + + | LOAD_ADDR FCARG1x, ((ptrdiff_t)opcodes) + | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.opcodes)] + | cmp TMP1, FCARG1x + | bne &exit_addr + } + + if (delayed_call_chain) { + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } + } + + if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, stack_check)) { + return 0; + } + + if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, trace)) { + if (!zend_jit_save_call_chain(Dst, call_level)) { + return 0; + } + } else { + delayed_call_chain = 1; + delayed_call_level = call_level; + } + + if (trace + && trace->op == ZEND_JIT_TRACE_END + && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + if (!zend_jit_set_valid_ip(Dst, opline + 1)) { + return 0; + } + } + + return 1; +} + +static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info) +{ + uint32_t num_args = 0; + zend_function *func = call_info->callee_func; + + /* It's okay to handle prototypes here, because they can only increase the accepted arguments. + * Anything legal for the parent method is also legal for the parent method. */ + while (num_args < call_info->num_args) { + zend_arg_info *arg_info = func->op_array.arg_info + num_args; + + if (ZEND_TYPE_IS_SET(arg_info->type)) { + if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) { + zend_op *opline = call_info->arg_info[num_args].opline; + zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; + uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type); + if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) { + break; + } + } else { + break; + } + } + num_args++; + } + return num_args; +} + +static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) +{ + zend_func_info *info = ZEND_FUNC_INFO(op_array); + zend_call_info *call_info = NULL; + const zend_function *func = NULL; + uint32_t i; + zend_jit_addr res_addr; + uint32_t call_num_args = 0; + bool unknown_num_args = 0; + const void *exit_addr = NULL; + const zend_op *prev_opline; + + if (RETURN_VALUE_USED(opline)) { + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + } else { + /* CPU stack allocated temporary zval */ + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RSP, TMP_ZVAL_OFFSET); + } + + prev_opline = opline - 1; + while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { + prev_opline--; + } + if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || + prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { + unknown_num_args = 1; + } + + if (info) { + call_info = info->callee_info; + while (call_info && call_info->caller_call_opline != opline) { + call_info = call_info->next_callee; + } + if (call_info && call_info->callee_func && !call_info->is_prototype) { + func = call_info->callee_func; + } + } + if (!func) { + /* resolve function at run time */ + } else if (func->type == ZEND_USER_FUNCTION) { + ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); + call_num_args = call_info->num_args; + } else if (func->type == ZEND_INTERNAL_FUNCTION) { + ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); + call_num_args = call_info->num_args; + } else { + ZEND_UNREACHABLE(); + } + + if (trace && !func) { + if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { + ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); +#ifndef ZEND_WIN32 + // TODO: ASLR may cause different addresses in different workers ??? + func = trace->func; + if (JIT_G(current_frame) && + JIT_G(current_frame)->call && + TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { + call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); + } else { + unknown_num_args = 1; + } +#endif + } else if (trace->op == ZEND_JIT_TRACE_ENTER) { + ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); + if (zend_accel_in_shm(trace->func->op_array.opcodes)) { + func = trace->func; + if (JIT_G(current_frame) && + JIT_G(current_frame)->call && + TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { + call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); + } else { + unknown_num_args = 1; + } + } + } + } + + bool may_have_extra_named_params = + opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && + (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); + + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | ldr RX, EX->call + } + zend_jit_stop_reuse_ip(); + + | // fbc = call->func; + | // mov r2, EX:RX->func ??? + | // SAVE_OPLINE(); + | SET_EX_OPLINE opline, REG0 + + if (opline->opcode == ZEND_DO_FCALL) { + if (!func) { + if (trace) { + uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | ldr REG0, EX:RX->func + | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] + | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w + | bne &exit_addr + } + } + } + + if (!delayed_call_chain) { + if (call_level == 1) { + | str xzr, EX->call + } else { + | //EX(call) = call->prev_execute_data; + | ldr REG0, EX:RX->prev_execute_data + | str REG0, EX->call + } + } + delayed_call_chain = 0; + + | //call->prev_execute_data = execute_data; + | str EX, EX:RX->prev_execute_data + + if (!func) { + | ldr REG0, EX:RX->func + } + + if (opline->opcode == ZEND_DO_FCALL) { + if (!func) { + if (!trace) { + | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] + | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w + | bne >1 + |.cold_code + |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, RX + } + | EXT_CALL zend_jit_deprecated_helper, REG0 + | GET_LOW_8BITS RETVALw, RETVALw + | ldr REG0, EX:RX->func // reload + | cbnz RETVALw, >1 // Result is 0 on exception + | b ->exception_handler + |.code + |1: + } + } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, RX + } + | EXT_CALL zend_jit_deprecated_helper, REG0 + | cbz RETVALw, ->exception_handler + } + } + + if (!func + && opline->opcode != ZEND_DO_UCALL + && opline->opcode != ZEND_DO_ICALL) { + | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] + | cmp TMP1w, #ZEND_USER_FUNCTION + | bne >8 + } + + if ((!func || func->type == ZEND_USER_FUNCTION) + && opline->opcode != ZEND_DO_ICALL) { + | // EX(call) = NULL; + | str xzr, EX:RX->call + + if (RETURN_VALUE_USED(opline)) { + | // EX(return_value) = EX_VAR(opline->result.var); + | LOAD_ZVAL_ADDR REG2, res_addr + | str REG2, EX:RX->return_value + } else { + | // EX(return_value) = 0; + | str xzr, EX:RX->return_value + } + + //EX_LOAD_RUN_TIME_CACHE(op_array); + if (!func || func->op_array.cache_size) { + if (func && op_array == &func->op_array) { + /* recursive call */ + if (trace || func->op_array.cache_size > sizeof(void*)) { + | ldr REG2, EX->run_time_cache + | str REG2, EX:RX->run_time_cache + } + } else { + if (func) { + | ldr REG0, EX:RX->func + } + | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] +// Always defined as ZEND_MAP_PTR_KIND_PTR_OR_OFFSET. See Zend/zend_map_ptr.h. +#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR + | ldr REG2, [REG2] +#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET + if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { + if (ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { + | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 + } else if ((func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) + && (!func->op_array.scope || (func->op_array.scope->ce_flags & ZEND_ACC_LINKED))) { + | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 + } else { + /* the called op_array may be not persisted yet */ + | TST_64_WITH_ONE REG2 + | beq >1 + | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 + |1: + } + | ldr REG2, [REG2] + } else { + | TST_64_WITH_ONE REG2 + | beq >1 + | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 + |1: + | ldr REG2, [REG2] + } +#else +# error "Unknown ZEND_MAP_PTR_KIND" +#endif + | str REG2, EX:RX->run_time_cache + } + } + + | // EG(current_execute_data) = execute_data; + | MEM_STORE_ZTS str, RX, executor_globals, current_execute_data, REG1 + | mov FP, RX + + | // opline = op_array->opcodes; + if (func && !unknown_num_args) { + | ADD_SUB_64_WITH_CONST_32 add, TMP1, RX, (EX_NUM_TO_VAR(call_num_args) + offsetof(zval, u1.type_info)), TMP1 // induction variable + for (i = call_num_args; i < func->op_array.last_var; i++) { + | // ZVAL_UNDEF(EX_VAR(n)) + | str wzr, [TMP1], #16 + } + + if (call_num_args <= func->op_array.num_args) { + if (!trace || (trace->op == ZEND_JIT_TRACE_END + && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { + uint32_t num_args; + + if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { + if (trace) { + num_args = 0; + } else if (call_info) { + num_args = skip_valid_arguments(op_array, ssa, call_info); + } else { + num_args = call_num_args; + } + } else { + num_args = call_num_args; + } + if (zend_accel_in_shm(func->op_array.opcodes)) { + | LOAD_IP_ADDR (func->op_array.opcodes + num_args) + } else { + | ldr REG0, EX->func + || ZEND_ASSERT((num_args * sizeof(zend_op)) <= ADD_SUB_IMM); + if (GCC_GLOBAL_REGS) { + | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] + if (num_args) { + | add IP, IP, #(num_args * sizeof(zend_op)) + } + } else { + | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] + if (num_args) { + | add FCARG1x, FCARG1x, #(num_args * sizeof(zend_op)) + } + | str FCARG1x, EX->opline + } + } + + if (!trace && op_array == &func->op_array) { + /* recursive call */ + if (ZEND_OBSERVER_ENABLED) { + | SAVE_IP + | mov FCARG1x, FP + | EXT_CALL zend_observer_fcall_begin, REG0 + } +#ifdef CONTEXT_THREADED_JIT + | NIY // TODO +#else + | b =>num_args +#endif + return 1; + } + } + } else { + if (!trace || (trace->op == ZEND_JIT_TRACE_END + && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { + if (func && zend_accel_in_shm(func->op_array.opcodes)) { + | LOAD_IP_ADDR (func->op_array.opcodes) + } else if (GCC_GLOBAL_REGS) { + | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] + } else { + | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] + | str FCARG1x, EX->opline + } + } + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, FP + } + | EXT_CALL zend_jit_copy_extra_args_helper, REG0 + } + } else { + | // opline = op_array->opcodes + if (func && zend_accel_in_shm(func->op_array.opcodes)) { + | LOAD_IP_ADDR (func->op_array.opcodes) + } else if (GCC_GLOBAL_REGS) { + | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] + } else { + | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] + | str FCARG1x, EX->opline + } + if (func) { + | // num_args = EX_NUM_ARGS(); + | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] + | // if (UNEXPECTED(num_args > first_extra_arg)) + | CMP_32_WITH_CONST REG1w, (func->op_array.num_args), TMP1w + } else { + | // first_extra_arg = op_array->num_args; + | ldr REG2w, [REG0, #offsetof(zend_op_array, num_args)] + | // num_args = EX_NUM_ARGS(); + | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] + | // if (UNEXPECTED(num_args > first_extra_arg)) + | cmp REG1w, REG2w + } + | bgt >1 + |.cold_code + |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, FP + } + | EXT_CALL zend_jit_copy_extra_args_helper, REG0 + if (!func) { + | ldr REG0, EX->func // reload + } + | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] // reload + | b >1 + |.code + if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { + if (!func) { + | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) + | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] + | TST_32_WITH_CONST TMP1w, ZEND_ACC_HAS_TYPE_HINTS, TMP2w + | bne >1 + } + | // opline += num_args; + || ZEND_ASSERT(sizeof(zend_op) == 32); + | mov REG2w, REG1w + | ADD_IP_SHIFT REG2, lsl #5, TMP1 + } + |1: + | // if (EXPECTED((int)num_args < op_array->last_var)) { + if (func) { + | LOAD_32BIT_VAL REG2w, func->op_array.last_var + } else { + | ldr REG2w, [REG0, #offsetof(zend_op_array, last_var)] + } + | subs REG2w, REG2w, REG1w + | ble >3 + | // zval *var = EX_VAR_NUM(num_args); + | add REG1, FP, REG1, lsl #4 + || ZEND_ASSERT(ZEND_CALL_FRAME_SLOT * sizeof(zval) <= ADD_SUB_IMM); + | add REG1, REG1, #(ZEND_CALL_FRAME_SLOT * sizeof(zval)) + |2: + | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w + | add REG1, REG1, #16 + | subs REG2w, REG2w, #1 + | bne <2 + |3: + } + + if (ZEND_OBSERVER_ENABLED) { + | SAVE_IP + | mov FCARG1x, FP + | EXT_CALL zend_observer_fcall_begin, REG0 + } + + if (trace) { + if (!func && (opline->opcode != ZEND_DO_UCALL)) { + | b >9 + } + } else { +#ifdef CONTEXT_THREADED_JIT + | NIY // TODO: CONTEXT_THREADED_JIT is always undefined. +#else + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + | JMP_IP TMP1 + } else if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], # SPAD // stack alignment + | JMP_IP TMP1 + } else { + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | mov RETVALx, #1 // ZEND_VM_ENTER + | ret + } + } +#endif + } + + if ((!func || func->type == ZEND_INTERNAL_FUNCTION) + && (opline->opcode != ZEND_DO_UCALL)) { + if (!func && (opline->opcode != ZEND_DO_ICALL)) { + |8: + } + if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { + if (!func) { + if (trace) { + uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] + | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w + | bne &exit_addr + } else { + | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] + | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w + | bne >1 + |.cold_code + |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, RX + } + | EXT_CALL zend_jit_deprecated_helper, REG0 + | GET_LOW_8BITS RETVALw, RETVALw + | ldr REG0, EX:RX->func // reload + | cbnz RETVALw, >1 // Result is 0 on exception + | b ->exception_handler + |.code + |1: + } + } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, RX + } + | EXT_CALL zend_jit_deprecated_helper, REG0 + | cbz RETVALw, ->exception_handler + | ldr REG0, EX:RX->func // reload + } + } + + | // ZVAL_NULL(EX_VAR(opline->result.var)); + | LOAD_ZVAL_ADDR FCARG2x, res_addr + | SET_Z_TYPE_INFO FCARG2x, IS_NULL, TMP1w + + | // EG(current_execute_data) = execute_data; + | MEM_STORE_ZTS str, RX, executor_globals, current_execute_data, REG1 + + zend_jit_reset_last_valid_opline(); + + | // fbc->internal_function.handler(call, ret); + | mov FCARG1x, RX + if (func) { + | EXT_CALL func->internal_function.handler, REG0 + } else { + | ldr TMP1, [REG0, #offsetof(zend_internal_function, handler)] + | blr TMP1 + } + + | // EG(current_execute_data) = execute_data; + | MEM_STORE_ZTS str, FP, executor_globals, current_execute_data, REG0 + + | // zend_vm_stack_free_args(call); + if (func && !unknown_num_args) { + for (i = 0; i < call_num_args; i++ ) { + uint32_t offset = EX_NUM_TO_VAR(i); + zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset); + | ZVAL_PTR_DTOR arg_addr, (MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN), 0, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + } else { + | mov FCARG1x, RX + | EXT_CALL zend_jit_vm_stack_free_args_helper, REG0 + } + if (may_have_extra_named_params) { + | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] + | TST_32_WITH_CONST TMP1w, (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24), TMP2w + | bne >1 + |.cold_code + |1: + | ldr FCARG1x, [RX, #offsetof(zend_execute_data, extra_named_params)] + | EXT_CALL zend_free_extra_named_params, REG0 + | b >2 + |.code + |2: + } + + |8: + if (opline->opcode == ZEND_DO_FCALL) { + // TODO: optimize ??? + | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) + | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] + | TST_32_WITH_CONST TMP1w, (ZEND_CALL_RELEASE_THIS >> 16), TMP2w + | bne >1 + |.cold_code + |1: + | add TMP1, RX, #offsetof(zend_execute_data, This) + | GET_Z_PTR FCARG1x, TMP1 + | // OBJ_RELEASE(object); + | OBJ_RELEASE ZREG_FCARG1x, >2, ZREG_TMP1, ZREG_TMP2 + | b >2 + |.code + |2: + } + + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + !JIT_G(current_frame) || + !JIT_G(current_frame)->call || + !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || + prev_opline->opcode == ZEND_SEND_UNPACK || + prev_opline->opcode == ZEND_SEND_ARRAY || + prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { + + | // zend_vm_stack_free_call_frame(call); + | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] + | TST_32_WITH_CONST TMP1w, ((ZEND_CALL_ALLOCATED >> 16) & 0xff), TMP2w + | bne >1 + |.cold_code + |1: + | mov FCARG1x, RX + | EXT_CALL zend_jit_free_call_frame, REG0 + | b >1 + |.code + } + | MEM_STORE_ZTS str, RX, executor_globals, vm_stack_top, REG0 + |1: + + if (!RETURN_VALUE_USED(opline)) { + zend_class_entry *ce; + bool ce_is_instanceof; + uint32_t func_info = call_info ? + zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : + (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); + + /* If an exception is thrown, the return_value may stay at the + * original value of null. */ + func_info |= MAY_BE_NULL; + + if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + } + + | // if (UNEXPECTED(EG(exception) != NULL)) { + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbnz REG0, ->icall_throw_handler + + // TODO: Can we avoid checking for interrupts after each call ??? + if (trace && last_valid_opline != opline) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + } else { + exit_addr = NULL; + } + if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { + return 0; + } + + if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { + | LOAD_IP_ADDR (opline + 1) + } else if (trace + && trace->op == ZEND_JIT_TRACE_END + && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + | LOAD_IP_ADDR (opline + 1) + } + } + + if (!func) { + |9: + } + + return 1; +} + +static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) +{ + uint32_t arg_num = opline->op2.num; + zend_jit_addr arg_addr; + + ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); + + if (!zend_jit_reuse_ip(Dst)) { + return 0; + } + + if (opline->opcode == ZEND_SEND_VAL_EX) { + uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); + + ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + /* Don't generate code that always throws exception */ + return 0; + } + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | ldr REG0, EX:RX->func + | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] + | TST_32_WITH_CONST TMP1w, mask, TMP2w + | bne &exit_addr + } else { + | ldr REG0, EX:RX->func + | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] + | TST_32_WITH_CONST TMP1w, mask, TMP2w + | bne >1 + |.cold_code + |1: + | SET_EX_OPLINE opline, REG0 + | b ->throw_cannot_pass_by_ref + |.code + } + } + + arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(opline, opline->op1); + + | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 + if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, REG0, TMP1 + } + } else { + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + + return 1; +} + +static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) +{ + | ldr FCARG1x, EX->call + | ldrb TMP1w, [FCARG1x, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] + | TST_32_WITH_CONST TMP1w, (ZEND_CALL_MAY_HAVE_UNDEF >> 24), TMP2w + | bne >1 + |.cold_code + |1: + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_handle_undef_args, REG0 + | cbz RETVALw, >2 + | b ->exception_handler + |.code + |2: + + return 1; +} + +static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) +{ + zend_jit_addr op1_addr, arg_addr, ref_addr; + + op1_addr = OP1_ADDR(); + arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + + if (!zend_jit_reuse_ip(Dst)) { + return 0; + } + + if (opline->op1_type == IS_VAR) { + if (op1_info & MAY_BE_INDIRECT) { + | LOAD_ZVAL_ADDR REG0, op1_addr + | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { + | IF_NOT_Z_TYPE REG0, IS_INDIRECT, >1, TMP1w + | // ret = Z_INDIRECT_P(ret); + | GET_Z_PTR REG0, REG0 + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + } + } else if (opline->op1_type == IS_CV) { + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 + | b >2 + |1: + } + op1_info &= ~MAY_BE_UNDEF; + op1_info |= MAY_BE_NULL; + } + } else { + ZEND_UNREACHABLE(); + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2, ZREG_TMP1 + | GET_ZVAL_PTR REG1, op1_addr, TMP1 + | GC_ADDREF REG1, TMP1w + | SET_ZVAL_PTR arg_addr, REG1, TMP1 + | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 + | b >6 + } + |2: + | // ZVAL_NEW_REF(arg, varptr); + if (opline->op1_type == IS_VAR) { + if (Z_REG(op1_addr) != ZREG_REG0 || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR REG0, op1_addr + } + | str REG0, T1 // save + } + | EMALLOC sizeof(zend_reference), op_array, opline // Allocate space in REG0 + | mov TMP1w, #2 + | str TMP1w, [REG0] + || ZEND_ASSERT(GC_REFERENCE <= MOVZ_IMM); + | movz TMP1w, #GC_REFERENCE + | str TMP1w, [REG0, #offsetof(zend_reference, gc.u.type_info)] + | str xzr, [REG0, #offsetof(zend_reference, sources.ptr)] + ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); + if (opline->op1_type == IS_VAR) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); + + | ldr REG1, T1 // restore + | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | SET_ZVAL_PTR val_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX, TMP1w, TMP2 + } else { + | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | SET_ZVAL_PTR op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 + } + | SET_ZVAL_PTR arg_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 + } + + |6: + | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline, ZREG_TMP1, ZREG_TMP2 + |7: + + return 1; +} + +static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) +{ + uint32_t arg_num = opline->op2.num; + zend_jit_addr arg_addr; + + ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && + opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || + arg_num <= MAX_ARG_FLAG_NUM); + + arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + + if (!zend_jit_reuse_ip(Dst)) { + return 0; + } + + if (opline->opcode == ZEND_SEND_VAR_EX) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { + return 0; + } + return 1; + } + } else { + uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); + + | ldr REG0, EX:RX->func + | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] + | TST_32_WITH_CONST TMP1w, mask, TMP2w + | bne >1 + |.cold_code + |1: + if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { + return 0; + } + | b >7 + |.code + } + } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + + if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + if (!(op1_info & MAY_BE_REF)) { + /* Don't generate code that always throws exception */ + return 0; + } else { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | GET_LOW_8BITS TMP1w, REG1w + | cmp TMP1w, #IS_REFERENCE + | bne &exit_addr + } + } + return 1; + } + } else { + uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); + + | ldr REG0, EX:RX->func + | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] + | TST_32_WITH_CONST TMP1w, mask, TMP2w + | bne >1 + |.cold_code + |1: + + mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); + + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + if (op1_info & MAY_BE_REF) { + | GET_LOW_8BITS TMP1w, REG1w + | cmp TMP1w, #IS_REFERENCE + | beq >7 + } + | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] + | TST_32_WITH_CONST TMP1w, mask, TMP2w + | bne >7 + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | b &exit_addr + } else { + | SET_EX_OPLINE opline, REG0 + | LOAD_ZVAL_ADDR FCARG1x, arg_addr + | EXT_CALL zend_jit_only_vars_by_reference, REG0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + | b >7 + } + + |.code + } + } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { + return 0; + } + return 1; + } + } else { + | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + | TST_32_WITH_CONST TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w + | bne >1 + |.cold_code + |1: + if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { + return 0; + } + | b >7 + |.code + } + } + + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 + |.cold_code + |1: + } + + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL, TMP1w, TMP2 + | cbz RETVALx, ->exception_handler + + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | b >7 + |.code + } else { + |7: + return 1; + } + } + + if (opline->opcode == ZEND_SEND_VAR_NO_REF) { + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + if (op1_info & MAY_BE_REF) { + | GET_LOW_8BITS TMP1w, REG1w + | cmp TMP1w, #IS_REFERENCE + | beq >7 + } + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | b &exit_addr + } else { + | SET_EX_OPLINE opline, REG0 + | LOAD_ZVAL_ADDR FCARG1x, arg_addr + | EXT_CALL zend_jit_only_vars_by_reference, REG0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + } else { + if (op1_info & MAY_BE_REF) { + if (opline->op1_type == IS_CV) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF op1_info, REG0w, REG2, TMP1w + } else { + zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 8); + + | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 + |.cold_code + |1: + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | // ZVAL_COPY_VALUE(return_value, &ref->value); + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | GC_DELREF FCARG1x, TMP1w + | beq >1 + | IF_NOT_REFCOUNTED REG0w, >2, TMP1w + | GC_ADDREF REG2, TMP1w + | b >2 + |1: + | EFREE_REFERENCE + | b >2 + |.code + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + |2: + } + } else { + if (op1_addr != op1_def_addr) { + if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { + return 0; + } + if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { + op1_addr= op1_def_addr; + } + } + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + if (opline->op1_type == IS_CV) { + | TRY_ADDREF op1_info, REG0w, REG2, TMP1w + } + } + } + |7: + + return 1; +} + +static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) +{ + uint32_t arg_num = opline->op2.num; + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { + TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); + | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); + || if (reuse_ip) { + | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w + | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + || } else { + | ldr REG0, EX->call + | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] + | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w + | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] + || } + } + } else { + if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { + TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); + | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); + || if (reuse_ip) { + | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w + | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + || } else { + | ldr REG0, EX->call + | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] + | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w + | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] + || } + } + } + } else { + // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { + uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); + + if (!zend_jit_reuse_ip(Dst)) { + return 0; + } + + | ldr REG0, EX:RX->func + | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] + | TST_32_WITH_CONST TMP1w, mask, TMP2w + | bne >1 + |.cold_code + |1: + | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); + | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w + | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + | b >1 + |.code + | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); + | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~(ZEND_CALL_SEND_ARG_BY_REF)), TMP2w + | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + |1: + } + + return 1; +} + +static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +{ + if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + if (jmp) { + | b >7 + } + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | b =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + if (jmp) { + | b >7 + } + } + + return 1; +} + +static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) +{ + if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + if (jmp) { + | b >7 + } + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | b =>target_label + } else { + ZEND_UNREACHABLE(); + } + } else { + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + if (jmp) { + | b >7 + } + } + + return 1; +} + +static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + uint32_t defined_label = (uint32_t)-1; + uint32_t undefined_label = (uint32_t)-1; + zval *zv = RT_CONSTANT(opline, opline->op1); + zend_jit_addr res_addr = 0; + + if (smart_branch_opcode && !exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + undefined_label = target_label; + } else if (smart_branch_opcode == ZEND_JMPNZ) { + defined_label = target_label; + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + undefined_label = target_label; + defined_label = target_label2; + } else { + ZEND_UNREACHABLE(); + } + } + + | // if (CACHED_PTR(opline->extended_value)) { + | ldr REG0, EX->run_time_cache + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1 + | cbz REG0, >1 + | TST_64_WITH_ONE REG0 + | bne >4 + |.cold_code + |4: + | MEM_LOAD_ZTS ldr, FCARG1x, executor_globals, zend_constants, FCARG1x + | lsr REG0, REG0, #1 + | ldr TMP1w, [FCARG1x, #offsetof(HashTable, nNumOfElements)] + | cmp TMP1, REG0 + + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | beq &exit_addr + } else { + | beq >3 + } + } else if (undefined_label != (uint32_t)-1) { + | beq =>undefined_label + } else { + | beq >3 + } + } else { + | beq >2 + } + |1: + | SET_EX_OPLINE opline, REG0 + | LOAD_ADDR FCARG1x, zv + | EXT_CALL zend_jit_check_constant, REG0 + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | cbz RETVALx, >3 + } else { + | cbnz RETVALx, >3 + } + | b &exit_addr + } else if (smart_branch_opcode) { + if (undefined_label != (uint32_t)-1) { + | cbz RETVALx, =>undefined_label + } else { + | cbz RETVALx, >3 + } + if (defined_label != (uint32_t)-1) { + | b =>defined_label + } else { + | b >3 + } + } else { + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + | cbnz RETVALx, >1 + |2: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | b >3 + } + |.code + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | b &exit_addr + } + } else if (defined_label != (uint32_t)-1) { + | b =>defined_label + } + } else { + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + |3: + + return 1; +} + +static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + uint32_t mask; + zend_jit_addr op1_addr = OP1_ADDR(); + + // TODO: support for is_resource() ??? + ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); + + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 + |.cold_code + |1: + } + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + zend_jit_check_exception_undef_result(Dst, opline); + if (opline->extended_value & MAY_BE_NULL) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | b &exit_addr + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { + | b >7 + } + } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { + return 0; + } + } else { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b &exit_addr + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { + | b >7 + } + } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { + return 0; + } + } + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + |.code + } + } + + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + mask = opline->extended_value; + if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | b &exit_addr + } + } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { + return 0; + } + } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b &exit_addr + } + } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { + return 0; + } + } else { + bool invert = 0; + zend_uchar type; + + switch (mask) { + case MAY_BE_NULL: type = IS_NULL; break; + case MAY_BE_FALSE: type = IS_FALSE; break; + case MAY_BE_TRUE: type = IS_TRUE; break; + case MAY_BE_LONG: type = IS_LONG; break; + case MAY_BE_DOUBLE: type = IS_DOUBLE; break; + case MAY_BE_STRING: type = IS_STRING; break; + case MAY_BE_ARRAY: type = IS_ARRAY; break; + case MAY_BE_OBJECT: type = IS_OBJECT; break; + case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; + case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; + case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; + case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; + case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; + case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; + case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; + case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; + case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; + default: + type = 0; + } + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR REG0, op1_addr + | ZVAL_DEREF REG0, op1_info, TMP1w + } + if (type == 0) { + if (smart_branch_opcode && + (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | // if (Z_REFCOUNTED_P(cv)) { + | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 + |.cold_code + |1: + } + | // if (!Z_DELREF_P(cv)) { + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | GC_DELREF FCARG1x, TMP1w + if (RC_MAY_BE_1(op1_info)) { + if (RC_MAY_BE_N(op1_info)) { + | bne >3 + } + if (op1_info & MAY_BE_REF) { + | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] + } else { + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 + } + | str REG0w, T1 // save + | // zval_dtor_func(r); + | ZVAL_DTOR_FUNC op1_info, opline, TMP1 + | ldr REG1w, T1 // restore + | b >2 + } + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (!RC_MAY_BE_1(op1_info)) { + | b >3 + } + |.code + } + |3: + if (op1_info & MAY_BE_REF) { + | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] + } else { + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 + } + |2: + } else { + if (op1_info & MAY_BE_REF) { + | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] + } else { + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 + } + } + | mov REG0w, #1 + | lsl REG0w, REG0w, REG1w + | TST_32_WITH_CONST REG0w, mask, TMP1w + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | bne &exit_addr + } else { + | beq &exit_addr + } + } else if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + | beq =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | bne =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | beq =>target_label + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + | cset REG0w, ne + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + } else { + if (smart_branch_opcode && + (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | // if (Z_REFCOUNTED_P(cv)) { + | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 + |.cold_code + |1: + } + | // if (!Z_DELREF_P(cv)) { + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | GC_DELREF FCARG1x, TMP1w + if (RC_MAY_BE_1(op1_info)) { + if (RC_MAY_BE_N(op1_info)) { + | bne >3 + } + if (op1_info & MAY_BE_REF) { + | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] + } else { + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 + } + | str REG0w, T1 // save + | // zval_dtor_func(r); + | ZVAL_DTOR_FUNC op1_info, opline, TMP1 + | ldr REG1w, T1 // restore + | b >2 + } + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (!RC_MAY_BE_1(op1_info)) { + | b >3 + } + |.code + } + |3: + if (op1_info & MAY_BE_REF) { + | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] + } else { + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 + } + |2: + // Note: 'type' is of uchar type and holds a positive value, + // hence it's safe to directly encode it as the imm field of 'cmp' instruction. + | cmp REG1w, #type + } else { + if (op1_info & MAY_BE_REF) { + | ldrb TMP1w, [REG0, #offsetof(zval,u1.v.type)] + | cmp TMP1w, #type + } else { + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 + | cmp TMP1w, #type + } + } + if (exit_addr) { + if (invert) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | bne &exit_addr + } else { + | beq &exit_addr + } + } else { + if (smart_branch_opcode == ZEND_JMPNZ) { + | beq &exit_addr + } else { + | bne &exit_addr + } + } + } else if (smart_branch_opcode) { + if (invert) { + if (smart_branch_opcode == ZEND_JMPZ) { + | beq =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | bne =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | beq =>target_label + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + if (smart_branch_opcode == ZEND_JMPZ) { + | bne =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | beq =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | bne =>target_label + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } + } + } else { + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + if (invert) { + | cset REG0w, ne + } else { + | cset REG0w, eq + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + } + } + } + + |7: + + return 1; +} + +static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var) +{ + uint32_t j, info; + + if (ssa->vars && ssa->var_info) { + info = ssa->var_info[var].type; + for (j = op_array->last_var; j < ssa->vars_count; j++) { + if (ssa->vars[j].var == var) { + info |= ssa->var_info[j].type; + } + } + } else { + info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF | + MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; + } + +#ifdef ZEND_JIT_USE_RC_INFERENCE + /* Refcount may be increased by RETURN opcode */ + if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) { + for (j = 0; j < ssa->cfg.blocks_count; j++) { + if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) && + ssa->cfg.blocks[j].len > 0) { + const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1; + + if (opline->opcode == ZEND_RETURN) { + if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) { + info |= MAY_BE_RCN; + break; + } + } + } + } + } +#endif + + return info; +} + +static int zend_jit_leave_frame(dasm_State **Dst) +{ + | // EG(current_execute_data) = EX(prev_execute_data); + | ldr REG0, EX->prev_execute_data + | MEM_STORE_ZTS str, REG0, executor_globals, current_execute_data, REG2 + return 1; +} + +static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) +{ + if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + uint32_t offset = EX_NUM_TO_VAR(var); + zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset); + | ZVAL_PTR_DTOR addr, info, 1, 1, NULL, ZREG_TMP1, ZREG_TMP2 + } + return 1; +} + +static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) +{ + if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset); + | ZVAL_PTR_DTOR addr, info, 0, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + return 1; +} + +static int zend_jit_leave_func(dasm_State **Dst, + const zend_op_array *op_array, + const zend_op *opline, + uint32_t op1_info, + bool left_frame, + zend_jit_trace_rec *trace, + zend_jit_trace_info *trace_info, + int indirect_var_access, + int may_throw) +{ + bool may_be_top_frame = + JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + !JIT_G(current_frame) || + !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); + bool may_need_call_helper = + indirect_var_access || /* may have symbol table */ + !op_array->function_name || /* may have symbol table */ + may_be_top_frame || + (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ + JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + !JIT_G(current_frame) || + TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ + (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ + bool may_need_release_this = + !(op_array->fn_flags & ZEND_ACC_CLOSURE) && + op_array->scope && + !(op_array->fn_flags & ZEND_ACC_STATIC) && + (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + !JIT_G(current_frame) || + !TRACE_FRAME_NO_NEED_REKEASE_THIS(JIT_G(current_frame))); + + if (may_need_call_helper || may_need_release_this) { + | ldr FCARG1w, [FP, #offsetof(zend_execute_data, This.u1.type_info)] + } + if (may_need_call_helper) { + if (!left_frame) { + left_frame = 1; + if (!zend_jit_leave_frame(Dst)) { + return 0; + } + } + /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ + + | TST_32_WITH_CONST FCARG1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE), TMP1w + if (trace && trace->op != ZEND_JIT_TRACE_END) { + | bne >1 + |.cold_code + |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG2x, FP + } + | EXT_CALL zend_jit_leave_func_helper, REG0 + + if (may_be_top_frame) { + // TODO: try to avoid this check ??? + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { +#if 0 + /* this check should be handled by the following OPLINE guard */ + | LOAD_ADDR TMP1, zend_jit_halt_op + | cmp IP, TMP1 + | beq ->trace_halt +#endif + } else if (GCC_GLOBAL_REGS) { + | cbz IP, ->trace_halt + } else { + | tst RETVALw, RETVALw + | blt ->trace_halt + } + } + + if (!GCC_GLOBAL_REGS) { + | // execute_data = EG(current_execute_data) + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 + } + | b >8 + |.code + } else { + | bne ->leave_function_handler + } + } + + if (op_array->fn_flags & ZEND_ACC_CLOSURE) { + if (!left_frame) { + left_frame = 1; + if (!zend_jit_leave_frame(Dst)) { + return 0; + } + } + | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); + | ldr FCARG1x, EX->func + | sub FCARG1x, FCARG1x, #sizeof(zend_object) + | OBJ_RELEASE ZREG_FCARG1x, >4, ZREG_TMP1, ZREG_TMP2 + |4: + } else if (may_need_release_this) { + if (!left_frame) { + left_frame = 1; + if (!zend_jit_leave_frame(Dst)) { + return 0; + } + } + | // if (call_info & ZEND_CALL_RELEASE_THIS) + | TST_32_WITH_CONST FCARG1w, ZEND_CALL_RELEASE_THIS, TMP1w + | beq >4 + | // zend_object *object = Z_OBJ(execute_data->This); + | ldr FCARG1x, EX->This.value.obj + | // OBJ_RELEASE(object); + | OBJ_RELEASE ZREG_FCARG1x, >4, ZREG_TMP1, ZREG_TMP2 + |4: + // TODO: avoid EG(excption) check for $this->foo() calls + may_throw = 1; + } + + | // EG(vm_stack_top) = (zval*)execute_data; + | MEM_STORE_ZTS str, FP, executor_globals, vm_stack_top, REG0 + | // execute_data = EX(prev_execute_data); + | ldr FP, EX->prev_execute_data + + if (!left_frame) { + | // EG(current_execute_data) = execute_data; + | MEM_STORE_ZTS str, FP, executor_globals, current_execute_data, REG0 + } + + |9: + if (trace) { + if (trace->op != ZEND_JIT_TRACE_END + && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { + zend_jit_reset_last_valid_opline(); + } else { + | LOAD_IP + | ADD_IP_FROM_CST sizeof(zend_op), TMP1 + } + + |8: + + if (trace->op == ZEND_JIT_TRACE_BACK + && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { + const zend_op *next_opline = trace->opline; + + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) + && (op1_info & MAY_BE_RC1) + && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { + /* exception might be thrown during destruction of unused return value */ + | // if (EG(exception)) + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbnz REG0, ->leave_throw_handler + } + do { + trace++; + } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); + ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); + next_opline = trace->opline; + ZEND_ASSERT(next_opline != NULL); + + if (trace->op == ZEND_JIT_TRACE_END + && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { + trace_info->flags |= ZEND_JIT_TRACE_LOOP; + | CMP_IP next_opline, TMP1, TMP2 + | beq =>0 // LOOP +#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE + | JMP_IP TMP1 +#else + | b ->trace_escape +#endif + } else { + | CMP_IP next_opline, TMP1, TMP2 + | bne ->trace_escape + } + + zend_jit_set_last_valid_opline(trace->opline); + + return 1; + } else if (may_throw || + (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) + && (op1_info & MAY_BE_RC1) + && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) + && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { + | // if (EG(exception)) + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbnz REG0, ->leave_throw_handler + } + + return 1; + } else { + | // if (EG(exception)) + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | LOAD_IP + | cbnz REG0, ->leave_throw_handler + | // opline = EX(opline) + 1 + | ADD_IP_FROM_CST sizeof(zend_op), TMP1 + } + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD +#ifdef CONTEXT_THREADED_JIT + | NIY // TODO: CONTEXT_THREADED_JIT is always undefined +#else + | JMP_IP TMP1 +#endif + } else if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], # SPAD // stack alignment +#ifdef CONTEXT_THREADED_JIT + | NIY // TODO +#else + | JMP_IP TMP1 +#endif + } else { +#ifdef CONTEXT_THREADED_JIT + ZEND_UNREACHABLE(); + // TODO: context threading can't work without GLOBAL REGS because we have to change + // the value of execute_data in execute_ex() + | NIY // TODO +#else + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | mov RETVALx, #2 // ZEND_VM_LEAVE ???? + | ret +#endif + } + + return 1; +} + +static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) +{ + zend_jit_addr ret_addr; + int8_t return_value_used; + + ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); + ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { + if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { + return_value_used = 1; + } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { + return_value_used = 0; + } else { + return_value_used = -1; + } + } else { + return_value_used = -1; + } + + if (ZEND_OBSERVER_ENABLED) { + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + + if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { + return 0; + } + op1_addr = dst; + } + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + | mov FCARG1x, FP + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_observer_fcall_end, REG0 + } + + // if (!EX(return_value)) + if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_REG1) { + if (return_value_used != 0) { + | ldr REG2, EX->return_value + } + ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); + } else { + if (return_value_used != 0) { + | ldr REG1, EX->return_value + } + ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); + } + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (return_value_used == -1) { + | cbz Rx(Z_REG(ret_addr)), >1 + |.cold_code + |1: + } + if (return_value_used != 1) { + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (jit_return_label >= 0) { + | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label, ZREG_TMP1, ZREG_TMP2 + } else { + | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9, ZREG_TMP1, ZREG_TMP2 + } + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | GC_DELREF FCARG1x, TMP1w + if (RC_MAY_BE_1(op1_info)) { + if (RC_MAY_BE_N(op1_info)) { + if (jit_return_label >= 0) { + | bne =>jit_return_label + } else { + | bne >9 + } + } + | //SAVE_OPLINE() + | ZVAL_DTOR_FUNC op1_info, opline, TMP1 + | //????ldr REG1, EX->return_value // reload ??? + } + if (return_value_used == -1) { + if (jit_return_label >= 0) { + | b =>jit_return_label + } else { + | b >9 + } + |.code + } + } + } else if (return_value_used == -1) { + if (jit_return_label >= 0) { + | cbz Rx(Z_REG(ret_addr)), =>jit_return_label + } else { + | cbz Rx(Z_REG(ret_addr)), >9 + } + } + + if (return_value_used == 0) { + |9: + return 1; + } + + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(opline, opline->op1); + | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 + if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, REG0, TMP1 + } + } else if (opline->op1_type == IS_TMP_VAR) { + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } else if (opline->op1_type == IS_CV) { + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR REG0, op1_addr + | ZVAL_DEREF REG0, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + } + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || + !op_array->function_name) { + | TRY_ADDREF op1_info, REG0w, REG2, TMP1w + } else if (return_value_used != 1) { + | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); + | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 + } + } + } else { + if (op1_info & MAY_BE_REF) { + zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); + + | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 + |.cold_code + |1: + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | GET_ZVAL_PTR REG0, op1_addr, TMP1 + | // ZVAL_COPY_VALUE(return_value, &ref->value); + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | GC_DELREF REG0, TMP1w + | beq >2 + | // if (IS_REFCOUNTED()) + if (jit_return_label >= 0) { + | IF_NOT_REFCOUNTED REG2w, =>jit_return_label, TMP1w + } else { + | IF_NOT_REFCOUNTED REG2w, >9, TMP1w + } + | // ADDREF + | GET_ZVAL_PTR REG2, ret_addr, TMP1 // reload + | GC_ADDREF REG2, TMP1w + if (jit_return_label >= 0) { + | b =>jit_return_label + } else { + | b >9 + } + |2: + | mov FCARG1x, REG0 + | EFREE_REFERENCE + if (jit_return_label >= 0) { + | b =>jit_return_label + } else { + | b >9 + } + |.code + } + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + + |9: + return 1; +} + +static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) +{ + ZEND_ASSERT(type_reg == ZREG_REG2); + + | GET_ZVAL_PTR REG1, val_addr, TMP1 + | IF_NOT_REFCOUNTED REG2w, >2, TMP1w + | GET_LOW_8BITS TMP2w, REG2w + | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 + | add REG1, REG1, #offsetof(zend_reference, val) + | GET_Z_TYPE_INFO REG2w, REG1 + | GET_Z_PTR REG1, REG1 + | IF_NOT_REFCOUNTED REG2w, >2, TMP1w + |1: + | GC_ADDREF REG1, TMP2w + |2: + | SET_ZVAL_PTR res_addr, REG1, TMP1 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 + + return 1; +} + +static bool zend_jit_may_avoid_refcounting(const zend_op *opline) +{ + switch (opline->opcode) { + case ZEND_FETCH_OBJ_FUNC_ARG: + if (!JIT_G(current_frame) || + !JIT_G(current_frame)->call->func || + !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { + return 0; + } + /* break missing intentionally */ + case ZEND_FETCH_OBJ_R: + case ZEND_FETCH_OBJ_IS: + if (opline->op2_type == IS_CONST + && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING + && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') { + return 1; + } + break; + case ZEND_FETCH_DIM_FUNC_ARG: + if (!JIT_G(current_frame) || + !JIT_G(current_frame)->call->func || + !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { + return 0; + } + /* break missing intentionally */ + case ZEND_FETCH_DIM_R: + case ZEND_FETCH_DIM_IS: + return 1; + case ZEND_ISSET_ISEMPTY_DIM_OBJ: + if (!(opline->extended_value & ZEND_ISEMPTY)) { + return 1; + } + break; + } + return 0; +} + +static int zend_jit_fetch_dim_read(dasm_State **Dst, + const zend_op *opline, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + uint32_t op1_info, + zend_jit_addr op1_addr, + bool op1_avoid_refcounting, + uint32_t op2_info, + uint32_t res_info, + zend_jit_addr res_addr, + int may_throw) +{ + zend_jit_addr orig_op1_addr, op2_addr; + const void *exit_addr = NULL; + const void *not_found_exit_addr = NULL; + const void *res_exit_addr = NULL; + bool result_avoid_refcounting = 0; + uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; + + orig_op1_addr = OP1_ADDR(); + op2_addr = OP2_ADDR(); + + if (opline->opcode != ZEND_FETCH_DIM_IS + && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + } + + if ((res_info & MAY_BE_GUARD) + && JIT_G(current_frame) + && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { + uint32_t flags = 0; + uint32_t old_op1_info = 0; + uint32_t old_info; + zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; + int32_t exit_point; + + if (opline->opcode != ZEND_FETCH_LIST_R + && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) + && !op1_avoid_refcounting) { + flags |= ZEND_JIT_EXIT_FREE_OP1; + } + if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) + && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + flags |= ZEND_JIT_EXIT_FREE_OP2; + } + if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) + && !(flags & ZEND_JIT_EXIT_FREE_OP1) + && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) + && (ssa_op+1)->op1_use == ssa_op->result_def + && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) + && zend_jit_may_avoid_refcounting(opline+1)) { + result_avoid_refcounting = 1; + ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; + } + + if (op1_avoid_refcounting) { + old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); + } + + if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { + old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); + exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); + res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!res_exit_addr) { + return 0; + } + res_info &= ~MAY_BE_GUARD; + ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; + } + + if (opline->opcode == ZEND_FETCH_DIM_IS + && !(res_info & MAY_BE_NULL)) { + old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); + exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); + not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!not_found_exit_addr) { + return 0; + } + } + + if (op1_avoid_refcounting) { + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); + } + } + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 + } + } + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, res_exit_addr, not_found_exit_addr, exit_addr)) { + return 0; + } + } + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + + if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { + if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 + } + } + | SET_EX_OPLINE opline, REG0 + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 + if (opline->opcode != ZEND_FETCH_DIM_IS) { + if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, REG0 + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + | EXT_CALL zend_jit_fetch_dim_str_r_helper, REG0 + } + | SET_ZVAL_PTR res_addr, RETVALx, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + | LOAD_ZVAL_ADDR CARG3, res_addr + | EXT_CALL zend_jit_fetch_dim_str_is_helper, REG0 + } + if ((op1_info & MAY_BE_ARRAY) || + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { + | b >9 // END + } + |6: + } + + if (op1_info & MAY_BE_OBJECT) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { + if (exit_addr) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6, ZREG_TMP1 + } + } + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + | LOAD_ZVAL_ADDR CARG3, res_addr + if (opline->opcode != ZEND_FETCH_DIM_IS) { + | EXT_CALL zend_jit_fetch_dim_obj_r_helper, REG0 + } else { + | EXT_CALL zend_jit_fetch_dim_obj_is_helper, REG0 + } + if ((op1_info & MAY_BE_ARRAY) || + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { + | b >9 // END + } + |6: + } + + if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { + | SET_EX_OPLINE opline, REG0 + if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + |1: + } + + if (op2_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 + | LOAD_32BIT_VAL FCARG1w, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + |1: + } + } + + if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) + && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_STRING)))) { + if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr + } else { + | SET_EX_OPLINE opline, REG0 + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || + Z_REG(op1_addr) != ZREG_FCARG1x || + Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + } + | EXT_CALL zend_jit_invalid_array_access, REG0 + } + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + if (op1_info & MAY_BE_ARRAY) { + | b >9 // END + } + } + + if (op1_info & MAY_BE_ARRAY) { + |.code + } + } + + if (op1_info & MAY_BE_ARRAY) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + |8: + if (res_exit_addr) { + uint32_t type = concrete_type(res_info); + if (op1_info & MAY_BE_ARRAY_OF_REF) { + | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w + } + if (type < IS_STRING) { + | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr, ZREG_TMP1 + } else { + | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 + | GET_LOW_8BITS TMP1w, REG2w + | IF_NOT_TYPE TMP1w, type, &res_exit_addr + } + | // ZVAL_COPY + |7: + | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (type < IS_STRING) { + if (Z_REG(res_addr) != ZREG_FP || + JIT_G(current_frame) == NULL || + STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { + | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 + } + } else { + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 + if (!result_avoid_refcounting) { + | TRY_ADDREF res_info, REG2w, REG1, TMP1w + } + } + } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { + return 0; + } + } else if (op1_info & MAY_BE_ARRAY_OF_REF) { + | // ZVAL_COPY_DEREF + | GET_ZVAL_TYPE_INFO Rw(ZREG_REG2), val_addr, TMP1 + if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_REG2)) { + return 0; + } + } else { + | // ZVAL_COPY + | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF res_info, REG1w, REG2, TMP1w + } + } + |9: // END + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { + /* Magic offsetGet() may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + + return 1; +} + +static int zend_jit_fetch_dim(dasm_State **Dst, + const zend_op *opline, + uint32_t op1_info, + zend_jit_addr op1_addr, + uint32_t op2_info, + zend_jit_addr res_addr, + int may_throw) +{ + zend_jit_addr op2_addr; + + op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w + | GET_Z_PTR FCARG2x, FCARG1x + | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] + | cmp TMP1w, #IS_ARRAY + | bne >2 + | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) + | b >3 + |.cold_code + |2: + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 + | mov FCARG1x, RETVALx + | cbnz FCARG1x, >1 + | b ->exception_handler_undef + |.code + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 + } + |3: + | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 + } + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | CMP_ZVAL_TYPE op1_addr, IS_FALSE, ZREG_TMP1 + | bgt >7 + } + if ((op1_info & MAY_BE_UNDEF) + && opline->opcode == ZEND_FETCH_DIM_RW) { + if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 + } + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + |1: + } + | // ZVAL_ARR(container, zend_new_array(8)); + if (Z_REG(op1_addr) != ZREG_FP) { + | str Rx(Z_REG(op1_addr)), T1 // save + } + | EXT_CALL _zend_new_array_0, REG0 + | mov REG0, RETVALx + if (Z_REG(op1_addr) != ZREG_FP) { + | ldr Rx(Z_REG(op1_addr)), T1 // restore + } + | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 + | mov FCARG1x, REG0 + if (op1_info & MAY_BE_ARRAY) { + | b >1 + |.code + |1: + } + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |6: + if (opline->op2_type == IS_UNUSED) { + | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); + | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval + | EXT_CALL zend_hash_next_index_insert, REG0 + | // if (UNEXPECTED(!var_ptr)) { + | cbz RETVALx, >1 + |.cold_code + |1: + | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); + | CANNOT_ADD_ELEMENT opline + | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 + | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); + | b >8 + |.code + | SET_ZVAL_PTR res_addr, RETVALx, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 + } else { + uint32_t type; + + switch (opline->opcode) { + case ZEND_FETCH_DIM_W: + case ZEND_FETCH_LIST_W: + type = BP_VAR_W; + break; + case ZEND_FETCH_DIM_RW: + type = BP_VAR_RW; + break; + case ZEND_FETCH_DIM_UNSET: + type = BP_VAR_UNSET; + break; + default: + ZEND_UNREACHABLE(); + } + + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, NULL, NULL, NULL)) { + return 0; + } + + |8: + | SET_ZVAL_PTR res_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 + + if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { + |.cold_code + |9: + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + | b >8 + |.code + } + } + } + + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |.cold_code + |7: + } + + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type == IS_UNUSED) { + | mov FCARG2x, xzr + } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + | LOAD_ZVAL_ADDR CARG3, res_addr + switch (opline->opcode) { + case ZEND_FETCH_DIM_W: + case ZEND_FETCH_LIST_W: + | EXT_CALL zend_jit_fetch_dim_obj_w_helper, REG0 + break; + case ZEND_FETCH_DIM_RW: + | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, REG0 + break; +// case ZEND_FETCH_DIM_UNSET: +// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, REG0 +// break; + default: + ZEND_UNREACHABLE(); + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + | b >8 // END + |.code + } + } + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { + /* ASSIGN_DIM may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + + |8: + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + return 1; +} + +static int zend_jit_isset_isempty_dim(dasm_State **Dst, + const zend_op *opline, + uint32_t op1_info, + zend_jit_addr op1_addr, + bool op1_avoid_refcounting, + uint32_t op2_info, + int may_throw, + zend_uchar smart_branch_opcode, + uint32_t target_label, + uint32_t target_label2, + const void *exit_addr) +{ + zend_jit_addr op2_addr, res_addr; + + // TODO: support for empty() ??? + ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); + + op2_addr = OP2_ADDR(); + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + const void *found_exit_addr = NULL; + const void *not_found_exit_addr = NULL; + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 + } + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 + if (exit_addr + && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) + && !may_throw + && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) + && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { + if (smart_branch_opcode == ZEND_JMPNZ) { + found_exit_addr = exit_addr; + } else { + not_found_exit_addr = exit_addr; + } + } + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, found_exit_addr, not_found_exit_addr, NULL)) { + return 0; + } + + if (found_exit_addr) { + |9: + return 1; + } else if (not_found_exit_addr) { + |8: + return 1; + } + } + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + + if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + | EXT_CALL zend_jit_isset_dim_helper, REG0 + | cbz RETVALw, >9 + if (op1_info & MAY_BE_ARRAY) { + | b >8 + |.code + } + } else { + if (op2_info & MAY_BE_UNDEF) { + if (op2_info & MAY_BE_ANY) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 + } + | LOAD_32BIT_VAL FCARG1w, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + |1: + } + if (op1_info & MAY_BE_ARRAY) { + | b >9 + |.code + } + } + } + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { + /* Magic offsetExists() may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + + if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { + |8: + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + if (!op1_avoid_refcounting) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + if (may_throw) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { + return 0; + } + } + if (!(opline->extended_value & ZEND_ISEMPTY)) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | b &exit_addr + } else { + | b >8 + } + } else if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b =>target_label2 + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | b =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | b >8 + } + } else { + | NIY // TODO: support for empty() + } + } + + |9: // not found + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + if (!op1_avoid_refcounting) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + if (may_throw) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { + return 0; + } + } + if (!(opline->extended_value & ZEND_ISEMPTY)) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b &exit_addr + } + } else if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | b =>target_label + } else { + ZEND_UNREACHABLE(); + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + } else { + | NIY // TODO: support for empty() + } + + |8: + + return 1; +} + +static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) +{ + zend_jit_addr op1_addr = OP1_ADDR(); + zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); + + | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; + | ldr REG0, EX->run_time_cache + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1 + | sub REG0, REG0, #1 + | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) + | MEM_LOAD_32_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1 + | cmp REG0, REG1, lsl #5 + | bhs >9 + | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); + | MEM_LOAD_ZTS ldr, TMP1, executor_globals, symbol_table.arData, REG1 + | add REG0, REG0, TMP1 + | IF_NOT_Z_TYPE REG0, IS_REFERENCE, >9, TMP1w + | // (EXPECTED(p->key == varname)) + | ldr TMP1, [REG0, #offsetof(Bucket, key)] + | LOAD_ADDR TMP2, varname + | cmp TMP1, TMP2 + | bne >9 + | GET_Z_PTR REG0, REG0 + | GC_ADDREF REG0, TMP1w + |1: + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) + | IF_ZVAL_REFCOUNTED op1_addr, >2, ZREG_TMP1, ZREG_TMP2 + |.cold_code + |2: + } + | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | // ZVAL_REF(variable_ptr, ref) + | SET_ZVAL_PTR op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 + | // if (GC_DELREF(garbage) == 0) + | GC_DELREF FCARG1x, TMP1w + if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { + | bne >3 + } else { + | bne >5 + } + | ZVAL_DTOR_FUNC op1_info, opline, TMP1 + | b >5 + if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { + |3: + | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) + | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w + | EXT_CALL gc_possible_root, REG0 + | b >5 + } + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + |.code + } + } + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | // ZVAL_REF(variable_ptr, ref) + | SET_ZVAL_PTR op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 + } + |5: + //END of handler + + |.cold_code + |9: + | LOAD_ADDR FCARG1x, (ptrdiff_t)varname + | ldr FCARG2x, EX->run_time_cache + if (opline->extended_value) { + | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, FCARG2x, opline->extended_value, TMP1 + } + | EXT_CALL zend_jit_fetch_global_helper, REG0 + | mov REG0, RETVALx + | b <1 + |.code + + return 1; +} + +static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) +{ + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + bool in_cold = 0; + uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; + zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1x : ZREG_REG0; + + if (ZEND_ARG_SEND_MODE(arg_info)) { + if (opline->opcode == ZEND_RECV_INIT) { + | LOAD_ZVAL_ADDR Rx(tmp_reg), res_addr + | ZVAL_DEREF Rx(tmp_reg), MAY_BE_REF, TMP1w + res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); + } else { + | GET_ZVAL_PTR Rx(tmp_reg), res_addr, TMP1 + res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); + } + } + + if (type_mask != 0) { + if (is_power_of_two(type_mask)) { + uint32_t type_code = concrete_type(type_mask); + | IF_NOT_ZVAL_TYPE res_addr, type_code, >1, ZREG_TMP1 + } else { + | mov REG2w, #1 + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1 + | lsl REG2w, REG2w, REG1w + | TST_32_WITH_CONST REG2w, type_mask, TMP1w + | beq >1 + } + + |.cold_code + |1: + + in_cold = 1; + } + + if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, res_addr + } + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | SET_EX_OPLINE opline, REG0 + } else { + | ADDR_STORE EX->opline, opline, REG0 + } + | LOAD_ADDR FCARG2x, (ptrdiff_t)arg_info + | EXT_CALL zend_jit_verify_arg_slow, REG0 + + if (check_exception) { + | GET_LOW_8BITS REG0w, RETVALw + if (in_cold) { + | cbnz REG0w, >1 + | b ->exception_handler + |.code + |1: + } else { + | cbz REG0w, ->exception_handler + } + } else if (in_cold) { + | b >1 + |.code + |1: + } + + return 1; +} + +static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) +{ + uint32_t arg_num = opline->op1.num; + zend_arg_info *arg_info = NULL; + + if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + if (EXPECTED(arg_num <= op_array->num_args)) { + arg_info = &op_array->arg_info[arg_num-1]; + } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { + arg_info = &op_array->arg_info[op_array->num_args]; + } + if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { + arg_info = NULL; + } + } + + if (arg_info || (opline+1)->opcode != ZEND_RECV) { + | ldr TMP1w, EX->This.u2.num_args + | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | blo &exit_addr + } else { + | blo >1 + |.cold_code + |1: + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | SET_EX_OPLINE opline, REG0 + } else { + | ADDR_STORE EX->opline, opline, REG0 + } + | mov FCARG1x, FP + | EXT_CALL zend_missing_arg_error, REG0 + | b ->exception_handler + |.code + } + } + + if (arg_info) { + if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { + return 0; + } + } + + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { + if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { + | LOAD_IP_ADDR (opline + 1) + zend_jit_set_last_valid_opline(opline + 1); + } + } + + return 1; +} + +static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) +{ + uint32_t arg_num = opline->op1.num; + zval *zv = RT_CONSTANT(opline, opline->op2); + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + | ldr TMP1w, EX->This.u2.num_args + | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w + | bhs >5 + } + | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 + if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, REG0, TMP1 + } + + if (Z_CONSTANT_P(zv)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | SET_EX_OPLINE opline, REG0 + } else { + | ADDR_STORE EX->opline, opline, REG0 + } + | LOAD_ZVAL_ADDR FCARG1x, res_addr + | ldr REG0, EX->func + | ldr FCARG2x, [REG0, #offsetof(zend_op_array, scope)] + | EXT_CALL zval_update_constant_ex, REG0 + | cbnz RETVALw, >1 + |.cold_code + |1: + | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline, ZREG_TMP1, ZREG_TMP2 + | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 + | b ->exception_handler + |.code + } + + |5: + + if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + do { + zend_arg_info *arg_info; + + if (arg_num <= op_array->num_args) { + arg_info = &op_array->arg_info[arg_num-1]; + } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { + arg_info = &op_array->arg_info[op_array->num_args]; + } else { + break; + } + if (!ZEND_TYPE_IS_SET(arg_info->type)) { + break; + } + if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { + return 0; + } + } while (0); + } + + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { + if (is_last) { + | LOAD_IP_ADDR (opline + 1) + zend_jit_set_last_valid_opline(opline + 1); + } + } + return 1; +} + +static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename) +{ + zend_property_info *info = NULL; + + if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) || + !ce || + !(ce->ce_flags & ZEND_ACC_LINKED) || + (ce->ce_flags & ZEND_ACC_TRAIT) || + ce->create_object) { + return NULL; + } + + if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) { + if (ce->info.user.filename != filename) { + /* class declaration might be changed independently */ + return NULL; + } + + if (ce->parent) { + zend_class_entry *parent = ce->parent; + + do { + if (parent->type == ZEND_INTERNAL_CLASS) { + break; + } else if (parent->info.user.filename != filename) { + /* some of parents class declarations might be changed independently */ + /* TODO: this check may be not enough, because even + * in the same it's possible to conditionally define + * few classes with the same name, and "parent" may + * change from request to request. + */ + return NULL; + } + parent = parent->parent; + } while (parent); + } + } + + info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); + if (info == NULL || + !IS_VALID_PROPERTY_OFFSET(info->offset) || + (info->flags & ZEND_ACC_STATIC)) { + return NULL; + } + + if (!(info->flags & ZEND_ACC_PUBLIC) && + (!on_this || info->ce != ce)) { + return NULL; + } + + return info; +} + +static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename) +{ + zend_property_info *info; + + if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) { + return 1; + } + + if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) { + if (ce->info.user.filename != filename) { + /* class declaration might be changed independently */ + return 1; + } + } + + info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); + if (info == NULL || + !IS_VALID_PROPERTY_OFFSET(info->offset) || + (info->flags & ZEND_ACC_STATIC)) { + return 1; + } + + if (!(info->flags & ZEND_ACC_PUBLIC) && + (!on_this || info->ce != ce)) { + return 1; + } + + return 0; +} + +static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) +{ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | LOAD_ADDR TMP1, ((ptrdiff_t)ce) + | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] + | cmp TMP2, TMP1 + | bne &exit_addr + + return 1; +} + +static int zend_jit_fetch_obj(dasm_State **Dst, + const zend_op *opline, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + uint32_t op1_info, + zend_jit_addr op1_addr, + bool op1_indirect, + zend_class_entry *ce, + bool ce_is_instanceof, + bool use_this, + bool op1_avoid_refcounting, + zend_class_entry *trace_ce, + int may_throw) +{ + zval *member; + zend_property_info *prop_info; + bool may_be_dynamic = 1; + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); + zend_jit_addr prop_addr; + uint32_t res_info = RES_INFO(); + + ZEND_ASSERT(opline->op2_type == IS_CONST); + ZEND_ASSERT(op1_info & MAY_BE_OBJECT); + + member = RT_CONSTANT(opline, opline->op2); + ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); + prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); + + if (opline->op1_type == IS_UNUSED || use_this) { + | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 + } else { + if (opline->op1_type == IS_VAR + && opline->opcode == ZEND_FETCH_OBJ_W + && (op1_info & MAY_BE_INDIRECT) + && Z_REG(op1_addr) == ZREG_FP) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w + | GET_Z_PTR FCARG1x, FCARG1x + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & MAY_BE_REF) { + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, ZREG_TMP1 + } + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + } + + if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { + prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); + if (prop_info) { + ce = trace_ce; + ce_is_instanceof = 0; + if (!(op1_info & MAY_BE_CLASS_GUARD)) { + if (!zend_jit_class_guard(Dst, opline, trace_ce)) { + return 0; + } + if (ssa->var_info && ssa_op->op1_use >= 0) { + ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_use].ce = ce; + ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; + } + } + } + } + + if (!prop_info) { + | ldr REG0, EX->run_time_cache + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG2, REG0, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS), TMP1 + | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] + | cmp REG2, TMP1 + | bne >5 + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)), TMP1 + may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); + if (may_be_dynamic) { + | tst REG0, REG0 + if (opline->opcode == ZEND_FETCH_OBJ_W) { + | blt >5 + } else { + | blt >8 // dynamic property + } + } + | add TMP1, FCARG1x, REG0 + | ldr REG2w, [TMP1, #offsetof(zval,u1.type_info)] + | IF_UNDEF REG2w, >5 + | mov FCARG1x, TMP1 + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + if (opline->opcode == ZEND_FETCH_OBJ_W + && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) + && (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS))) { + uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; + + | ldr REG0, EX->run_time_cache + | SAFE_MEM_ACC_WITH_UOFFSET ldr, FCARG2x, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2), TMP1 + | cbnz FCARG2x, >1 + |.cold_code + |1: + if (flags == ZEND_FETCH_DIM_WRITE) { + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_check_array_promotion, REG0 + | b >9 + } else if (flags == ZEND_FETCH_REF) { + | LOAD_ZVAL_ADDR CARG3, res_addr + | EXT_CALL zend_jit_create_typed_ref, REG0 + | b >9 + } else { + ZEND_UNREACHABLE(); + } + |.code + } + } else { + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); + | SAFE_MEM_ACC_WITH_UOFFSET_32 ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { + /* perform IS_UNDEF check only after result type guard (during deoptimization) */ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | IF_UNDEF REG2w, &exit_addr + } + } else { + | IF_UNDEF REG2w, >5 + } + if (opline->opcode == ZEND_FETCH_OBJ_W + && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) + && ZEND_TYPE_IS_SET(prop_info->type)) { + uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; + + if (flags == ZEND_FETCH_DIM_WRITE) { + if ((ZEND_TYPE_FULL_MASK(prop_info->type) & (MAY_BE_ITERABLE|MAY_BE_ARRAY)) == 0) { + | cmp REG2w, #IS_FALSE + | ble >1 + |.cold_code + |1: + if (Z_REG(prop_addr) != ZREG_FCARG1x || Z_OFFSET(prop_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + } + | LOAD_ADDR FCARG2x, prop_info + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_check_array_promotion, REG0 + | b >9 + |.code + } + } else if (flags == ZEND_FETCH_REF) { + | GET_LOW_8BITS TMP1w, REG2w + | IF_TYPE TMP1w, IS_REFERENCE, >1 + if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { + | LOAD_ADDR FCARG2x, prop_info + } else { + int prop_info_offset = + (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); + + | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] + | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 + } + if (Z_REG(prop_addr) != ZREG_FCARG1x || Z_OFFSET(prop_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + } + | LOAD_ZVAL_ADDR CARG3, res_addr + | EXT_CALL zend_jit_create_typed_ref, REG0 + | b >9 + |1: + } else { + ZEND_UNREACHABLE(); + } + } + } + if (op1_avoid_refcounting) { + SET_STACK_REG(JIT_G(current_frame)->stack, + EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); + } + if (opline->opcode == ZEND_FETCH_OBJ_W) { + if (Z_REG(prop_addr) != ZREG_FCARG1x || Z_OFFSET(prop_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + } + | SET_ZVAL_PTR res_addr, FCARG1x, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { + ssa->var_info[ssa_op->result_def].indirect_reference = 1; + } + } else { + bool result_avoid_refcounting = 0; + + if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { + uint32_t flags = 0; + uint32_t old_info; + zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; + int32_t exit_point; + const void *exit_addr; + uint32_t type; + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) + && !use_this + && !op1_avoid_refcounting) { + flags = ZEND_JIT_EXIT_FREE_OP1; + } + + | LOAD_ZVAL_ADDR REG0, prop_addr + + if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) + && !(flags & ZEND_JIT_EXIT_FREE_OP1) + && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) + && (ssa_op+1)->op1_use == ssa_op->result_def + && zend_jit_may_avoid_refcounting(opline+1)) { + result_avoid_refcounting = 1; + ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; + } + + old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); + exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + + res_info &= ~MAY_BE_GUARD; + ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; + type = concrete_type(res_info); + + | // ZVAL_DEREF() + | GET_LOW_8BITS TMP1w, REG2w + | IF_NOT_TYPE TMP1w, IS_REFERENCE, >1 + | GET_Z_PTR REG0, REG0 + | add REG0, REG0, #offsetof(zend_reference, val) + if (type < IS_STRING) { + |1: + | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, ZREG_TMP1 + } else { + | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 + |1: + | GET_LOW_8BITS TMP1w, REG2w + | IF_NOT_TYPE TMP1w, type, &exit_addr + } + | // ZVAL_COPY + | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 + if (type < IS_STRING) { + if (Z_REG(res_addr) != ZREG_FP || + JIT_G(current_frame) == NULL || + STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { + | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 + } + } else { + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 + if (!result_avoid_refcounting) { + | TRY_ADDREF res_info, REG2w, REG1, TMP1w + } + } + } else { + if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_REG2)) { + return 0; + } + } + } + + |.cold_code + + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { + |5: + | SET_EX_OPLINE opline, REG0 + if (opline->opcode == ZEND_FETCH_OBJ_W) { + | EXT_CALL zend_jit_fetch_obj_w_slow, REG0 + } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { + | EXT_CALL zend_jit_fetch_obj_r_slow, REG0 + } else { + | EXT_CALL zend_jit_fetch_obj_is_slow, REG0 + } + | b >9 + } + + if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { + |7: + if (opline->opcode != ZEND_FETCH_OBJ_IS) { + | SET_EX_OPLINE opline, REG0 + if (opline->opcode != ZEND_FETCH_OBJ_W + && (op1_info & MAY_BE_UNDEF)) { + zend_jit_addr orig_op1_addr = OP1_ADDR(); + + if (op1_info & MAY_BE_ANY) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 + } + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + |1: + | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr + } else if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | LOAD_ADDR FCARG2x, Z_STRVAL_P(member) + if (opline->opcode == ZEND_FETCH_OBJ_W) { + | EXT_CALL zend_jit_invalid_property_write, REG0 + | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 + } else { + | EXT_CALL zend_jit_invalid_property_read, REG0 + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + } + | b >9 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + | b >9 + } + } + + if (!prop_info + && may_be_dynamic + && opline->opcode != ZEND_FETCH_OBJ_W) { + |8: + | mov FCARG2x, REG0 + | SET_EX_OPLINE opline, REG0 + if (opline->opcode != ZEND_FETCH_OBJ_IS) { + | EXT_CALL zend_jit_fetch_obj_r_dynamic, REG0 + } else { + | EXT_CALL zend_jit_fetch_obj_is_dynamic, REG0 + } + | b >9 + } + + |.code; + |9: // END + if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { + if (opline->op1_type == IS_VAR + && opline->opcode == ZEND_FETCH_OBJ_W + && (op1_info & MAY_BE_RC1)) { + zend_jit_addr orig_op1_addr = OP1_ADDR(); + + | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1, ZREG_TMP1, ZREG_TMP2 + | GET_ZVAL_PTR FCARG1x, orig_op1_addr, TMP1 + | GC_DELREF FCARG1x, TMP1w + | bne >1 + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_extract_helper, REG0 + |1: + } else if (!op1_avoid_refcounting) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + } + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && prop_info + && opline->op1_type != IS_VAR + && opline->op1_type != IS_TMP_VAR) { + may_throw = 0; + } + + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + + return 1; +} + +static int zend_jit_incdec_obj(dasm_State **Dst, + const zend_op *opline, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + uint32_t op1_info, + zend_jit_addr op1_addr, + bool op1_indirect, + zend_class_entry *ce, + bool ce_is_instanceof, + bool use_this, + zend_class_entry *trace_ce, + int may_throw) +{ + zval *member; + zend_string *name; + zend_property_info *prop_info; + zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); + zend_jit_addr res_addr = 0; + zend_jit_addr prop_addr; + bool needs_slow_path = 0; + + ZEND_ASSERT(opline->op2_type == IS_CONST); + ZEND_ASSERT(op1_info & MAY_BE_OBJECT); + + if (opline->result_type != IS_UNUSED) { + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + } + + member = RT_CONSTANT(opline, opline->op2); + ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); + name = Z_STR_P(member); + prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + + if (opline->op1_type == IS_UNUSED || use_this) { + | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 + } else { + if (opline->op1_type == IS_VAR + && (op1_info & MAY_BE_INDIRECT) + && Z_REG(op1_addr) == ZREG_FP) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w + | GET_Z_PTR FCARG1x, FCARG1x + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & MAY_BE_REF) { + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 + |.cold_code + |1: + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | LOAD_ADDR FCARG2x, ZSTR_VAL(name) + | EXT_CALL zend_jit_invalid_property_incdec, REG0 + | b ->exception_handler + |.code + } + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + } + + if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { + prop_info = zend_get_known_property_info(op_array, trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + if (prop_info) { + ce = trace_ce; + ce_is_instanceof = 0; + if (!(op1_info & MAY_BE_CLASS_GUARD)) { + if (!zend_jit_class_guard(Dst, opline, trace_ce)) { + return 0; + } + if (ssa->var_info && ssa_op->op1_use >= 0) { + ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_use].ce = ce; + ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; + } + if (ssa->var_info && ssa_op->op1_def >= 0) { + ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_def].ce = ce; + ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; + } + } + } + } + + if (!prop_info) { + needs_slow_path = 1; + + | ldr REG0, EX->run_time_cache + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 + | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] + | cmp REG2, TMP1 + | bne >7 + if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + | SAFE_MEM_ACC_WITH_UOFFSET ldr, TMP1, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 + | cbnz TMP1, >7 + } + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 + | tst REG0, REG0 + | blt >7 + | add TMP1, FCARG1x, REG0 + | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] + | IF_TYPE TMP2w , IS_UNDEF, >7 + | mov FCARG1x, TMP1 + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } else { + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 + | IF_TYPE TMP2w, IS_UNDEF, &exit_addr + } else { + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 + | IF_TYPE TMP2w, IS_UNDEF, >7 + needs_slow_path = 1; + } + if (ZEND_TYPE_IS_SET(prop_info->type)) { + | SET_EX_OPLINE opline, REG0 + if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { + | LOAD_ADDR FCARG2x, prop_info + } else { + int prop_info_offset = + (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); + + | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] + | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 + } + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + if (opline->result_type == IS_UNUSED) { + switch (opline->opcode) { + case ZEND_PRE_INC_OBJ: + case ZEND_POST_INC_OBJ: + | EXT_CALL zend_jit_inc_typed_prop, REG0 + break; + case ZEND_PRE_DEC_OBJ: + case ZEND_POST_DEC_OBJ: + | EXT_CALL zend_jit_dec_typed_prop, REG0 + break; + default: + ZEND_UNREACHABLE(); + } + } else { + | LOAD_ZVAL_ADDR CARG3, res_addr + switch (opline->opcode) { + case ZEND_PRE_INC_OBJ: + | EXT_CALL zend_jit_pre_inc_typed_prop, REG0 + break; + case ZEND_PRE_DEC_OBJ: + | EXT_CALL zend_jit_pre_dec_typed_prop, REG0 + break; + case ZEND_POST_INC_OBJ: + | EXT_CALL zend_jit_post_inc_typed_prop, REG0 + break; + case ZEND_POST_DEC_OBJ: + | EXT_CALL zend_jit_post_dec_typed_prop, REG0 + break; + default: + ZEND_UNREACHABLE(); + } + } + } + } + + if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { + zend_jit_addr var_addr = prop_addr; + + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + if (Z_REG(prop_addr) != ZREG_FCARG1x || Z_OFFSET(prop_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + } + + | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 + | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 + | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] + | cbnz TMP1, >1 + | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) + |.cold_code + |1: + if (opline) { + | SET_EX_OPLINE opline, REG0 + } + if (opline->result_type == IS_UNUSED) { + | mov FCARG2x, xzr + } else { + | LOAD_ZVAL_ADDR FCARG2x, res_addr + } + switch (opline->opcode) { + case ZEND_PRE_INC_OBJ: + | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 + break; + case ZEND_PRE_DEC_OBJ: + | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 + break; + case ZEND_POST_INC_OBJ: + | EXT_CALL zend_jit_post_inc_typed_ref, REG0 + break; + case ZEND_POST_DEC_OBJ: + | EXT_CALL zend_jit_post_dec_typed_ref, REG0 + break; + default: + ZEND_UNREACHABLE(); + } + | b >9 + |.code + + |2: + | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2, ZREG_TMP1 + if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { + if (opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + } + if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { + | LONG_ADD_SUB_WITH_IMM adds, var_addr, Z_L(1), TMP1, TMP2 + } else { + | LONG_ADD_SUB_WITH_IMM subs, var_addr, Z_L(1), TMP1, TMP2 + } + | bvs >3 + if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { + if (opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + } + |.cold_code + |2: + if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { + | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF MAY_BE_ANY, REG0w, REG2, TMP1w + } + if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { + if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { + | LOAD_ZVAL_ADDR FCARG2x, res_addr + | EXT_CALL zend_jit_pre_inc, REG0 + } else { + | EXT_CALL increment_function, REG0 + } + } else { + if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { + | LOAD_ZVAL_ADDR FCARG2x, res_addr + | EXT_CALL zend_jit_pre_dec, REG0 + } else { + | EXT_CALL decrement_function, REG0 + } + } + | b >4 + + |3: + if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { + uint64_t val = 0x43e0000000000000; + | LOAD_64BIT_VAL REG0, val + | SET_ZVAL_LVAL_FROM_REG var_addr, REG0, TMP1 + if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { + | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 + } + } else { + uint64_t val = 0xc3e0000000000000; + | LOAD_64BIT_VAL REG0, val + | SET_ZVAL_LVAL_FROM_REG var_addr, REG0, TMP1 + if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { + | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 + } + } + | b >4 + |.code + |4: + } + + if (needs_slow_path) { + |.cold_code + |7: + | SET_EX_OPLINE opline, REG0 + | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); + | LOAD_ADDR FCARG2x, name + | ldr CARG3, EX->run_time_cache + | ADD_SUB_64_WITH_CONST_32 add, CARG3, CARG3, opline->extended_value, TMP1 + if (opline->result_type == IS_UNUSED) { + | mov CARG4, xzr + } else { + | LOAD_ZVAL_ADDR CARG4, res_addr + } + + switch (opline->opcode) { + case ZEND_PRE_INC_OBJ: + | EXT_CALL zend_jit_pre_inc_obj_helper, REG0 + break; + case ZEND_PRE_DEC_OBJ: + | EXT_CALL zend_jit_pre_dec_obj_helper, REG0 + break; + case ZEND_POST_INC_OBJ: + | EXT_CALL zend_jit_post_inc_obj_helper, REG0 + break; + case ZEND_POST_DEC_OBJ: + | EXT_CALL zend_jit_post_dec_obj_helper, REG0 + break; + default: + ZEND_UNREACHABLE(); + } + + | b >9 + |.code + } + + |9: + if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + + return 1; +} + +static int zend_jit_assign_obj_op(dasm_State **Dst, + const zend_op *opline, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + uint32_t op1_info, + zend_jit_addr op1_addr, + uint32_t val_info, + zend_ssa_range *val_range, + bool op1_indirect, + zend_class_entry *ce, + bool ce_is_instanceof, + bool use_this, + zend_class_entry *trace_ce, + int may_throw) +{ + zval *member; + zend_string *name; + zend_property_info *prop_info; + zend_jit_addr val_addr = OP1_DATA_ADDR(); + zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); + zend_jit_addr prop_addr; + bool needs_slow_path = 0; + binary_op_type binary_op = get_binary_op(opline->extended_value); + + ZEND_ASSERT(opline->op2_type == IS_CONST); + ZEND_ASSERT(op1_info & MAY_BE_OBJECT); + ZEND_ASSERT(opline->result_type == IS_UNUSED); + + member = RT_CONSTANT(opline, opline->op2); + ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); + name = Z_STR_P(member); + prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + + if (opline->op1_type == IS_UNUSED || use_this) { + | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 + } else { + if (opline->op1_type == IS_VAR + && (op1_info & MAY_BE_INDIRECT) + && Z_REG(op1_addr) == ZREG_FP) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w + | GET_Z_PTR FCARG1x, FCARG1x + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & MAY_BE_REF) { + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 + |.cold_code + |1: + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | LOAD_ADDR FCARG2x, ZSTR_VAL(name) + if (op1_info & MAY_BE_UNDEF) { + | EXT_CALL zend_jit_invalid_property_assign_op, REG0 + } else { + | EXT_CALL zend_jit_invalid_property_assign, REG0 + } + if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) + && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | b >8 + } else { + | b ->exception_handler + } + |.code + } + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + } + + if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { + prop_info = zend_get_known_property_info(op_array, trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + if (prop_info) { + ce = trace_ce; + ce_is_instanceof = 0; + if (!(op1_info & MAY_BE_CLASS_GUARD)) { + if (!zend_jit_class_guard(Dst, opline, trace_ce)) { + return 0; + } + if (ssa->var_info && ssa_op->op1_use >= 0) { + ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_use].ce = ce; + ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; + } + if (ssa->var_info && ssa_op->op1_def >= 0) { + ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_def].ce = ce; + ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; + } + } + } + } + + if (!prop_info) { + needs_slow_path = 1; + + | ldr REG0, EX->run_time_cache + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG2, REG0, ((opline+1)->extended_value), TMP1 + | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] + | cmp REG2, TMP2 + | bne >7 + if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + | SAFE_MEM_ACC_WITH_UOFFSET ldr, TMP1, REG0, ((opline+1)->extended_value + sizeof(void*) * 2), TMP1 + | cbnz TMP1, >7 + } + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, ((opline+1)->extended_value + sizeof(void*)), TMP1 + | tst REG0, REG0 + | blt >7 + | add TMP1, FCARG1x, REG0 + | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] + | IF_TYPE TMP2w, IS_UNDEF, >7 + | mov FCARG1x, TMP1 + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } else { + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 + | IF_TYPE TMP2w, IS_UNDEF, &exit_addr + } else { + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 + | IF_TYPE TMP2w, IS_UNDEF, >7 + needs_slow_path = 1; + } + if (ZEND_TYPE_IS_SET(prop_info->type)) { + uint32_t info = val_info; + + if (opline) { + | SET_EX_OPLINE opline, REG0 + } + + | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1, ZREG_TMP1 + |.cold_code + |1: + | GET_ZVAL_PTR FCARG1x, prop_addr, TMP1 + if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2x || Z_OFFSET(val_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG2x, val_addr + } + | LOAD_ADDR CARG3, binary_op + | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 + | b >9 + |.code + + | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); + + if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { + | LOAD_ADDR FCARG2x, prop_info + } else { + int prop_info_offset = + (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); + + | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] + | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 + } + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + | LOAD_ZVAL_ADDR CARG3, val_addr + | LOAD_ADDR CARG4, binary_op + + | EXT_CALL zend_jit_assign_op_to_typed_prop, REG0 + + if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + info |= MAY_BE_RC1|MAY_BE_RCN; + } + + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + } + + if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { + zend_jit_addr var_addr = prop_addr; + uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; + uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; + + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + | LOAD_ZVAL_ADDR REG0, prop_addr + + | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 + | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 + | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] + | cbnz TMP1, >1 + | add REG0, FCARG1x, #offsetof(zend_reference, val) + |.cold_code + |1: + if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2x || Z_OFFSET(val_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG2x, val_addr + } + if (opline) { + | SET_EX_OPLINE opline, REG0 + } + | LOAD_ADDR CARG3, binary_op + | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 + | b >9 + |.code + |2: + + switch (opline->extended_value) { + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + case ZEND_DIV: + if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, + 1 /* may overflow */, 0)) { + return 0; + } + break; + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + case ZEND_SL: + case ZEND_SR: + case ZEND_MOD: + if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, + IS_CV, opline->op1, var_addr, var_info, NULL, + (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, + val_range, + 0, var_addr, var_def_info, var_info, 0)) { + return 0; + } + break; + case ZEND_CONCAT: + if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, + 0)) { + return 0; + } + break; + default: + ZEND_UNREACHABLE(); + } + } + + if (needs_slow_path) { + |.cold_code + |7: + | SET_EX_OPLINE opline, REG0 + | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); + | LOAD_ADDR FCARG2x, name + | LOAD_ZVAL_ADDR CARG3, val_addr + | ldr CARG4, EX->run_time_cache + | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, (opline+1)->extended_value, TMP1 + | LOAD_ADDR CARG5, binary_op + | EXT_CALL zend_jit_assign_obj_op_helper, REG0 + + if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + val_info |= MAY_BE_RC1|MAY_BE_RCN; + } + + |8: + | // FREE_OP_DATA(); + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + | b >9 + |.code + } + + |9: + if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + + return 1; +} + +static int zend_jit_assign_obj(dasm_State **Dst, + const zend_op *opline, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + uint32_t op1_info, + zend_jit_addr op1_addr, + uint32_t val_info, + bool op1_indirect, + zend_class_entry *ce, + bool ce_is_instanceof, + bool use_this, + zend_class_entry *trace_ce, + int may_throw) +{ + zval *member; + zend_string *name; + zend_property_info *prop_info; + zend_jit_addr val_addr = OP1_DATA_ADDR(); + zend_jit_addr res_addr = 0; + zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); + zend_jit_addr prop_addr; + bool needs_slow_path = 0; + + if (RETURN_VALUE_USED(opline)) { + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + } + + ZEND_ASSERT(opline->op2_type == IS_CONST); + ZEND_ASSERT(op1_info & MAY_BE_OBJECT); + + member = RT_CONSTANT(opline, opline->op2); + ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); + name = Z_STR_P(member); + prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + + if (opline->op1_type == IS_UNUSED || use_this) { + | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 + } else { + if (opline->op1_type == IS_VAR + && (op1_info & MAY_BE_INDIRECT) + && Z_REG(op1_addr) == ZREG_FP) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w + | GET_Z_PTR FCARG1x, FCARG1x + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & MAY_BE_REF) { + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 + |.cold_code + |1: + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | LOAD_ADDR FCARG2x, ZSTR_VAL(name) + | EXT_CALL zend_jit_invalid_property_assign, REG0 + if (RETURN_VALUE_USED(opline)) { + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + } + if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) + && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | b >8 + } else { + | b ->exception_handler + } + |.code + } + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + } + + if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { + prop_info = zend_get_known_property_info(op_array, trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + if (prop_info) { + ce = trace_ce; + ce_is_instanceof = 0; + if (!(op1_info & MAY_BE_CLASS_GUARD)) { + if (!zend_jit_class_guard(Dst, opline, trace_ce)) { + return 0; + } + if (ssa->var_info && ssa_op->op1_use >= 0) { + ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_use].ce = ce; + ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; + } + if (ssa->var_info && ssa_op->op1_def >= 0) { + ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_def].ce = ce; + ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; + } + } + } + } + + if (!prop_info) { + needs_slow_path = 1; + + | ldr REG0, EX->run_time_cache + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 + | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] + | cmp REG2, TMP1 + | bne >5 + if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + | SAFE_MEM_ACC_WITH_UOFFSET ldr, FCARG2x, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 + } + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 + | tst REG0, REG0 + | blt >5 + | add TMP2, FCARG1x, REG0 + | ldrb TMP1w, [TMP2, #offsetof(zval,u1.v.type)] + | IF_TYPE TMP1w, IS_UNDEF, >5 + | mov FCARG1x, TMP2 + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + | cbnz FCARG2x, >1 + |.cold_code + |1: + | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); + | SET_EX_OPLINE opline, REG0 + | LOAD_ZVAL_ADDR CARG3, val_addr + if (RETURN_VALUE_USED(opline)) { + | LOAD_ZVAL_ADDR CARG4, res_addr + } else { + | mov CARG4, xzr + } + + | EXT_CALL zend_jit_assign_to_typed_prop, REG0 + + if ((opline+1)->op1_type == IS_CONST) { + | // TODO: ??? + | // if (Z_TYPE_P(value) == orig_type) { + | // CACHE_PTR_EX(cache_slot + 2, NULL); + } + + if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) + && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | b >8 + } else { + | b >9 + } + |.code + } + } else { + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); + if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set) { + // Undefined property with magic __get()/__set() + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 + | IF_TYPE TMP2w, IS_UNDEF, &exit_addr + } else { + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 + | IF_TYPE TMP2w, IS_UNDEF, >5 + needs_slow_path = 1; + } + } + if (ZEND_TYPE_IS_SET(prop_info->type)) { + uint32_t info = val_info; + + | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); + | SET_EX_OPLINE opline, REG0 + if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { + | LOAD_ADDR FCARG2x, prop_info + } else { + int prop_info_offset = + (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); + + | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] + | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 + } + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + | LOAD_ZVAL_ADDR CARG3, val_addr + if (RETURN_VALUE_USED(opline)) { + | LOAD_ZVAL_ADDR CARG4, res_addr + } else { + | mov CARG4, xzr + } + + | EXT_CALL zend_jit_assign_to_typed_prop, REG0 + + if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + info |= MAY_BE_RC1|MAY_BE_RCN; + } + + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + } + + if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { + // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); + if (opline->result_type == IS_UNUSED) { + if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { + return 0; + } + } else { + if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { + return 0; + } + } + } + + if (needs_slow_path) { + |.cold_code + |5: + | SET_EX_OPLINE opline, REG0 + | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); + | LOAD_ADDR FCARG2x, name + + | LOAD_ZVAL_ADDR CARG3, val_addr + | ldr CARG4, EX->run_time_cache + | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, opline->extended_value, TMP1 + if (RETURN_VALUE_USED(opline)) { + | LOAD_ZVAL_ADDR CARG5, res_addr + } else { + | mov CARG5, xzr + } + + | EXT_CALL zend_jit_assign_obj_helper, REG0 + + if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + val_info |= MAY_BE_RC1|MAY_BE_RCN; + } + + |8: + | // FREE_OP_DATA(); + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + | b >9 + |.code + } + + |9: + if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + + return 1; +} + +static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) +{ + zend_jit_addr op1_addr = OP1_ADDR(); + + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + if (may_throw) { + | SET_EX_OPLINE opline, REG0 + } + if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { + if (op1_info & MAY_BE_ARRAY) { + | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 + } + | SAFE_MEM_ACC_WITH_UOFFSET_32 ldr, FCARG1w, FP, (opline->op1.var + offsetof(zval, u2.fe_iter_idx)), TMP1 + | mvn TMP1w, wzr // TODO: DynAsm fails loading #-1 + | cmp FCARG1w, TMP1w + | beq >7 + | EXT_CALL zend_hash_iterator_del, REG0 + |7: + } + | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + } + return 1; +} + +static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) +{ + if (opline->op1_type == IS_CONST) { + zval *zv; + size_t len; + + zv = RT_CONSTANT(opline, opline->op1); + ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); + len = Z_STRLEN_P(zv); + + if (len > 0) { + const char *str = Z_STRVAL_P(zv); + + | SET_EX_OPLINE opline, REG0 + | LOAD_ADDR CARG1, str + | LOAD_64BIT_VAL CARG2, len + | EXT_CALL zend_write, REG0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + } else { + zend_jit_addr op1_addr = OP1_ADDR(); + + ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); + + | SET_EX_OPLINE opline, REG0 + | GET_ZVAL_PTR REG0, op1_addr, TMP1 + | add CARG1, REG0, #offsetof(zend_string, val) + | ldr CARG2, [REG0, #offsetof(zend_string, len)] + | EXT_CALL zend_write, REG0 + if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { + | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + return 1; +} + +static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) +{ + zend_jit_addr res_addr = RES_ADDR(); + + if (opline->op1_type == IS_CONST) { + zval *zv; + size_t len; + + zv = RT_CONSTANT(opline, opline->op1); + ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); + len = Z_STRLEN_P(zv); + + | SET_ZVAL_LVAL res_addr, len, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + } else { + ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); + + | GET_ZVAL_PTR REG0, op1_addr, TMP1 + | ldr REG0, [REG0, #offsetof(zend_string, len)] + | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + return 1; +} + +static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, int may_throw) +{ + zend_jit_addr res_addr = RES_ADDR(); + + if (opline->op1_type == IS_CONST) { + zval *zv; + zend_long count; + + zv = RT_CONSTANT(opline, opline->op1); + ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); + count = zend_hash_num_elements(Z_ARRVAL_P(zv)); + + | SET_ZVAL_LVAL res_addr, count, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + } else { + ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); + // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. + + | GET_ZVAL_PTR REG0, op1_addr, TMP1 + // Sign-extend the 32-bit value to a potentially 64-bit zend_long + | ldr REG0w, [REG0, #offsetof(HashTable, nNumOfElements)] + | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + + if (may_throw) { + return zend_jit_check_exception(Dst); + } + return 1; +} + +static int zend_jit_load_this(dasm_State **Dst, uint32_t var) +{ + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); + + | ldr FCARG1x, EX->This.value.ptr + | SET_ZVAL_PTR var_addr, FCARG1x, TMP1 + | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX, TMP1w, TMP2 + | GC_ADDREF FCARG1x, TMP1w + return 1; +} + +static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) +{ + if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + if (!JIT_G(current_frame) || + !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { + + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | ldrb TMP1w, EX->This.u1.v.type + | cmp TMP1w, #IS_OBJECT + | bne &exit_addr + + if (JIT_G(current_frame)) { + TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); + } + } + } else { + + | ldrb TMP1w, EX->This.u1.v.type + | cmp TMP1w, #IS_OBJECT + | bne >1 + |.cold_code + |1: + | SET_EX_OPLINE opline, REG0 + | b ->invalid_this + |.code + } + } + + if (!check_only) { + if (!zend_jit_load_this(Dst, opline->result.var)) { + return 0; + } + } + return 1; +} + +static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) +{ + uint32_t count; + Bucket *p; + const zend_op *target; + int b; + int32_t exit_point; + const void *exit_addr; + + if (default_label) { + | cbz REG0, &default_label + } else if (next_opline) { + | cbz REG0, >3 + } else { + | cbz REG0, =>default_b + } + | LOAD_ADDR FCARG1x, jumptable + | ldr TMP1, [FCARG1x, #offsetof(HashTable, arData)] + | sub REG0, REG0, TMP1 + | mov FCARG1x, #(sizeof(Bucket) / sizeof(void*)) + | sdiv REG0, REG0, FCARG1x + | adr FCARG1x, >4 + | ldr TMP1, [FCARG1x, REG0] + | br TMP1 + + |.jmp_table + |.align 8 + |4: + if (trace_info) { + trace_info->jmp_table_size += zend_hash_num_elements(jumptable); + } + + count = jumptable->nNumUsed; + p = jumptable->arData; + do { + if (Z_TYPE(p->val) == IS_UNDEF) { + if (default_label) { + | .addr &default_label + } else if (next_opline) { + | .addr >3 + } else { + | .addr =>default_b + } + } else { + target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); + if (!next_opline) { + b = ssa->cfg.map[target - op_array->opcodes]; + | .addr =>b + } else if (next_opline == target) { + | .addr >3 + } else { + exit_point = zend_jit_trace_get_exit_point(target, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + | .addr &exit_addr + } + } + p++; + count--; + } while (count); + |.code + + return 1; +} + +static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) +{ + HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); + const zend_op *next_opline = NULL; + + if (trace) { + ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); + ZEND_ASSERT(trace->opline != NULL); + next_opline = trace->opline; + } + + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(opline, opline->op1); + zval *jump_zv = NULL; + int b; + + if (opline->opcode == ZEND_SWITCH_LONG) { + if (Z_TYPE_P(zv) == IS_LONG) { + jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); + } + } else if (opline->opcode == ZEND_SWITCH_STRING) { + if (Z_TYPE_P(zv) == IS_STRING) { + jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1); + } + } else if (opline->opcode == ZEND_MATCH) { + if (Z_TYPE_P(zv) == IS_LONG) { + jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); + } else if (Z_TYPE_P(zv) == IS_STRING) { + jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1); + } + } else { + ZEND_UNREACHABLE(); + } + if (next_opline) { + const zend_op *target; + + if (jump_zv != NULL) { + target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); + } else { + target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); + } + ZEND_ASSERT(target == next_opline); + } else { + if (jump_zv != NULL) { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; + } else { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; + } + | b =>b + } + } else { + zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; + uint32_t op1_info = OP1_INFO(); + zend_jit_addr op1_addr = OP1_ADDR(); + const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); + const zend_op *target; + int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; + int b; + int32_t exit_point; + const void *fallback_label = NULL; + const void *default_label = NULL; + const void *exit_addr; + + if (next_opline) { + if (next_opline != opline + 1) { + exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + fallback_label = zend_jit_trace_get_exit_addr(exit_point); + } + if (next_opline != default_opline) { + exit_point = zend_jit_trace_get_exit_point(default_opline, 0); + default_label = zend_jit_trace_get_exit_addr(exit_point); + } + } + + if (opline->opcode == ZEND_SWITCH_LONG) { + if (op1_info & MAY_BE_LONG) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1, ZREG_TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr, TMP1 + |.cold_code + |1: + | // ZVAL_DEREF(op) + if (fallback_label) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 + } + | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 + if (fallback_label) { + | add TMP1, FCARG2x, #offsetof(zend_reference, val) + | IF_NOT_Z_TYPE TMP1, IS_LONG, &fallback_label, TMP2w + } else { + | add TMP1, FCARG2x, #offsetof(zend_reference, val) + | IF_NOT_Z_TYPE TMP1, IS_LONG, >3, TMP2w + } + | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.lval)] + | b >2 + |.code + |2: + } else { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { + if (fallback_label) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 + } + } + | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr, TMP1 + } + if (HT_IS_PACKED(jumptable)) { + uint32_t count = jumptable->nNumUsed; + Bucket *p = jumptable->arData; + + | CMP_64_WITH_CONST_32 FCARG2x, jumptable->nNumUsed, TMP1 + if (default_label) { + | bhs &default_label + } else if (next_opline) { + | bhs >3 + } else { + | bhs =>default_b + } + | adr REG0, >4 + | ldr TMP1, [REG0, FCARG2x, lsl #3] + | br TMP1 + + |.jmp_table + |.align 8 + |4: + if (trace_info) { + trace_info->jmp_table_size += count; + } + p = jumptable->arData; + do { + if (Z_TYPE(p->val) == IS_UNDEF) { + if (default_label) { + | .addr &default_label + } else if (next_opline) { + | .addr >3 + } else { + | .addr =>default_b + } + } else { + target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); + if (!next_opline) { + b = ssa->cfg.map[target - op_array->opcodes]; + | .addr =>b + } else if (next_opline == target) { + | .addr >3 + } else { + exit_point = zend_jit_trace_get_exit_point(target, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + | .addr &exit_addr + } + } + p++; + count--; + } while (count); + |.code + |3: + } else { + | LOAD_ADDR FCARG1x, jumptable + | EXT_CALL zend_hash_index_find, REG0 + | mov REG0, RETVALx + if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { + return 0; + } + |3: + } + } + } else if (opline->opcode == ZEND_SWITCH_STRING) { + if (op1_info & MAY_BE_STRING) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1, ZREG_TMP1 + | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 + |.cold_code + |1: + | // ZVAL_DEREF(op) + if (fallback_label) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 + } + | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 + if (fallback_label) { + | add TMP1, FCARG2x, #offsetof(zend_reference, val) + | IF_NOT_Z_TYPE TMP1, IS_STRING, &fallback_label, TMP2w + } else { + | add TMP1, FCARG2x, #offsetof(zend_reference, val) + | IF_NOT_Z_TYPE TMP1, IS_STRING, >3, TMP2w + } + | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.ptr)] + | b >2 + |.code + |2: + } else { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { + if (fallback_label) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 + } + } + | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 + } + | LOAD_ADDR FCARG1x, jumptable + | EXT_CALL zend_hash_find, REG0 + | mov REG0, RETVALx + if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { + return 0; + } + |3: + } + } else if (opline->opcode == ZEND_MATCH) { + if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + | ZVAL_DEREF FCARG2x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + } + | LOAD_ADDR FCARG1x, jumptable + if (op1_info & MAY_BE_LONG) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { + if (op1_info & MAY_BE_STRING) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5, ZREG_TMP1 + } else if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 + } else if (default_label) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label, ZREG_TMP1 + } else if (next_opline) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b, ZREG_TMP1 + } + } + | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr, TMP1 + | EXT_CALL zend_hash_index_find, REG0 + | mov REG0, RETVALx + if (op1_info & MAY_BE_STRING) { + | b >2 + } + } + if (op1_info & MAY_BE_STRING) { + |5: + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { + if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 + } else if (default_label) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label, ZREG_TMP1 + } else if (next_opline) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b, ZREG_TMP1 + } + } + | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 + | EXT_CALL zend_hash_find, REG0 + | mov REG0, RETVALx + } + |2: + if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { + return 0; + } + } + if (op1_info & MAY_BE_UNDEF) { + |6: + if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { + if (default_label) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label, ZREG_TMP1 + } else if (next_opline) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3, ZREG_TMP1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b, ZREG_TMP1 + } + } + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + if (!zend_jit_check_exception_undef_result(Dst, opline)) { + return 0; + } + } + if (default_label) { + | b &default_label + } else if (next_opline) { + | b >3 + } else { + | b =>default_b + } + |3: + } else { + ZEND_UNREACHABLE(); + } + } + return 1; +} + +static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) +{ + zend_arg_info *arg_info = &op_array->arg_info[-1]; + ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); + zend_jit_addr op1_addr = OP1_ADDR(); + bool needs_slow_check = 1; + bool slow_check_in_cold = 1; + uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; + + if (type_mask == 0) { + slow_check_in_cold = 0; + } else { + if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { + slow_check_in_cold = 0; + } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { + needs_slow_check = 0; + } else if (is_power_of_two(type_mask)) { + uint32_t type_code = concrete_type(type_mask); + | IF_NOT_ZVAL_TYPE op1_addr, type_code, >7, ZREG_TMP1 + } else { + | mov REG2w, #1 + | GET_ZVAL_TYPE REG1w, op1_addr, TMP1 + | lsl REG2w, REG2w, REG1w + | TST_32_WITH_CONST REG2w, type_mask, TMP1w + | beq >7 + } + } + if (needs_slow_check) { + if (slow_check_in_cold) { + |.cold_code + |7: + } + | SET_EX_OPLINE opline, REG1 + if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >8, ZREG_TMP1 + | LOAD_32BIT_VAL FCARG1x, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + | LOAD_ADDR_ZTS REG0, executor_globals, uninitialized_zval + } + |8: + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | ldr FCARG2x, EX->func + | LOAD_ADDR CARG3, (ptrdiff_t)arg_info + | ldr REG0, EX->run_time_cache + | ADD_SUB_64_WITH_CONST_32 add, CARG4, REG0, opline->op2.num, TMP1 + | EXT_CALL zend_jit_verify_return_slow, REG0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + if (slow_check_in_cold) { + | b >9 + |.code + } + } + |9: + return 1; +} + +static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + // TODO: support for empty() ??? + ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); + + if (op1_info & MAY_BE_REF) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + |1: + } + + if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { + if (exit_addr) { + ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); + } else if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | b =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | b =>target_label2 + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { + if (exit_addr) { + ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); + } else if (smart_branch_opcode) { + if (smart_branch_opcode != ZEND_JMPNZ) { + | b =>target_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + } else { + ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP1w, Rx(Z_REG(op1_addr)), (Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)), TMP1 + | cmp TMP1w, #IS_NULL + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | bgt &exit_addr + } else { + | ble &exit_addr + } + } else if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + | ble =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | bgt =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | ble =>target_label + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + | cset REG0w, gt + | add REG0w, REG0w, #IS_FALSE + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + } + + return 1; +} + +static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) +{ + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(opline, opline->op1); + + | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 + if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, REG0, TMP1 + } + } else { + zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + + | // ZVAL_COPY(res, value); + | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_REG0, ZREG_FCARG1x, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + if (opline->op1_type == IS_CV) { + | TRY_ADDREF op1_info, REG0w, FCARG1x, TMP1w + } + } + | // Z_FE_POS_P(res) = 0; + | SAFE_MEM_ACC_WITH_UOFFSET_32 str, wzr, FP, (opline->result.var + offsetof(zval, u2.fe_pos)), TMP1 + + return 1; +} + +static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr) +{ + zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + + | // array = EX_VAR(opline->op1.var); + | // fe_ht = Z_ARRVAL_P(array); + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | // pos = Z_FE_POS_P(array); + | SAFE_MEM_ACC_WITH_UOFFSET_32 ldr, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 + | // p = fe_ht->arData + pos; + || ZEND_ASSERT(sizeof(Bucket) == 32); + | mov FCARG2w, REG0w + | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] + | add FCARG2x, TMP1, FCARG2x, lsl #5 + |1: + | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { + | ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)] + | cmp TMP1w, REG0w + | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); + | // ZEND_VM_CONTINUE(); + if (exit_addr) { + if (exit_opcode == ZEND_JMP) { + | bls &exit_addr + } else { + | bls >3 + } + } else { + | bls =>target_label + } + | // pos++; + | add REG0w, REG0w, #1 + | // value_type = Z_TYPE_INFO_P(value); + | // if (EXPECTED(value_type != IS_UNDEF)) { + if (!exit_addr || exit_opcode == ZEND_JMP) { + | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w + } else { + | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr, TMP1w + } + | // p++; + | add FCARG2x, FCARG2x, #sizeof(Bucket) + | b <1 + |3: + + if (!exit_addr || exit_opcode == ZEND_JMP) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + uint32_t val_info; + + | // Z_FE_POS_P(array) = pos + 1; + | SAFE_MEM_ACC_WITH_UOFFSET_32 str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 + + if (RETURN_VALUE_USED(opline)) { + zend_jit_addr res_addr = RES_ADDR(); + + if ((op1_info & MAY_BE_ARRAY_KEY_LONG) + && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { + | // if (!p->key) { + | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] + | cbz REG0, >2 + } + if (op1_info & MAY_BE_ARRAY_KEY_STRING) { + | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); + | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] + | SET_ZVAL_PTR res_addr, REG0, TMP1 + | ldr TMP1w, [REG0, #offsetof(zend_refcounted, gc.u.type_info)] + | TST_32_WITH_CONST TMP1w, IS_STR_INTERNED, TMP2w + | beq >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 + | b >3 + |1: + | GC_ADDREF REG0, TMP1w + | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2 + + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + | b >3 + |2: + } + } + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); + | ldr REG0, [FCARG2x, #offsetof(Bucket, h)] + | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + } + |3: + } + + val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); + if (val_info & MAY_BE_ARRAY) { + val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; + } + if (op1_info & MAY_BE_ARRAY_OF_REF) { + val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | + MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; + } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + val_info |= MAY_BE_RC1 | MAY_BE_RCN; + } + + if (opline->op2_type == IS_CV) { + | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); + if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { + return 0; + } + } else { + | // ZVAL_COPY(res, value); + | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_REG0, ZREG_FCARG1x, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF val_info, REG0w, FCARG1x, TMP1w + } + } + + return 1; +} + +static int zend_jit_fetch_constant(dasm_State **Dst, + const zend_op *opline, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op) +{ + zval *zv = RT_CONSTANT(opline, opline->op2) + 1; + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + uint32_t res_info = RES_INFO(); + + | // c = CACHED_PTR(opline->extended_value); + | ldr FCARG1x, EX->run_time_cache + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, FCARG1x, opline->extended_value, TMP1 + | // if (c != NULL) + | cbz REG0, >9 + | // if (!IS_SPECIAL_CACHE_VAL(c)) + || ZEND_ASSERT(CACHE_SPECIAL == 1); + | TST_64_WITH_ONE REG0 + | bne >9 + |8: + + if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { + zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; + uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + int32_t exit_point; + const void *exit_addr = NULL; + + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); + exit_point = zend_jit_trace_get_exit_point(opline+1, 0); + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + res_info &= ~MAY_BE_GUARD; + ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; + + uint32_t type = concrete_type(res_info); + + if (type < IS_STRING) { + | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr, ZREG_TMP1 + } else { + | GET_ZVAL_TYPE_INFO REG2w, const_addr, TMP1 + | IF_NOT_TYPE REG2w, type, &exit_addr + } + | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 + if (type < IS_STRING) { + | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 + | TRY_ADDREF res_info, REG2w, REG1, TMP1w + } + } else { + | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1w + } + + |.cold_code + |9: + | // SAVE_OPLINE(); + | SET_EX_OPLINE opline, REG0 + | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); + | LOAD_ADDR FCARG1x, zv + | LOAD_32BIT_VAL FCARG2w, opline->op1.num + | EXT_CALL zend_jit_get_constant, REG0 + | mov REG0, RETVALx + | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); + | cbnz REG0, <8 + | b ->exception_handler + |.code + return 1; +} + +static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); + ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); + + | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); + | LOAD_ADDR FCARG1x, ht + if (opline->op1_type != IS_CONST) { + | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 + | EXT_CALL zend_hash_find, REG0 + } else { + zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); + | LOAD_ADDR FCARG2x, str + | EXT_CALL _zend_hash_find_known_hash, REG0 + } + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | cbz RETVALx, &exit_addr + } else { + | cbnz RETVALx, &exit_addr + } + } else if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + | cbz RETVALx, =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | cbnz RETVALx, =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | cbz RETVALx, =>target_label + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + | tst RETVALx, RETVALx + | cset REG0w, ne + | add REG0w, REG0w, #IS_FALSE + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + + return 1; +} + +static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) +{ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 + + return 1; +} + +static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) +{ + zend_jit_addr var_addr = *var_addr_ptr; + uint32_t var_info = *var_info_ptr; + const void *exit_addr = NULL; + + if (add_ref_guard || add_type_guard) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + } + + if (add_ref_guard) { + | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 + } + if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { + /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ + if (Z_REG(var_addr) != ZREG_FCARG1x || Z_OFFSET(var_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, var_addr + } + | EXT_CALL zend_jit_unref_helper, REG0 + } else { + | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, offsetof(zend_reference, val)); + *var_addr_ptr = var_addr; + } + + if (var_type != IS_UNKNOWN) { + var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); + } + if (add_type_guard + && var_type != IS_UNKNOWN + && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { + | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr, ZREG_TMP1 + + ZEND_ASSERT(var_info & (1 << var_type)); + if (var_type < IS_STRING) { + var_info = (1 << var_type); + } else if (var_type != IS_ARRAY) { + var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); + } else { + var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); + } + + *var_info_ptr = var_info; + } else { + var_info &= ~MAY_BE_REF; + *var_info_ptr = var_info; + } + + return 1; +} + +static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) +{ + zend_jit_addr var_addr = *var_addr_ptr; + uint32_t var_info = *var_info_ptr; + int32_t exit_point; + const void *exit_addr; + + if (add_indirect_guard) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr, ZREG_TMP1 + | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 + } else { + /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ + if (opline->op1_type != IS_VAR || + (opline-1)->result_type != IS_VAR || + (opline-1)->result.var != opline->op1.var || + (opline-1)->op2_type == IS_VAR || + (opline-1)->op2_type == IS_TMP_VAR) { + | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 + } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { + | mov FCARG1x, REG0 + } + } + *var_info_ptr &= ~MAY_BE_INDIRECT; + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + *var_addr_ptr = var_addr; + + if (var_type != IS_UNKNOWN) { + var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); + } + if (!(var_type & IS_TRACE_REFERENCE) + && var_type != IS_UNKNOWN + && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { + exit_point = zend_jit_trace_get_exit_point(opline, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | IF_NOT_Z_TYPE FCARG1x, var_type, &exit_addr, TMP1w + + //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); + ZEND_ASSERT(var_info & (1 << var_type)); + if (var_type < IS_STRING) { + var_info = (1 << var_type); + } else if (var_type != IS_ARRAY) { + var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); + } else { + var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); + } + + *var_info_ptr = var_info; + } + + return 1; +} + +static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) +{ + if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { + return 0; + } + + switch (opline->opcode) { + case ZEND_QM_ASSIGN: + case ZEND_SEND_VAR: + case ZEND_ASSIGN: + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + return 1; + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + if (def_var == ssa_op->result_def && + use_var == ssa_op->op1_use) { + return 1; + } + break; + default: + break; + } + return 0; +} + +static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) +{ + uint32_t op1_info, op2_info; + + switch (opline->opcode) { + case ZEND_QM_ASSIGN: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + case ZEND_CASE: + case ZEND_RETURN: + return 1; + case ZEND_ASSIGN: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + return + opline->op1_type == IS_CV && + !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_REF)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + case ZEND_SL: + case ZEND_SR: + case ZEND_MOD: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + op1_info = OP1_INFO(); + op2_info = OP1_DEF_INFO(); + return opline->op1_type == IS_CV + && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)) + && (op2_info & MAY_BE_LONG); + case ZEND_BOOL: + case ZEND_BOOL_NOT: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + return 1; + case ZEND_FETCH_DIM_R: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (trace + && trace->op1_type != IS_UNKNOWN + && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { + op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); + } + return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && + (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && + (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || + (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && + (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); + } + return 0; +} + +static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) +{ + if (ssa->vars[var].no_val) { + /* we don't need the value */ + return 0; + } + + if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { + /* Disable global register allocation, + * register allocation for SSA variables connected through Phi functions + */ + if (ssa->vars[var].definition_phi) { + return 0; + } + if (ssa->vars[var].phi_use_chain) { + zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; + do { + if (!ssa->vars[phi->ssa_var].no_val) { + return 0; + } + phi = zend_ssa_next_use_phi(ssa, var, phi); + } while (phi); + } + } + + if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && + ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { + /* bad type */ + return 0; + } + + return 1; +} + +static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) +{ + if (!zend_jit_var_supports_reg(ssa, var)) { + return 0; + } + + if (ssa->vars[var].definition >= 0) { + uint32_t def = ssa->vars[var].definition; + if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { + return 0; + } + } + + if (ssa->vars[var].use_chain >= 0) { + int use = ssa->vars[var].use_chain; + + do { + if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && + !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { + return 0; + } + use = zend_ssa_next_use(ssa->ops, var, use); + } while (use >= 0); + } + + return 1; +} + +static bool zend_needs_extra_reg_for_const(const zend_op *opline, zend_uchar op_type, znode_op op) +{ +|| if (op_type == IS_CONST) { +|| zval *zv = RT_CONSTANT(opline, op); +|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0) { +|| return 1; +|| } else if (Z_TYPE_P(zv) == IS_LONG) { +|| return 1; +|| } +|| } + return 0; +} + +static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) +{ + uint32_t op1_info, op2_info; + + switch (opline->opcode) { + case ZEND_FETCH_DIM_R: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || + ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { + return ZEND_REGSET(ZREG_FCARG1x); + } + break; + default: + break; + } + + return ZEND_REGSET_EMPTY; +} + +static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) +{ + uint32_t op1_info, op2_info, res_info; + zend_regset regset = ZEND_REGSET_SCRATCH; + + switch (opline->opcode) { + case ZEND_NOP: + case ZEND_OP_DATA: + case ZEND_JMP: + case ZEND_RETURN: + regset = ZEND_REGSET_EMPTY; + break; + case ZEND_QM_ASSIGN: + if (ssa_op->op1_def == current_var || + ssa_op->result_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + /* break missing intentionally */ + case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: + if (ssa_op->op1_use == current_var) { + regset = ZEND_REGSET(ZREG_REG0); + break; + } + op1_info = OP1_INFO(); + if (!(op1_info & MAY_BE_UNDEF)) { + if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_FPR0); + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + regset = ZEND_REGSET(ZREG_REG0); + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); + } + } + break; + case ZEND_SEND_VAR: + if (ssa_op->op1_use == current_var || + ssa_op->op1_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + op1_info = OP1_INFO(); + if (!(op1_info & MAY_BE_UNDEF)) { + if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_FPR0); + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); + if (op1_info & MAY_BE_REF) { + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } + } + break; + case ZEND_ASSIGN: + if (ssa_op->op2_use == current_var || + ssa_op->op2_def == current_var || + ssa_op->op1_def == current_var || + ssa_op->result_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (opline->op1_type == IS_CV + && !(op2_info & MAY_BE_UNDEF) + && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { + if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_FPR0); + } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + regset = ZEND_REGSET(ZREG_REG0); + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); + } + } + break; + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + if (ssa_op->op1_use == current_var || + ssa_op->op1_def == current_var || + ssa_op->result_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + op1_info = OP1_INFO(); + if (opline->op1_type == IS_CV + && (op1_info & MAY_BE_LONG) + && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + regset = ZEND_REGSET_EMPTY; + if (op1_info & MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_FPR0); + } + } + break; + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + + regset = ZEND_REGSET_EMPTY; + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { + if (ssa_op->result_def != current_var && + (ssa_op->op1_use != current_var || !last_use)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + res_info = OP1_INFO(); + if (res_info & MAY_BE_DOUBLE) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + ZEND_REGSET_INCL(regset, ZREG_FPR1); + } + } + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { + if (ssa_op->result_def != current_var) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { + if (zend_is_commutative(opline->opcode)) { + if (ssa_op->result_def != current_var) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + } else { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + if (ssa_op->result_def != current_var && + (ssa_op->op1_use != current_var || !last_use)) { + ZEND_REGSET_INCL(regset, ZREG_FPR1); + } + } + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { + if (ssa_op->result_def != current_var && + (ssa_op->op1_use != current_var || !last_use) && + (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + } + if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || + zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { + if (!ZEND_REGSET_IN(regset, ZREG_REG0)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } else { + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } + } + break; + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + if (ssa_op->result_def != current_var && + (ssa_op->op1_use != current_var || !last_use)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || + zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { + if (!ZEND_REGSET_IN(regset, ZREG_REG0)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } else { + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } + } + break; + case ZEND_SL: + case ZEND_SR: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + if (ssa_op->result_def != current_var && + (ssa_op->op1_use != current_var || !last_use)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + if (opline->op2_type != IS_CONST && ssa_op->op2_use != current_var) { + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } + break; + case ZEND_MOD: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + if (opline->op2_type == IS_CONST && + Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG && + zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2))) && + OP1_HAS_RANGE() && + OP1_MIN_RANGE() >= 0) { + if (ssa_op->result_def != current_var && + (ssa_op->op1_use != current_var || !last_use)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + if (sizeof(void*) == 8 + && !IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) - 1)) { + if (!ZEND_REGSET_IN(regset, ZREG_REG0)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } else { + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } + } else { + ZEND_REGSET_INCL(regset, ZREG_REG0); + ZEND_REGSET_INCL(regset, ZREG_REG2); + if (opline->op2_type == IS_CONST) { + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } + } + break; + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + case ZEND_CASE: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + regset = ZEND_REGSET_EMPTY; + if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && + opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { + if (ssa_op->op1_use != current_var && + ssa_op->op2_use != current_var) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + } + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { + if (ssa_op->op1_use != current_var && + ssa_op->op2_use != current_var) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + } + if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || + zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + } + break; + case ZEND_BOOL: + case ZEND_BOOL_NOT: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + op1_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { + regset = ZEND_REGSET_EMPTY; + if (op1_info & MAY_BE_DOUBLE) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + if (opline->opcode == ZEND_BOOL || + opline->opcode == ZEND_BOOL_NOT || + opline->opcode == ZEND_JMPZ_EX || + opline->opcode == ZEND_JMPNZ_EX) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + } + break; + case ZEND_DO_UCALL: + case ZEND_DO_FCALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_INCLUDE_OR_EVAL: + case ZEND_GENERATOR_CREATE: + case ZEND_YIELD: + case ZEND_YIELD_FROM: + regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); + break; + default: + break; + } + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + if (ssa_op == ssa->ops + && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL + && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } + + /* %r0 is used to check EG(vm_interrupt) */ + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + if (ssa_op == ssa->ops + && (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP || + JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) { +#if ZTS + ZEND_REGSET_INCL(regset, ZREG_REG0); +#else + if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } +#endif + } + } else { + uint32_t b = ssa->cfg.map[ssa_op - ssa->ops]; + + if ((ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) != 0 + && ssa->cfg.blocks[b].start == ssa_op - ssa->ops) { +#if ZTS + ZEND_REGSET_INCL(regset, ZREG_REG0); +#else + if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } +#endif + } + } + + return regset; +} + +static size_t dasm_venners_size = 0; +void **dasm_labels_veneers = NULL; + +static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset) +{ + void *veneer; + ptrdiff_t na; + int n, m; + + /* try to reuse veneers for global labels */ + if ((ins >> 16) == DASM_REL_LG + && *(b-1) < 0 + && dasm_labels_veneers[-*(b-1)]) { + + veneer = dasm_labels_veneers[-*(b-1)]; + na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; + n = (int)na; + + /* check if we can jump to veneer */ + if ((ptrdiff_t)n != na) { + /* pass */ + } else if (!(ins & 0xf800)) { /* B, BL */ + if ((n & 3) == 0 && ((n+0x08000000) >> 28) == 0) { + return n; + } + } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ + if ((n & 3) == 0 && ((n+0x00100000) >> 21) == 0) { + return n; + } + } else if ((ins & 0x3000) == 0x2000) { /* ADR */ + /* pass */ + } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ + /* pass */ + } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ + if ((n & 3) == 0 && ((n+0x00008000) >> 16) == 0) { + return n; + } + } + } + + veneer = (char*)buffer + (Dst->codesize + dasm_venners_size); + + if (veneer > dasm_end) { + return 0; /* jit_buffer_size overflow */ + } + + na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; + n = (int)na; + + /* check if we can jump to veneer */ + if ((ptrdiff_t)n != na) { + ZEND_ASSERT(0); + return 0; + } else if (!(ins & 0xf800)) { /* B, BL */ + if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) { + ZEND_ASSERT(0); + return 0; + } + } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ + if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) { + ZEND_ASSERT(0); + return 0; + } + } else if ((ins & 0x3000) == 0x2000) { /* ADR */ + ZEND_ASSERT(0); + return 0; + } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ + ZEND_ASSERT(0); + return 0; + } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ + if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) { + ZEND_ASSERT(0); + return 0; + } + } else if ((ins & 0x8000)) { /* absolute */ + ZEND_ASSERT(0); + return 0; + } else { + ZEND_ASSERT(0); + return 0; + } + + // TODO: support for long veneers (above 128MB) ??? + + /* check if we can use B to jump from veneer */ + na = (ptrdiff_t)cp + offset - (ptrdiff_t)veneer - 4; + m = (int)na; + if ((ptrdiff_t)m != na) { + ZEND_ASSERT(0); + return 0; + } else if ((m & 3) != 0 || ((m+0x08000000) >> 28) != 0) { + ZEND_ASSERT(0); + return 0; + } + + /* generate B instruction */ + *(uint32_t*)veneer = 0x14000000 | ((m >> 2) & 0x03ffffff); + dasm_venners_size += 4; + + if ((ins >> 16) == DASM_REL_LG + && *(b-1) < 0) { + /* reuse this veneer for the future jumps to global label */ + dasm_labels_veneers[-*(b-1)] = veneer; + /* Dst->globals[*(b-1)] = veneer; */ + +#ifdef HAVE_DISASM + if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { + const char *name = zend_jit_disasm_find_symbol((ptrdiff_t)cp + offset - 4, &offset); + + if (name && !offset) { + if (strstr(name, "@veneer") == NULL) { + char *new_name; + + zend_spprintf(&new_name, 0, "%s@veneer", name); + zend_jit_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4); + efree(new_name); + } else { + zend_jit_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4); + } + } + } +#endif + } + + return n; +} + +#if defined(__clang__) +# pragma clang diagnostic pop +#endif + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/ext/opcache/jit/zend_jit_arm64.h b/ext/opcache/jit/zend_jit_arm64.h new file mode 100644 index 0000000000000..173df223e8f18 --- /dev/null +++ b/ext/opcache/jit/zend_jit_arm64.h @@ -0,0 +1,157 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + | Hao Sun | + +----------------------------------------------------------------------+ +*/ + +#ifndef HAVE_JIT_ARM64_H +#define HAVE_JIT_ARM64_H + +typedef enum _zend_reg { + ZREG_NONE = -1, + + ZREG_X0, + ZREG_X1, + ZREG_X2, + ZREG_X3, + ZREG_X4, + ZREG_X5, + ZREG_X6, + ZREG_X7, + ZREG_X8, + ZREG_X9, + ZREG_X10, + ZREG_X11, + ZREG_X12, + ZREG_X13, + ZREG_X14, + ZREG_X15, + ZREG_X16, + ZREG_X17, + ZREG_X18, + ZREG_X19, + ZREG_X20, + ZREG_X21, + ZREG_X22, + ZREG_X23, + ZREG_X24, + ZREG_X25, + ZREG_X26, + ZREG_X27, + ZREG_X28, + ZREG_X29, + ZREG_X30, + ZREG_X31, + + ZREG_V0, + ZREG_V1, + ZREG_V2, + ZREG_V3, + ZREG_V4, + ZREG_V5, + ZREG_V6, + ZREG_V7, + ZREG_V8, + ZREG_V9, + ZREG_V10, + ZREG_V11, + ZREG_V12, + ZREG_V13, + ZREG_V14, + ZREG_V15, + ZREG_V16, + ZREG_V17, + ZREG_V18, + ZREG_V19, + ZREG_V20, + ZREG_V21, + ZREG_V22, + ZREG_V23, + ZREG_V24, + ZREG_V25, + ZREG_V26, + ZREG_V27, + ZREG_V28, + ZREG_V29, + ZREG_V30, + ZREG_V31, + + ZREG_NUM, + + ZREG_THIS, /* used for delayed FETCH_THIS deoptimization */ + + /* pseudo constants used by deoptimizer */ + ZREG_LONG_MIN_MINUS_1, + ZREG_LONG_MIN, + ZREG_LONG_MAX, + ZREG_LONG_MAX_PLUS_1, + ZREG_NULL, + + ZREG_ZVAL_TRY_ADDREF, + ZREG_ZVAL_COPY_GPR0, +} zend_reg; + +typedef struct _zend_jit_registers_buf { + uint64_t gpr[32]; /* general purpose integer register */ + double fpr[32]; /* floating point registers */ +} zend_jit_registers_buf; + +#define ZREG_RSP ZREG_X31 +#define ZREG_RLR ZREG_X30 +#define ZREG_RFP ZREG_X29 +#define ZREG_RPR ZREG_X18 + +#define ZREG_FP ZREG_X27 +#define ZREG_IP ZREG_X28 +#define ZREG_RX ZREG_IP + +#define ZREG_REG0 ZREG_X8 +#define ZREG_REG1 ZREG_X9 +#define ZREG_REG2 ZREG_X10 + +#define ZREG_FPR0 ZREG_V0 +#define ZREG_FPR1 ZREG_V1 + +#define ZREG_TMP1 ZREG_X15 +#define ZREG_TMP2 ZREG_X16 +#define ZREG_TMP3 ZREG_X17 +#define ZREG_FPTMP ZREG_V16 + +#define ZREG_COPY ZREG_REG0 +#define ZREG_FIRST_FPR ZREG_V0 + +typedef uint64_t zend_regset; + +#define ZEND_REGSET_64BIT 1 + +# define ZEND_REGSET_FIXED \ + (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_RLR) | ZEND_REGSET(ZREG_RFP) | \ + ZEND_REGSET(ZREG_RPR) | ZEND_REGSET(ZREG_FP) | ZEND_REGSET(ZREG_IP) | \ + ZEND_REGSET_INTERVAL(ZREG_TMP1, ZREG_TMP3) | ZEND_REGSET(ZREG_FPTMP)) +# define ZEND_REGSET_GP \ + ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_X0, ZREG_X30), ZEND_REGSET_FIXED) +# define ZEND_REGSET_FP \ + ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_V0, ZREG_V31), ZEND_REGSET_FIXED) +# define ZEND_REGSET_SCRATCH \ + (ZEND_REGSET_INTERVAL(ZREG_X0, ZREG_X17) | ZEND_REGSET_INTERVAL(ZREG_V0, ZREG_V7) | \ + ZEND_REGSET_INTERVAL(ZREG_V16, ZREG_V31)) +# define ZEND_REGSET_PRESERVED \ + (ZEND_REGSET_INTERVAL(ZREG_X19, ZREG_X28) | ZEND_REGSET_INTERVAL(ZREG_V8, ZREG_V15)) + +#define ZEND_REGSET_LOW_PRIORITY \ + (ZEND_REGSET(ZREG_REG0) | ZEND_REGSET(ZREG_REG1) | ZEND_REGSET(ZREG_FPR0) | ZEND_REGSET(ZREG_FPR1)) + +#endif /* ZEND_JIT_ARM64_H */ diff --git a/ext/opcache/jit/zend_jit_disasm.c b/ext/opcache/jit/zend_jit_disasm.c index 451b511793926..8bbbea6e74b0f 100644 --- a/ext/opcache/jit/zend_jit_disasm.c +++ b/ext/opcache/jit/zend_jit_disasm.c @@ -225,6 +225,7 @@ static uint64_t zend_jit_disasm_branch_target(csh cs, const cs_insn *insn) { unsigned int i; +#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32) if (cs_insn_group(cs, insn, X86_GRP_JUMP)) { for (i = 0; i < insn->detail->x86.op_count; i++) { if (insn->detail->x86.operands[i].type == X86_OP_IMM) { @@ -232,6 +233,16 @@ static uint64_t zend_jit_disasm_branch_target(csh cs, const cs_insn *insn) } } } +#elif defined(__aarch64__) + if (cs_insn_group(cs, insn, ARM64_GRP_JUMP) + || insn->id == ARM64_INS_BL + || insn->id == ARM64_INS_ADR) { + for (i = 0; i < insn->detail->arm64.op_count; i++) { + if (insn->detail->arm64.operands[i].type == ARM64_OP_IMM) + return insn->detail->arm64.operands[i].imm; + } + } +#endif return 0; } @@ -310,7 +321,7 @@ static int zend_jit_disasm(const char *name, #endif #ifdef HAVE_CAPSTONE -# if defined(__x86_64__) || defined(_WIN64) +# if defined(__x86_64__) || defined(_WIN64) || defined(ZEND_WIN32) if (cs_open(CS_ARCH_X86, CS_MODE_64, &cs) != CS_ERR_OK) return 0; cs_option(cs, CS_OPT_DETAIL, CS_OPT_ON); @@ -319,6 +330,11 @@ static int zend_jit_disasm(const char *name, # else cs_option(cs, CS_OPT_SYNTAX, CS_OPT_SYNTAX_ATT); # endif +# elif defined(__aarch64__) + if (cs_open(CS_ARCH_ARM64, CS_MODE_ARM, &cs) != CS_ERR_OK) + return 0; + cs_option(cs, CS_OPT_DETAIL, CS_OPT_ON); + cs_option(cs, CS_OPT_SYNTAX, CS_OPT_SYNTAX_ATT); # else if (cs_open(CS_ARCH_X86, CS_MODE_32, &cs) != CS_ERR_OK) return 0; @@ -431,9 +447,15 @@ static int zend_jit_disasm(const char *name, } # ifdef HAVE_CAPSTONE_ITER + if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) { + fprintf(stderr, " %" PRIx64 ":", insn->address); + } fprintf(stderr, "\t%s ", insn->mnemonic); p = insn->op_str; # else + if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) { + fprintf(stderr, " %" PRIx64 ":", insn[i].address); + } fprintf(stderr, "\t%s ", insn[i].mnemonic); p = insn[i].op_str; # endif @@ -533,6 +555,9 @@ static int zend_jit_disasm(const char *name, } } } + if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) { + fprintf(stderr, " %" PRIx64 ":", ud_insn_off(&ud)); + } fprintf(stderr, "\t%s\n", ud_insn_asm(&ud)); } #endif diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 0717ee32364bb..eb91dda317e56 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -20,9 +20,8 @@ +----------------------------------------------------------------------+ */ -#define HAVE_GDB -#ifdef HAVE_GDB +#define HAVE_GDB #include "zend_elf.h" #include "zend_gdb.h" @@ -92,7 +91,9 @@ enum { DW_REG_8, DW_REG_9, DW_REG_10, DW_REG_11, DW_REG_12, DW_REG_13, DW_REG_14, DW_REG_15, DW_REG_RA, - /* TODO: ARM supports? */ +#elif defined(__aarch64__) + DW_REG_SP = 31, + DW_REG_RA = 30, #else #error "Unsupported target architecture" #endif @@ -160,6 +161,8 @@ static const zend_elf_header zend_elfhdr_template = { .machine = 3, #elif defined(__x86_64__) .machine = 62, +#elif defined(__aarch64__) + .machine = 183, #else # error "Unsupported target architecture" #endif @@ -278,21 +281,26 @@ static void zend_gdbjit_symtab(zend_gdbjit_ctx *ctx) sym->info = ELFSYM_INFO(ELFSYM_BIND_GLOBAL, ELFSYM_TYPE_FUNC); } +typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); +typedef ZEND_SET_ALIGNED(1, uint32_t unaligned_uint32_t); +typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); +typedef ZEND_SET_ALIGNED(1, uintptr_t unaligned_uintptr_t); + #define SECTALIGN(p, a) \ ((p) = (uint8_t *)(((uintptr_t)(p) + ((a)-1)) & ~(uintptr_t)((a)-1))) /* Shortcuts to generate DWARF structures. */ #define DB(x) (*p++ = (x)) #define DI8(x) (*(int8_t *)p = (x), p++) -#define DU16(x) (*(uint16_t *)p = (x), p += 2) -#define DU32(x) (*(uint32_t *)p = (x), p += 4) -#define DADDR(x) (*(uintptr_t *)p = (x), p += sizeof(uintptr_t)) +#define DU16(x) (*(unaligned_uint16_t *)p = (x), p += 2) +#define DU32(x) (*(unaligned_uint32_t *)p = (x), p += 4) +#define DADDR(x) (*(unaligned_uintptr_t *)p = (x), p += sizeof(uintptr_t)) #define DUV(x) (ctx->p = p, zend_gdbjit_uleb128(ctx, (x)), p = ctx->p) #define DSV(x) (ctx->p = p, zend_gdbjit_sleb128(ctx, (x)), p = ctx->p) #define DSTR(str) (ctx->p = p, zend_gdbjit_strz(ctx, (str)), p = ctx->p) #define DALIGNNOP(s) while ((uintptr_t)p & ((s)-1)) *p++ = DW_CFA_nop #define DSECT(name, stmt) \ - { uint32_t *szp_##name = (uint32_t *)p; p += 4; stmt \ + { unaligned_uint32_t *szp_##name = (uint32_t *)p; p += 4; stmt \ *szp_##name = (uint32_t)((p-(uint8_t *)szp_##name)-4); } static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) @@ -327,6 +335,9 @@ static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) #elif defined(__x86_64__) DB(DW_CFA_advance_loc|4); /* sub $0x8,%rsp */ DB(DW_CFA_def_cfa_offset); DUV(16); /* Aligned stack frame size. */ +#elif defined(__aarch64__) + DB(DW_CFA_advance_loc|1); /* Only an approximation. */ + DB(DW_CFA_def_cfa_offset); DUV(32); /* Aligned stack frame size. */ #else # error "Unsupported target architecture" #endif @@ -491,5 +502,3 @@ static void zend_jit_gdb_init(void) } #endif } - -#endif diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 83bf939e5eed6..9beb453b309c9 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -14,23 +14,39 @@ +----------------------------------------------------------------------+ | Authors: Dmitry Stogov | | Xinchen Hui | + | Hao Sun | +----------------------------------------------------------------------+ */ #ifndef ZEND_JIT_INTERNAL_H #define ZEND_JIT_INTERNAL_H +#include "zend_bitset.h" + /* Register Set */ #define ZEND_REGSET_EMPTY 0 #define ZEND_REGSET_IS_EMPTY(regset) \ (regset == ZEND_REGSET_EMPTY) +#define ZEND_REGSET_IS_SINGLETON(regset) \ + (regset && !(regset & (regset - 1))) + +#if (!ZEND_REGSET_64BIT) #define ZEND_REGSET(reg) \ (1u << (reg)) +#else +#define ZEND_REGSET(reg) \ + (1ull << (reg)) +#endif +#if (!ZEND_REGSET_64BIT) #define ZEND_REGSET_INTERVAL(reg1, reg2) \ (((1u << ((reg2) - (reg1) + 1)) - 1) << (reg1)) +#else +#define ZEND_REGSET_INTERVAL(reg1, reg2) \ + (((1ull << ((reg2) - (reg1) + 1)) - 1) << (reg1)) +#endif #define ZEND_REGSET_IN(regset, reg) \ (((regset) & ZEND_REGSET(reg)) != 0) @@ -50,15 +66,13 @@ #define ZEND_REGSET_DIFFERENCE(set1, set2) \ ((set1) & ~(set2)) -#ifndef _WIN32 -# if (ZREG_NUM <= 32) +#if !defined(_WIN32) +# if (!ZEND_REGSET_64BIT) # define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctz(set)) # define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clz(set)^31)) -# elif(ZREG_NUM <= 64) +# else # define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctzll(set)) # define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clzll(set)^63)) -# else -# errir "Too many registers" # endif #else # include @@ -77,7 +91,7 @@ uint32_t __inline __zend_jit_clz(uint32_t value) { return 32; } # define ZEND_REGSET_FIRST(set) ((zend_reg)__zend_jit_ctz(set)) -# define ZEND_REGSET_LAST(set) ((zend_reg)(__zend_jit_clz(set)^31))) +# define ZEND_REGSET_LAST(set) ((zend_reg)(__zend_jit_clz(set)^31)) #endif #define ZEND_REGSET_FOREACH(set, reg) \ @@ -711,4 +725,72 @@ static zend_always_inline bool zend_jit_may_be_polymorphic_call(const zend_op *o } } +/* Instruction cache flush */ +#ifndef JIT_CACHE_FLUSH +# if defined (__aarch64__) +# if ((defined(__GNUC__) && ZEND_GCC_VERSION >= 4003) || __has_builtin(__builtin___clear_cache)) +# define JIT_CACHE_FLUSH(from, to) __builtin___clear_cache((char*)(from), (char*)(to)) +# else +# error "Missing builtin to flush instruction cache for AArch64" +# endif +# else /* Not required to implement on archs with unified caches */ +# define JIT_CACHE_FLUSH(from, to) +# endif +#endif /* !JIT_CACHE_FLUSH */ + +/* bit helpers */ + +static zend_always_inline bool zend_long_is_power_of_two(zend_long x) +{ + return (x > 0) && !(x & (x - 1)); +} + +static zend_always_inline uint32_t zend_long_floor_log2(zend_long x) +{ + ZEND_ASSERT(zend_long_is_power_of_two(x)); + return zend_ulong_ntz(x); +} + +/* from http://aggregate.org/MAGIC/ */ +static zend_always_inline uint32_t ones32(uint32_t x) +{ + x -= ((x >> 1) & 0x55555555); + x = (((x >> 2) & 0x33333333) + (x & 0x33333333)); + x = (((x >> 4) + x) & 0x0f0f0f0f); + x += (x >> 8); + x += (x >> 16); + return x & 0x0000003f; +} + +static zend_always_inline uint32_t floor_log2(uint32_t x) +{ + ZEND_ASSERT(x != 0); + x |= (x >> 1); + x |= (x >> 2); + x |= (x >> 4); + x |= (x >> 8); + x |= (x >> 16); + return ones32(x) - 1; +} + +static zend_always_inline bool is_power_of_two(uint32_t x) +{ + return !(x & (x - 1)) && x != 0; +} + +static zend_always_inline bool has_concrete_type(uint32_t value_type) +{ + return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); +} + +static zend_always_inline uint32_t concrete_type(uint32_t value_type) +{ + return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); +} + +static zend_always_inline bool is_signed(double d) +{ + return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0; +} + #endif /* ZEND_JIT_INTERNAL_H */ diff --git a/ext/opcache/jit/zend_jit_perf_dump.c b/ext/opcache/jit/zend_jit_perf_dump.c index d8f2d6130e8cc..ace998fe9d9ad 100644 --- a/ext/opcache/jit/zend_jit_perf_dump.c +++ b/ext/opcache/jit/zend_jit_perf_dump.c @@ -22,6 +22,9 @@ #include #include #include +#include +#include +#include #if defined(__linux__) #include diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 0803605769896..2b379e2429ea1 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -6947,7 +6947,7 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t) } else if (STACK_REG(stack, j) == ZREG_ZVAL_COPY_GPR0) { fprintf(stderr, " "); zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j); - fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name[0]); + fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name[ZREG_COPY]); } } fprintf(stderr, "\n"); @@ -7478,7 +7478,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf } else if (STACK_REG(stack, i) == ZREG_ZVAL_TRY_ADDREF) { Z_TRY_ADDREF_P(EX_VAR_NUM(i)); } else if (STACK_REG(stack, i) == ZREG_ZVAL_COPY_GPR0) { - zval *val = (zval*)regs->gpr[0]; + zval *val = (zval*)regs->gpr[ZREG_COPY]; if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) { /* Undefined array index or property */ @@ -7519,7 +7519,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf } } if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { - zend_function *func = (zend_function*)regs->gpr[0]; + zend_function *func = (zend_function*)regs->gpr[ZREG_COPY]; if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) { zend_string_release_ex(func->common.function_name, 0); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index f27cf39b4cf29..bb92d2b687147 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -28,7 +28,12 @@ #include "Optimizer/zend_func_info.h" #include "Optimizer/zend_call_graph.h" #include "zend_jit.h" +#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32) #include "zend_jit_x86.h" +#elif defined(__aarch64__) +#include "zend_jit_arm64.h" +#endif + #include "zend_jit_internal.h" #ifdef HAVE_GCC_GLOBAL_REGS @@ -36,9 +41,12 @@ # if defined(__x86_64__) register zend_execute_data* volatile execute_data __asm__("%r14"); register const zend_op* volatile opline __asm__("%r15"); -# else +# elif defined(i386) register zend_execute_data* volatile execute_data __asm__("%esi"); register const zend_op* volatile opline __asm__("%edi"); +# elif defined(__aarch64__) +register zend_execute_data* volatile execute_data __asm__("x27"); +register const zend_op* volatile opline __asm__("x28"); # endif # pragma GCC diagnostic warning "-Wvolatile-register-var" #endif diff --git a/ext/opcache/jit/zend_jit_vtune.c b/ext/opcache/jit/zend_jit_vtune.c index 1f71bd741eb0d..35fd3a031022d 100644 --- a/ext/opcache/jit/zend_jit_vtune.c +++ b/ext/opcache/jit/zend_jit_vtune.c @@ -16,6 +16,8 @@ +----------------------------------------------------------------------+ */ +#if defined(__x86_64__) || defined(i386) + #define HAVE_VTUNE 1 #include "jit/vtune/jitprofiling.h" @@ -40,3 +42,5 @@ static void zend_jit_vtune_register(const char *name, iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&jmethod); } + +#endif /* defined(__x86_64__) || defined(i386) */ diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 01910e4c0b306..52efe13cd7ea0 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1703,50 +1703,6 @@ static void zend_jit_stop_reuse_ip(void) reuse_ip = 0; } -/* bit helpers */ - -/* from http://aggregate.org/MAGIC/ */ -static uint32_t ones32(uint32_t x) -{ - x -= ((x >> 1) & 0x55555555); - x = (((x >> 2) & 0x33333333) + (x & 0x33333333)); - x = (((x >> 4) + x) & 0x0f0f0f0f); - x += (x >> 8); - x += (x >> 16); - return x & 0x0000003f; -} - -static uint32_t floor_log2(uint32_t x) -{ - ZEND_ASSERT(x != 0); - x |= (x >> 1); - x |= (x >> 2); - x |= (x >> 4); - x |= (x >> 8); - x |= (x >> 16); - return ones32(x) - 1; -} - -static bool is_power_of_two(uint32_t x) -{ - return !(x & (x - 1)) && x != 0; -} - -static bool has_concrete_type(uint32_t value_type) -{ - return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); -} - -static uint32_t concrete_type(uint32_t value_type) -{ - return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); -} - -static inline bool is_signed(double d) -{ - return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0; -} - static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { |->interrupt_handler: @@ -4233,7 +4189,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, } else { result_reg = Z_REG(res_addr); } - } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { result_reg = Z_REG(op1_addr); } else if (Z_REG(res_addr) != ZREG_R0) { result_reg = ZREG_R0; @@ -4244,32 +4200,42 @@ static int zend_jit_math_long_long(dasm_State **Dst, } if (opcode == ZEND_MUL && - Z_MODE(op2_addr) == IS_CONST_ZVAL && - IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && - is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { - - if (Z_MODE(op1_addr) == IS_REG && Z_LVAL_P(Z_ZV(op2_addr)) == 2) { + Z_MODE(op2_addr) == IS_CONST_ZVAL && + Z_LVAL_P(Z_ZV(op2_addr)) == 2) { + if (Z_MODE(op1_addr) == IS_REG && !may_overflow) { | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] } else { | GET_ZVAL_LVAL result_reg, op1_addr - | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | add Ra(result_reg), Ra(result_reg) } } else if (opcode == ZEND_MUL && - Z_MODE(op1_addr) == IS_CONST_ZVAL && - IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && - is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { - - if (Z_MODE(op2_addr) == IS_REG && Z_LVAL_P(Z_ZV(op1_addr)) == 2) { + Z_MODE(op2_addr) == IS_CONST_ZVAL && + !may_overflow && + Z_LVAL_P(Z_ZV(op2_addr)) > 0 && + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { + | GET_ZVAL_LVAL result_reg, op1_addr + | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + } else if (opcode == ZEND_MUL && + Z_MODE(op1_addr) == IS_CONST_ZVAL && + Z_LVAL_P(Z_ZV(op1_addr)) == 2) { + if (Z_MODE(op2_addr) == IS_REG && !may_overflow) { | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Ra(Z_REG(op2_addr))] } else { | GET_ZVAL_LVAL result_reg, op2_addr - | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) + | add Ra(result_reg), Ra(result_reg) } + } else if (opcode == ZEND_MUL && + Z_MODE(op1_addr) == IS_CONST_ZVAL && + !may_overflow && + Z_LVAL_P(Z_ZV(op1_addr)) > 0 && + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { + | GET_ZVAL_LVAL result_reg, op2_addr + | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && - is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { | GET_ZVAL_LVAL result_reg, op1_addr - | shr Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | shr Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) } else if (opcode == ZEND_ADD && !may_overflow && Z_MODE(op1_addr) == IS_REG && diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index 795569517d2b8..924db409c3289 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -88,6 +88,7 @@ typedef struct _zend_jit_registers_buf { } zend_jit_registers_buf; #define ZREG_FIRST_FPR ZREG_XMM0 +#define ZREG_COPY ZREG_R0 #define ZREG_RAX ZREG_R0 #define ZREG_RCX ZREG_R1 @@ -113,6 +114,8 @@ typedef struct _zend_jit_registers_buf { typedef uint32_t zend_regset; +#define ZEND_REGSET_64BIT 0 + #ifdef _WIN64 # define ZEND_REGSET_FIXED \ (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_R14) | ZEND_REGSET(ZREG_R15)) diff --git a/ext/opcache/tests/jit/add_001.phpt b/ext/opcache/tests/jit/add_001.phpt new file mode 100644 index 0000000000000..25fb1e7b4c71f --- /dev/null +++ b/ext/opcache/tests/jit/add_001.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT ADD: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(2) diff --git a/ext/opcache/tests/jit/add_002.phpt b/ext/opcache/tests/jit/add_002.phpt new file mode 100644 index 0000000000000..bce4b6b38937b --- /dev/null +++ b/ext/opcache/tests/jit/add_002.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT ADD: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(4097) diff --git a/ext/opcache/tests/jit/add_003.phpt b/ext/opcache/tests/jit/add_003.phpt new file mode 100644 index 0000000000000..70e814968b124 --- /dev/null +++ b/ext/opcache/tests/jit/add_003.phpt @@ -0,0 +1,24 @@ +--TEST-- +JIT ADD: 003 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--SKIPIF-- + +--FILE-- + +--EXPECT-- +float(9.223372036854776E+18) diff --git a/ext/opcache/tests/jit/add_004.phpt b/ext/opcache/tests/jit/add_004.phpt new file mode 100644 index 0000000000000..761ead9e8aba4 --- /dev/null +++ b/ext/opcache/tests/jit/add_004.phpt @@ -0,0 +1,24 @@ +--TEST-- +JIT ADD: 004 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--SKIPIF-- + +--FILE-- + +--EXPECT-- +float(9.223372036854776E+18) diff --git a/ext/opcache/tests/jit/add_005.phpt b/ext/opcache/tests/jit/add_005.phpt new file mode 100644 index 0000000000000..7958bb97d2893 --- /dev/null +++ b/ext/opcache/tests/jit/add_005.phpt @@ -0,0 +1,24 @@ +--TEST-- +JIT ADD: 005 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECTF-- +Fatal error: Uncaught TypeError: Unsupported operand types: string + int in %s:%d +Stack trace: +#0 %s(%d): foo('hello') +#1 {main} + thrown in %s on line %d diff --git a/ext/opcache/tests/jit/add_006.phpt b/ext/opcache/tests/jit/add_006.phpt new file mode 100644 index 0000000000000..f15bdf2c4c761 --- /dev/null +++ b/ext/opcache/tests/jit/add_006.phpt @@ -0,0 +1,26 @@ +--TEST-- +JIT ADD: 006 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(8) +float(8) +float(8) +float(8) diff --git a/ext/opcache/tests/jit/assign_037.phpt b/ext/opcache/tests/jit/assign_037.phpt new file mode 100644 index 0000000000000..292837ebdae16 --- /dev/null +++ b/ext/opcache/tests/jit/assign_037.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT ASSIGN: Assign refcounted string (with result) +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(2) "aa" +string(2) "aa" diff --git a/ext/opcache/tests/jit/assign_038.phpt b/ext/opcache/tests/jit/assign_038.phpt new file mode 100644 index 0000000000000..bcbda7b02798b --- /dev/null +++ b/ext/opcache/tests/jit/assign_038.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT ASSIGN: Assign constant (with result) +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +opcache.optimization_level=0 ; disable optimizer to produce ASSIGN with result +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(2) "bb" +string(2) "bb" diff --git a/ext/opcache/tests/jit/assign_039.phpt b/ext/opcache/tests/jit/assign_039.phpt new file mode 100644 index 0000000000000..737d37a15378c --- /dev/null +++ b/ext/opcache/tests/jit/assign_039.phpt @@ -0,0 +1,18 @@ +--TEST-- +JIT ASSIGN: Assign reference IS_VAR +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +NULL diff --git a/ext/opcache/tests/jit/concat_001.phpt b/ext/opcache/tests/jit/concat_001.phpt new file mode 100644 index 0000000000000..85d4633765d31 --- /dev/null +++ b/ext/opcache/tests/jit/concat_001.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT CONCAT: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(2) "ab" +string(2) "a5" diff --git a/ext/opcache/tests/jit/hot_func_001.phpt b/ext/opcache/tests/jit/hot_func_001.phpt new file mode 100644 index 0000000000000..6c0346d9b479d --- /dev/null +++ b/ext/opcache/tests/jit/hot_func_001.phpt @@ -0,0 +1,24 @@ +--TEST-- +JIT HOT_FUNC: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +opcache.jit=tracing +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(5) "hello" +string(5) "hello" +string(5) "hello" diff --git a/ext/opcache/tests/jit/hot_func_002.phpt b/ext/opcache/tests/jit/hot_func_002.phpt new file mode 100644 index 0000000000000..3090a8cdcc285 --- /dev/null +++ b/ext/opcache/tests/jit/hot_func_002.phpt @@ -0,0 +1,25 @@ +--TEST-- +JIT HOT_FUNC: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +opcache.jit=tracing +opcache.jit_hot_func=2 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(5) "hello" +string(5) "hello" +string(5) "hello" diff --git a/ext/opcache/tests/jit/icall_001.phpt b/ext/opcache/tests/jit/icall_001.phpt new file mode 100644 index 0000000000000..b52dcdddee990 --- /dev/null +++ b/ext/opcache/tests/jit/icall_001.phpt @@ -0,0 +1,24 @@ +--TEST-- +JIT ICALL: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +bool(true) +int(0) +int(42) +int(-42) +float(0) +float(2) +string(5) "hello" +array(0) { +} diff --git a/ext/opcache/tests/jit/identical_001.phpt b/ext/opcache/tests/jit/identical_001.phpt new file mode 100644 index 0000000000000..03b1f4ea7f068 --- /dev/null +++ b/ext/opcache/tests/jit/identical_001.phpt @@ -0,0 +1,31 @@ +--TEST-- +JIT IDENTICAL: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +bool(true) +bool(false) +bool(false) +bool(true) +bool(false) +bool(true) +bool(false) diff --git a/ext/opcache/tests/jit/identical_002.phpt b/ext/opcache/tests/jit/identical_002.phpt new file mode 100644 index 0000000000000..789a3f8d36efb --- /dev/null +++ b/ext/opcache/tests/jit/identical_002.phpt @@ -0,0 +1,131 @@ +--TEST-- +JIT IDENTICAL: 002 Comparison with NaN +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +bool(true) +bool(false) +bool(false) +bool(false) +bool(true) +bool(true) +int(1) +int(0) +int(0) +int(0) +int(1) +int(1) +1 +5 +6 +8 +9 +A +!bool(true) +bool(false) +bool(false) +bool(false) +!bool(true) +!bool(true) +bool(true) +!bool(false) +!bool(false) +!bool(false) +bool(true) +bool(true) +bool(false) +bool(true) +int(0) +int(1) diff --git a/ext/opcache/tests/jit/inc_021.phpt b/ext/opcache/tests/jit/inc_021.phpt new file mode 100644 index 0000000000000..bfc1e73f8217f --- /dev/null +++ b/ext/opcache/tests/jit/inc_021.phpt @@ -0,0 +1,33 @@ +--TEST-- +JIT INC: 021 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--SKIPIF-- + +--FILE-- + +--EXPECT-- +float(9.223372036854776E+18) +float(2.1) +float(-9.223372036854776E+18) +float(0.10000000000000009) diff --git a/ext/opcache/tests/jit/inc_022.phpt b/ext/opcache/tests/jit/inc_022.phpt new file mode 100644 index 0000000000000..75971cff6c2e9 --- /dev/null +++ b/ext/opcache/tests/jit/inc_022.phpt @@ -0,0 +1,31 @@ +--TEST-- +JIT INC: 022 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(3) "abd" +int(6) +float(2.1) +int(4) +float(0.10000000000000009) diff --git a/ext/opcache/tests/jit/loop_001.phpt b/ext/opcache/tests/jit/loop_001.phpt new file mode 100644 index 0000000000000..ed2a918b49cc1 --- /dev/null +++ b/ext/opcache/tests/jit/loop_001.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT LOOP: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +10 diff --git a/ext/opcache/tests/jit/loop_002.phpt b/ext/opcache/tests/jit/loop_002.phpt new file mode 100644 index 0000000000000..499fd670d1881 --- /dev/null +++ b/ext/opcache/tests/jit/loop_002.phpt @@ -0,0 +1,26 @@ +--TEST-- +JIT HOT LOOP: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +opcache.jit=tracing +opcache.jit_hot_func=2 +opcache.jit_hot_loop=2 +opcache.jit_hot_side_exit=0 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +10 diff --git a/ext/opcache/tests/jit/mul_001.phpt b/ext/opcache/tests/jit/mul_001.phpt new file mode 100644 index 0000000000000..94a07bfce4e71 --- /dev/null +++ b/ext/opcache/tests/jit/mul_001.phpt @@ -0,0 +1,32 @@ +--TEST-- +JIT MUL: 001 integer multiplay +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(6) +int(12) +int(333) diff --git a/ext/opcache/tests/jit/mul_002.phpt b/ext/opcache/tests/jit/mul_002.phpt new file mode 100644 index 0000000000000..eecedc11a844a --- /dev/null +++ b/ext/opcache/tests/jit/mul_002.phpt @@ -0,0 +1,24 @@ +--TEST-- +JIT MUL: 002 integer overflow +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--SKIPIF-- + +--FILE-- + +--EXPECT-- +float(1.343250910680478E+23) diff --git a/ext/opcache/tests/jit/mul_003.phpt b/ext/opcache/tests/jit/mul_003.phpt new file mode 100644 index 0000000000000..5042cd6c3acb4 --- /dev/null +++ b/ext/opcache/tests/jit/mul_003.phpt @@ -0,0 +1,33 @@ +--TEST-- +JIT MUL: 003 boundary value for optmizing MUL to SHIFT +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--SKIPIF-- + +--FILE-- + +--EXPECT-- +int(-6442450944) +int(-6442450944) \ No newline at end of file diff --git a/ext/opcache/tests/jit/mul_004.phpt b/ext/opcache/tests/jit/mul_004.phpt new file mode 100644 index 0000000000000..b1855a0ee489e --- /dev/null +++ b/ext/opcache/tests/jit/mul_004.phpt @@ -0,0 +1,72 @@ +--TEST-- +JIT MUL: 004 Overflow check for optmizing MUL to SHIFT +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--SKIPIF-- + +--FILE-- + +--EXPECT-- +int(24) +int(-88) +float(7.378697629483821E+19) +int(48) +int(-208) +float(1.4757395258967641E+20) +int(805306368) +int(-805306368) +float(2.9514790517935283E+20) +int(12884901888) +int(-12884901888) +float(1.8446744073709552E+19) +int(20) +float(1.8446744073709552E+19) diff --git a/ext/opcache/tests/jit/not_001.phpt b/ext/opcache/tests/jit/not_001.phpt new file mode 100644 index 0000000000000..0820b7e942ba6 --- /dev/null +++ b/ext/opcache/tests/jit/not_001.phpt @@ -0,0 +1,26 @@ +--TEST-- +JIT NOT: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +bool(false) +bool(true) +bool(false) +bool(true) diff --git a/ext/opcache/tests/jit/not_002.phpt b/ext/opcache/tests/jit/not_002.phpt new file mode 100644 index 0000000000000..68df4ab774db2 --- /dev/null +++ b/ext/opcache/tests/jit/not_002.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT NOT: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +bool(false) +bool(true) diff --git a/ext/opcache/tests/jit/recv_002.phpt b/ext/opcache/tests/jit/recv_002.phpt new file mode 100644 index 0000000000000..0f38edc4400eb --- /dev/null +++ b/ext/opcache/tests/jit/recv_002.phpt @@ -0,0 +1,27 @@ +--TEST-- +JIT RECV: too few arguments +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECTF-- +Fatal error: Uncaught ArgumentCountError: Too few arguments to function test(), 0 passed in %srecv_002.php on line 7 and exactly 1 expected in %s:3 +Stack trace: +#0 %s(7): test() +#1 {main} + thrown in %s on line 3 \ No newline at end of file diff --git a/ext/opcache/tests/jit/recv_003.phpt b/ext/opcache/tests/jit/recv_003.phpt new file mode 100644 index 0000000000000..0eb214e9cac6e --- /dev/null +++ b/ext/opcache/tests/jit/recv_003.phpt @@ -0,0 +1,36 @@ +--TEST-- +JIT RECV: slow type check +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECTF-- +ok + +Fatal error: Uncaught TypeError: test(): Argument #1 ($x) must be of type A, C given, called in %s:9 +Stack trace: +#0 %s(15): test(Object(C)) +#1 {main} + thrown in %s on line 9 diff --git a/ext/opcache/tests/jit/recv_004.phpt b/ext/opcache/tests/jit/recv_004.phpt new file mode 100644 index 0000000000000..4e540f29cb008 --- /dev/null +++ b/ext/opcache/tests/jit/recv_004.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT RECV: default arguments with type checks +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(3) +int(2) diff --git a/ext/opcache/tests/jit/recv_005.phpt b/ext/opcache/tests/jit/recv_005.phpt new file mode 100644 index 0000000000000..239cbc938cf03 --- /dev/null +++ b/ext/opcache/tests/jit/recv_005.phpt @@ -0,0 +1,26 @@ +--TEST-- +JIT RECV: 005 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(1) +float(1) +string(5) "hello" +array(0) { +} diff --git a/ext/opcache/tests/jit/ret_001.phpt b/ext/opcache/tests/jit/ret_001.phpt new file mode 100644 index 0000000000000..59408c72a4b78 --- /dev/null +++ b/ext/opcache/tests/jit/ret_001.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT RET: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(1) diff --git a/ext/opcache/tests/jit/ret_002.phpt b/ext/opcache/tests/jit/ret_002.phpt new file mode 100644 index 0000000000000..a755bd78a45af --- /dev/null +++ b/ext/opcache/tests/jit/ret_002.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT RET: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +float(1) diff --git a/ext/opcache/tests/jit/ret_003.phpt b/ext/opcache/tests/jit/ret_003.phpt new file mode 100644 index 0000000000000..1bc716b9e5bee --- /dev/null +++ b/ext/opcache/tests/jit/ret_003.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT RET: 003 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(5) "hello" diff --git a/ext/opcache/tests/jit/sub_001.phpt b/ext/opcache/tests/jit/sub_001.phpt new file mode 100644 index 0000000000000..4c98c14738868 --- /dev/null +++ b/ext/opcache/tests/jit/sub_001.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT SUB: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(41) diff --git a/ext/opcache/tests/jit/ucall_001.phpt b/ext/opcache/tests/jit/ucall_001.phpt new file mode 100644 index 0000000000000..df69fc7018d6e --- /dev/null +++ b/ext/opcache/tests/jit/ucall_001.phpt @@ -0,0 +1,19 @@ +--TEST-- +JIT UCALL: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(5) "hello" diff --git a/ext/opcache/tests/jit/ucall_002.phpt b/ext/opcache/tests/jit/ucall_002.phpt new file mode 100644 index 0000000000000..14787e7737501 --- /dev/null +++ b/ext/opcache/tests/jit/ucall_002.phpt @@ -0,0 +1,21 @@ +--TEST-- +JIT UCALL: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(5) "hello" +string(6) "world!" diff --git a/ext/opcache/tests/jit/ucall_003.phpt b/ext/opcache/tests/jit/ucall_003.phpt new file mode 100644 index 0000000000000..89b0776c9b55d --- /dev/null +++ b/ext/opcache/tests/jit/ucall_003.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT UCALL: 003 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(5) "hello" diff --git a/ext/opcache/tests/jit/ucall_004.phpt b/ext/opcache/tests/jit/ucall_004.phpt new file mode 100644 index 0000000000000..0f29e4d22f1d5 --- /dev/null +++ b/ext/opcache/tests/jit/ucall_004.phpt @@ -0,0 +1,23 @@ +--TEST-- +JIT UCALL: 004 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(5) "hello" +string(5) "hello" +string(5) "hello" diff --git a/ext/opcache/tests/jit/xor_001.phpt b/ext/opcache/tests/jit/xor_001.phpt new file mode 100644 index 0000000000000..abae4cd1beee4 --- /dev/null +++ b/ext/opcache/tests/jit/xor_001.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT XOR: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(4) diff --git a/ext/opcache/tests/jit/xor_002.phpt b/ext/opcache/tests/jit/xor_002.phpt new file mode 100644 index 0000000000000..2f8ff8461300c --- /dev/null +++ b/ext/opcache/tests/jit/xor_002.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT XOR: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(0) diff --git a/ext/opcache/tests/jit/xor_003.phpt b/ext/opcache/tests/jit/xor_003.phpt new file mode 100644 index 0000000000000..9d0277b4b3583 --- /dev/null +++ b/ext/opcache/tests/jit/xor_003.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT XOR: 003 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(3) "```"