Jit arm64

TSRM/TSRM.c

@@ -741,6 +741,14 @@ TSRM_API size_t tsrm_get_ls_cache_tcb_offset(void)
 	asm ("leal _tsrm_ls_cache@ntpoff,%0"
           : "=r" (ret));
 	return ret;
+#elif defined(__aarch64__)
+	size_t ret;
+
+	asm("mov %0, xzr\n\t"
+	    "add %0, %0, #:tprel_hi12:_tsrm_ls_cache, lsl #12\n\t"
+	    "add %0, %0, #:tprel_lo12_nc:_tsrm_ls_cache"
+	     : "=r" (ret));
+	return ret;
 #else
 	return 0;
 #endif

build/Makefile.global

@@ -117,6 +117,7 @@ clean:
 	find . -name .libs -a -type d|xargs rm -rf
 	rm -f libphp.la $(SAPI_CLI_PATH) $(SAPI_CGI_PATH) $(SAPI_LITESPEED_PATH) $(SAPI_FPM_PATH) $(OVERALL_TARGET) modules/* libs/*
 	rm -f ext/opcache/jit/zend_jit_x86.c
+	rm -f ext/opcache/jit/zend_jit_arm64.c
 
 distclean: clean
 	rm -f Makefile config.cache config.log config.status Makefile.objects Makefile.fragments libtool main/php_config.h main/internal_functions_cli.c main/internal_functions.c Zend/zend_dtrace_gen.h Zend/zend_dtrace_gen.h.bak Zend/zend_config.h

ext/opcache/config.m4

@@ -29,7 +29,7 @@ if test "$PHP_OPCACHE" != "no"; then
 
   if test "$PHP_OPCACHE_JIT" = "yes"; then
     case $host_cpu in
-      i[[34567]]86*|x86*)
+      i[[34567]]86*|x86*|aarch64)
         ;;
       *)
         AC_MSG_WARN([JIT not supported by host architecture])
@@ -77,6 +77,7 @@ if test "$PHP_OPCACHE" != "no"; then
     fi
 
     PHP_SUBST(DASM_FLAGS)
+    PHP_SUBST(DASM_ARCH)
 
     AC_MSG_CHECKING(for opagent in default path)
     for i in /usr/local /usr; do

ext/opcache/config.w32

@@ -25,6 +25,7 @@ if (PHP_OPCACHE != "no") {
 				dasm_flags += " -D ZTS=1";
 			}
 			DEFINE("DASM_FLAGS", dasm_flags);
+			DEFINE("DASM_ARCH", "x86");
 
 			AC_DEFINE('HAVE_JIT', 1, 'Define to enable JIT');
 			/* XXX read this dynamically */

ext/opcache/jit/Makefile.frag

@@ -2,11 +2,11 @@
 $(builddir)/minilua: $(srcdir)/jit/dynasm/minilua.c
 	$(BUILD_CC) $(srcdir)/jit/dynasm/minilua.c -lm -o $@
 
-$(builddir)/jit/zend_jit_x86.c: $(srcdir)/jit/zend_jit_x86.dasc $(srcdir)/jit/dynasm/*.lua $(builddir)/minilua
-	$(builddir)/minilua $(srcdir)/jit/dynasm/dynasm.lua  $(DASM_FLAGS) -o $@ $(srcdir)/jit/zend_jit_x86.dasc
+$(builddir)/jit/zend_jit_$(DASM_ARCH).c: $(srcdir)/jit/zend_jit_$(DASM_ARCH).dasc $(srcdir)/jit/dynasm/*.lua $(builddir)/minilua
+	$(builddir)/minilua $(srcdir)/jit/dynasm/dynasm.lua  $(DASM_FLAGS) -o $@ $(srcdir)/jit/zend_jit_$(DASM_ARCH).dasc
 
 $(builddir)/jit/zend_jit.lo: \
-	$(builddir)/jit/zend_jit_x86.c \
+	$(builddir)/jit/zend_jit_$(DASM_ARCH).c \
 	$(srcdir)/jit/zend_jit_helpers.c \
 	$(srcdir)/jit/zend_jit_disasm.c \
 	$(srcdir)/jit/zend_jit_gdb.c \

ext/opcache/jit/dynasm/dasm_arm64.h

@@ -404,6 +404,15 @@ int dasm_link(Dst_DECL, size_t *szp)
   return DASM_S_OK;
 }
 
+#ifdef DASM_ADD_VENEER
+#define CK_REL(x, o) \
+  do { if (!(x) && !(n = DASM_ADD_VENEER(D, buffer, ins, b, cp, o))) \
+    return DASM_S_RANGE_REL|(p-D->actionlist-1); \
+  } while (0)
+#else
+#define CK_REL(x, o) CK(x, RANGE_REL)
+#endif
+
 #ifdef DASM_CHECKS
 #define CK(x, st) \
   do { if (!(x)) return DASM_S_##st|(p-D->actionlist-1); } while (0)
@@ -444,7 +453,7 @@ int dasm_encode(Dst_DECL, void *buffer)
 	  if (n < 0) {
 	    ptrdiff_t na = (ptrdiff_t)D->globals[-n] - (ptrdiff_t)cp + 4;
 	    n = (int)na;
-	    CK((ptrdiff_t)n == na, RANGE_REL);
+	    CK_REL((ptrdiff_t)n == na, na);
 	    goto patchrel;
 	  }
 	  /* fallthrough */
@@ -453,18 +462,18 @@ int dasm_encode(Dst_DECL, void *buffer)
 	  n = *DASM_POS2PTR(D, n) - (int)((char *)cp - base) + 4;
 	patchrel:
 	  if (!(ins & 0xf800)) {  /* B, BL */
-	    CK((n & 3) == 0 && ((n+0x08000000) >> 28) == 0, RANGE_REL);
+	    CK_REL((n & 3) == 0 && ((n+0x08000000) >> 28) == 0, n);
 	    cp[-1] |= ((n >> 2) & 0x03ffffff);
 	  } else if ((ins & 0x800)) {  /* B.cond, CBZ, CBNZ, LDR* literal */
-	    CK((n & 3) == 0 && ((n+0x00100000) >> 21) == 0, RANGE_REL);
+	    CK_REL((n & 3) == 0 && ((n+0x00100000) >> 21) == 0, n);
 	    cp[-1] |= ((n << 3) & 0x00ffffe0);
 	  } else if ((ins & 0x3000) == 0x2000) {  /* ADR */
-	    CK(((n+0x00100000) >> 21) == 0, RANGE_REL);
+	    CK_REL(((n+0x00100000) >> 21) == 0, n);
 	    cp[-1] |= ((n << 3) & 0x00ffffe0) | ((n & 3) << 29);
 	  } else if ((ins & 0x3000) == 0x3000) {  /* ADRP */
 	    cp[-1] |= ((n >> 9) & 0x00ffffe0) | (((n >> 12) & 3) << 29);
 	  } else if ((ins & 0x1000)) {  /* TBZ, TBNZ */
-	    CK((n & 3) == 0 && ((n+0x00008000) >> 16) == 0, RANGE_REL);
+	    CK_REL((n & 3) == 0 && ((n+0x00008000) >> 16) == 0, n);
 	    cp[-1] |= ((n << 3) & 0x0007ffe0);
 	  } else if ((ins & 0x8000)) {  /* absolute */
 	    cp[0] = (unsigned int)((ptrdiff_t)cp - 4 + n);
@@ -475,7 +484,7 @@ int dasm_encode(Dst_DECL, void *buffer)
 	case DASM_REL_A: {
 	  ptrdiff_t na = (((ptrdiff_t)(*b++) << 32) | (unsigned int)n) - (ptrdiff_t)cp + 4;
 	  n = (int)na;
-	  CK((ptrdiff_t)n == na, RANGE_REL);
+	  CK_REL((ptrdiff_t)n == na, na);
 	  goto patchrel;
 	}
 	case DASM_LABEL_LG:

ext/opcache/jit/zend_jit.c

@@ -39,7 +39,14 @@
 #include "Optimizer/zend_call_graph.h"
 #include "Optimizer/zend_dump.h"
 
+#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32)
 #include "jit/zend_jit_x86.h"
+#elif defined (__aarch64__)
+#include "jit/zend_jit_arm64.h"
+#else
+#error "JIT not supported on this platform"
+#endif
+
 #include "jit/zend_jit_internal.h"
 
 #ifdef ZTS
@@ -188,11 +195,6 @@ static bool zend_is_commutative(zend_uchar opcode)
 		opcode == ZEND_BW_XOR;
 }
 
-static bool zend_long_is_power_of_two(zend_long x)
-{
-	return (x > 0) && !(x & (x - 1));
-}
-
 #define OP_RANGE(ssa_op, opN) \
 	(((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
 	  ssa->var_info && \
@@ -204,19 +206,30 @@ static bool zend_long_is_power_of_two(zend_long x)
 #define OP2_RANGE()      OP_RANGE(ssa_op, op2)
 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
 
+#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32)
 #include "dynasm/dasm_x86.h"
+#elif defined(__aarch64__)
+static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset);
+#define DASM_ADD_VENEER zend_jit_add_veneer
+#include "dynasm/dasm_arm64.h"
+#endif
+
 #include "jit/zend_jit_helpers.c"
 #include "jit/zend_jit_disasm.c"
 #ifndef _WIN32
-#include "jit/zend_jit_gdb.c"
-#include "jit/zend_jit_perf_dump.c"
+# include "jit/zend_jit_gdb.c"
+# include "jit/zend_jit_perf_dump.c"
 #endif
 #ifdef HAVE_OPROFILE
 # include "jit/zend_jit_oprofile.c"
 #endif
-#include "jit/zend_jit_vtune.c"
 
+#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32)
+#include "jit/zend_jit_vtune.c"
 #include "jit/zend_jit_x86.c"
+#elif defined(__aarch64__)
+#include "jit/zend_jit_arm64.c"
+#endif
 
 #if _WIN32
 # include <Windows.h>
@@ -298,15 +311,32 @@ static void handle_dasm_error(int ret) {
 		case DASM_S_RANGE_PC:
 			fprintf(stderr, "DASM_S_RANGE_PC %d\n", ret & 0xffffffu);
 			break;
+#ifdef DASM_S_RANGE_VREG
 		case DASM_S_RANGE_VREG:
 			fprintf(stderr, "DASM_S_RANGE_VREG\n");
 			break;
+#endif
+#ifdef DASM_S_UNDEF_L
 		case DASM_S_UNDEF_L:
 			fprintf(stderr, "DASM_S_UNDEF_L\n");
 			break;
+#endif
+#ifdef DASM_S_UNDEF_LG
+		case DASM_S_UNDEF_LG:
+			fprintf(stderr, "DASM_S_UNDEF_LG\n");
+			break;
+#endif
+#ifdef DASM_S_RANGE_REL
+		case DASM_S_RANGE_REL:
+			fprintf(stderr, "DASM_S_RANGE_REL\n");
+			break;
+#endif
 		case DASM_S_UNDEF_PC:
 			fprintf(stderr, "DASM_S_UNDEF_PC\n");
 			break;
+		default:
+			fprintf(stderr, "DASM_S_%0x\n", ret & 0xff000000u);
+			break;
 	}
 	ZEND_UNREACHABLE();
 }
@@ -380,6 +410,10 @@ static void *dasm_link_and_encode(dasm_State             **dasm_state,
 		return NULL;
 	}
 
+#ifdef __aarch64__
+	dasm_venners_size = 0;
+#endif
+
 	ret = dasm_encode(dasm_state, *dasm_ptr);
 	if (ret != DASM_S_OK) {
 #if ZEND_DEBUG
@@ -388,9 +422,16 @@ static void *dasm_link_and_encode(dasm_State             **dasm_state,
 		return NULL;
 	}
 
+#ifdef __aarch64__
+	size += dasm_venners_size;
+#endif
+
 	entry = *dasm_ptr;
 	*dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT));
 
+	/* flush the hardware I-cache */
+	JIT_CACHE_FLUSH(entry, entry + size);
+
 	if (trace_num) {
 		zend_jit_trace_add_code(entry, size);
 	}
@@ -4365,8 +4406,14 @@ ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached)
 		return FAILURE;
 	}
 
-	/* save JIT buffer pos */
 	zend_jit_unprotect();
+#ifdef __aarch64__
+	/* reserve space for global labels veneers */
+	dasm_labels_veneers = *dasm_ptr;
+	*dasm_ptr = (void**)*dasm_ptr + zend_lb_MAX;
+	memset(dasm_labels_veneers, 0, sizeof(void*) * zend_lb_MAX);
+#endif
+	/* save JIT buffer pos */
 	dasm_ptr[1] = dasm_ptr[0];
 	zend_jit_protect();
 
@@ -4376,7 +4423,7 @@ ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached)
 ZEND_EXT_API void zend_jit_shutdown(void)
 {
 	if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE) {
-		fprintf(stderr, "\nJIT memory usage: %td\n", (char*)*dasm_ptr - (char*)dasm_buf);
+		fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf));
 	}
 
 #ifdef HAVE_OPROFILE

ext/opcache/jit/zend_jit.h

@@ -53,6 +53,7 @@
 
 #define ZEND_JIT_DEBUG_GDB       (1<<8)
 #define ZEND_JIT_DEBUG_SIZE      (1<<9)
+#define ZEND_JIT_DEBUG_ASM_ADDR  (1<<10)
 
 #define ZEND_JIT_DEBUG_TRACE_START     (1<<12)
 #define ZEND_JIT_DEBUG_TRACE_STOP      (1<<13)