Generate veneers to perform long jumps.

This makes symfony_demo app working with -d opcacge.jit=1205.
(21 MB of JIT-ed code).

Author: dstogov

ext/opcache/jit/zend_jit.c

@@ -209,6 +209,8 @@ static bool zend_is_commutative(zend_uchar opcode)
 #if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32)
 #include "dynasm/dasm_x86.h"
 #elif defined(__aarch64__)
+static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset);
+#define DASM_ADD_VENEER zend_jit_add_veneer
 #include "dynasm/dasm_arm64.h"
 #endif
 
@@ -408,6 +410,10 @@ static void *dasm_link_and_encode(dasm_State             **dasm_state,
 		return NULL;
 	}
 
+#ifdef __aarch64__
+	dasm_venners_size = 0;
+#endif
+
 	ret = dasm_encode(dasm_state, *dasm_ptr);
 	if (ret != DASM_S_OK) {
 #if ZEND_DEBUG
@@ -416,6 +422,10 @@ static void *dasm_link_and_encode(dasm_State             **dasm_state,
 		return NULL;
 	}
 
+#ifdef __aarch64__
+	size += dasm_venners_size;
+#endif
+
 	entry = *dasm_ptr;
 	*dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT));
 
@@ -4396,8 +4406,14 @@ ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached)
 		return FAILURE;
 	}
 
-	/* save JIT buffer pos */
 	zend_jit_unprotect();
+#ifdef __aarch64__
+	/* reserve space for global labels veneers */
+	dasm_labels_veneers = *dasm_ptr;
+	*dasm_ptr = (void**)*dasm_ptr + zend_lb_MAX;
+	memset(dasm_labels_veneers, 0, sizeof(void*) * zend_lb_MAX);
+#endif
+	/* save JIT buffer pos */
 	dasm_ptr[1] = dasm_ptr[0];
 	zend_jit_protect();
 

ext/opcache/jit/zend_jit_arm64.dasc

@@ -15017,6 +15017,133 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend
 	return regset;
 }
 
+static size_t dasm_venners_size = 0;
+void **dasm_labels_veneers = NULL;
+
+static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset)
+{
+	void *veneer;
+	ptrdiff_t na;
+	int n, m;
+
+	/* try to reuse veneers for global labels */
+	if ((ins >> 16) == DASM_REL_LG
+	 && *(b-1) < 0
+	 && dasm_labels_veneers[-*(b-1)]) {
+
+		veneer = dasm_labels_veneers[-*(b-1)];
+		na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4;
+		n = (int)na;
+
+		/* check if we can jump to veneer */
+		if ((ptrdiff_t)n != na) {
+			/* pass */
+		} else if (!(ins & 0xf800)) {  /* B, BL */
+			if ((n & 3) == 0 && ((n+0x08000000) >> 28) == 0) {
+				return n;
+			}
+		} else if ((ins & 0x800)) {  /* B.cond, CBZ, CBNZ, LDR* literal */
+			if ((n & 3) == 0 && ((n+0x00100000) >> 21) == 0) {
+				return n;
+			}
+		} else if ((ins & 0x3000) == 0x2000) {  /* ADR */
+			/* pass */
+		} else if ((ins & 0x3000) == 0x3000) {  /* ADRP */
+			/* pass */
+		} else if ((ins & 0x1000)) {  /* TBZ, TBNZ */
+			if ((n & 3) == 0 && ((n+0x00008000) >> 16) == 0) {
+				return n;
+			}
+		}
+	}
+
+	veneer = (char*)buffer + (Dst->codesize + dasm_venners_size);
+
+	if (veneer > dasm_end) {
+		return 0; /* jit_buffer_size overflow */
+	}
+
+	na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4;
+	n = (int)na;
+
+	/* check if we can jump to veneer */
+	if ((ptrdiff_t)n != na) {
+		ZEND_ASSERT(0);
+		return 0;
+	} else if (!(ins & 0xf800)) {  /* B, BL */
+		if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) {
+			ZEND_ASSERT(0);
+			return 0;
+		}
+	} else if ((ins & 0x800)) {  /* B.cond, CBZ, CBNZ, LDR* literal */
+		if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) {
+			ZEND_ASSERT(0);
+			return 0;
+		}
+	} else if ((ins & 0x3000) == 0x2000) {  /* ADR */
+		ZEND_ASSERT(0);
+		return 0;
+	} else if ((ins & 0x3000) == 0x3000) {  /* ADRP */
+		ZEND_ASSERT(0);
+		return 0;
+	} else if ((ins & 0x1000)) {  /* TBZ, TBNZ */
+		if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) {
+			ZEND_ASSERT(0);
+			return 0;
+		}
+	} else if ((ins & 0x8000)) {  /* absolute */
+		ZEND_ASSERT(0);
+		return 0;
+	} else {
+		ZEND_ASSERT(0);
+		return 0;
+	}
+
+	// TODO: support for long veneers (above 128MB) ???
+
+	/* check if we can use B to jump from veneer */
+	na = (ptrdiff_t)cp + offset - (ptrdiff_t)veneer - 4;
+	m = (int)na;
+	if ((ptrdiff_t)m != na) {
+		ZEND_ASSERT(0);
+		return 0;
+	} else if ((m & 3) != 0 || ((m+0x08000000) >> 28) != 0) {
+		ZEND_ASSERT(0);
+		return 0;
+	}
+
+	/* generate B instruction */
+	*(uint32_t*)veneer = 0x14000000 | ((m >> 2) & 0x03ffffff);
+	dasm_venners_size += 4;
+
+	if ((ins >> 16) == DASM_REL_LG
+	 && *(b-1) < 0) {
+		/* reuse this veneer for the future jumps to global label */
+		dasm_labels_veneers[-*(b-1)] = veneer;
+		/* Dst->globals[*(b-1)] = veneer; */
+
+#ifdef HAVE_DISASM
+	    if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
+			const char *name = zend_jit_disasm_find_symbol((ptrdiff_t)cp + offset - 4, &offset);
+
+			if (name && !offset) {
+				if (strstr(name, "@veneer") == NULL) {
+					char *new_name;
+
+					zend_spprintf(&new_name, 0, "%s@veneer", name);
+					zend_jit_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4);
+					efree(new_name);
+				} else {
+					zend_jit_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4);
+				}
+			}
+		}
+#endif
+	}
+
+	return n;
+}
+
 #if defined(__clang__)
 # pragma clang diagnostic pop
 #endif