Optimize observers

Inline the lookup whether a function is observed at all.
This strategy is also used for FRAMELESS calls. If the frameless call is observed, we instead allocate a call frame and push the arguments, to call the the function afterwards.
Doing so is still a performance benefit as opposed to executing individual INIT_FCALL+SEND_VAL ops. Thus, even if the frameless call turns out to be observed, the call overhead is slightly lower than before.
If the internal function is not observed at all, the unavoidable overhead is fetching the FLF zend_function pointer and the run-time cache needs to be inspected.

As part of this work, it turned out to be most viable to put the result operand on the ZEND_OP_DATA instead of ZEND_FRAMELESS_ICALL_3, allowing seamless interoperability with the DO_ICALL opcode.
This is a bit unusual in comparison to all other ZEND_OP_DATA usages, but seems to not pose problems overall.

There is also a small issue resolved: trampolines would always use the ZEND_CALL_TRAMPOLINE_SPEC_OBSERVER function due to zend_observer_fcall_op_array_extension being set to -1 too late.

Signed-off-by: Bob Weinand <bobwei9@hotmail.com>

Author: bwoebi

Zend/Optimizer/optimize_temp_vars_5.c

@@ -49,6 +49,7 @@ void zend_optimize_temporary_variables(zend_op_array *op_array, zend_optimizer_c
 	int *map_T;				/* Map's the T to its new index */
 	zend_op *opline, *end;
 	int currT;
+	int exclNextT = -1;
 	int i;
 	int max = -1;
 	void *checkpoint = zend_arena_checkpoint(ctx->arena);
@@ -144,6 +145,11 @@ void zend_optimize_temporary_variables(zend_op_array *op_array, zend_optimizer_c
 			opline->op2.var = NUM_VAR(map_T[currT] + offset);
 		}
 
+		if (exclNextT != -1) {
+			zend_bitset_excl(taken_T, exclNextT);
+			exclNextT = -1;
+		}
+
 		if (opline->result_type & (IS_VAR | IS_TMP_VAR)) {
 			currT = VAR_NUM(opline->result.var) - offset;
 			if (map_T[currT] == INVALID_VAR) {
@@ -157,7 +163,12 @@ void zend_optimize_temporary_variables(zend_op_array *op_array, zend_optimizer_c
 				 * since the fast_var could also be set by ZEND_HANDLE_EXCEPTION
 				 * which could be ahead of it */
 				if (opline->opcode != ZEND_FAST_CALL) {
-					zend_bitset_excl(taken_T, map_T[currT]);
+					if (opline->opcode == ZEND_OP_DATA) {
+						/* ZEND_OP_DATA may contain a result, which must not be equal to any of its prior opcode vars */
+						exclNextT = map_T[currT];
+					} else {
+						zend_bitset_excl(taken_T, map_T[currT]);
+					}
 				}
 				if (opline->opcode == ZEND_ROPE_INIT) {
 					uint32_t num = ((opline->extended_value * sizeof(zend_string*)) + (sizeof(zval) - 1)) / sizeof(zval);

Zend/Optimizer/sccp.c

@@ -2139,6 +2139,10 @@ static int try_remove_definition(sccp_ctx *ctx, int var_num, zend_ssa_var *var,
 				}
 				return 0;
 			}
+			if (opline->opcode == ZEND_OP_DATA) {
+				/* Consider the primary opline for matching */
+				--opline;
+			}
 			if (ssa_op->op1_def >= 0 || ssa_op->op2_def >= 0) {
 				if (var->use_chain < 0 && var->phi_use_chain == NULL) {
 					switch (opline->opcode) {
@@ -2209,6 +2213,8 @@ static int try_remove_definition(sccp_ctx *ctx, int var_num, zend_ssa_var *var,
 						zend_ssa_remove_instr(ssa, opline, ssa_op);
 						removed_ops++;
 						if (has_op_data) {
+							old_type = opline[1].result_type;
+							old_var = opline[1].result.var;
 							zend_ssa_remove_instr(ssa, opline + 1, ssa_op + 1);
 							removed_ops++;
 						}

Zend/Optimizer/scdf.c

@@ -124,7 +124,10 @@ void scdf_solve(scdf_ctx *scdf, const char *name) {
 				zend_ssa_op *ssa_op = &ssa->ops[i];
 				if (opline->opcode == ZEND_OP_DATA) {
 					opline--;
-					ssa_op--;
+					if (opline->opcode != ZEND_FRAMELESS_ICALL_3) {
+						// result op not on OP_DATA
+						ssa_op--;
+					}
 				}
 				scdf->handlers.visit_instr(scdf, opline, ssa_op);
 				if (i == block->start + block->len - 1) {
@@ -162,7 +165,7 @@ void scdf_solve(scdf_ctx *scdf, const char *name) {
 				for (j = block->start; j < end; j++) {
 					opline = &scdf->op_array->opcodes[j];
 					zend_bitset_excl(scdf->instr_worklist, j);
-					if (opline->opcode != ZEND_OP_DATA) {
+					if (opline->opcode != ZEND_OP_DATA || opline[-1].opcode == ZEND_FRAMELESS_ICALL_3) {
 						scdf->handlers.visit_instr(scdf, opline, &ssa->ops[j]);
 					}
 				}
@@ -172,7 +175,10 @@ void scdf_solve(scdf_ctx *scdf, const char *name) {
 					ZEND_ASSERT(opline && "Should have opline in non-empty block");
 					if (opline->opcode == ZEND_OP_DATA) {
 						opline--;
-						j--;
+						if (opline->opcode != ZEND_FRAMELESS_ICALL_3) {
+							// result op not on OP_DATA
+							j--;
+						}
 					}
 					scdf->handlers.mark_feasible_successors(scdf, i, block, opline, &ssa->ops[j-1]);
 				}

Zend/Optimizer/zend_dfg.c

@@ -130,6 +130,9 @@ static zend_always_inline void _zend_dfg_add_use_def_op(const zend_op_array *op_
 					zend_bitset_incl(use, var_num);
 				}
 			}
+			if (opline->opcode == ZEND_FRAMELESS_ICALL_3) {
+				++opline; // OP_DATA has result
+			}
 			break;
 		case ZEND_ASSIGN_DIM_OP:
 		case ZEND_ASSIGN_OBJ_OP:

Zend/Optimizer/zend_inference.c

@@ -3869,7 +3869,7 @@ static zend_always_inline zend_result _zend_update_type_info(
 				UPDATE_SSA_TYPE(tmp, ssa_op->op2_def);
 			}
 			if (opline->opcode == ZEND_FRAMELESS_ICALL_3) {
-				zend_ssa_op *next_ssa_op = ssa_op + 1;
+				zend_ssa_op *next_ssa_op = ++ssa_op; // OP_DATA has result op
 				if (next_ssa_op->op1_def >= 0) {
 					ZEND_ASSERT(next_ssa_op->op1_use >= 0);
 					tmp = ssa->var_info[next_ssa_op->op1_use].type;

Zend/Optimizer/zend_ssa.c

@@ -782,12 +782,14 @@ static zend_always_inline int _zend_ssa_rename_op(const zend_op_array *op_array,
 				//NEW_SSA_VAR(opline->op2.var)
 			}
 			if (opline->opcode == ZEND_FRAMELESS_ICALL_3) {
-				next = opline + 1;
+				// Result is on following OP_DATA
+				next = ++opline;
+				++k;
 				if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
-					ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
+					ssa_ops[k].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
 					//USE_SSA_VAR(op_array->last_var + next->op1.var);
 					if ((build_flags & ZEND_SSA_RC_INFERENCE) && next->op1_type == IS_CV) {
-						ssa_ops[k + 1].op1_def = ssa_vars_count;
+						ssa_ops[k].op1_def = ssa_vars_count;
 						var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count;
 						ssa_vars_count++;
 						//NEW_SSA_VAR(next->op1.var)

Zend/zend_compile.c

@@ -4542,7 +4542,7 @@ static uint32_t find_frameless_function_offset(uint32_t arity, void *handler)
 
 static const zend_frameless_function_info *find_frameless_function_info(zend_ast_list *args, zend_function *fbc, uint32_t type)
 {
-	if (ZEND_OBSERVER_ENABLED || zend_execute_internal) {
+	if (zend_execute_internal) {
 		return NULL;
 	}
 
@@ -4608,7 +4608,12 @@ static uint32_t zend_compile_frameless_icall_ex(znode *result, zend_ast_list *ar
 		SET_NODE(opline->op2, &arg_zvs[1]);
 	}
 	if (num_args >= 3) {
-		zend_emit_op_data(&arg_zvs[2]);
+		// Put result znode on OP_DATA to ensure dispatch in observer fallback can be aligned with the last opcode of the call
+		zend_op *op_data = zend_emit_op_data(&arg_zvs[2]);
+		op_data->result = opline->result;
+		op_data->result_type = opline->result_type;
+		opline->result_type = IS_UNUSED;
+		++opnum;
 	}
 	return opnum;
 }

Zend/zend_frameless_function.h

@@ -115,7 +115,7 @@ typedef void (*zend_frameless_function_3)(zval *return_value, zval *op1, zval *o
 extern size_t zend_flf_count;
 extern size_t zend_flf_capacity;
 ZEND_API extern void **zend_flf_handlers;
-extern zend_function **zend_flf_functions;
+ZEND_API extern zend_function **zend_flf_functions;
 
 typedef struct {
 	void *handler;

Zend/zend_globals.h

@@ -193,6 +193,8 @@ struct _zend_executor_globals {
 
 	uint32_t jit_trace_num; /* Used by tracing JIT to reference the currently running trace */
 
+	zend_execute_data *current_observed_frame;
+
 	int ticks_count;
 
 	zend_long precision;