Optimize observers

Inline the lookup whether a function is observed at all.
This strategy is also used for FRAMELESS calls. If the frameless call is observed, we instead allocate a call frame and push the arguments, to call the the function afterwards.
Doing so is still a performance benefit as opposed to executing individual INIT_FCALL+SEND_VAL ops. Thus, even if the frameless call turns out to be observed, the call overhead is slightly lower than before.
If the internal function is not observed at all, the unavoidable overhead is fetching the FLF zend_function pointer and the run-time cache needs to be inspected.

As part of this work, it turned out to be most viable to put the result operand on the ZEND_OP_DATA instead of ZEND_FRAMELESS_ICALL_3, allowing seamless interoperability with the DO_ICALL opcode.
This is a bit unusual in comparison to all other ZEND_OP_DATA usages, but seems to not pose problems overall.

Author: bwoebi

Zend/Optimizer/sccp.c

@@ -2211,6 +2211,8 @@ static int try_remove_definition(sccp_ctx *ctx, int var_num, zend_ssa_var *var,
 						if (has_op_data) {
 							zend_ssa_remove_instr(ssa, opline + 1, ssa_op + 1);
 							removed_ops++;
+							old_type = opline[1].result_type;
+							old_var = opline[1].result.var;
 						}
 					}
 					ssa_op->result_def = var_num;

Zend/Optimizer/zend_inference.c

@@ -3869,7 +3869,7 @@ static zend_always_inline zend_result _zend_update_type_info(
 				UPDATE_SSA_TYPE(tmp, ssa_op->op2_def);
 			}
 			if (opline->opcode == ZEND_FRAMELESS_ICALL_3) {
-				zend_ssa_op *next_ssa_op = ssa_op + 1;
+				zend_ssa_op *next_ssa_op = ++ssa_op;
 				if (next_ssa_op->op1_def >= 0) {
 					ZEND_ASSERT(next_ssa_op->op1_use >= 0);
 					tmp = ssa->var_info[next_ssa_op->op1_use].type;

Zend/Optimizer/zend_ssa.c

@@ -782,12 +782,14 @@ static zend_always_inline int _zend_ssa_rename_op(const zend_op_array *op_array,
 				//NEW_SSA_VAR(opline->op2.var)
 			}
 			if (opline->opcode == ZEND_FRAMELESS_ICALL_3) {
-				next = opline + 1;
+				// Result is on following OP_DATA
+				next = ++opline;
+				++k;
 				if (next->op1_type & (IS_CV|IS_VAR|IS_TMP_VAR)) {
-					ssa_ops[k + 1].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
+					ssa_ops[k].op1_use = var[EX_VAR_TO_NUM(next->op1.var)];
 					//USE_SSA_VAR(op_array->last_var + next->op1.var);
 					if ((build_flags & ZEND_SSA_RC_INFERENCE) && next->op1_type == IS_CV) {
-						ssa_ops[k + 1].op1_def = ssa_vars_count;
+						ssa_ops[k].op1_def = ssa_vars_count;
 						var[EX_VAR_TO_NUM(next->op1.var)] = ssa_vars_count;
 						ssa_vars_count++;
 						//NEW_SSA_VAR(next->op1.var)

Zend/zend_compile.c

@@ -4542,7 +4542,7 @@ static uint32_t find_frameless_function_offset(uint32_t arity, void *handler)
 
 static const zend_frameless_function_info *find_frameless_function_info(zend_ast_list *args, zend_function *fbc, uint32_t type)
 {
-	if (ZEND_OBSERVER_ENABLED || zend_execute_internal) {
+	if (zend_execute_internal) {
 		return NULL;
 	}
 
@@ -4608,7 +4608,12 @@ static uint32_t zend_compile_frameless_icall_ex(znode *result, zend_ast_list *ar
 		SET_NODE(opline->op2, &arg_zvs[1]);
 	}
 	if (num_args >= 3) {
-		zend_emit_op_data(&arg_zvs[2]);
+		// Put result znode on OP_DATA to ensure dispatch in observer fallback can be aligned with the last opcode of the call
+		zend_op *op_data = zend_emit_op_data(&arg_zvs[2]);
+		op_data->result = opline->result;
+		op_data->result_type = opline->result_type;
+		opline->result_type = IS_UNUSED;
+		++opnum;
 	}
 	return opnum;
 }

Zend/zend_observer.c

@@ -14,6 +14,7 @@
    +----------------------------------------------------------------------+
    | Authors: Levi Morrison <levim@php.net>                               |
    |          Sammy Kaye Powers <sammyk@php.net>                          |
+   |          Bob Weinand <bobwei9@hotmail.com>                           |
    +----------------------------------------------------------------------+
 */
 
@@ -23,14 +24,8 @@
 #include "zend_llist.h"
 #include "zend_vm.h"
 
-#define ZEND_OBSERVER_DATA(function) \
-	ZEND_OP_ARRAY_EXTENSION((&(function)->common), zend_observer_fcall_op_array_extension)
-
 #define ZEND_OBSERVER_NOT_OBSERVED ((void *) 2)
 
-#define ZEND_OBSERVABLE_FN(function) \
-	(ZEND_MAP_PTR(function->common.run_time_cache) && !(function->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
-
 zend_llist zend_observers_fcall_list;
 zend_llist zend_observer_function_declared_callbacks;
 zend_llist zend_observer_class_linked_callbacks;
@@ -44,7 +39,7 @@ bool zend_observer_errors_observed;
 bool zend_observer_function_declared_observed;
 bool zend_observer_class_linked_observed;
 
-ZEND_TLS zend_execute_data *current_observed_frame;
+ZEND_EXT_TLS zend_execute_data *zend_observer_current_frame;
 
 // Call during minit/startup ONLY
 ZEND_API void zend_observer_fcall_register(zend_observer_fcall_init init)
@@ -101,7 +96,7 @@ ZEND_API void zend_observer_post_startup(void)
 
 ZEND_API void zend_observer_activate(void)
 {
-	current_observed_frame = NULL;
+    zend_observer_current_frame = NULL;
 }
 
 ZEND_API void zend_observer_shutdown(void)
@@ -121,21 +116,24 @@ static void zend_observer_fcall_install(zend_execute_data *execute_data)
 	zend_function *function = execute_data->func;
 
 	ZEND_ASSERT(RUN_TIME_CACHE(&function->common));
-	zend_observer_fcall_begin_handler *begin_handlers = (zend_observer_fcall_begin_handler *)&ZEND_OBSERVER_DATA(function);
+	zend_observer_fcall_begin_handler *begin_handlers = ZEND_OBSERVER_DATA(function), *begin_handlers_start = begin_handlers;
 	zend_observer_fcall_end_handler *end_handlers = (zend_observer_fcall_end_handler *)begin_handlers + list->count, *end_handlers_start = end_handlers;
 
 	*begin_handlers = ZEND_OBSERVER_NOT_OBSERVED;
 	*end_handlers = ZEND_OBSERVER_NOT_OBSERVED;
+	bool has_handlers = false;
 
 	for (zend_llist_element *element = list->head; element; element = element->next) {
 		zend_observer_fcall_init init;
 		memcpy(&init, element->data, sizeof init);
 		zend_observer_fcall_handlers handlers = init(execute_data);
 		if (handlers.begin) {
 			*(begin_handlers++) = handlers.begin;
+			has_handlers = true;
 		}
 		if (handlers.end) {
 			*(end_handlers++) = handlers.end;
+			has_handlers = true;
 		}
 	}
 
@@ -145,6 +143,10 @@ static void zend_observer_fcall_install(zend_execute_data *execute_data)
 		*end_handlers = *end_handlers_start;
 		*end_handlers_start = tmp;
 	}
+
+	if (!has_handlers) {
+		*begin_handlers_start = ZEND_OBSERVER_NONE_OBSERVED;
+	}
 }
 
 static bool zend_observer_remove_handler(void **first_handler, void *old_handler) {
@@ -169,8 +171,8 @@ static bool zend_observer_remove_handler(void **first_handler, void *old_handler
 
 ZEND_API void zend_observer_add_begin_handler(zend_function *function, zend_observer_fcall_begin_handler begin) {
 	size_t registered_observers = zend_observers_fcall_list.count;
-	zend_observer_fcall_begin_handler *first_handler = (void *)&ZEND_OBSERVER_DATA(function), *last_handler = first_handler + registered_observers - 1;
-	if (*first_handler == ZEND_OBSERVER_NOT_OBSERVED) {
+	zend_observer_fcall_begin_handler *first_handler = ZEND_OBSERVER_DATA(function), *last_handler = first_handler + registered_observers - 1;
+	if (*first_handler == ZEND_OBSERVER_NOT_OBSERVED || *first_handler == ZEND_OBSERVER_NONE_OBSERVED) {
 		*first_handler = begin;
 	} else {
 		for (zend_observer_fcall_begin_handler *cur_handler = first_handler + 1; cur_handler <= last_handler; ++cur_handler) {
@@ -185,24 +187,45 @@ ZEND_API void zend_observer_add_begin_handler(zend_function *function, zend_obse
 }
 
 ZEND_API bool zend_observer_remove_begin_handler(zend_function *function, zend_observer_fcall_begin_handler begin) {
-	return zend_observer_remove_handler((void **)&ZEND_OBSERVER_DATA(function), begin);
+	void **begin_handlers = (void **)ZEND_OBSERVER_DATA(function);
+	if (zend_observer_remove_handler(begin_handlers, begin)) {
+		if (*begin_handlers == ZEND_OBSERVER_NOT_OBSERVED) {
+			size_t registered_observers = zend_observers_fcall_list.count;
+			if (begin_handlers[registered_observers] /* first end handler */ == ZEND_OBSERVER_NOT_OBSERVED) {
+				*begin_handlers = ZEND_OBSERVER_NONE_OBSERVED;
+			}
+		}
+		return true;
+	}
+	return false;
 }
 
 ZEND_API void zend_observer_add_end_handler(zend_function *function, zend_observer_fcall_end_handler end) {
 	size_t registered_observers = zend_observers_fcall_list.count;
-	zend_observer_fcall_end_handler *end_handler = (zend_observer_fcall_end_handler *)&ZEND_OBSERVER_DATA(function) + registered_observers;
+	void **begin_handler = (void **)ZEND_OBSERVER_DATA(function);
+	zend_observer_fcall_end_handler *end_handler = (zend_observer_fcall_end_handler *)begin_handler + registered_observers;
 	// to allow to preserve the invariant that end handlers are in reverse order of begin handlers, push the new end handler in front
 	if (*end_handler != ZEND_OBSERVER_NOT_OBSERVED) {
 		// there's no space for new handlers, then it's forbidden to call this function
 		ZEND_ASSERT(end_handler[registered_observers - 1] == NULL);
 		memmove(end_handler + 1, end_handler, sizeof(end_handler) * (registered_observers - 1));
+	} else if (*begin_handler == ZEND_OBSERVER_NONE_OBSERVED) {
+		*begin_handler = ZEND_OBSERVER_NOT_OBSERVED;
 	}
 	*end_handler = end;
 }
 
 ZEND_API bool zend_observer_remove_end_handler(zend_function *function, zend_observer_fcall_end_handler end) {
 	size_t registered_observers = zend_observers_fcall_list.count;
-	return zend_observer_remove_handler((void **)&ZEND_OBSERVER_DATA(function) + registered_observers, end);
+	void **begin_handlers = (void **)ZEND_OBSERVER_DATA(function);
+	void **end_handlers = begin_handlers + registered_observers;
+	if (zend_observer_remove_handler(end_handlers, end)) {
+		if (*begin_handlers == ZEND_OBSERVER_NOT_OBSERVED && *end_handlers == ZEND_OBSERVER_NOT_OBSERVED) {
+			*begin_handlers = ZEND_OBSERVER_NONE_OBSERVED;
+		}
+		return true;
+	}
+	return false;
 }
 
 static inline zend_execute_data **prev_observed_frame(zend_execute_data *execute_data) {
@@ -211,33 +234,33 @@ static inline zend_execute_data **prev_observed_frame(zend_execute_data *execute
 	return (zend_execute_data **)&Z_PTR_P(EX_VAR_NUM((ZEND_USER_CODE(func->type) ? func->op_array.last_var : ZEND_CALL_NUM_ARGS(execute_data)) + func->common.T - 1));
 }
 
-static void ZEND_FASTCALL _zend_observe_fcall_begin(zend_execute_data *execute_data)
-{
+static void ZEND_FASTCALL _zend_observe_fcall_begin(zend_execute_data *execute_data) {
 	if (!ZEND_OBSERVER_ENABLED) {
 		return;
 	}
 
-	zend_function *function = execute_data->func;
+	zend_observer_fcall_begin_specialized(execute_data, true);
+}
 
-	if (!ZEND_OBSERVABLE_FN(function)) {
-		return;
-	}
+void ZEND_FASTCALL zend_observer_fcall_begin_prechecked(zend_execute_data *execute_data, zend_observer_fcall_begin_handler *handler)
+{
+	zend_observer_fcall_begin_handler *possible_handlers_end = handler + zend_observers_fcall_list.count;
 
-	zend_observer_fcall_begin_handler *handler = (zend_observer_fcall_begin_handler *)&ZEND_OBSERVER_DATA(function);
 	if (!*handler) {
 		zend_observer_fcall_install(execute_data);
+		if (zend_observer_handler_is_unobserved(handler)) {
+			return;
+		}
 	}
 
-	zend_observer_fcall_begin_handler *possible_handlers_end = handler + zend_observers_fcall_list.count;
-
 	zend_observer_fcall_end_handler *end_handler = (zend_observer_fcall_end_handler *)possible_handlers_end;
 	if (*end_handler != ZEND_OBSERVER_NOT_OBSERVED) {
-		*prev_observed_frame(execute_data) = current_observed_frame;
-		current_observed_frame = execute_data;
-	}
+		*prev_observed_frame(execute_data) = zend_observer_current_frame;
+		zend_observer_current_frame = execute_data;
 
-	if (*handler == ZEND_OBSERVER_NOT_OBSERVED) {
-		return;
+		if (*handler == ZEND_OBSERVER_NOT_OBSERVED) { // this function must not be called if ZEND_OBSERVER_NONE_OBSERVED, hence sufficient to check
+			return;
+		}
 	}
 
 	do {
@@ -252,17 +275,17 @@ ZEND_API void ZEND_FASTCALL zend_observer_generator_resume(zend_execute_data *ex
 
 ZEND_API void ZEND_FASTCALL zend_observer_fcall_begin(zend_execute_data *execute_data)
 {
-	ZEND_ASSUME(execute_data->func);
-	if (!(execute_data->func->common.fn_flags & ZEND_ACC_GENERATOR)) {
+	ZEND_ASSUME(EX(func));
+	if (!(EX(func)->common.fn_flags & ZEND_ACC_GENERATOR)) {
 		_zend_observe_fcall_begin(execute_data);
 	}
 }
 
 static inline void call_end_observers(zend_execute_data *execute_data, zval *return_value) {
-	zend_function *func = execute_data->func;
+	zend_function *func = EX(func);
 	ZEND_ASSERT(func);
 
-	zend_observer_fcall_end_handler *handler = (zend_observer_fcall_end_handler *)&ZEND_OBSERVER_DATA(func) + zend_observers_fcall_list.count;
+	zend_observer_fcall_end_handler *handler = (zend_observer_fcall_end_handler *)ZEND_OBSERVER_DATA(func) + zend_observers_fcall_list.count;
 	// TODO: Fix exceptions from generators
 	// ZEND_ASSERT(fcall_data);
 	if (!*handler || *handler == ZEND_OBSERVER_NOT_OBSERVED) {
@@ -275,19 +298,16 @@ static inline void call_end_observers(zend_execute_data *execute_data, zval *ret
 	} while (++handler != possible_handlers_end && *handler != NULL);
 }
 
-ZEND_API void ZEND_FASTCALL zend_observer_fcall_end(zend_execute_data *execute_data, zval *return_value)
+ZEND_API void ZEND_FASTCALL zend_observer_fcall_end_prechecked(zend_execute_data *execute_data, zval *return_value)
 {
-	if (execute_data != current_observed_frame) {
-		return;
-	}
 	call_end_observers(execute_data, return_value);
-	current_observed_frame = *prev_observed_frame(execute_data);
+	zend_observer_current_frame = *prev_observed_frame(execute_data);
 }
 
 ZEND_API void zend_observer_fcall_end_all(void)
 {
-	zend_execute_data *execute_data = current_observed_frame, *original_execute_data = EG(current_execute_data);
-	current_observed_frame = NULL;
+	zend_execute_data *execute_data = zend_observer_current_frame, *original_execute_data = EG(current_execute_data);
+    zend_observer_current_frame = NULL;
 	while (execute_data) {
 		EG(current_execute_data) = execute_data;
 		call_end_observers(execute_data, NULL);
@@ -388,8 +408,8 @@ ZEND_API void ZEND_FASTCALL zend_observer_fiber_switch_notify(zend_fiber_context
 		callback(from, to);
 	}
 
-	from->top_observed_frame = current_observed_frame;
-	current_observed_frame = to->top_observed_frame;
+	from->top_observed_frame = zend_observer_current_frame;
+    zend_observer_current_frame = to->top_observed_frame;
 }
 
 ZEND_API void ZEND_FASTCALL zend_observer_fiber_destroy_notify(zend_fiber_context *destroying)

Zend/zend_observer.h

@@ -14,6 +14,7 @@
    +----------------------------------------------------------------------+
    | Authors: Levi Morrison <levim@php.net>                               |
    |          Sammy Kaye Powers <sammyk@php.net>                          |
+   |          Bob Weinand <bobwei9@hotmail.com>                           |
    +----------------------------------------------------------------------+
 */
 
@@ -31,6 +32,21 @@ extern ZEND_API bool zend_observer_errors_observed;
 extern ZEND_API bool zend_observer_function_declared_observed;
 extern ZEND_API bool zend_observer_class_linked_observed;
 
+extern ZEND_API ZEND_EXT_TLS zend_execute_data *zend_observer_current_frame;
+
+typedef void (*zend_observer_fcall_begin_handler)(zend_execute_data *execute_data);
+typedef void (*zend_observer_fcall_end_handler)(zend_execute_data *execute_data, zval *retval);
+
+typedef struct _zend_observer_fcall_handlers {
+	zend_observer_fcall_begin_handler begin;
+	zend_observer_fcall_end_handler end;
+} zend_observer_fcall_handlers;
+
+#define ZEND_OBSERVER_DATA(function) \
+	((zend_observer_fcall_begin_handler *)&ZEND_OP_ARRAY_EXTENSION((&(function)->common), zend_observer_fcall_op_array_extension))
+
+#define ZEND_OBSERVER_NONE_OBSERVED ((void *) 3)
+
 #define ZEND_OBSERVER_ENABLED (zend_observer_fcall_op_array_extension != -1)
 
 #define ZEND_OBSERVER_FCALL_BEGIN(execute_data) do { \
@@ -45,14 +61,6 @@ extern ZEND_API bool zend_observer_class_linked_observed;
 		} \
 	} while (0)
 
-typedef void (*zend_observer_fcall_begin_handler)(zend_execute_data *execute_data);
-typedef void (*zend_observer_fcall_end_handler)(zend_execute_data *execute_data, zval *retval);
-
-typedef struct _zend_observer_fcall_handlers {
-	zend_observer_fcall_begin_handler begin;
-	zend_observer_fcall_end_handler end;
-} zend_observer_fcall_handlers;
-
 /* If the fn should not be observed then return {NULL, NULL} */
 typedef zend_observer_fcall_handlers (*zend_observer_fcall_init)(zend_execute_data *execute_data);
 
@@ -71,15 +79,44 @@ ZEND_API void zend_observer_post_startup(void); // Called by engine after MINITs
 ZEND_API void zend_observer_activate(void);
 ZEND_API void zend_observer_shutdown(void);
 
-ZEND_API void ZEND_FASTCALL zend_observer_fcall_begin(
-	zend_execute_data *execute_data);
+ZEND_API void ZEND_FASTCALL zend_observer_fcall_begin(zend_execute_data *execute_data);
+void ZEND_FASTCALL zend_observer_fcall_begin_prechecked(zend_execute_data *execute_data, zend_observer_fcall_begin_handler *observer_data);
+
+static zend_always_inline bool zend_observer_handler_is_unobserved(zend_observer_fcall_begin_handler *handler) {
+	return *handler == ZEND_OBSERVER_NONE_OBSERVED;
+}
+
+static zend_always_inline bool zend_observer_fcall_has_no_observers(zend_execute_data *execute_data, bool allow_generator, zend_observer_fcall_begin_handler **handler) {
+	zend_function *function = EX(func);
+	void *ZEND_MAP_PTR(runtime_cache) = ZEND_MAP_PTR((function)->common.run_time_cache);
+
+	if (function->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | (allow_generator ? 0 : ZEND_ACC_GENERATOR))) {
+		return true;
+	}
+
+	if (!ZEND_MAP_PTR(runtime_cache)) {
+		return true;
+	}
+
+	*handler = (zend_observer_fcall_begin_handler *)ZEND_MAP_PTR_GET(runtime_cache) + zend_observer_fcall_op_array_extension;
+	return zend_observer_handler_is_unobserved(*handler);
+}
+
+static zend_always_inline void zend_observer_fcall_begin_specialized(zend_execute_data *execute_data, bool allow_generator) {
+	zend_observer_fcall_begin_handler *handler;
+	if (!zend_observer_fcall_has_no_observers(execute_data, allow_generator, &handler)) {
+		zend_observer_fcall_begin_prechecked(execute_data, handler);
+	}
+}
 
-ZEND_API void ZEND_FASTCALL zend_observer_generator_resume(
-	zend_execute_data *execute_data);
+ZEND_API void ZEND_FASTCALL zend_observer_generator_resume(zend_execute_data *execute_data);
 
-ZEND_API void ZEND_FASTCALL zend_observer_fcall_end(
-	zend_execute_data *execute_data,
-	zval *return_value);
+ZEND_API void ZEND_FASTCALL zend_observer_fcall_end_prechecked(zend_execute_data *execute_data, zval *return_value);
+static zend_always_inline void zend_observer_fcall_end(zend_execute_data *execute_data, zval *return_value) {
+	if (execute_data == zend_observer_current_frame) {
+		zend_observer_fcall_end_prechecked(execute_data, return_value);
+	}
+}
 
 ZEND_API void zend_observer_fcall_end_all(void);