diff --git a/micrometer-support/src/main/java/io/javaoperatorsdk/operator/monitoring/micrometer/MicrometerMetrics.java b/micrometer-support/src/main/java/io/javaoperatorsdk/operator/monitoring/micrometer/MicrometerMetrics.java index fb7d064c26..6bc4f22fdc 100644 --- a/micrometer-support/src/main/java/io/javaoperatorsdk/operator/monitoring/micrometer/MicrometerMetrics.java +++ b/micrometer-support/src/main/java/io/javaoperatorsdk/operator/monitoring/micrometer/MicrometerMetrics.java @@ -5,28 +5,56 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.OperatorException; import io.javaoperatorsdk.operator.api.monitoring.Metrics; import io.javaoperatorsdk.operator.api.reconciler.Constants; import io.javaoperatorsdk.operator.api.reconciler.RetryInfo; +import io.javaoperatorsdk.operator.processing.Controller; import io.javaoperatorsdk.operator.processing.GroupVersionKind; import io.javaoperatorsdk.operator.processing.event.Event; import io.javaoperatorsdk.operator.processing.event.ResourceID; import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Timer; +import static io.javaoperatorsdk.operator.api.reconciler.Constants.CONTROLLER_NAME; + public class MicrometerMetrics implements Metrics { private static final String PREFIX = "operator.sdk."; private static final String RECONCILIATIONS = "reconciliations."; + private static final String RECONCILIATIONS_EXECUTIONS = PREFIX + RECONCILIATIONS + "executions."; + private static final String RECONCILIATIONS_QUEUE_SIZE = PREFIX + RECONCILIATIONS + "queue.size."; private final MeterRegistry registry; + private final Map gauges = new ConcurrentHashMap<>(); public MicrometerMetrics(MeterRegistry registry) { this.registry = registry; } + @Override + public void controllerRegistered(Controller controller) { + String executingThreadsName = + RECONCILIATIONS_EXECUTIONS + controller.getConfiguration().getName(); + AtomicInteger executingThreads = + registry.gauge(executingThreadsName, + gvkTags(controller.getConfiguration().getResourceClass()), + new AtomicInteger(0)); + gauges.put(executingThreadsName, executingThreads); + + String controllerQueueName = + RECONCILIATIONS_QUEUE_SIZE + controller.getConfiguration().getName(); + AtomicInteger controllerQueueSize = + registry.gauge(controllerQueueName, + gvkTags(controller.getConfiguration().getResourceClass()), + new AtomicInteger(0)); + gauges.put(controllerQueueName, controllerQueueSize); + } + public T timeControllerExecution(ControllerExecution execution) { final var name = execution.controllerName(); final var execName = PREFIX + "controllers.execution." + execution.name(); @@ -94,6 +122,10 @@ public void reconcileCustomResource(HasMetadata resource, RetryInfo retryInfoNul "" + retryInfo.map(RetryInfo::getAttemptCount).orElse(0), RECONCILIATIONS + "retries.last", "" + retryInfo.map(RetryInfo::isLastAttempt).orElse(true)); + + AtomicInteger controllerQueueSize = + gauges.get(RECONCILIATIONS_QUEUE_SIZE + metadata.get(CONTROLLER_NAME)); + controllerQueueSize.incrementAndGet(); } @Override @@ -101,6 +133,24 @@ public void finishedReconciliation(HasMetadata resource, Map met incrementCounter(ResourceID.fromResource(resource), RECONCILIATIONS + "success", metadata); } + @Override + public void reconciliationExecutionStarted(HasMetadata resource, Map metadata) { + AtomicInteger reconcilerExecutions = + gauges.get(RECONCILIATIONS_EXECUTIONS + metadata.get(CONTROLLER_NAME)); + reconcilerExecutions.incrementAndGet(); + } + + @Override + public void reconciliationExecutionFinished(HasMetadata resource, Map metadata) { + AtomicInteger reconcilerExecutions = + gauges.get(RECONCILIATIONS_EXECUTIONS + metadata.get(CONTROLLER_NAME)); + reconcilerExecutions.decrementAndGet(); + + AtomicInteger controllerQueueSize = + gauges.get(RECONCILIATIONS_QUEUE_SIZE + metadata.get(CONTROLLER_NAME)); + controllerQueueSize.decrementAndGet(); + } + public void failedReconciliation(HasMetadata resource, Exception exception, Map metadata) { var cause = exception.getCause(); @@ -118,6 +168,12 @@ public void failedReconciliation(HasMetadata resource, Exception exception, return registry.gaugeMapSize(PREFIX + name + ".size", Collections.emptyList(), map); } + public static List gvkTags(Class resourceClass) { + final var gvk = GroupVersionKind.gvkFor(resourceClass); + return List.of(Tag.of("group", gvk.group), Tag.of("version", gvk.version), + Tag.of("kind", gvk.kind)); + } + private void incrementCounter(ResourceID id, String counterName, Map metadata, String... additionalTags) { final var additionalTagsNb = diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/monitoring/Metrics.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/monitoring/Metrics.java index 31ffda96f8..c134a5522f 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/monitoring/Metrics.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/monitoring/Metrics.java @@ -6,6 +6,7 @@ import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.api.reconciler.Context; import io.javaoperatorsdk.operator.api.reconciler.RetryInfo; +import io.javaoperatorsdk.operator.processing.Controller; import io.javaoperatorsdk.operator.processing.event.Event; import io.javaoperatorsdk.operator.processing.event.ResourceID; @@ -20,6 +21,11 @@ public interface Metrics { */ Metrics NOOP = new Metrics() {}; + /** + * Do initialization if necessary; + */ + default void controllerRegistered(Controller controller) {} + /** * Called when an event has been accepted by the SDK from an event source, which would result in * potentially triggering the associated Reconciler. @@ -63,6 +69,12 @@ default void failedReconciliation(HasMetadata resource, Exception exception, failedReconciliation(ResourceID.fromResource(resource), exception, metadata); } + + default void reconciliationExecutionStarted(HasMetadata resource, Map metadata) {} + + default void reconciliationExecutionFinished(HasMetadata resource, + Map metadata) {} + /** * * @deprecated Use (and implement) {@link #cleanupDoneFor(ResourceID, Map)} instead diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Constants.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Constants.java index 55a9cbdbed..08aeb16f90 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Constants.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Constants.java @@ -21,6 +21,7 @@ public final class Constants { public static final String SAME_AS_CONTROLLER = "JOSDK_SAME_AS_CONTROLLER"; public static final String RESOURCE_GVK_KEY = "josdk.resource.gvk"; + public static final String CONTROLLER_NAME = "controller.name"; private Constants() {} } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/Controller.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/Controller.java index e9790b63e0..4eea206cb4 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/Controller.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/Controller.java @@ -90,10 +90,10 @@ public Controller(Reconciler

reconciler, eventProcessor = new EventProcessor<>(eventSourceManager); eventSourceManager.postProcessDefaultEventSourcesAfterProcessorInitializer(); controllerHealthInfo = new ControllerHealthInfo(eventSourceManager); - final var context = new EventSourceContext<>( eventSourceManager.getControllerResourceEventSource(), configuration, kubernetesClient); initAndRegisterEventSources(context); + ConfigurationServiceProvider.instance().getMetrics().controllerRegistered(this); } @Override diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java index 57679f48ca..c3cc524969 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java @@ -1,7 +1,7 @@ package io.javaoperatorsdk.operator.processing.event; import java.time.Duration; -import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.ExecutorService; @@ -18,7 +18,6 @@ import io.javaoperatorsdk.operator.api.config.ExecutorServiceManager; import io.javaoperatorsdk.operator.api.monitoring.Metrics; import io.javaoperatorsdk.operator.api.reconciler.Constants; -import io.javaoperatorsdk.operator.processing.Controller; import io.javaoperatorsdk.operator.processing.LifecycleAware; import io.javaoperatorsdk.operator.processing.MDCUtils; import io.javaoperatorsdk.operator.processing.event.rate.RateLimiter; @@ -98,9 +97,10 @@ private EventProcessor( this.rateLimiter = controllerConfiguration.getRateLimiter(); metricsMetadata = Optional.ofNullable(eventSourceManager.getController()) - .map(Controller::getAssociatedGroupVersionKind) - .map(gvk -> Map.of(Constants.RESOURCE_GVK_KEY, (Object) gvk)) - .orElse(Collections.emptyMap()); + .map(c -> Map.of( + Constants.RESOURCE_GVK_KEY, c.getAssociatedGroupVersionKind(), + Constants.CONTROLLER_NAME, controllerConfiguration.getName())) + .orElse(new HashMap<>()); } @Override @@ -400,11 +400,13 @@ public void run() { final var name = thread.getName(); try { MDCUtils.addResourceInfo(executionScope.getResource()); + metrics.reconciliationExecutionStarted(executionScope.getResource(), metricsMetadata); thread.setName("ReconcilerExecutor-" + controllerName() + "-" + thread.getId()); PostExecutionControl

postExecutionControl = reconciliationDispatcher.handleExecution(executionScope); eventProcessingFinished(executionScope, postExecutionControl); } finally { + metrics.reconciliationExecutionFinished(executionScope.getResource(), metricsMetadata); // restore original name thread.setName(name); MDCUtils.removeResourceInfo(); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/MockControllerConfiguration.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/MockControllerConfiguration.java index 0e3fc5df22..baf680b677 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/MockControllerConfiguration.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/MockControllerConfiguration.java @@ -14,6 +14,7 @@ public static ControllerConfiguration forResource( when(configuration.getResourceClass()).thenReturn(resourceType); when(configuration.getNamespaces()).thenReturn(DEFAULT_NAMESPACES_SET); when(configuration.getEffectiveNamespaces()).thenCallRealMethod(); + when(configuration.getName()).thenReturn(resourceType.getSimpleName()); return configuration; } } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/ResourceEventFilterTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/ResourceEventFilterTest.java index 7cc5a20781..7a8be0cc0c 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/ResourceEventFilterTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/ResourceEventFilterTest.java @@ -135,7 +135,7 @@ public ControllerConfig(String finalizer, boolean generationAware, ResourceEventFilter eventFilter, Class customResourceClass) { super( null, - null, + "testController", null, finalizer, generationAware, diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/controller/ControllerResourceEventSourceTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/controller/ControllerResourceEventSourceTest.java index 00980743e0..3c7f528d41 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/controller/ControllerResourceEventSourceTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/controller/ControllerResourceEventSourceTest.java @@ -178,7 +178,7 @@ public TestConfiguration(boolean generationAware, OnAddFilter genericFilter) { super( null, - null, + "testController", null, FINALIZER, generationAware,