diff --git a/operator-framework-core/pom.xml b/operator-framework-core/pom.xml index 14851dc779..2c5fcf9053 100644 --- a/operator-framework-core/pom.xml +++ b/operator-framework-core/pom.xml @@ -15,6 +15,13 @@ Core framework for implementing Kubernetes operators jar + + 11 + 11 + 11 + 1.7.3 + + @@ -92,5 +99,10 @@ log4j-core test + + io.micrometer + micrometer-core + ${micrometer-core.version} + diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Metrics.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Metrics.java new file mode 100644 index 0000000000..6edc599e47 --- /dev/null +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Metrics.java @@ -0,0 +1,180 @@ +package io.javaoperatorsdk.operator; + +import io.fabric8.kubernetes.client.CustomResource; +import io.javaoperatorsdk.operator.api.Context; +import io.javaoperatorsdk.operator.api.DeleteControl; +import io.javaoperatorsdk.operator.api.ResourceController; +import io.javaoperatorsdk.operator.api.UpdateControl; +import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; +import io.micrometer.core.instrument.*; +import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; +import io.micrometer.core.instrument.distribution.pause.PauseDetector; +import io.micrometer.core.instrument.noop.*; +import java.util.concurrent.TimeUnit; +import java.util.function.ToDoubleFunction; +import java.util.function.ToLongFunction; + +public class Metrics { + public static final Metrics NOOP = new Metrics(new NoopMeterRegistry(Clock.SYSTEM)); + private final MeterRegistry registry; + + public Metrics(MeterRegistry registry) { + this.registry = registry; + } + + public UpdateControl timeControllerCreateOrUpdate( + ResourceController controller, + ControllerConfiguration configuration, + R resource, + Context context) { + final var name = configuration.getName(); + final var timer = + Timer.builder("operator.sdk.controllers.execution.createorupdate") + .tags("controller", name) + .publishPercentiles(0.3, 0.5, 0.95) + .publishPercentileHistogram() + .register(registry); + try { + final var result = timer.record(() -> controller.createOrUpdateResource(resource, context)); + String successType = "cr"; + if (result.isUpdateStatusSubResource()) { + successType = "status"; + } + if (result.isUpdateCustomResourceAndStatusSubResource()) { + successType = "both"; + } + registry + .counter( + "operator.sdk.controllers.execution.success", "controller", name, "type", successType) + .increment(); + return result; + } catch (Exception e) { + registry + .counter( + "operator.sdk.controllers.execution.failure", + "controller", + name, + "exception", + e.getClass().getSimpleName()) + .increment(); + throw e; + } + } + + public DeleteControl timeControllerDelete( + ResourceController controller, + ControllerConfiguration configuration, + CustomResource resource, + Context context) { + final var name = configuration.getName(); + final var timer = + Timer.builder("operator.sdk.controllers.execution.delete") + .tags("controller", name) + .publishPercentiles(0.3, 0.5, 0.95) + .publishPercentileHistogram() + .register(registry); + try { + final var result = timer.record(() -> controller.deleteResource(resource, context)); + String successType = "notDelete"; + if (result == DeleteControl.DEFAULT_DELETE) { + successType = "delete"; + } + registry + .counter( + "operator.sdk.controllers.execution.success", "controller", name, "type", successType) + .increment(); + return result; + } catch (Exception e) { + registry + .counter( + "operator.sdk.controllers.execution.failure", + "controller", + name, + "exception", + e.getClass().getSimpleName()) + .increment(); + throw e; + } + } + + public void timeControllerRetry() { + + registry + .counter( + "operator.sdk.retry.on.exception", "retry", "retryCounter", "type", + "retryException") + .increment(); + + } + + public void timeControllerEvents() { + + registry + .counter( + "operator.sdk.total.events.received", "events", "totalEvents", "type", + "eventsReceived") + .increment(); + + } + + public static class NoopMeterRegistry extends MeterRegistry { + public NoopMeterRegistry(Clock clock) { + super(clock); + } + + @Override + protected Gauge newGauge(Meter.Id id, T t, ToDoubleFunction toDoubleFunction) { + return new NoopGauge(id); + } + + @Override + protected Counter newCounter(Meter.Id id) { + return new NoopCounter(id); + } + + @Override + protected Timer newTimer( + Meter.Id id, + DistributionStatisticConfig distributionStatisticConfig, + PauseDetector pauseDetector) { + return new NoopTimer(id); + } + + @Override + protected DistributionSummary newDistributionSummary( + Meter.Id id, DistributionStatisticConfig distributionStatisticConfig, double v) { + return new NoopDistributionSummary(id); + } + + @Override + protected Meter newMeter(Meter.Id id, Meter.Type type, Iterable iterable) { + return new NoopMeter(id); + } + + @Override + protected FunctionTimer newFunctionTimer( + Meter.Id id, + T t, + ToLongFunction toLongFunction, + ToDoubleFunction toDoubleFunction, + TimeUnit timeUnit) { + return new NoopFunctionTimer(id); + } + + @Override + protected FunctionCounter newFunctionCounter( + Meter.Id id, T t, ToDoubleFunction toDoubleFunction) { + return new NoopFunctionCounter(id); + } + + @Override + protected TimeUnit getBaseTimeUnit() { + return TimeUnit.SECONDS; + } + + @Override + protected DistributionStatisticConfig defaultHistogramConfig() { + return DistributionStatisticConfig.NONE; + } + } +} diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java index 0894d9ac70..dc0de9b415 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java @@ -25,14 +25,17 @@ public class Operator implements AutoCloseable { private final Object lock; private final List controllers; private volatile boolean started; + private final Metrics metrics; - public Operator(KubernetesClient k8sClient, ConfigurationService configurationService) { + public Operator( + KubernetesClient k8sClient, ConfigurationService configurationService, Metrics metrics) { this.k8sClient = k8sClient; this.configurationService = configurationService; this.closeables = new ArrayList<>(); this.lock = new Object(); this.controllers = new ArrayList<>(); this.started = false; + this.metrics = metrics; } /** Adds a shutdown hook that automatically calls {@link #close()} when the app shuts down. */ @@ -40,6 +43,10 @@ public void installShutdownHook() { Runtime.getRuntime().addShutdownHook(new Thread(this::close)); } + public Operator(KubernetesClient k8sClient, ConfigurationService configurationService) { + this(k8sClient, configurationService, Metrics.NOOP); + } + public KubernetesClient getKubernetesClient() { return k8sClient; } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java index 7482080148..126bb46b93 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.fabric8.kubernetes.client.Config; import io.fabric8.kubernetes.client.CustomResource; +import io.javaoperatorsdk.operator.Metrics; import io.javaoperatorsdk.operator.api.ResourceController; import java.util.Set; @@ -93,4 +94,8 @@ default ObjectMapper getObjectMapper() { default int getTerminationTimeoutSeconds() { return DEFAULT_TERMINATION_TIMEOUT_SECONDS; } + + default Metrics getMetrics() { + return Metrics.NOOP; + } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/DefaultEventHandler.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/DefaultEventHandler.java index ddd6cc8e4b..c59ca2dc9e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/DefaultEventHandler.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/DefaultEventHandler.java @@ -6,6 +6,7 @@ import io.fabric8.kubernetes.client.CustomResource; import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.javaoperatorsdk.operator.Metrics; import io.javaoperatorsdk.operator.api.ResourceController; import io.javaoperatorsdk.operator.api.RetryInfo; import io.javaoperatorsdk.operator.api.config.ConfigurationService; @@ -16,6 +17,7 @@ import io.javaoperatorsdk.operator.processing.retry.GenericRetry; import io.javaoperatorsdk.operator.processing.retry.Retry; import io.javaoperatorsdk.operator.processing.retry.RetryExecution; +import io.micrometer.core.instrument.Clock; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -46,6 +48,7 @@ public class DefaultEventHandler implements EventHandler { private final int terminationTimeout; private final ReentrantLock lock = new ReentrantLock(); private DefaultEventSourceManager eventSourceManager; + private ControllerConfiguration configuration; public DefaultEventHandler( ResourceController controller, ControllerConfiguration configuration, MixedOperation client) { @@ -54,20 +57,20 @@ public DefaultEventHandler( configuration.getName(), GenericRetry.fromConfiguration(configuration.getRetryConfiguration()), configuration.getConfigurationService().concurrentReconciliationThreads(), - configuration.getConfigurationService().getTerminationTimeoutSeconds()); + configuration.getConfigurationService().getTerminationTimeoutSeconds(), configuration); } DefaultEventHandler( EventDispatcher eventDispatcher, String relatedControllerName, Retry retry, - int concurrentReconciliationThreads) { + int concurrentReconciliationThreads, ControllerConfiguration configuration) { this( eventDispatcher, relatedControllerName, retry, concurrentReconciliationThreads, - ConfigurationService.DEFAULT_TERMINATION_TIMEOUT_SECONDS); + ConfigurationService.DEFAULT_TERMINATION_TIMEOUT_SECONDS, configuration); } private DefaultEventHandler( @@ -75,7 +78,7 @@ private DefaultEventHandler( String relatedControllerName, Retry retry, int concurrentReconciliationThreads, - int terminationTimeout) { + int terminationTimeout, ControllerConfiguration configuration) { this.eventDispatcher = eventDispatcher; this.retry = retry; this.controllerName = relatedControllerName; @@ -85,6 +88,7 @@ private DefaultEventHandler( new ScheduledThreadPoolExecutor( concurrentReconciliationThreads, runnable -> new Thread(runnable, "EventHandler-" + relatedControllerName)); + this.configuration = configuration; } @Override @@ -113,6 +117,10 @@ public void handleEvent(Event event) { final Predicate selector = event.getCustomResourcesSelector(); for (String uid : eventSourceManager.getLatestResourceUids(selector)) { eventBuffer.addEvent(uid, event); + configuration + .getConfigurationService() + .getMetrics() + .timeControllerEvents(); executeBufferedEvents(uid); } } finally { @@ -162,6 +170,10 @@ void eventProcessingFinished( if (retry != null && postExecutionControl.exceptionDuringExecution()) { handleRetryOnException(executionScope); + configuration + .getConfigurationService() + .getMetrics() + .timeControllerRetry(); return; } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/EventDispatcher.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/EventDispatcher.java index 382a973ea5..e69d69ffb7 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/EventDispatcher.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/EventDispatcher.java @@ -123,7 +123,12 @@ private PostExecutionControl handleCreateOrUpdate( getName(resource), getVersion(resource), executionScope); - UpdateControl updateControl = controller.createOrUpdateResource(resource, context); + + UpdateControl updateControl = + configuration + .getConfigurationService() + .getMetrics() + .timeControllerCreateOrUpdate(controller, configuration, resource, context); R updatedCustomResource = null; if (updateControl.isUpdateCustomResourceAndStatusSubResource()) { updatedCustomResource = updateCustomResource(updateControl.getCustomResource()); @@ -153,8 +158,12 @@ private PostExecutionControl handleDelete(R resource, Context context) { "Executing delete for resource: {} with version: {}", getName(resource), getVersion(resource)); - // todo: this is be executed in a try-catch statement, in case this fails - DeleteControl deleteControl = controller.deleteResource(resource, context); + + DeleteControl deleteControl = + configuration + .getConfigurationService() + .getMetrics() + .timeControllerDelete(controller, configuration, resource, context); final var useFinalizer = configuration.useFinalizer(); if (useFinalizer) { if (deleteControl == DeleteControl.DEFAULT_DELETE diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/CustomResourceSelectorTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/CustomResourceSelectorTest.java index 5bc2eb20e9..72c681b857 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/CustomResourceSelectorTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/CustomResourceSelectorTest.java @@ -11,10 +11,13 @@ import static org.mockito.Mockito.when; import io.fabric8.kubernetes.client.Watcher; +import io.javaoperatorsdk.operator.Metrics; import io.javaoperatorsdk.operator.api.config.ConfigurationService; +import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; import io.javaoperatorsdk.operator.processing.event.DefaultEvent; import io.javaoperatorsdk.operator.processing.event.DefaultEventSourceManager; import io.javaoperatorsdk.operator.processing.event.internal.CustomResourceEvent; +import io.javaoperatorsdk.operator.processing.event.internal.TimerEventSource; import io.javaoperatorsdk.operator.sample.simple.TestCustomResource; import java.util.Objects; import java.util.UUID; @@ -33,12 +36,17 @@ class CustomResourceSelectorTest { private final DefaultEventSourceManager defaultEventSourceManagerMock = mock(DefaultEventSourceManager.class); + private TimerEventSource retryTimerEventSourceMock = mock(TimerEventSource.class); + private ControllerConfiguration configuration = + mock(ControllerConfiguration.class); + private final ConfigurationService configService = mock(ConfigurationService.class); + private final DefaultEventHandler defaultEventHandler = new DefaultEventHandler( eventDispatcherMock, "Test", null, - ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER); + ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER, configuration); @BeforeEach public void setup() { @@ -58,6 +66,10 @@ public void setup() { }) .when(defaultEventSourceManagerMock) .cleanup(any()); + + when(configuration.getName()).thenReturn("DefaultEventHandlerTest"); + when(configService.getMetrics()).thenReturn(Metrics.NOOP); + when(configuration.getConfigurationService()).thenReturn(configService); } @Test diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/DefaultEventHandlerTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/DefaultEventHandlerTest.java index b4bd89dd18..699784180f 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/DefaultEventHandlerTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/DefaultEventHandlerTest.java @@ -13,8 +13,11 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import io.fabric8.kubernetes.client.CustomResource; import io.fabric8.kubernetes.client.Watcher; +import io.javaoperatorsdk.operator.Metrics; import io.javaoperatorsdk.operator.api.config.ConfigurationService; +import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; import io.javaoperatorsdk.operator.processing.event.DefaultEventSourceManager; import io.javaoperatorsdk.operator.processing.event.Event; import io.javaoperatorsdk.operator.processing.event.internal.CustomResourceEvent; @@ -44,20 +47,25 @@ class DefaultEventHandlerTest { mock(DefaultEventSourceManager.class); private TimerEventSource retryTimerEventSourceMock = mock(TimerEventSource.class); + private ControllerConfiguration configuration = + mock(ControllerConfiguration.class); + private final ConfigurationService configService = mock(ConfigurationService.class); private DefaultEventHandler defaultEventHandler = new DefaultEventHandler( eventDispatcherMock, "Test", null, - ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER); + ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER, + configuration); private DefaultEventHandler defaultEventHandlerWithRetry = new DefaultEventHandler( eventDispatcherMock, "Test", GenericRetry.defaultLimitedExponentialRetry(), - ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER); + ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER, + configuration); @BeforeEach public void setup() { @@ -66,6 +74,10 @@ public void setup() { defaultEventHandler.setEventSourceManager(defaultEventSourceManagerMock); defaultEventHandlerWithRetry.setEventSourceManager(defaultEventSourceManagerMock); + when(configuration.getName()).thenReturn("DefaultEventHandlerTest"); + when(configService.getMetrics()).thenReturn(Metrics.NOOP); + when(configuration.getConfigurationService()).thenReturn(configService); + // todo: remove when(defaultEventSourceManagerMock.getCache()).thenReturn(customResourceCache); doCallRealMethod().when(defaultEventSourceManagerMock).getLatestResource(any()); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/EventDispatcherTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/EventDispatcherTest.java index 1d00ebfcac..fa44508227 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/EventDispatcherTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/EventDispatcherTest.java @@ -15,12 +15,14 @@ import io.fabric8.kubernetes.client.CustomResource; import io.fabric8.kubernetes.client.Watcher; +import io.javaoperatorsdk.operator.Metrics; import io.javaoperatorsdk.operator.TestUtils; import io.javaoperatorsdk.operator.api.Context; import io.javaoperatorsdk.operator.api.DeleteControl; import io.javaoperatorsdk.operator.api.ResourceController; import io.javaoperatorsdk.operator.api.RetryInfo; import io.javaoperatorsdk.operator.api.UpdateControl; +import io.javaoperatorsdk.operator.api.config.ConfigurationService; import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; import io.javaoperatorsdk.operator.processing.event.Event; import io.javaoperatorsdk.operator.processing.event.internal.CustomResourceEvent; @@ -40,6 +42,7 @@ class EventDispatcherTest { private final ResourceController controller = mock(ResourceController.class); private ControllerConfiguration configuration = mock(ControllerConfiguration.class); + private final ConfigurationService configService = mock(ConfigurationService.class); private final EventDispatcher.CustomResourceFacade customResourceFacade = mock(EventDispatcher.CustomResourceFacade.class); @@ -51,6 +54,9 @@ void setup() { when(configuration.getFinalizer()).thenReturn(DEFAULT_FINALIZER); when(configuration.useFinalizer()).thenCallRealMethod(); + when(configuration.getName()).thenReturn("EventDispatcherTestController"); + when(configService.getMetrics()).thenReturn(Metrics.NOOP); + when(configuration.getConfigurationService()).thenReturn(configService); when(controller.createOrUpdateResource(eq(testCustomResource), any())) .thenReturn(UpdateControl.updateCustomResource(testCustomResource)); when(controller.deleteResource(eq(testCustomResource), any())) @@ -135,7 +141,6 @@ void callsDeleteIfObjectHasFinalizerAndMarkedForDelete() { @Test void callDeleteOnControllerIfMarkedForDeletionWhenNoFinalizerIsConfigured() { configureToNotUseFinalizer(); - markForDeletion(testCustomResource); eventDispatcher.handleExecution( @@ -156,6 +161,9 @@ void doNotCallDeleteIfMarkedForDeletionWhenFinalizerHasAlreadyBeenRemoved() { private void configureToNotUseFinalizer() { ControllerConfiguration configuration = mock(ControllerConfiguration.class); + when(configuration.getName()).thenReturn("EventDispatcherTestController"); + when(configService.getMetrics()).thenReturn(Metrics.NOOP); + when(configuration.getConfigurationService()).thenReturn(configService); when(configuration.useFinalizer()).thenReturn(false); eventDispatcher = new EventDispatcher(controller, configuration, customResourceFacade); } diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/IntegrationTestSupport.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/IntegrationTestSupport.java index fc1cd343cb..bbdb510d8e 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/IntegrationTestSupport.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/IntegrationTestSupport.java @@ -59,7 +59,7 @@ public void initialize(KubernetesClient k8sClient, ResourceController controller namespaces.create( new NamespaceBuilder().withNewMetadata().withName(TEST_NAMESPACE).endMetadata().build()); } - operator = new Operator(k8sClient, configurationService); + operator = new Operator(k8sClient, configurationService, Metrics.NOOP); final var overriddenConfig = ControllerConfigurationOverrider.override(config).settingNamespace(TEST_NAMESPACE); if (retry != null) {