diff --git a/operator-framework-core/pom.xml b/operator-framework-core/pom.xml
index 14851dc779..2c5fcf9053 100644
--- a/operator-framework-core/pom.xml
+++ b/operator-framework-core/pom.xml
@@ -15,6 +15,13 @@
Core framework for implementing Kubernetes operators
jar
+
+ 11
+ 11
+ 11
+ 1.7.3
+
+
@@ -92,5 +99,10 @@
log4j-core
test
+
+ io.micrometer
+ micrometer-core
+ ${micrometer-core.version}
+
diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Metrics.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Metrics.java
new file mode 100644
index 0000000000..6edc599e47
--- /dev/null
+++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Metrics.java
@@ -0,0 +1,180 @@
+package io.javaoperatorsdk.operator;
+
+import io.fabric8.kubernetes.client.CustomResource;
+import io.javaoperatorsdk.operator.api.Context;
+import io.javaoperatorsdk.operator.api.DeleteControl;
+import io.javaoperatorsdk.operator.api.ResourceController;
+import io.javaoperatorsdk.operator.api.UpdateControl;
+import io.javaoperatorsdk.operator.api.config.ControllerConfiguration;
+import io.micrometer.core.instrument.*;
+import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
+import io.micrometer.core.instrument.distribution.pause.PauseDetector;
+import io.micrometer.core.instrument.noop.*;
+import java.util.concurrent.TimeUnit;
+import java.util.function.ToDoubleFunction;
+import java.util.function.ToLongFunction;
+
+public class Metrics {
+ public static final Metrics NOOP = new Metrics(new NoopMeterRegistry(Clock.SYSTEM));
+ private final MeterRegistry registry;
+
+ public Metrics(MeterRegistry registry) {
+ this.registry = registry;
+ }
+
+ public UpdateControl timeControllerCreateOrUpdate(
+ ResourceController controller,
+ ControllerConfiguration configuration,
+ R resource,
+ Context context) {
+ final var name = configuration.getName();
+ final var timer =
+ Timer.builder("operator.sdk.controllers.execution.createorupdate")
+ .tags("controller", name)
+ .publishPercentiles(0.3, 0.5, 0.95)
+ .publishPercentileHistogram()
+ .register(registry);
+ try {
+ final var result = timer.record(() -> controller.createOrUpdateResource(resource, context));
+ String successType = "cr";
+ if (result.isUpdateStatusSubResource()) {
+ successType = "status";
+ }
+ if (result.isUpdateCustomResourceAndStatusSubResource()) {
+ successType = "both";
+ }
+ registry
+ .counter(
+ "operator.sdk.controllers.execution.success", "controller", name, "type", successType)
+ .increment();
+ return result;
+ } catch (Exception e) {
+ registry
+ .counter(
+ "operator.sdk.controllers.execution.failure",
+ "controller",
+ name,
+ "exception",
+ e.getClass().getSimpleName())
+ .increment();
+ throw e;
+ }
+ }
+
+ public DeleteControl timeControllerDelete(
+ ResourceController controller,
+ ControllerConfiguration configuration,
+ CustomResource resource,
+ Context context) {
+ final var name = configuration.getName();
+ final var timer =
+ Timer.builder("operator.sdk.controllers.execution.delete")
+ .tags("controller", name)
+ .publishPercentiles(0.3, 0.5, 0.95)
+ .publishPercentileHistogram()
+ .register(registry);
+ try {
+ final var result = timer.record(() -> controller.deleteResource(resource, context));
+ String successType = "notDelete";
+ if (result == DeleteControl.DEFAULT_DELETE) {
+ successType = "delete";
+ }
+ registry
+ .counter(
+ "operator.sdk.controllers.execution.success", "controller", name, "type", successType)
+ .increment();
+ return result;
+ } catch (Exception e) {
+ registry
+ .counter(
+ "operator.sdk.controllers.execution.failure",
+ "controller",
+ name,
+ "exception",
+ e.getClass().getSimpleName())
+ .increment();
+ throw e;
+ }
+ }
+
+ public void timeControllerRetry() {
+
+ registry
+ .counter(
+ "operator.sdk.retry.on.exception", "retry", "retryCounter", "type",
+ "retryException")
+ .increment();
+
+ }
+
+ public void timeControllerEvents() {
+
+ registry
+ .counter(
+ "operator.sdk.total.events.received", "events", "totalEvents", "type",
+ "eventsReceived")
+ .increment();
+
+ }
+
+ public static class NoopMeterRegistry extends MeterRegistry {
+ public NoopMeterRegistry(Clock clock) {
+ super(clock);
+ }
+
+ @Override
+ protected Gauge newGauge(Meter.Id id, T t, ToDoubleFunction toDoubleFunction) {
+ return new NoopGauge(id);
+ }
+
+ @Override
+ protected Counter newCounter(Meter.Id id) {
+ return new NoopCounter(id);
+ }
+
+ @Override
+ protected Timer newTimer(
+ Meter.Id id,
+ DistributionStatisticConfig distributionStatisticConfig,
+ PauseDetector pauseDetector) {
+ return new NoopTimer(id);
+ }
+
+ @Override
+ protected DistributionSummary newDistributionSummary(
+ Meter.Id id, DistributionStatisticConfig distributionStatisticConfig, double v) {
+ return new NoopDistributionSummary(id);
+ }
+
+ @Override
+ protected Meter newMeter(Meter.Id id, Meter.Type type, Iterable iterable) {
+ return new NoopMeter(id);
+ }
+
+ @Override
+ protected FunctionTimer newFunctionTimer(
+ Meter.Id id,
+ T t,
+ ToLongFunction toLongFunction,
+ ToDoubleFunction toDoubleFunction,
+ TimeUnit timeUnit) {
+ return new NoopFunctionTimer(id);
+ }
+
+ @Override
+ protected FunctionCounter newFunctionCounter(
+ Meter.Id id, T t, ToDoubleFunction toDoubleFunction) {
+ return new NoopFunctionCounter(id);
+ }
+
+ @Override
+ protected TimeUnit getBaseTimeUnit() {
+ return TimeUnit.SECONDS;
+ }
+
+ @Override
+ protected DistributionStatisticConfig defaultHistogramConfig() {
+ return DistributionStatisticConfig.NONE;
+ }
+ }
+}
diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java
index 0894d9ac70..dc0de9b415 100644
--- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java
+++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java
@@ -25,14 +25,17 @@ public class Operator implements AutoCloseable {
private final Object lock;
private final List controllers;
private volatile boolean started;
+ private final Metrics metrics;
- public Operator(KubernetesClient k8sClient, ConfigurationService configurationService) {
+ public Operator(
+ KubernetesClient k8sClient, ConfigurationService configurationService, Metrics metrics) {
this.k8sClient = k8sClient;
this.configurationService = configurationService;
this.closeables = new ArrayList<>();
this.lock = new Object();
this.controllers = new ArrayList<>();
this.started = false;
+ this.metrics = metrics;
}
/** Adds a shutdown hook that automatically calls {@link #close()} when the app shuts down. */
@@ -40,6 +43,10 @@ public void installShutdownHook() {
Runtime.getRuntime().addShutdownHook(new Thread(this::close));
}
+ public Operator(KubernetesClient k8sClient, ConfigurationService configurationService) {
+ this(k8sClient, configurationService, Metrics.NOOP);
+ }
+
public KubernetesClient getKubernetesClient() {
return k8sClient;
}
diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java
index 7482080148..126bb46b93 100644
--- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java
+++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ConfigurationService.java
@@ -3,6 +3,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import io.fabric8.kubernetes.client.Config;
import io.fabric8.kubernetes.client.CustomResource;
+import io.javaoperatorsdk.operator.Metrics;
import io.javaoperatorsdk.operator.api.ResourceController;
import java.util.Set;
@@ -93,4 +94,8 @@ default ObjectMapper getObjectMapper() {
default int getTerminationTimeoutSeconds() {
return DEFAULT_TERMINATION_TIMEOUT_SECONDS;
}
+
+ default Metrics getMetrics() {
+ return Metrics.NOOP;
+ }
}
diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/DefaultEventHandler.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/DefaultEventHandler.java
index ddd6cc8e4b..c59ca2dc9e 100644
--- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/DefaultEventHandler.java
+++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/DefaultEventHandler.java
@@ -6,6 +6,7 @@
import io.fabric8.kubernetes.client.CustomResource;
import io.fabric8.kubernetes.client.dsl.MixedOperation;
+import io.javaoperatorsdk.operator.Metrics;
import io.javaoperatorsdk.operator.api.ResourceController;
import io.javaoperatorsdk.operator.api.RetryInfo;
import io.javaoperatorsdk.operator.api.config.ConfigurationService;
@@ -16,6 +17,7 @@
import io.javaoperatorsdk.operator.processing.retry.GenericRetry;
import io.javaoperatorsdk.operator.processing.retry.Retry;
import io.javaoperatorsdk.operator.processing.retry.RetryExecution;
+import io.micrometer.core.instrument.Clock;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -46,6 +48,7 @@ public class DefaultEventHandler implements EventHandler {
private final int terminationTimeout;
private final ReentrantLock lock = new ReentrantLock();
private DefaultEventSourceManager eventSourceManager;
+ private ControllerConfiguration configuration;
public DefaultEventHandler(
ResourceController controller, ControllerConfiguration configuration, MixedOperation client) {
@@ -54,20 +57,20 @@ public DefaultEventHandler(
configuration.getName(),
GenericRetry.fromConfiguration(configuration.getRetryConfiguration()),
configuration.getConfigurationService().concurrentReconciliationThreads(),
- configuration.getConfigurationService().getTerminationTimeoutSeconds());
+ configuration.getConfigurationService().getTerminationTimeoutSeconds(), configuration);
}
DefaultEventHandler(
EventDispatcher eventDispatcher,
String relatedControllerName,
Retry retry,
- int concurrentReconciliationThreads) {
+ int concurrentReconciliationThreads, ControllerConfiguration configuration) {
this(
eventDispatcher,
relatedControllerName,
retry,
concurrentReconciliationThreads,
- ConfigurationService.DEFAULT_TERMINATION_TIMEOUT_SECONDS);
+ ConfigurationService.DEFAULT_TERMINATION_TIMEOUT_SECONDS, configuration);
}
private DefaultEventHandler(
@@ -75,7 +78,7 @@ private DefaultEventHandler(
String relatedControllerName,
Retry retry,
int concurrentReconciliationThreads,
- int terminationTimeout) {
+ int terminationTimeout, ControllerConfiguration configuration) {
this.eventDispatcher = eventDispatcher;
this.retry = retry;
this.controllerName = relatedControllerName;
@@ -85,6 +88,7 @@ private DefaultEventHandler(
new ScheduledThreadPoolExecutor(
concurrentReconciliationThreads,
runnable -> new Thread(runnable, "EventHandler-" + relatedControllerName));
+ this.configuration = configuration;
}
@Override
@@ -113,6 +117,10 @@ public void handleEvent(Event event) {
final Predicate selector = event.getCustomResourcesSelector();
for (String uid : eventSourceManager.getLatestResourceUids(selector)) {
eventBuffer.addEvent(uid, event);
+ configuration
+ .getConfigurationService()
+ .getMetrics()
+ .timeControllerEvents();
executeBufferedEvents(uid);
}
} finally {
@@ -162,6 +170,10 @@ void eventProcessingFinished(
if (retry != null && postExecutionControl.exceptionDuringExecution()) {
handleRetryOnException(executionScope);
+ configuration
+ .getConfigurationService()
+ .getMetrics()
+ .timeControllerRetry();
return;
}
diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/EventDispatcher.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/EventDispatcher.java
index 382a973ea5..e69d69ffb7 100644
--- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/EventDispatcher.java
+++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/EventDispatcher.java
@@ -123,7 +123,12 @@ private PostExecutionControl handleCreateOrUpdate(
getName(resource),
getVersion(resource),
executionScope);
- UpdateControl updateControl = controller.createOrUpdateResource(resource, context);
+
+ UpdateControl updateControl =
+ configuration
+ .getConfigurationService()
+ .getMetrics()
+ .timeControllerCreateOrUpdate(controller, configuration, resource, context);
R updatedCustomResource = null;
if (updateControl.isUpdateCustomResourceAndStatusSubResource()) {
updatedCustomResource = updateCustomResource(updateControl.getCustomResource());
@@ -153,8 +158,12 @@ private PostExecutionControl handleDelete(R resource, Context context) {
"Executing delete for resource: {} with version: {}",
getName(resource),
getVersion(resource));
- // todo: this is be executed in a try-catch statement, in case this fails
- DeleteControl deleteControl = controller.deleteResource(resource, context);
+
+ DeleteControl deleteControl =
+ configuration
+ .getConfigurationService()
+ .getMetrics()
+ .timeControllerDelete(controller, configuration, resource, context);
final var useFinalizer = configuration.useFinalizer();
if (useFinalizer) {
if (deleteControl == DeleteControl.DEFAULT_DELETE
diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/CustomResourceSelectorTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/CustomResourceSelectorTest.java
index 5bc2eb20e9..72c681b857 100644
--- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/CustomResourceSelectorTest.java
+++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/CustomResourceSelectorTest.java
@@ -11,10 +11,13 @@
import static org.mockito.Mockito.when;
import io.fabric8.kubernetes.client.Watcher;
+import io.javaoperatorsdk.operator.Metrics;
import io.javaoperatorsdk.operator.api.config.ConfigurationService;
+import io.javaoperatorsdk.operator.api.config.ControllerConfiguration;
import io.javaoperatorsdk.operator.processing.event.DefaultEvent;
import io.javaoperatorsdk.operator.processing.event.DefaultEventSourceManager;
import io.javaoperatorsdk.operator.processing.event.internal.CustomResourceEvent;
+import io.javaoperatorsdk.operator.processing.event.internal.TimerEventSource;
import io.javaoperatorsdk.operator.sample.simple.TestCustomResource;
import java.util.Objects;
import java.util.UUID;
@@ -33,12 +36,17 @@ class CustomResourceSelectorTest {
private final DefaultEventSourceManager defaultEventSourceManagerMock =
mock(DefaultEventSourceManager.class);
+ private TimerEventSource retryTimerEventSourceMock = mock(TimerEventSource.class);
+ private ControllerConfiguration configuration =
+ mock(ControllerConfiguration.class);
+ private final ConfigurationService configService = mock(ConfigurationService.class);
+
private final DefaultEventHandler defaultEventHandler =
new DefaultEventHandler(
eventDispatcherMock,
"Test",
null,
- ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER);
+ ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER, configuration);
@BeforeEach
public void setup() {
@@ -58,6 +66,10 @@ public void setup() {
})
.when(defaultEventSourceManagerMock)
.cleanup(any());
+
+ when(configuration.getName()).thenReturn("DefaultEventHandlerTest");
+ when(configService.getMetrics()).thenReturn(Metrics.NOOP);
+ when(configuration.getConfigurationService()).thenReturn(configService);
}
@Test
diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/DefaultEventHandlerTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/DefaultEventHandlerTest.java
index b4bd89dd18..699784180f 100644
--- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/DefaultEventHandlerTest.java
+++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/DefaultEventHandlerTest.java
@@ -13,8 +13,11 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import io.fabric8.kubernetes.client.CustomResource;
import io.fabric8.kubernetes.client.Watcher;
+import io.javaoperatorsdk.operator.Metrics;
import io.javaoperatorsdk.operator.api.config.ConfigurationService;
+import io.javaoperatorsdk.operator.api.config.ControllerConfiguration;
import io.javaoperatorsdk.operator.processing.event.DefaultEventSourceManager;
import io.javaoperatorsdk.operator.processing.event.Event;
import io.javaoperatorsdk.operator.processing.event.internal.CustomResourceEvent;
@@ -44,20 +47,25 @@ class DefaultEventHandlerTest {
mock(DefaultEventSourceManager.class);
private TimerEventSource retryTimerEventSourceMock = mock(TimerEventSource.class);
+ private ControllerConfiguration configuration =
+ mock(ControllerConfiguration.class);
+ private final ConfigurationService configService = mock(ConfigurationService.class);
private DefaultEventHandler defaultEventHandler =
new DefaultEventHandler(
eventDispatcherMock,
"Test",
null,
- ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER);
+ ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER,
+ configuration);
private DefaultEventHandler defaultEventHandlerWithRetry =
new DefaultEventHandler(
eventDispatcherMock,
"Test",
GenericRetry.defaultLimitedExponentialRetry(),
- ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER);
+ ConfigurationService.DEFAULT_RECONCILIATION_THREADS_NUMBER,
+ configuration);
@BeforeEach
public void setup() {
@@ -66,6 +74,10 @@ public void setup() {
defaultEventHandler.setEventSourceManager(defaultEventSourceManagerMock);
defaultEventHandlerWithRetry.setEventSourceManager(defaultEventSourceManagerMock);
+ when(configuration.getName()).thenReturn("DefaultEventHandlerTest");
+ when(configService.getMetrics()).thenReturn(Metrics.NOOP);
+ when(configuration.getConfigurationService()).thenReturn(configService);
+
// todo: remove
when(defaultEventSourceManagerMock.getCache()).thenReturn(customResourceCache);
doCallRealMethod().when(defaultEventSourceManagerMock).getLatestResource(any());
diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/EventDispatcherTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/EventDispatcherTest.java
index 1d00ebfcac..fa44508227 100644
--- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/EventDispatcherTest.java
+++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/EventDispatcherTest.java
@@ -15,12 +15,14 @@
import io.fabric8.kubernetes.client.CustomResource;
import io.fabric8.kubernetes.client.Watcher;
+import io.javaoperatorsdk.operator.Metrics;
import io.javaoperatorsdk.operator.TestUtils;
import io.javaoperatorsdk.operator.api.Context;
import io.javaoperatorsdk.operator.api.DeleteControl;
import io.javaoperatorsdk.operator.api.ResourceController;
import io.javaoperatorsdk.operator.api.RetryInfo;
import io.javaoperatorsdk.operator.api.UpdateControl;
+import io.javaoperatorsdk.operator.api.config.ConfigurationService;
import io.javaoperatorsdk.operator.api.config.ControllerConfiguration;
import io.javaoperatorsdk.operator.processing.event.Event;
import io.javaoperatorsdk.operator.processing.event.internal.CustomResourceEvent;
@@ -40,6 +42,7 @@ class EventDispatcherTest {
private final ResourceController controller = mock(ResourceController.class);
private ControllerConfiguration configuration =
mock(ControllerConfiguration.class);
+ private final ConfigurationService configService = mock(ConfigurationService.class);
private final EventDispatcher.CustomResourceFacade customResourceFacade =
mock(EventDispatcher.CustomResourceFacade.class);
@@ -51,6 +54,9 @@ void setup() {
when(configuration.getFinalizer()).thenReturn(DEFAULT_FINALIZER);
when(configuration.useFinalizer()).thenCallRealMethod();
+ when(configuration.getName()).thenReturn("EventDispatcherTestController");
+ when(configService.getMetrics()).thenReturn(Metrics.NOOP);
+ when(configuration.getConfigurationService()).thenReturn(configService);
when(controller.createOrUpdateResource(eq(testCustomResource), any()))
.thenReturn(UpdateControl.updateCustomResource(testCustomResource));
when(controller.deleteResource(eq(testCustomResource), any()))
@@ -135,7 +141,6 @@ void callsDeleteIfObjectHasFinalizerAndMarkedForDelete() {
@Test
void callDeleteOnControllerIfMarkedForDeletionWhenNoFinalizerIsConfigured() {
configureToNotUseFinalizer();
-
markForDeletion(testCustomResource);
eventDispatcher.handleExecution(
@@ -156,6 +161,9 @@ void doNotCallDeleteIfMarkedForDeletionWhenFinalizerHasAlreadyBeenRemoved() {
private void configureToNotUseFinalizer() {
ControllerConfiguration configuration = mock(ControllerConfiguration.class);
+ when(configuration.getName()).thenReturn("EventDispatcherTestController");
+ when(configService.getMetrics()).thenReturn(Metrics.NOOP);
+ when(configuration.getConfigurationService()).thenReturn(configService);
when(configuration.useFinalizer()).thenReturn(false);
eventDispatcher = new EventDispatcher(controller, configuration, customResourceFacade);
}
diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/IntegrationTestSupport.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/IntegrationTestSupport.java
index fc1cd343cb..bbdb510d8e 100644
--- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/IntegrationTestSupport.java
+++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/IntegrationTestSupport.java
@@ -59,7 +59,7 @@ public void initialize(KubernetesClient k8sClient, ResourceController controller
namespaces.create(
new NamespaceBuilder().withNewMetadata().withName(TEST_NAMESPACE).endMetadata().build());
}
- operator = new Operator(k8sClient, configurationService);
+ operator = new Operator(k8sClient, configurationService, Metrics.NOOP);
final var overriddenConfig =
ControllerConfigurationOverrider.override(config).settingNamespace(TEST_NAMESPACE);
if (retry != null) {