diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java index 14c7cf6d1e..e94f5f3a7e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessor.java @@ -1,5 +1,6 @@ package io.javaoperatorsdk.operator.processing.event; +import java.net.HttpURLConnection; import java.time.Duration; import java.util.HashMap; import java.util.Map; @@ -10,6 +11,7 @@ import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.client.KubernetesClientException; import io.javaoperatorsdk.operator.OperatorException; import io.javaoperatorsdk.operator.api.config.ConfigurationService; import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; @@ -224,6 +226,7 @@ synchronized void eventProcessingFinished( postExecutionControl); unsetUnderExecution(resourceID); + logErrorIfNoRetryConfigured(executionScope, postExecutionControl); // If a delete event present at this phase, it was received during reconciliation. // So we either removed the finalizer during reconciliation or we don't use finalizers. // Either way we don't want to retry. @@ -261,6 +264,17 @@ synchronized void eventProcessingFinished( } } + /** + * In case retry is configured more complex error logging takes place, see handleRetryOnException + */ + private void logErrorIfNoRetryConfigured(ExecutionScope

executionScope, + PostExecutionControl

postExecutionControl) { + if (!isRetryConfigured() && postExecutionControl.exceptionDuringExecution()) { + log.error("Error during event processing {}", executionScope, + postExecutionControl.getRuntimeException().orElseThrow()); + } + } + private void reScheduleExecutionIfInstructed( PostExecutionControl

postExecutionControl, P customResource) { @@ -302,6 +316,7 @@ private void handleRetryOnException( boolean eventPresent = state.eventPresent(); state.markEventReceived(); + retryAwareErrorLogging(state.getRetry(), eventPresent, exception, executionScope); if (eventPresent) { log.debug("New events exists for for resource id: {}", resourceID); submitReconciliationExecution(state); @@ -319,11 +334,29 @@ private void handleRetryOnException( retryEventSource().scheduleOnce(resourceID, delay); }, () -> { - log.error("Exhausted retries for {}", executionScope); + log.error("Exhausted retries for scope {}.", executionScope); scheduleExecutionForMaxReconciliationInterval(executionScope.getResource()); }); } + private void retryAwareErrorLogging(RetryExecution retry, boolean eventPresent, + Exception exception, + ExecutionScope

executionScope) { + if (!eventPresent && !retry.isLastAttempt() && exception instanceof KubernetesClientException) { + KubernetesClientException ex = (KubernetesClientException) exception; + if (ex.getCode() == HttpURLConnection.HTTP_CONFLICT) { + log.debug("Full client conflict error during event processing {}", executionScope, + exception); + log.warn( + "Resource Kubernetes Resource Creator/Update Conflict during reconciliation. Message: {} Resource name: {}", + ex.getMessage(), ex.getFullResourceName()); + return; + } + } + log.error("Error during event processing {}", executionScope, + exception); + } + private void cleanupOnSuccessfulExecution(ExecutionScope

executionScope) { log.debug( "Cleanup for successful execution for resource: {}", getName(executionScope.getResource())); diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java index 9a110a7ba8..a03ab8b399 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/ReconciliationDispatcher.java @@ -63,7 +63,6 @@ public PostExecutionControl

handleExecution(ExecutionScope

executionScope) try { return handleDispatch(executionScope); } catch (Exception e) { - log.error("Error during event processing {} failed.", executionScope, e); return PostExecutionControl.exceptionDuringExecution(e); } }