From c0b043cbe24a10b2c8927e94838eb9f393ff834a Mon Sep 17 00:00:00 2001 From: csviri Date: Tue, 22 Nov 2022 10:37:00 +0100 Subject: [PATCH 1/8] fix: leader election stop deadlock --- .../java/io/javaoperatorsdk/operator/Operator.java | 10 ++++------ .../event/source/informer/InformerManager.java | 1 + .../operator/sample/WebPageOperator.java | 1 + 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java index 3f09fe77ed..41213ac736 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java @@ -112,12 +112,10 @@ public void stop() throws OperatorException { final var configurationService = ConfigurationServiceProvider.instance(); log.info( "Operator SDK {} is shutting down...", configurationService.getVersion().getSdkVersion()); - - controllerManager.stop(); - ExecutorServiceManager.stop(); - leaderElectionManager.stop(); - if (configurationService.closeClientOnStop()) { - kubernetesClient.close(); + controllerManager.stop(); + ExecutorServiceManager.stop(); + if (configurationService.closeClientOnStop()) { + kubernetesClient.close(); } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerManager.java index f5f52d1c0e..e994b7301e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerManager.java @@ -110,6 +110,7 @@ public void stop() { try { log.debug("Stopping informer for namespace: {} -> {}", ns, source); source.stop(); + log.debug("Informer stopped: {} -> {}", ns, source); } catch (Exception e) { log.warn("Error stopping informer for namespace: {} -> {}", ns, source, e); } diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java index 9b137649fc..4bde6b75e9 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java @@ -11,6 +11,7 @@ import io.fabric8.kubernetes.client.*; import io.javaoperatorsdk.operator.Operator; +import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration; public class WebPageOperator { public static final String WEBPAGE_RECONCILER_ENV = "WEBPAGE_RECONCILER"; From c68282a7f8589456ac58a33cba470f874e57cbdb Mon Sep 17 00:00:00 2001 From: csviri Date: Tue, 22 Nov 2022 10:38:45 +0100 Subject: [PATCH 2/8] cleanup --- .../main/java/io/javaoperatorsdk/operator/Operator.java | 8 ++++---- .../processing/event/source/informer/InformerManager.java | 1 - .../javaoperatorsdk/operator/sample/WebPageOperator.java | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java index 41213ac736..37ec5883e4 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java @@ -112,10 +112,10 @@ public void stop() throws OperatorException { final var configurationService = ConfigurationServiceProvider.instance(); log.info( "Operator SDK {} is shutting down...", configurationService.getVersion().getSdkVersion()); - controllerManager.stop(); - ExecutorServiceManager.stop(); - if (configurationService.closeClientOnStop()) { - kubernetesClient.close(); + controllerManager.stop(); + ExecutorServiceManager.stop(); + if (configurationService.closeClientOnStop()) { + kubernetesClient.close(); } } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerManager.java index e994b7301e..f5f52d1c0e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerManager.java @@ -110,7 +110,6 @@ public void stop() { try { log.debug("Stopping informer for namespace: {} -> {}", ns, source); source.stop(); - log.debug("Informer stopped: {} -> {}", ns, source); } catch (Exception e) { log.warn("Error stopping informer for namespace: {} -> {}", ns, source, e); } diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java index 4bde6b75e9..e86bcbb120 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java @@ -9,9 +9,9 @@ import org.takes.http.Exit; import org.takes.http.FtBasic; -import io.fabric8.kubernetes.client.*; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientBuilder; import io.javaoperatorsdk.operator.Operator; -import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration; public class WebPageOperator { public static final String WEBPAGE_RECONCILER_ENV = "WEBPAGE_RECONCILER"; From 785422b8309bbb1fdcc61e89425ce33f0db2d488 Mon Sep 17 00:00:00 2001 From: csviri Date: Tue, 22 Nov 2022 13:51:50 +0100 Subject: [PATCH 3/8] fix: leader election stop deadlock --- .../main/java/io/javaoperatorsdk/operator/Operator.java | 7 ++++++- .../javaoperatorsdk/operator/sample/WebPageOperator.java | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java index 37ec5883e4..e54bcd3ef8 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/Operator.java @@ -67,7 +67,11 @@ public Operator(KubernetesClient kubernetesClient, ConfigurationService configur /** Adds a shutdown hook that automatically calls {@link #stop()} when the app shuts down. */ public void installShutdownHook() { - Runtime.getRuntime().addShutdownHook(new Thread(this::stop)); + if (!leaderElectionManager.isLeaderElectionEnabled()) { + Runtime.getRuntime().addShutdownHook(new Thread(this::stop)); + } else { + log.warn("Leader election is on, shutdown hook will not be installed."); + } } public KubernetesClient getKubernetesClient() { @@ -114,6 +118,7 @@ public void stop() throws OperatorException { "Operator SDK {} is shutting down...", configurationService.getVersion().getSdkVersion()); controllerManager.stop(); ExecutorServiceManager.stop(); + leaderElectionManager.stop(); if (configurationService.closeClientOnStop()) { kubernetesClient.close(); } diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java index e86bcbb120..ed201f6a5c 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java @@ -2,6 +2,7 @@ import java.io.IOException; +import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.takes.facets.fork.FkRegex; @@ -24,7 +25,7 @@ public static void main(String[] args) throws IOException { log.info("WebServer Operator starting!"); KubernetesClient client = new KubernetesClientBuilder().build(); - Operator operator = new Operator(client); + Operator operator = new Operator(client,o->o.withLeaderElectionConfiguration(new LeaderElectionConfiguration("nameaa"))); String reconcilerEnvVar = System.getenv(WEBPAGE_RECONCILER_ENV); if (WEBPAGE_CLASSIC_RECONCILER_ENV_VALUE.equals(reconcilerEnvVar)) { operator.register(new WebPageReconciler(client)); From c0d3e15917e569de6a4c17e89ddaaf8c3de9ee07 Mon Sep 17 00:00:00 2001 From: csviri Date: Tue, 22 Nov 2022 13:57:06 +0100 Subject: [PATCH 4/8] format --- .../io/javaoperatorsdk/operator/sample/WebPageOperator.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java index ed201f6a5c..ca12380533 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java @@ -2,7 +2,6 @@ import java.io.IOException; -import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.takes.facets.fork.FkRegex; @@ -13,6 +12,7 @@ import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientBuilder; import io.javaoperatorsdk.operator.Operator; +import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration; public class WebPageOperator { public static final String WEBPAGE_RECONCILER_ENV = "WEBPAGE_RECONCILER"; @@ -25,7 +25,8 @@ public static void main(String[] args) throws IOException { log.info("WebServer Operator starting!"); KubernetesClient client = new KubernetesClientBuilder().build(); - Operator operator = new Operator(client,o->o.withLeaderElectionConfiguration(new LeaderElectionConfiguration("nameaa"))); + Operator operator = new Operator(client, + o -> o.withLeaderElectionConfiguration(new LeaderElectionConfiguration("nameaa"))); String reconcilerEnvVar = System.getenv(WEBPAGE_RECONCILER_ENV); if (WEBPAGE_CLASSIC_RECONCILER_ENV_VALUE.equals(reconcilerEnvVar)) { operator.register(new WebPageReconciler(client)); From cf196f5dc94fb5302618cbc4086fb662e13564a7 Mon Sep 17 00:00:00 2001 From: csviri Date: Wed, 23 Nov 2022 13:18:59 +0100 Subject: [PATCH 5/8] fix issue --- .../io/javaoperatorsdk/operator/sample/WebPageOperator.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java index ca12380533..9573d4992e 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java @@ -25,8 +25,7 @@ public static void main(String[] args) throws IOException { log.info("WebServer Operator starting!"); KubernetesClient client = new KubernetesClientBuilder().build(); - Operator operator = new Operator(client, - o -> o.withLeaderElectionConfiguration(new LeaderElectionConfiguration("nameaa"))); + Operator operator = new Operator(client); String reconcilerEnvVar = System.getenv(WEBPAGE_RECONCILER_ENV); if (WEBPAGE_CLASSIC_RECONCILER_ENV_VALUE.equals(reconcilerEnvVar)) { operator.register(new WebPageReconciler(client)); From 6639cdd8723476d1a6b790f3b2a0e8e06f82bf0c Mon Sep 17 00:00:00 2001 From: csviri Date: Wed, 23 Nov 2022 13:20:16 +0100 Subject: [PATCH 6/8] format --- .../java/io/javaoperatorsdk/operator/sample/WebPageOperator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java index 9573d4992e..e86bcbb120 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java @@ -12,7 +12,6 @@ import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientBuilder; import io.javaoperatorsdk.operator.Operator; -import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration; public class WebPageOperator { public static final String WEBPAGE_RECONCILER_ENV = "WEBPAGE_RECONCILER"; From 25ba08c74fc3b5bfabd384fb09df612e8ffa3e2c Mon Sep 17 00:00:00 2001 From: csviri Date: Wed, 23 Nov 2022 13:20:32 +0100 Subject: [PATCH 7/8] imports --- .../javaoperatorsdk/operator/sample/WebPageOperator.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java index e86bcbb120..9418e4e34b 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java @@ -1,7 +1,8 @@ package io.javaoperatorsdk.operator.sample; -import java.io.IOException; - +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientBuilder; +import io.javaoperatorsdk.operator.Operator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.takes.facets.fork.FkRegex; @@ -9,9 +10,7 @@ import org.takes.http.Exit; import org.takes.http.FtBasic; -import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClientBuilder; -import io.javaoperatorsdk.operator.Operator; +import java.io.IOException; public class WebPageOperator { public static final String WEBPAGE_RECONCILER_ENV = "WEBPAGE_RECONCILER"; From 22e6f2dd04a6a2e9df35b7f76a6406ba1e9ee2e9 Mon Sep 17 00:00:00 2001 From: csviri Date: Wed, 23 Nov 2022 13:41:44 +0100 Subject: [PATCH 8/8] format --- .../javaoperatorsdk/operator/sample/WebPageOperator.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java index 9418e4e34b..e86bcbb120 100644 --- a/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java +++ b/sample-operators/webpage/src/main/java/io/javaoperatorsdk/operator/sample/WebPageOperator.java @@ -1,8 +1,7 @@ package io.javaoperatorsdk.operator.sample; -import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClientBuilder; -import io.javaoperatorsdk.operator.Operator; +import java.io.IOException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.takes.facets.fork.FkRegex; @@ -10,7 +9,9 @@ import org.takes.http.Exit; import org.takes.http.FtBasic; -import java.io.IOException; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientBuilder; +import io.javaoperatorsdk.operator.Operator; public class WebPageOperator { public static final String WEBPAGE_RECONCILER_ENV = "WEBPAGE_RECONCILER";