From a4f91546605ba03243ced1dca45dcd6fbd41b996 Mon Sep 17 00:00:00 2001 From: csviri Date: Thu, 23 Jun 2022 15:49:35 +0200 Subject: [PATCH 01/12] feat: primary to secondary mapper --- .../informer/InformerConfiguration.java | 18 ++++++ .../InformerPrimaryToSecondaryMapper.java | 11 ++++ .../source/PrimaryToSecondaryMapper.java | 11 ++++ .../source/informer/InformerEventSource.java | 45 ++++++++++---- .../operator/PrimaryToSecondaryIT.java | 59 ++++++++++++++++++ .../sample/primarytosecondary/Cluster.java | 15 +++++ .../primarytosecondary/ClusterStatus.java | 5 ++ .../sample/primarytosecondary/Job.java | 15 +++++ .../primarytosecondary/JobReconciler.java | 62 +++++++++++++++++++ .../sample/primarytosecondary/JobSpec.java | 15 +++++ .../sample/primarytosecondary/JobStatus.java | 5 ++ 11 files changed, 249 insertions(+), 12 deletions(-) create mode 100644 operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/InformerPrimaryToSecondaryMapper.java create mode 100644 operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/PrimaryToSecondaryMapper.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/Cluster.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/ClusterStatus.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/Job.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobReconciler.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobSpec.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobStatus.java diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerConfiguration.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerConfiguration.java index 9cb0f1ee1d..4daa6e5644 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerConfiguration.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerConfiguration.java @@ -8,6 +8,7 @@ import io.javaoperatorsdk.operator.api.config.ResourceConfiguration; import io.javaoperatorsdk.operator.api.config.Utils; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; +import io.javaoperatorsdk.operator.processing.event.source.InformerPrimaryToSecondaryMapper; import io.javaoperatorsdk.operator.processing.event.source.SecondaryToPrimaryMapper; import io.javaoperatorsdk.operator.processing.event.source.informer.Mappers; @@ -19,15 +20,18 @@ public interface InformerConfiguration class DefaultInformerConfiguration extends DefaultResourceConfiguration implements InformerConfiguration { + private final InformerPrimaryToSecondaryMapper primaryToSecondaryMapper; private final SecondaryToPrimaryMapper secondaryToPrimaryMapper; private final boolean followControllerNamespaceChanges; protected DefaultInformerConfiguration(String labelSelector, Class resourceClass, + InformerPrimaryToSecondaryMapper primaryToSecondaryMapper, SecondaryToPrimaryMapper secondaryToPrimaryMapper, Set namespaces, boolean followControllerNamespaceChanges) { super(labelSelector, resourceClass, namespaces); this.followControllerNamespaceChanges = followControllerNamespaceChanges; + this.primaryToSecondaryMapper = primaryToSecondaryMapper; this.secondaryToPrimaryMapper = Objects.requireNonNullElse(secondaryToPrimaryMapper, Mappers.fromOwnerReference()); @@ -41,6 +45,10 @@ public SecondaryToPrimaryMapper getSecondaryToPrimaryMapper() { return secondaryToPrimaryMapper; } + @Override + public

InformerPrimaryToSecondaryMapper

getPrimaryToSecondaryMapper() { + return (InformerPrimaryToSecondaryMapper

) primaryToSecondaryMapper; + } } /** @@ -53,9 +61,12 @@ public SecondaryToPrimaryMapper getSecondaryToPrimaryMapper() { SecondaryToPrimaryMapper getSecondaryToPrimaryMapper(); +

InformerPrimaryToSecondaryMapper

getPrimaryToSecondaryMapper(); + @SuppressWarnings("unused") class InformerConfigurationBuilder { + private InformerPrimaryToSecondaryMapper primaryToSecondaryMapper; private SecondaryToPrimaryMapper secondaryToPrimaryMapper; private Set namespaces; private String labelSelector; @@ -66,6 +77,12 @@ private InformerConfigurationBuilder(Class resourceClass) { this.resourceClass = resourceClass; } + public

InformerConfigurationBuilder withPrimaryToSecondaryMapper( + InformerPrimaryToSecondaryMapper

primaryToSecondaryMapper) { + this.primaryToSecondaryMapper = primaryToSecondaryMapper; + return this; + } + public InformerConfigurationBuilder withSecondaryToPrimaryMapper( SecondaryToPrimaryMapper secondaryToPrimaryMapper) { this.secondaryToPrimaryMapper = secondaryToPrimaryMapper; @@ -136,6 +153,7 @@ public InformerConfigurationBuilder withLabelSelector(String labelSelector) { public InformerConfiguration build() { return new DefaultInformerConfiguration<>(labelSelector, resourceClass, + primaryToSecondaryMapper, secondaryToPrimaryMapper, namespaces, inheritControllerNamespacesOnChange); } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/InformerPrimaryToSecondaryMapper.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/InformerPrimaryToSecondaryMapper.java new file mode 100644 index 0000000000..ca3cd2a339 --- /dev/null +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/InformerPrimaryToSecondaryMapper.java @@ -0,0 +1,11 @@ +package io.javaoperatorsdk.operator.processing.event.source; + +import java.util.Set; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.javaoperatorsdk.operator.processing.event.ResourceID; + +public interface InformerPrimaryToSecondaryMapper

{ + + Set toSecondaryResourceIDs(P primary); +} diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/PrimaryToSecondaryMapper.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/PrimaryToSecondaryMapper.java new file mode 100644 index 0000000000..069163001b --- /dev/null +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/PrimaryToSecondaryMapper.java @@ -0,0 +1,11 @@ +package io.javaoperatorsdk.operator.processing.event.source; + +import java.util.Set; + +import io.fabric8.kubernetes.api.model.HasMetadata; + +public interface PrimaryToSecondaryMapper

{ + + Set toSecondaryResourceIdentifiers(P primary); + +} diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java index 47fe4abcf3..9978473a64 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java @@ -16,6 +16,7 @@ import io.javaoperatorsdk.operator.processing.event.Event; import io.javaoperatorsdk.operator.processing.event.EventHandler; import io.javaoperatorsdk.operator.processing.event.ResourceID; +import io.javaoperatorsdk.operator.processing.event.source.InformerPrimaryToSecondaryMapper; /** *

@@ -74,20 +75,23 @@ public class InformerEventSource private final EventRecorder eventRecorder = new EventRecorder<>(); // we need direct control for the indexer to propagate the just update resource also to the index private final PrimaryToSecondaryIndex primaryToSecondaryIndex; + private final InformerPrimaryToSecondaryMapper

primaryToSecondaryMapper; public InformerEventSource( InformerConfiguration configuration, EventSourceContext

context) { - super(context.getClient().resources(configuration.getResourceClass()), configuration); - this.configuration = configuration; - primaryToSecondaryIndex = - new PrimaryToSecondaryIndex<>(configuration.getSecondaryToPrimaryMapper()); + this(configuration, context.getClient()); } public InformerEventSource(InformerConfiguration configuration, KubernetesClient client) { super(client.resources(configuration.getResourceClass()), configuration); this.configuration = configuration; - primaryToSecondaryIndex = - new PrimaryToSecondaryIndex<>(configuration.getSecondaryToPrimaryMapper()); + primaryToSecondaryMapper = configuration.getPrimaryToSecondaryMapper(); + if (primaryToSecondaryMapper == null) { + primaryToSecondaryIndex = + new PrimaryToSecondaryIndex<>(configuration.getSecondaryToPrimaryMapper()); + } else { + primaryToSecondaryIndex = null; + } } @Override @@ -95,7 +99,9 @@ public void onAdd(R resource) { if (log.isDebugEnabled()) { log.debug("On add event received for resource id: {}", ResourceID.fromResource(resource)); } - primaryToSecondaryIndex.onAddOrUpdate(resource); + if (useSecondaryToPrimaryIndex()) { + primaryToSecondaryIndex.onAddOrUpdate(resource); + } onAddOrUpdate("add", resource, () -> InformerEventSource.super.onAdd(resource)); } @@ -104,7 +110,9 @@ public void onUpdate(R oldObject, R newObject) { if (log.isDebugEnabled()) { log.debug("On update event received for resource id: {}", ResourceID.fromResource(newObject)); } - primaryToSecondaryIndex.onAddOrUpdate(newObject); + if (useSecondaryToPrimaryIndex()) { + primaryToSecondaryIndex.onAddOrUpdate(newObject); + } onAddOrUpdate("update", newObject, () -> InformerEventSource.super.onUpdate(oldObject, newObject)); } @@ -114,7 +122,9 @@ public void onDelete(R resource, boolean b) { if (log.isDebugEnabled()) { log.debug("On delete event received for resource id: {}", ResourceID.fromResource(resource)); } - primaryToSecondaryIndex.onDelete(resource); + if (useSecondaryToPrimaryIndex()) { + primaryToSecondaryIndex.onDelete(resource); + } super.onDelete(resource, b); propagateEvent(resource); } @@ -177,8 +187,13 @@ private void propagateEvent(R object) { @Override public Set getSecondaryResources(P primary) { - var secondaryIDs = - primaryToSecondaryIndex.getSecondaryResources(ResourceID.fromResource(primary)); + Set secondaryIDs; + if (useSecondaryToPrimaryIndex()) { + secondaryIDs = + primaryToSecondaryIndex.getSecondaryResources(ResourceID.fromResource(primary)); + } else { + secondaryIDs = primaryToSecondaryMapper.toSecondaryResourceIDs(primary); + } return secondaryIDs.stream().map(this::get).flatMap(Optional::stream) .collect(Collectors.toSet()); } @@ -201,7 +216,9 @@ public synchronized void handleRecentResourceCreate(ResourceID resourceID, R res } private void handleRecentCreateOrUpdate(R resource, Runnable runnable) { - primaryToSecondaryIndex.onAddOrUpdate(resource); + if (useSecondaryToPrimaryIndex()) { + primaryToSecondaryIndex.onAddOrUpdate(resource); + } if (eventRecorder.isRecordingFor(ResourceID.fromResource(resource))) { handleRecentResourceOperationAndStopEventRecording(resource); } else { @@ -247,6 +264,10 @@ private void handleRecentResourceOperationAndStopEventRecording(R resource) { } } + private boolean useSecondaryToPrimaryIndex() { + return this.primaryToSecondaryMapper == null; + } + @Override public synchronized void prepareForCreateOrUpdateEventFiltering(ResourceID resourceID, R resource) { diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java new file mode 100644 index 0000000000..cfba078293 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java @@ -0,0 +1,59 @@ +package io.javaoperatorsdk.operator; + +import java.time.Duration; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.junit.LocalOperatorExtension; +import io.javaoperatorsdk.operator.sample.primarytosecondary.Cluster; +import io.javaoperatorsdk.operator.sample.primarytosecondary.Job; +import io.javaoperatorsdk.operator.sample.primarytosecondary.JobReconciler; +import io.javaoperatorsdk.operator.sample.primarytosecondary.JobSpec; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +class PrimaryToSecondaryIT { + + public static final String CLUSTER_NAME = "cluster1"; + public static final int MIN_DELAY = 150; + @RegisterExtension + LocalOperatorExtension operator = + LocalOperatorExtension.builder() + + .withReconciler(new JobReconciler()) + .build(); + + // todo load other CRD + @Test + void readsSecondaryInManyToOneCases() throws InterruptedException { + operator.create(Cluster.class, cluster()); + Thread.sleep(MIN_DELAY); + operator.create(Job.class, job()); + + await().pollDelay(Duration.ofMillis(400)).untilAsserted( + () -> assertThat(operator.getReconcilerOfType(JobReconciler.class).getNumberOfExecutions()) + .isEqualTo(1)); + } + + Job job() { + var job = new Job(); + job.setMetadata(new ObjectMetaBuilder() + .withName("job1") + .build()); + job.setSpec(new JobSpec()); + job.getSpec().setClusterName(CLUSTER_NAME); + return job; + } + + Cluster cluster() { + Cluster cluster = new Cluster(); + cluster.setMetadata(new ObjectMetaBuilder() + .withName(CLUSTER_NAME) + .build()); + return cluster; + } + +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/Cluster.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/Cluster.java new file mode 100644 index 0000000000..ffc43141a4 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/Cluster.java @@ -0,0 +1,15 @@ +package io.javaoperatorsdk.operator.sample.primarytosecondary; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.ShortNames; +import io.fabric8.kubernetes.model.annotation.Version; + +@Group("sample.javaoperatorsdk") +@Version("v1") +@ShortNames("clu") +public class Cluster + extends CustomResource + implements Namespaced { +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/ClusterStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/ClusterStatus.java new file mode 100644 index 0000000000..eac5c48b70 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/ClusterStatus.java @@ -0,0 +1,5 @@ +package io.javaoperatorsdk.operator.sample.primarytosecondary; + +public class ClusterStatus { + +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/Job.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/Job.java new file mode 100644 index 0000000000..5a3d43de79 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/Job.java @@ -0,0 +1,15 @@ +package io.javaoperatorsdk.operator.sample.primarytosecondary; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.ShortNames; +import io.fabric8.kubernetes.model.annotation.Version; + +@Group("sample.javaoperatorsdk") +@Version("v1") +@ShortNames("cjo") +public class Job + extends CustomResource + implements Namespaced { +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobReconciler.java new file mode 100644 index 0000000000..c0c5188ea5 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobReconciler.java @@ -0,0 +1,62 @@ +package io.javaoperatorsdk.operator.sample.primarytosecondary; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import io.javaoperatorsdk.operator.api.config.informer.InformerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.*; +import io.javaoperatorsdk.operator.processing.event.ResourceID; +import io.javaoperatorsdk.operator.processing.event.source.EventSource; +import io.javaoperatorsdk.operator.processing.event.source.InformerPrimaryToSecondaryMapper; +import io.javaoperatorsdk.operator.processing.event.source.informer.InformerEventSource; + +@ControllerConfiguration() +public class JobReconciler + implements Reconciler, EventSourceInitializer { + + private static final String JOB_CLUSTER_INDEX = "job-cluster-index"; + + private final AtomicInteger numberOfExecutions = new AtomicInteger(0); + + @Override + public UpdateControl reconcile( + Job resource, Context context) { + + context.getSecondaryResource(Cluster.class) + .orElseThrow(() -> new IllegalStateException("Secondary resource should be present")); + numberOfExecutions.addAndGet(1); + return UpdateControl.noUpdate(); + } + + @Override + public Map prepareEventSources(EventSourceContext context) { + context.getPrimaryCache().addIndexer(JOB_CLUSTER_INDEX, (job -> List + .of(indexKey(job.getSpec().getClusterName(), job.getMetadata().getNamespace())))); + + InformerConfiguration informerConfiguration = + InformerConfiguration.from(Cluster.class, context) + .withSecondaryToPrimaryMapper(cluster -> context.getPrimaryCache() + .byIndex(JOB_CLUSTER_INDEX, indexKey(cluster.getMetadata().getName(), + cluster.getMetadata().getNamespace())) + .stream().map(ResourceID::fromResource).collect(Collectors.toSet())) + .withPrimaryToSecondaryMapper( + (InformerPrimaryToSecondaryMapper) primary -> Set.of(new ResourceID( + primary.getSpec().getClusterName(), primary.getMetadata().getNamespace()))) + .withNamespacesInheritedFromController(context) + .build(); + + return EventSourceInitializer + .nameEventSources(new InformerEventSource<>(informerConfiguration, context)); + } + + private String indexKey(String clusterName, String namespace) { + return clusterName + "#" + namespace; + } + + public int getNumberOfExecutions() { + return numberOfExecutions.get(); + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobSpec.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobSpec.java new file mode 100644 index 0000000000..c7546dea71 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobSpec.java @@ -0,0 +1,15 @@ +package io.javaoperatorsdk.operator.sample.primarytosecondary; + +public class JobSpec { + + private String clusterName; + + public String getClusterName() { + return clusterName; + } + + public JobSpec setClusterName(String clusterName) { + this.clusterName = clusterName; + return this; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobStatus.java new file mode 100644 index 0000000000..b2d7cf6259 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobStatus.java @@ -0,0 +1,5 @@ +package io.javaoperatorsdk.operator.sample.primarytosecondary; + +public class JobStatus { + +} From 4df223b6ac5a4aafe46b6cf5398506de0dd26c54 Mon Sep 17 00:00:00 2001 From: csviri Date: Thu, 23 Jun 2022 15:53:59 +0200 Subject: [PATCH 02/12] cleanup --- .../config/informer/InformerConfiguration.java | 16 ++++++++-------- .../source/InformerPrimaryToSecondaryMapper.java | 11 ----------- .../event/source/PrimaryToSecondaryMapper.java | 4 ++-- .../source/informer/InformerEventSource.java | 4 ++-- .../sample/primarytosecondary/JobReconciler.java | 4 ++-- 5 files changed, 14 insertions(+), 25 deletions(-) delete mode 100644 operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/InformerPrimaryToSecondaryMapper.java diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerConfiguration.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerConfiguration.java index 4daa6e5644..ddfae2919e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerConfiguration.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerConfiguration.java @@ -8,7 +8,7 @@ import io.javaoperatorsdk.operator.api.config.ResourceConfiguration; import io.javaoperatorsdk.operator.api.config.Utils; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; -import io.javaoperatorsdk.operator.processing.event.source.InformerPrimaryToSecondaryMapper; +import io.javaoperatorsdk.operator.processing.event.source.PrimaryToSecondaryMapper; import io.javaoperatorsdk.operator.processing.event.source.SecondaryToPrimaryMapper; import io.javaoperatorsdk.operator.processing.event.source.informer.Mappers; @@ -20,13 +20,13 @@ public interface InformerConfiguration class DefaultInformerConfiguration extends DefaultResourceConfiguration implements InformerConfiguration { - private final InformerPrimaryToSecondaryMapper primaryToSecondaryMapper; + private final PrimaryToSecondaryMapper primaryToSecondaryMapper; private final SecondaryToPrimaryMapper secondaryToPrimaryMapper; private final boolean followControllerNamespaceChanges; protected DefaultInformerConfiguration(String labelSelector, Class resourceClass, - InformerPrimaryToSecondaryMapper primaryToSecondaryMapper, + PrimaryToSecondaryMapper primaryToSecondaryMapper, SecondaryToPrimaryMapper secondaryToPrimaryMapper, Set namespaces, boolean followControllerNamespaceChanges) { super(labelSelector, resourceClass, namespaces); @@ -46,8 +46,8 @@ public SecondaryToPrimaryMapper getSecondaryToPrimaryMapper() { } @Override - public

InformerPrimaryToSecondaryMapper

getPrimaryToSecondaryMapper() { - return (InformerPrimaryToSecondaryMapper

) primaryToSecondaryMapper; + public

PrimaryToSecondaryMapper

getPrimaryToSecondaryMapper() { + return (PrimaryToSecondaryMapper

) primaryToSecondaryMapper; } } @@ -61,12 +61,12 @@ public

InformerPrimaryToSecondaryMapper

getPrimaryToS SecondaryToPrimaryMapper getSecondaryToPrimaryMapper(); -

InformerPrimaryToSecondaryMapper

getPrimaryToSecondaryMapper(); +

PrimaryToSecondaryMapper

getPrimaryToSecondaryMapper(); @SuppressWarnings("unused") class InformerConfigurationBuilder { - private InformerPrimaryToSecondaryMapper primaryToSecondaryMapper; + private PrimaryToSecondaryMapper primaryToSecondaryMapper; private SecondaryToPrimaryMapper secondaryToPrimaryMapper; private Set namespaces; private String labelSelector; @@ -78,7 +78,7 @@ private InformerConfigurationBuilder(Class resourceClass) { } public

InformerConfigurationBuilder withPrimaryToSecondaryMapper( - InformerPrimaryToSecondaryMapper

primaryToSecondaryMapper) { + PrimaryToSecondaryMapper

primaryToSecondaryMapper) { this.primaryToSecondaryMapper = primaryToSecondaryMapper; return this; } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/InformerPrimaryToSecondaryMapper.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/InformerPrimaryToSecondaryMapper.java deleted file mode 100644 index ca3cd2a339..0000000000 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/InformerPrimaryToSecondaryMapper.java +++ /dev/null @@ -1,11 +0,0 @@ -package io.javaoperatorsdk.operator.processing.event.source; - -import java.util.Set; - -import io.fabric8.kubernetes.api.model.HasMetadata; -import io.javaoperatorsdk.operator.processing.event.ResourceID; - -public interface InformerPrimaryToSecondaryMapper

{ - - Set toSecondaryResourceIDs(P primary); -} diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/PrimaryToSecondaryMapper.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/PrimaryToSecondaryMapper.java index 069163001b..37f31e2e3e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/PrimaryToSecondaryMapper.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/PrimaryToSecondaryMapper.java @@ -3,9 +3,9 @@ import java.util.Set; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.javaoperatorsdk.operator.processing.event.ResourceID; public interface PrimaryToSecondaryMapper

{ - Set toSecondaryResourceIdentifiers(P primary); - + Set toSecondaryResourceIDs(P primary); } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java index 9978473a64..7cb71597ba 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java @@ -16,7 +16,7 @@ import io.javaoperatorsdk.operator.processing.event.Event; import io.javaoperatorsdk.operator.processing.event.EventHandler; import io.javaoperatorsdk.operator.processing.event.ResourceID; -import io.javaoperatorsdk.operator.processing.event.source.InformerPrimaryToSecondaryMapper; +import io.javaoperatorsdk.operator.processing.event.source.PrimaryToSecondaryMapper; /** *

@@ -75,7 +75,7 @@ public class InformerEventSource private final EventRecorder eventRecorder = new EventRecorder<>(); // we need direct control for the indexer to propagate the just update resource also to the index private final PrimaryToSecondaryIndex primaryToSecondaryIndex; - private final InformerPrimaryToSecondaryMapper

primaryToSecondaryMapper; + private final PrimaryToSecondaryMapper

primaryToSecondaryMapper; public InformerEventSource( InformerConfiguration configuration, EventSourceContext

context) { diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobReconciler.java index c0c5188ea5..1ac7676265 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobReconciler.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/sample/primarytosecondary/JobReconciler.java @@ -10,7 +10,7 @@ import io.javaoperatorsdk.operator.api.reconciler.*; import io.javaoperatorsdk.operator.processing.event.ResourceID; import io.javaoperatorsdk.operator.processing.event.source.EventSource; -import io.javaoperatorsdk.operator.processing.event.source.InformerPrimaryToSecondaryMapper; +import io.javaoperatorsdk.operator.processing.event.source.PrimaryToSecondaryMapper; import io.javaoperatorsdk.operator.processing.event.source.informer.InformerEventSource; @ControllerConfiguration() @@ -43,7 +43,7 @@ public Map prepareEventSources(EventSourceContext cont cluster.getMetadata().getNamespace())) .stream().map(ResourceID::fromResource).collect(Collectors.toSet())) .withPrimaryToSecondaryMapper( - (InformerPrimaryToSecondaryMapper) primary -> Set.of(new ResourceID( + (PrimaryToSecondaryMapper) primary -> Set.of(new ResourceID( primary.getSpec().getClusterName(), primary.getMetadata().getNamespace()))) .withNamespacesInheritedFromController(context) .build(); From cabafc7881e10945abc954f22e984c8dc3ea56e5 Mon Sep 17 00:00:00 2001 From: csviri Date: Thu, 23 Jun 2022 16:48:55 +0200 Subject: [PATCH 03/12] additional CRD support in IT --- .../junit/LocalOperatorExtension.java | 58 +++++++++++++------ .../operator/PrimaryToSecondaryIT.java | 3 +- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/LocalOperatorExtension.java b/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/LocalOperatorExtension.java index edccd025a6..2eb0fa5900 100644 --- a/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/LocalOperatorExtension.java +++ b/operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/LocalOperatorExtension.java @@ -15,8 +15,10 @@ import org.slf4j.LoggerFactory; import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.client.CustomResource; import io.fabric8.kubernetes.client.LocalPortForward; import io.javaoperatorsdk.operator.Operator; +import io.javaoperatorsdk.operator.ReconcilerUtils; import io.javaoperatorsdk.operator.RegisteredController; import io.javaoperatorsdk.operator.api.config.ConfigurationService; import io.javaoperatorsdk.operator.api.config.ControllerConfigurationOverrider; @@ -32,15 +34,17 @@ public class LocalOperatorExtension extends AbstractOperatorExtension { private final Operator operator; private final List reconcilers; - private final List portForwards; + private final List portForwards; private final List localPortForwards; + private final List> additionalCustomResourceDefinitions; private final Map registeredControllers; private LocalOperatorExtension( ConfigurationService configurationService, List reconcilers, List infrastructure, - List portForwards, + List portForwards, + List> additionalCustomResourceDefinitions, Duration infrastructureTimeout, boolean preserveNamespaceOnError, boolean waitForNamespaceDeletion, @@ -55,6 +59,7 @@ private LocalOperatorExtension( this.reconcilers = reconcilers; this.portForwards = portForwards; this.localPortForwards = new ArrayList<>(portForwards.size()); + this.additionalCustomResourceDefinitions = additionalCustomResourceDefinitions; this.operator = new Operator(getKubernetesClient(), this.configurationService); this.registeredControllers = new HashMap<>(); } @@ -114,10 +119,12 @@ protected void before(ExtensionContext context) { .withName(podName).portForward(ref.getPort(), ref.getLocalPort())); } + additionalCustomResourceDefinitions + .forEach(cr -> applyCrd(ReconcilerUtils.getResourceTypeName(cr))); + for (var ref : reconcilers) { final var config = configurationService.getConfigurationFor(ref.reconciler); final var oconfig = override(config).settingNamespace(namespace); - final var path = "/META-INF/fabric8/" + config.getResourceTypeName() + "-v1.yml"; if (ref.retry != null) { oconfig.withRetry(ref.retry); @@ -126,17 +133,7 @@ protected void before(ExtensionContext context) { ref.controllerConfigurationOverrider.accept(oconfig); } - try (InputStream is = getClass().getResourceAsStream(path)) { - final var crd = kubernetesClient.load(is); - crd.createOrReplace(); - Thread.sleep(CRD_READY_WAIT); // readiness is not applicable for CRD, just wait a little - LOGGER.debug("Applied CRD with name: {}", config.getResourceTypeName()); - } catch (InterruptedException ex) { - LOGGER.error("Interrupted.", ex); - Thread.currentThread().interrupt(); - } catch (Exception ex) { - throw new IllegalStateException("Cannot apply CRD yaml: " + path, ex); - } + applyCrd(config.getResourceTypeName()); if (ref.reconciler instanceof KubernetesClientAware) { ((KubernetesClientAware) ref.reconciler).setKubernetesClient(kubernetesClient); @@ -150,6 +147,21 @@ protected void before(ExtensionContext context) { this.operator.start(); } + private void applyCrd(String resourceTypeName) { + String path = "/META-INF/fabric8/" + resourceTypeName + "-v1.yml"; + try (InputStream is = getClass().getResourceAsStream(path)) { + final var crd = getKubernetesClient().load(is); + crd.createOrReplace(); + Thread.sleep(CRD_READY_WAIT); // readiness is not applicable for CRD, just wait a little + LOGGER.debug("Applied CRD with path: {}", path); + } catch (InterruptedException ex) { + LOGGER.error("Interrupted.", ex); + Thread.currentThread().interrupt(); + } catch (Exception ex) { + throw new IllegalStateException("Cannot apply CRD yaml: " + path, ex); + } + } + protected void after(ExtensionContext context) { super.after(context); @@ -172,12 +184,14 @@ protected void after(ExtensionContext context) { @SuppressWarnings("rawtypes") public static class Builder extends AbstractBuilder { private final List reconcilers; - private final List portForwards; + private final List portForwards; + private final List> additionalCustomResourceDefinitions; protected Builder() { super(); this.reconcilers = new ArrayList<>(); this.portForwards = new ArrayList<>(); + this.additionalCustomResourceDefinitions = new ArrayList<>(); } public Builder withReconciler( @@ -217,16 +231,24 @@ public Builder withReconciler(Class value) { public Builder withPortForward(String namespace, String labelKey, String labelValue, int port, int localPort) { - portForwards.add(new PortFowardSpec(namespace, labelKey, labelValue, port, localPort)); + portForwards.add(new PortForwardSpec(namespace, labelKey, labelValue, port, localPort)); return this; } + public Builder withAdditionalCustomResourceDefinition( + Class customResource) { + additionalCustomResourceDefinitions.add(customResource); + return this; + } + + public LocalOperatorExtension build() { return new LocalOperatorExtension( configurationService, reconcilers, infrastructure, portForwards, + additionalCustomResourceDefinitions, infrastructureTimeout, preserveNamespaceOnError, waitForNamespaceDeletion, @@ -234,14 +256,14 @@ public LocalOperatorExtension build() { } } - private static class PortFowardSpec { + private static class PortForwardSpec { final String namespace; final String labelKey; final String labelValue; final int port; final int localPort; - public PortFowardSpec(String namespace, String labelKey, String labelValue, int port, + public PortForwardSpec(String namespace, String labelKey, String labelValue, int port, int localPort) { this.namespace = namespace; this.labelKey = labelKey; diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java index cfba078293..7da755113e 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java @@ -22,11 +22,10 @@ class PrimaryToSecondaryIT { @RegisterExtension LocalOperatorExtension operator = LocalOperatorExtension.builder() - + .withAdditionalCustomResourceDefinition(Cluster.class) .withReconciler(new JobReconciler()) .build(); - // todo load other CRD @Test void readsSecondaryInManyToOneCases() throws InterruptedException { operator.create(Cluster.class, cluster()); From 80fb6fc4430f2e1ba9a874e0327f148b4a0d1873 Mon Sep 17 00:00:00 2001 From: csviri Date: Thu, 23 Jun 2022 16:57:05 +0200 Subject: [PATCH 04/12] IT improvements --- .../operator/MappingNoPrimaryToSecondaryIT.java | 4 ++++ .../io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/MappingNoPrimaryToSecondaryIT.java diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/MappingNoPrimaryToSecondaryIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/MappingNoPrimaryToSecondaryIT.java new file mode 100644 index 0000000000..e5d921a752 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/MappingNoPrimaryToSecondaryIT.java @@ -0,0 +1,4 @@ +package io.javaoperatorsdk.operator; + +public class MappingNoPrimaryToSecondaryIT { +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java index 7da755113e..b54923c6e6 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java @@ -19,6 +19,7 @@ class PrimaryToSecondaryIT { public static final String CLUSTER_NAME = "cluster1"; public static final int MIN_DELAY = 150; + @RegisterExtension LocalOperatorExtension operator = LocalOperatorExtension.builder() @@ -32,7 +33,7 @@ void readsSecondaryInManyToOneCases() throws InterruptedException { Thread.sleep(MIN_DELAY); operator.create(Job.class, job()); - await().pollDelay(Duration.ofMillis(400)).untilAsserted( + await().pollDelay(Duration.ofMillis(300)).untilAsserted( () -> assertThat(operator.getReconcilerOfType(JobReconciler.class).getNumberOfExecutions()) .isEqualTo(1)); } From 74e265f0b51a37f511c2a3e249d0fee649155902 Mon Sep 17 00:00:00 2001 From: csviri Date: Thu, 23 Jun 2022 17:30:25 +0200 Subject: [PATCH 05/12] cleanup --- .../operator/MappingNoPrimaryToSecondaryIT.java | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/MappingNoPrimaryToSecondaryIT.java diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/MappingNoPrimaryToSecondaryIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/MappingNoPrimaryToSecondaryIT.java deleted file mode 100644 index e5d921a752..0000000000 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/MappingNoPrimaryToSecondaryIT.java +++ /dev/null @@ -1,4 +0,0 @@ -package io.javaoperatorsdk.operator; - -public class MappingNoPrimaryToSecondaryIT { -} From c18f14f9df166480b591e4e1f3063cddb2a524a2 Mon Sep 17 00:00:00 2001 From: csviri Date: Fri, 24 Jun 2022 09:13:44 +0200 Subject: [PATCH 06/12] docs wip --- docs/documentation/features.md | 4 ++++ .../processing/event/source/PrimaryToSecondaryMapper.java | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/docs/documentation/features.md b/docs/documentation/features.md index 8cbf9ec841..a68f091073 100644 --- a/docs/documentation/features.md +++ b/docs/documentation/features.md @@ -520,6 +520,10 @@ for this feature. ## Monitoring with Micrometer +## Managing Relation between Primary and Secondary Resources + + + ## Automatic Generation of CRDs Note that this is feature of [Fabric8 Kubernetes Client](https://github.com/fabric8io/kubernetes-client) not the JOSDK. diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/PrimaryToSecondaryMapper.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/PrimaryToSecondaryMapper.java index 37f31e2e3e..866a2b2251 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/PrimaryToSecondaryMapper.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/PrimaryToSecondaryMapper.java @@ -5,6 +5,14 @@ import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.processing.event.ResourceID; +/** + * Primary to Secondary mapper only needed in some cases, typically when there it many-to-one or + * many-to-many relation between primary and secondary resources. If there is owner reference (or + * reference with annotations) from secondary to primary this is not needed. See + * PrimaryToSecondaryIT integration tests that handles many-to-many relationship. + * + * @param

primary resource type + */ public interface PrimaryToSecondaryMapper

{ Set toSecondaryResourceIDs(P primary); From b7720f974ad2c450c9df6ac5297847af150b33d1 Mon Sep 17 00:00:00 2001 From: csviri Date: Fri, 24 Jun 2022 09:43:02 +0200 Subject: [PATCH 07/12] docs --- docs/documentation/features.md | 55 ++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/docs/documentation/features.md b/docs/documentation/features.md index a68f091073..7a604dc14e 100644 --- a/docs/documentation/features.md +++ b/docs/documentation/features.md @@ -377,22 +377,12 @@ public class TomcatReconciler implements Reconciler, EventSourceInitiali @Override public List prepareEventSources(EventSourceContext context) { - SharedIndexInformer deploymentInformer = - kubernetesClient.apps() - .deployments() - .inAnyNamespace() - .withLabel("app.kubernetes.io/managed-by", "tomcat-operator") - .runnableInformer(0); - - return List.of( - new InformerEventSource<>(deploymentInformer, d -> { - var ownerReferences = d.getMetadata().getOwnerReferences(); - if (!ownerReferences.isEmpty()) { - return Set.of(new ResourceID(ownerReferences.get(0).getName(), d.getMetadata().getNamespace())); - } else { - return EMPTY_SET; - } - })); + var configMapEventSource = + new InformerEventSource<>(InformerConfiguration.from(Deployment.class, context) + .withLabelSelector(SELECTOR) + .withSecondaryToPrimaryMapper(Mappers.fromAnnotation(ANNOTATION_NAME,ANNOTATION_NAMESPACE) + .build(), context)); + return EventSourceInitializer.nameEventSources(configMapEventSource); } ... } @@ -401,21 +391,38 @@ public class TomcatReconciler implements Reconciler, EventSourceInitiali In the example above an `InformerEventSource` is registered (more on this specific eventsource later). Multiple things are going on here: -1. An `SharedIndexInformer` (class from fabric8 Kubernetes client) is created. This will watch and produce events for +1. In the background `SharedIndexInformer` (class from fabric8 Kubernetes client) is created. This will watch and produce events for `Deployments` in every namespace, but will filter them based on label. So `Deployments` which are not managed by `tomcat-operator` (the label is not present on them) will not trigger a reconciliation. 2. In the next step an [InformerEventSource](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java) - is created, which wraps the `SharedIndexInformer`. In addition to that a mapping functions is provided, **this maps - the event of the watched resource (in this case `Deployment`) to the custom resources to reconcile**. Not that in - this case this is a simple task, since `Deployment` is already created with an owner reference. Therefore, - the `ResourceID` - what identifies the custom resource to reconcile is created from the owner reference. + is created, which wraps the `SharedIndexInformer`. In addition to that a mapping functions is provided, + with `withSecondaryToPrimaryMapper`, this maps the event of the watched resource (in this case `Deployment`) to the + custom resources to reconcile. Note that usually this is covered by a default mapper , when `Deployment` + is created with an owner reference, the default mapper gets the mapping information from there. Thus, + the `ResourceID` what identifies the custom resource to reconcile is created from the owner reference. + For sake of the example a mapper is added that maps secondary to primary resource based on annotations. Note that a set of `ResourceID` is returned, this is usually just a set with one element. The possibility to specify multiple values are there to cover some rare corner cases. If an irrelevant resource is observed, an empty set can be returned to not reconcile any custom resource. +### Managing Relation between Primary and Secondary Resources + +As already touched in previous section, a `SecondaryToPrimaryMapper` is required to map events to trigger reconciliation +of the primary resource. By default, this is handled with a mapper that utilizes owner references. If an owner reference +cannot be used (for example resources are in different namespace), other mapper can be provided, typically an annotation +based on is provided. + +Adding a `SecondaryToPrimaryMapper` is typically sufficient when there is a one-to-many relationship between primary and +secondary resources. The secondary resources can be mapped to its primary owner, and this is enough information to also +get the resource using the API from the context in reconciler: `context.getSecondaryResources(...)`. There are however +cases when to map the other way around this mapper is not enough, a `PrimaryToSecondaryMapper` is required. +This is typically when there is a many-to-one or many-to-many relationship between resources, thus the primary resource +is referencing a secondary resources. In these cases the mentioned reverse mapper is required to work properly. +See [PrimaryToSecondaryIT](https://github.com/java-operator-sdk/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/PrimaryToSecondaryIT.java) +integration test for a sample. + ### Built-in EventSources There are multiple event-sources provided out of the box, the following are some more central ones: @@ -520,10 +527,6 @@ for this feature. ## Monitoring with Micrometer -## Managing Relation between Primary and Secondary Resources - - - ## Automatic Generation of CRDs Note that this is feature of [Fabric8 Kubernetes Client](https://github.com/fabric8io/kubernetes-client) not the JOSDK. From 92540da2db4bac8127848c10f36aed61b2a84ffe Mon Sep 17 00:00:00 2001 From: csviri Date: Fri, 24 Jun 2022 13:05:06 +0200 Subject: [PATCH 08/12] NO OP primary to secondary index --- .../DefaultPrimaryToSecondaryIndex.java | 52 +++++++++++++++++++ .../source/informer/InformerEventSource.java | 20 +++---- .../informer/NOOPPrimaryToSecondaryIndex.java | 32 ++++++++++++ .../informer/PrimaryToSecondaryIndex.java | 44 ++-------------- ...> DefaultPrimaryToSecondaryIndexTest.java} | 6 +-- 5 files changed, 98 insertions(+), 56 deletions(-) create mode 100644 operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/DefaultPrimaryToSecondaryIndex.java create mode 100644 operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java rename operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/{PrimaryToSecondaryIndexTest.java => DefaultPrimaryToSecondaryIndexTest.java} (94%) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/DefaultPrimaryToSecondaryIndex.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/DefaultPrimaryToSecondaryIndex.java new file mode 100644 index 0000000000..65434cf53d --- /dev/null +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/DefaultPrimaryToSecondaryIndex.java @@ -0,0 +1,52 @@ +package io.javaoperatorsdk.operator.processing.event.source.informer; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.javaoperatorsdk.operator.processing.event.ResourceID; +import io.javaoperatorsdk.operator.processing.event.source.SecondaryToPrimaryMapper; + +class DefaultPrimaryToSecondaryIndex implements PrimaryToSecondaryIndex { + + private SecondaryToPrimaryMapper secondaryToPrimaryMapper; + private Map> index = new HashMap<>(); + + public DefaultPrimaryToSecondaryIndex(SecondaryToPrimaryMapper secondaryToPrimaryMapper) { + this.secondaryToPrimaryMapper = secondaryToPrimaryMapper; + } + + @Override + public synchronized void onAddOrUpdate(R resource) { + Set primaryResources = secondaryToPrimaryMapper.toPrimaryResourceIDs(resource); + primaryResources.forEach( + primaryResource -> { + var resourceSet = + index.computeIfAbsent(primaryResource, pr -> ConcurrentHashMap.newKeySet()); + resourceSet.add(ResourceID.fromResource(resource)); + }); + } + + @Override + public synchronized void onDelete(R resource) { + Set primaryResources = secondaryToPrimaryMapper.toPrimaryResourceIDs(resource); + primaryResources.forEach( + primaryResource -> { + var secondaryResources = index.get(primaryResource); + secondaryResources.remove(ResourceID.fromResource(resource)); + if (secondaryResources.isEmpty()) { + index.remove(primaryResource); + } + }); + } + + @Override + public synchronized Set getSecondaryResources(ResourceID primary) { + var resourceIDs = index.get(primary); + if (resourceIDs == null) { + return Collections.emptySet(); + } else { + return Collections.unmodifiableSet(resourceIDs); + } + } +} diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java index 7cb71597ba..4e0d96d5d9 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerEventSource.java @@ -88,9 +88,9 @@ public InformerEventSource(InformerConfiguration configuration, KubernetesCli primaryToSecondaryMapper = configuration.getPrimaryToSecondaryMapper(); if (primaryToSecondaryMapper == null) { primaryToSecondaryIndex = - new PrimaryToSecondaryIndex<>(configuration.getSecondaryToPrimaryMapper()); + new DefaultPrimaryToSecondaryIndex<>(configuration.getSecondaryToPrimaryMapper()); } else { - primaryToSecondaryIndex = null; + primaryToSecondaryIndex = NOOPPrimaryToSecondaryIndex.getInstance(); } } @@ -99,9 +99,7 @@ public void onAdd(R resource) { if (log.isDebugEnabled()) { log.debug("On add event received for resource id: {}", ResourceID.fromResource(resource)); } - if (useSecondaryToPrimaryIndex()) { - primaryToSecondaryIndex.onAddOrUpdate(resource); - } + primaryToSecondaryIndex.onAddOrUpdate(resource); onAddOrUpdate("add", resource, () -> InformerEventSource.super.onAdd(resource)); } @@ -110,9 +108,7 @@ public void onUpdate(R oldObject, R newObject) { if (log.isDebugEnabled()) { log.debug("On update event received for resource id: {}", ResourceID.fromResource(newObject)); } - if (useSecondaryToPrimaryIndex()) { - primaryToSecondaryIndex.onAddOrUpdate(newObject); - } + primaryToSecondaryIndex.onAddOrUpdate(newObject); onAddOrUpdate("update", newObject, () -> InformerEventSource.super.onUpdate(oldObject, newObject)); } @@ -122,9 +118,7 @@ public void onDelete(R resource, boolean b) { if (log.isDebugEnabled()) { log.debug("On delete event received for resource id: {}", ResourceID.fromResource(resource)); } - if (useSecondaryToPrimaryIndex()) { - primaryToSecondaryIndex.onDelete(resource); - } + primaryToSecondaryIndex.onDelete(resource); super.onDelete(resource, b); propagateEvent(resource); } @@ -216,9 +210,7 @@ public synchronized void handleRecentResourceCreate(ResourceID resourceID, R res } private void handleRecentCreateOrUpdate(R resource, Runnable runnable) { - if (useSecondaryToPrimaryIndex()) { - primaryToSecondaryIndex.onAddOrUpdate(resource); - } + primaryToSecondaryIndex.onAddOrUpdate(resource); if (eventRecorder.isRecordingFor(ResourceID.fromResource(resource))) { handleRecentResourceOperationAndStopEventRecording(resource); } else { diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java new file mode 100644 index 0000000000..46e7cd0f63 --- /dev/null +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java @@ -0,0 +1,32 @@ +package io.javaoperatorsdk.operator.processing.event.source.informer; + +import java.util.Set; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.javaoperatorsdk.operator.processing.event.ResourceID; + + +class NOOPPrimaryToSecondaryIndex + implements PrimaryToSecondaryIndex { + + @SuppressWarnings("rawtypes") + private final static NOOPPrimaryToSecondaryIndex instance = new NOOPPrimaryToSecondaryIndex(); + + public static NOOPPrimaryToSecondaryIndex getInstance() { + return instance; + } + + private NOOPPrimaryToSecondaryIndex() { + } + + @Override + public void onAddOrUpdate(R resource) {} + + @Override + public void onDelete(R resource) {} + + @Override + public Set getSecondaryResources(ResourceID primary) { + throw new IllegalStateException("Should not be called"); + } +} diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndex.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndex.java index 8918895aa1..7a87b23272 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndex.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndex.java @@ -1,49 +1,15 @@ package io.javaoperatorsdk.operator.processing.event.source.informer; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; +import java.util.Set; import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.processing.event.ResourceID; -import io.javaoperatorsdk.operator.processing.event.source.SecondaryToPrimaryMapper; -class PrimaryToSecondaryIndex { +public interface PrimaryToSecondaryIndex { - private SecondaryToPrimaryMapper secondaryToPrimaryMapper; - private Map> index = new HashMap<>(); + void onAddOrUpdate(R resource); - public PrimaryToSecondaryIndex(SecondaryToPrimaryMapper secondaryToPrimaryMapper) { - this.secondaryToPrimaryMapper = secondaryToPrimaryMapper; - } + void onDelete(R resource); - public synchronized void onAddOrUpdate(R resource) { - Set primaryResources = secondaryToPrimaryMapper.toPrimaryResourceIDs(resource); - primaryResources.forEach( - primaryResource -> { - var resourceSet = - index.computeIfAbsent(primaryResource, pr -> ConcurrentHashMap.newKeySet()); - resourceSet.add(ResourceID.fromResource(resource)); - }); - } - - public synchronized void onDelete(R resource) { - Set primaryResources = secondaryToPrimaryMapper.toPrimaryResourceIDs(resource); - primaryResources.forEach( - primaryResource -> { - var secondaryResources = index.get(primaryResource); - secondaryResources.remove(ResourceID.fromResource(resource)); - if (secondaryResources.isEmpty()) { - index.remove(primaryResource); - } - }); - } - - public synchronized Set getSecondaryResources(ResourceID primary) { - var resourceIDs = index.get(primary); - if (resourceIDs == null) { - return Collections.emptySet(); - } else { - return Collections.unmodifiableSet(resourceIDs); - } - } + Set getSecondaryResources(ResourceID primary); } diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/DefaultPrimaryToSecondaryIndexTest.java similarity index 94% rename from operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexTest.java rename to operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/DefaultPrimaryToSecondaryIndexTest.java index ca73b135a7..da2d1b7cf0 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/source/informer/DefaultPrimaryToSecondaryIndexTest.java @@ -15,12 +15,12 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -class PrimaryToSecondaryIndexTest { +class DefaultPrimaryToSecondaryIndexTest { private SecondaryToPrimaryMapper secondaryToPrimaryMapperMock = mock(SecondaryToPrimaryMapper.class); - private PrimaryToSecondaryIndex primaryToSecondaryIndex = - new PrimaryToSecondaryIndex<>(secondaryToPrimaryMapperMock); + private DefaultPrimaryToSecondaryIndex primaryToSecondaryIndex = + new DefaultPrimaryToSecondaryIndex<>(secondaryToPrimaryMapperMock); private ResourceID primaryID1 = new ResourceID("id1", "default"); private ResourceID primaryID2 = new ResourceID("id2", "default"); From 8d785d9d9750967c92ed324bbca7247aef9cc2aa Mon Sep 17 00:00:00 2001 From: csviri Date: Fri, 24 Jun 2022 13:30:30 +0200 Subject: [PATCH 09/12] format --- .../event/source/informer/NOOPPrimaryToSecondaryIndex.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java index 46e7cd0f63..2163f4e1ac 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java @@ -5,7 +5,6 @@ import io.fabric8.kubernetes.api.model.HasMetadata; import io.javaoperatorsdk.operator.processing.event.ResourceID; - class NOOPPrimaryToSecondaryIndex implements PrimaryToSecondaryIndex { @@ -16,8 +15,7 @@ public static NOOPPrimaryToSecondaryIndex getInstance return instance; } - private NOOPPrimaryToSecondaryIndex() { - } + private NOOPPrimaryToSecondaryIndex() {} @Override public void onAddOrUpdate(R resource) {} From 42ecef7186c1647363b9ffd4137d9f5dc4811483 Mon Sep 17 00:00:00 2001 From: csviri Date: Fri, 24 Jun 2022 13:41:05 +0200 Subject: [PATCH 10/12] improvements --- .../source/informer/NOOPPrimaryToSecondaryIndex.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java index 2163f4e1ac..0aedf1dc1e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java @@ -9,7 +9,7 @@ class NOOPPrimaryToSecondaryIndex implements PrimaryToSecondaryIndex { @SuppressWarnings("rawtypes") - private final static NOOPPrimaryToSecondaryIndex instance = new NOOPPrimaryToSecondaryIndex(); + private static final NOOPPrimaryToSecondaryIndex instance = new NOOPPrimaryToSecondaryIndex(); public static NOOPPrimaryToSecondaryIndex getInstance() { return instance; @@ -18,13 +18,17 @@ public static NOOPPrimaryToSecondaryIndex getInstance private NOOPPrimaryToSecondaryIndex() {} @Override - public void onAddOrUpdate(R resource) {} + public void onAddOrUpdate(R resource) { + throw new UnsupportedOperationException(); + } @Override - public void onDelete(R resource) {} + public void onDelete(R resource) { + throw new UnsupportedOperationException(); + } @Override public Set getSecondaryResources(ResourceID primary) { - throw new IllegalStateException("Should not be called"); + throw new UnsupportedOperationException(); } } From b3216a3c5b33f638f42ef7c534632ff76e61aab2 Mon Sep 17 00:00:00 2001 From: csviri Date: Fri, 24 Jun 2022 13:41:26 +0200 Subject: [PATCH 11/12] revert no op --- .../event/source/informer/NOOPPrimaryToSecondaryIndex.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java index 0aedf1dc1e..d894aaf07a 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java @@ -19,12 +19,10 @@ private NOOPPrimaryToSecondaryIndex() {} @Override public void onAddOrUpdate(R resource) { - throw new UnsupportedOperationException(); } @Override public void onDelete(R resource) { - throw new UnsupportedOperationException(); } @Override From 75fabcc8a34dfef6cdd6bffb014f2b44ace70f3c Mon Sep 17 00:00:00 2001 From: csviri Date: Fri, 24 Jun 2022 13:44:35 +0200 Subject: [PATCH 12/12] comments --- .../event/source/informer/NOOPPrimaryToSecondaryIndex.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java index d894aaf07a..ddc8cbec18 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/NOOPPrimaryToSecondaryIndex.java @@ -19,10 +19,12 @@ private NOOPPrimaryToSecondaryIndex() {} @Override public void onAddOrUpdate(R resource) { + // empty method because of noop implementation } @Override public void onDelete(R resource) { + // empty method because of noop implementation } @Override