Skip to content

feat: multi cluster InformerEventSource support #2495

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 72 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
72 commits
Select commit Hold shift + click to select a range
de1ba28
chore: set version to 5.0.0-SNAPSHOT (#2200)
csviri Jan 18, 2024
82e4632
improve: java version minimal 11, tested on 21 (#2207)
csviri Jan 18, 2024
30e233c
improve: remove deprecated EventFilter (#2208)
csviri Jan 18, 2024
8ff4cee
docs: 5.0 migration guide skeleton (#2210)
csviri Jan 19, 2024
77a3eee
improve: managed dependent reconciliation results not optional (#2212)
csviri Jan 22, 2024
15c2661
improve: remove deprecated RetryConfiguration (#2211)
csviri Jan 22, 2024
32f4c3a
feat: JDK client is now the default (#2235)
csviri Feb 6, 2024
affdd07
fix: update SCM information (#2237)
metacosm Feb 7, 2024
128499a
feat: move name is directly to dependent resource (#2250)
csviri Mar 4, 2024
af9311a
format
csviri Mar 7, 2024
5ca2afa
fix: test after rebase on master (#2270)
csviri Mar 7, 2024
c3ce02e
feat: use java 17 as baseline (#2271)
csviri Mar 8, 2024
c5a7a77
improve: matcher always considers metadata (#2273)
csviri Mar 9, 2024
5ab13d4
feat: API to check if next reconciliation is imminent (#2272)
csviri Mar 11, 2024
3794bea
feat: Workflow extracted to a separate annotation (#2274)
csviri Mar 11, 2024
4138c9c
fix: pom properties cleanup (#2280)
csviri Mar 11, 2024
658f3f6
improve: remove EventSourceInitializer (#2257)
csviri Mar 12, 2024
ab5a827
refactor: make loadFromProperties private (#2287)
metacosm Mar 13, 2024
6e4cf34
feat: distinguish resources based on desired state (#2252)
csviri Mar 13, 2024
c0d687b
feat: explicit workflow invocation (#2289)
csviri Mar 18, 2024
8e6a18e
feat: use ssa for status update by default from `UpdateControl` (#2278)
csviri Mar 18, 2024
7cb4302
refactor: make WorkflowSpec an interface (#2300)
metacosm Mar 20, 2024
6108f9b
improve: replace current formatting plugins with spotless plugin (#2…
csviri Mar 21, 2024
5d4e9fc
refactor: make ManagedWorkflowSupport public to be able to reuse it (…
metacosm Mar 21, 2024
67b3b31
fix: package fix leader election sample (#2303)
csviri Mar 24, 2024
1edfc8c
feat: remove resource discriminator (#2299)
csviri Mar 25, 2024
c083dd4
refactor: remove deprecated getTerminationTimeoutSeconds and associat…
metacosm Mar 26, 2024
41dc0f6
improve: additional logging to mysql schema e2e test (#2320)
csviri Mar 27, 2024
170774d
feat: handle clustered resource on secondary to primary mapper init (…
metacosm Mar 27, 2024
d8e2636
feat: using SSA for finalizer and primary patch (#2305)
csviri Apr 8, 2024
672d6d8
fix: error with java docs validation (#2336)
csviri Apr 8, 2024
8d92f3d
fix: format after rebase
csviri Apr 10, 2024
6c2d6fe
fix: remove now unneeded okhttp dependency override, wrong comment (#…
metacosm Apr 11, 2024
56a347e
feat: run spotless on compile (#2330)
metacosm Apr 12, 2024
bd1f2c2
refactor: clean-up (#2325)
metacosm Apr 12, 2024
159c2ab
improve: samples correctly create status object for SSA (#2365)
csviri Apr 25, 2024
1b5d8d0
improve: named event sources and related changes (#2340)
csviri Apr 28, 2024
fa18a50
feat: silent exception handling in managed workflows (#2363)
csviri Apr 29, 2024
a99dc2a
feat: feature flag to not clone secondary resource on access (#2364)
csviri May 2, 2024
1cd9f43
fix: unit test for event processor stopping (#2373)
csviri May 2, 2024
f5723fb
feat: all event source is resource event source (#2367)
csviri May 2, 2024
eb7becd
feat: read-only bulk dependent (#2372)
csviri May 14, 2024
1c7a094
feat: remove automatic observed generation handling (#2382)
csviri May 14, 2024
7e68d02
docs: bulk dependent resource migration (#2383)
csviri May 15, 2024
aa5f2fc
fix: correct default secondary to primary mapper (#2386)
csviri May 15, 2024
1a756d6
improve: secondaryToPrimary check type of owner reference (#2371)
csviri May 16, 2024
8a3a644
improve: remove deprecated APIs (#2375)
csviri May 16, 2024
f7c928b
improve: ensure unique name on event sources (#2370)
csviri May 16, 2024
443ebdc
docs: migrate v5 changes to docsy (#2397)
csviri May 21, 2024
1e1002a
docs: @Workflow usage (#2413)
csviri Jun 12, 2024
76de032
improve: remove ErrorStatusHandler interface (#2438)
csviri Jun 17, 2024
5569ad5
improve: dependent configuration improvements - context independent (…
csviri Jun 19, 2024
c3698c0
feat: general crd checking activation condition (#2433)
csviri Jun 19, 2024
160efcc
improve: generics for prepare event source (#2407)
csviri Jun 19, 2024
444a538
fix: improper javadoc preventing release (#2450)
metacosm Jun 20, 2024
de824d7
docs: use docs dir instead of docsy (#2443)
csviri Jun 20, 2024
f49afea
refactor: move getResourceClassResolver to BaseConfigurationService (…
metacosm Jun 20, 2024
19ebdb2
feat: @ControllerConfiguration annotation is optional (#2412)
csviri Jun 21, 2024
1031300
bump: chore use slf4j v2 (#2406)
csviri Jun 29, 2024
0865103
feat: integration test to show multiple dependents with activation co…
csviri Jul 1, 2024
811964f
refactor: `updateMatcher` removal (#2431)
csviri Jul 2, 2024
bfa756a
chore(docs): add some javadoc (#2452)
metacosm Jul 3, 2024
c8ef10d
fix: improper use of @link (#2464)
metacosm Jul 3, 2024
eabdb20
feat: move controller informer-related configuration to InformerConfi…
metacosm Jul 8, 2024
657301f
feat: allow returning additional information from conditions (#2426)
metacosm Jul 8, 2024
dd104ab
feat: support for graceful shutdown based on configuration (#2479)
10000-ki Aug 6, 2024
02b876b
refactor: move integration tests to feature packages and sub packages…
csviri Aug 8, 2024
a6621c2
refactor: clean up informer configuration classes (#2470)
metacosm Aug 9, 2024
fdd96b5
feat: support to handle different cluster for InformerEventSource
csviri Aug 13, 2024
3ea8aa1
docs
csviri Aug 13, 2024
3e3c118
format
csviri Aug 13, 2024
a7087fa
wip
csviri Aug 13, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
8 changes: 3 additions & 5 deletions .github/workflows/e2e-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
sample_operators_tests:
strategy:
matrix:
sample_dir:
sample:
- "sample-operators/mysql-schema"
- "sample-operators/tomcat-operator"
- "sample-operators/webpage"
Expand Down Expand Up @@ -48,12 +48,10 @@ jobs:
run: mvn install -DskipTests

- name: Run integration tests in local mode
working-directory: ${{ matrix.sample_dir }}
run: |
mvn test -P end-to-end-tests
mvn test -P end-to-end-tests -pl ${{ matrix.sample }}

- name: Run E2E tests as a deployment
working-directory: ${{ matrix.sample_dir }}
run: |
eval $(minikube -p minikube docker-env)
mvn jib:dockerBuild test -P end-to-end-tests -Dtest.deployment=remote
mvn jib:dockerBuild test -P end-to-end-tests -Dtest.deployment=remote -pl ${{ matrix.sample }}
6 changes: 3 additions & 3 deletions .github/workflows/hugo.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ jobs:
id: pages
uses: actions/configure-pages@v5
- name: Install Node.js dependencies
working-directory: ./docsy
working-directory: ./docs
run: |
[[ -f package-lock.json || -f npm-shrinkwrap.json ]] && npm ci || true
npm install -D autoprefixer
Expand All @@ -61,7 +61,7 @@ jobs:
HUGO_ENVIRONMENT: production
HUGO_ENV: production
TZ: America/Los_Angeles
working-directory: ./docsy
working-directory: ./docs
run: |
hugo \
--gc \
Expand All @@ -70,7 +70,7 @@ jobs:
- name: Upload artifact
uses: actions/upload-pages-artifact@v3
with:
path: ./docsy/public
path: ./docs/public

# Deployment job
deploy:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ on:
http-client:
type: string
required: false
default: 'okhttp'
default: 'jdk'
experimental:
type: boolean
required: false
Expand Down
9 changes: 4 additions & 5 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,15 @@ jobs:
cache: 'maven'
- name: Check code format
run: |
./mvnw ${MAVEN_ARGS} formatter:validate -Dconfigfile=$PWD/contributing/eclipse-google-style.xml -pl '!operator-framework-bom' --file pom.xml
./mvnw ${MAVEN_ARGS} impsort:check -pl '!operator-framework-bom' --file pom.xml
./mvnw ${MAVEN_ARGS} spotless:check --file pom.xml
- name: Run unit tests
run: ./mvnw ${MAVEN_ARGS} clean install --file pom.xml

integration_tests:
strategy:
matrix:
java: [ 11, 17 ]
kubernetes: ['v1.27.14', 'v1.28.10', 'v1.29.5','1.30.1']
java: [ 17, 21 ]
kubernetes: [ 'v1.26.13', 'v1.27.10', 'v1.28.6', 'v1.29.1' ]
uses: ./.github/workflows/integration-tests.yml
with:
java-version: ${{ matrix.java }}
Expand All @@ -56,7 +55,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
java: [ 11, 17 ]
java: [ 17, 21 ]
steps:
- uses: actions/checkout@v4
- name: Set up Java and Maven
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/release-project-in-dir.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
- name: Set up Java and Maven
uses: actions/setup-java@v4
with:
java-version: 11
java-version: 17
distribution: temurin
cache: 'maven'

Expand Down Expand Up @@ -61,7 +61,7 @@ jobs:
- name: Set up Java and Maven
uses: actions/setup-java@v4
with:
java-version: 11
java-version: 17
distribution: temurin
cache: 'maven'

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/snapshot-releases.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: 11
java-version: 17
cache: 'maven'
- name: Build and test project
run: ./mvnw ${MAVEN_ARGS} clean install --file pom.xml
Expand All @@ -34,7 +34,7 @@ jobs:
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: 11
java-version: 17
cache: 'maven'
- name: Release Maven package
uses: samuelmeuli/action-maven-publish@v1
Expand Down
3 changes: 1 addition & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# ![java-operator-sdk](docsy/static/images/full_logo.png)
# ![java-operator-sdk](docs/static/images/full_logo.png)

![Java CI with Maven](https://github.com/operator-framework/java-operator-sdk/actions/workflows/snapshot-releases.yml/badge.svg)
[![Slack](https://img.shields.io/badge/Slack-4A154B?style=flat-square&logo=slack&logoColor=white)](https://kubernetes.slack.com/archives/CAW0GV7A5 "get invite here: https://communityinviter.com/apps/kubernetes/community" )
Expand Down Expand Up @@ -40,7 +40,6 @@ It makes it easy to implement best practices and patterns for an Operator. Featu
* Handling dependent resources, related events, and caching.
* Automatic Retries
* Smart event scheduling
* Handling Observed Generations automatically
* Easy to use Error Handling
* ... and everything that a batteries included framework needs

Expand Down
56 changes: 28 additions & 28 deletions bootstrapper-maven-plugin/pom.xml
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<artifactId>java-operator-sdk</artifactId>
<groupId>io.javaoperatorsdk</groupId>
<version>4.9.3-SNAPSHOT</version>
<artifactId>java-operator-sdk</artifactId>
<version>5.0.0-SNAPSHOT</version>
</parent>

<artifactId>bootstrapper</artifactId>
Expand Down Expand Up @@ -39,7 +39,7 @@
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<artifactId>log4j-slf4j2-impl</artifactId>
<scope>test</scope>
</dependency>
<dependency>
Expand Down Expand Up @@ -72,29 +72,29 @@
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-plugin-plugin</artifactId>
<version>${maven-plugin-plugin.version}</version>
<configuration>
<goalPrefix>josdk-bootstrapper</goalPrefix>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>templating-maven-plugin</artifactId>
<version>${templating-maven-plugin.version}</version>
<executions>
<execution>
<id>filtering-java-templates</id>
<goals>
<goal>filter-sources</goal>
</goals>
</execution>
</executions>
</plugin>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-plugin-plugin</artifactId>
<version>${maven-plugin-plugin.version}</version>
<configuration>
<goalPrefix>josdk-bootstrapper</goalPrefix>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>templating-maven-plugin</artifactId>
<version>${templating-maven-plugin.version}</version>
<executions>
<execution>
<id>filtering-java-templates</id>
<goals>
<goal>filter-sources</goal>
</goals>
</execution>
</executions>
</plugin>

</plugins>
</plugins>
</build>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public class Bootstrapper {

private static final Logger log = LoggerFactory.getLogger(Bootstrapper.class);

private MustacheFactory mustacheFactory = new DefaultMustacheFactory();
private final MustacheFactory mustacheFactory = new DefaultMustacheFactory();

// .gitignore gets excluded from resource, using here a prefixed version
private static final Map<String, String> TOP_LEVEL_STATIC_FILES =
Expand Down
Original file line number Diff line number Diff line change
@@ -1,23 +1,15 @@
package {{groupId}};

import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.ConfigMapBuilder;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.javaoperatorsdk.operator.api.config.informer.InformerConfiguration;
import io.javaoperatorsdk.operator.api.reconciler.Reconciler;
import io.javaoperatorsdk.operator.api.reconciler.EventSourceInitializer;
import io.javaoperatorsdk.operator.api.reconciler.UpdateControl;
import io.javaoperatorsdk.operator.api.reconciler.Context;
import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext;
import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration;
import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent;
import io.javaoperatorsdk.operator.processing.event.source.EventSource;
import io.javaoperatorsdk.operator.processing.event.source.informer.InformerEventSource;
import io.javaoperatorsdk.operator.api.reconciler.Workflow;

import java.util.Map;
import java.util.Optional;

@ControllerConfiguration(dependents = {@Dependent(type = ConfigMapDependentResource.class)})
@Workflow(dependents = {@Dependent(type = ConfigMapDependentResource.class)})
public class {{artifactClassId}}Reconciler implements Reconciler<{{artifactClassId}}CustomResource> {

public UpdateControl<{{artifactClassId}}CustomResource> reconcile({{artifactClassId}}CustomResource primary,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package {{groupId}};

import io.javaoperatorsdk.operator.api.ObservedGenerationAwareStatus;

public class {{artifactClassId}}Status extends ObservedGenerationAwareStatus {
public class {{artifactClassId}}Status {

}
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
<packaging>jar</packaging>

<properties>
<java.version>11</java.version>
<java.version>17</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<josdk.version>{{josdkVersion}}</josdk.version>
Expand Down Expand Up @@ -58,7 +58,7 @@
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<artifactId>log4j-slf4j2-impl</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
Expand Down
21 changes: 7 additions & 14 deletions caffeine-bounded-cache-support/pom.xml
Original file line number Diff line number Diff line change
@@ -1,22 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>java-operator-sdk</artifactId>
<groupId>io.javaoperatorsdk</groupId>
<version>4.9.3-SNAPSHOT</version>
<artifactId>java-operator-sdk</artifactId>
<version>5.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>caffeine-bounded-cache-support</artifactId>
<name>Operator SDK - Caffeine Bounded Cache Support</name>

<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
</properties>

<dependencies>
<dependency>
<groupId>io.javaoperatorsdk</groupId>
Expand Down Expand Up @@ -44,7 +37,7 @@
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<artifactId>log4j-slf4j2-impl</artifactId>
<scope>test</scope>
</dependency>
<dependency>
Expand All @@ -70,10 +63,10 @@
However, this is needed to compile the tests so let's disable apt just for the compile phase -->
<execution>
<id>default-compile</id>
<phase>compile</phase>
<goals>
<goal>compile</goal>
</goals>
<phase>compile</phase>
<configuration>
<compilerArgs>
<arg>-proc:none</arg>
Expand All @@ -85,4 +78,4 @@
</plugins>
</build>

</project>
</project>
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.javaoperatorsdk.operator.processing.event.source.cache.sample;

import java.time.Duration;
import java.util.List;
import java.util.Map;

import org.slf4j.Logger;
Expand All @@ -13,7 +14,7 @@
import io.fabric8.kubernetes.client.CustomResource;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClientBuilder;
import io.javaoperatorsdk.operator.api.config.informer.InformerConfiguration;
import io.javaoperatorsdk.operator.api.config.informer.InformerEventSourceConfiguration;
import io.javaoperatorsdk.operator.api.reconciler.*;
import io.javaoperatorsdk.operator.processing.event.source.EventSource;
import io.javaoperatorsdk.operator.processing.event.source.cache.BoundedItemStore;
Expand All @@ -28,7 +29,7 @@
import com.github.benmanes.caffeine.cache.Caffeine;

public abstract class AbstractTestReconciler<P extends CustomResource<BoundedCacheTestSpec, BoundedCacheTestStatus>>
implements Reconciler<P>, EventSourceInitializer<P> {
implements Reconciler<P> {

private static final Logger log =
LoggerFactory.getLogger(BoundedCacheClusterScopeTestReconciler.class);
Expand Down Expand Up @@ -69,20 +70,23 @@ protected void createConfigMap(P resource, Context<P> context) {
}

@Override
public Map<String, EventSource> prepareEventSources(
public List<EventSource<?, P>> prepareEventSources(
EventSourceContext<P> context) {

var boundedItemStore =
boundedItemStore(new KubernetesClientBuilder().build(),
ConfigMap.class, Duration.ofMinutes(1), 1); // setting max size for testing purposes

var es = new InformerEventSource<>(InformerConfiguration.from(ConfigMap.class, context)
.withItemStore(boundedItemStore)
.withSecondaryToPrimaryMapper(
Mappers.fromOwnerReference(this instanceof BoundedCacheClusterScopeTestReconciler))
.build(), context);
var es = new InformerEventSource<>(
InformerEventSourceConfiguration.from(ConfigMap.class, primaryClass())
.withInformerConfiguration(c -> c.withItemStore(boundedItemStore))
.withSecondaryToPrimaryMapper(
Mappers.fromOwnerReferences(context.getPrimaryResourceClass(),
this instanceof BoundedCacheClusterScopeTestReconciler))
.build(),
context);

return EventSourceInitializer.nameEventSources(es);
return List.of(es);
}

private void ensureStatus(P resource) {
Expand All @@ -102,4 +106,7 @@ public static <R extends HasMetadata> BoundedItemStore<R> boundedItemStore(
.build();
return CaffeineBoundedItemStores.boundedItemStore(client, rClass, cache);
}

protected abstract Class<P> primaryClass();

}
Loading
Loading