Intermittent java.util.ConcurrentModificationException on cleanup()

[morhidi]

Getting these errors sometimes, hard to repro :/ (v3.0.1)

2022-05-26 13:43:28,345 i.j.o.p.e.ReconciliationDispatcher [ERROR] [default.basic-session-cluster] Error during event processing ExecutionScope{ resource id: ResourceID{name='basic-session-cluster', namespace='default'}, version: 969441} failed.
io.javaoperatorsdk.operator.OperatorException: java.util.ConcurrentModificationException
	at io.javaoperatorsdk.operator.processing.Controller.cleanup(Controller.java:146)
	at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleCleanup(ReconciliationDispatcher.java:276)
	at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleDispatch(ReconciliationDispatcher.java:78)
	at io.javaoperatorsdk.operator.processing.event.ReconciliationDispatcher.handleExecution(ReconciliationDispatcher.java:55)
	at io.javaoperatorsdk.operator.processing.event.EventProcessor$ControllerExecution.run(EventProcessor.java:356)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: java.util.ConcurrentModificationException
	at java.base/java.util.HashMap$KeySpliterator.forEachRemaining(HashMap.java:1626)
	at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484)
	at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
	at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913)
	at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
	at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578)
	at io.javaoperatorsdk.operator.processing.event.source.informer.InformerEventSource.getSecondaryResources(InformerEventSource.java:183)
	at io.javaoperatorsdk.operator.api.reconciler.DefaultContext.lambda$getSecondaryResources$0(DefaultContext.java:37)
	at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
	at java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1655)
	at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484)
	at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
	at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913)
	at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
	at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578)
	at io.javaoperatorsdk.operator.api.reconciler.DefaultContext.getSecondaryResources(DefaultContext.java:39)
	at org.apache.flink.kubernetes.operator.reconciler.deployment.SessionReconciler.cleanup(SessionReconciler.java:151)
	at org.apache.flink.kubernetes.operator.reconciler.deployment.SessionReconciler.cleanup(SessionReconciler.java:48)
	at org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:105)
	at org.apache.flink.kubernetes.operator.controller.FlinkDeploymentController.cleanup(FlinkDeploymentController.java:58)
	at io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:139)
	at io.javaoperatorsdk.operator.processing.Controller$1.execute(Controller.java:113)
	at io.javaoperatorsdk.operator.api.monitoring.Metrics.timeControllerExecution(Metrics.java:34)
	at io.javaoperatorsdk.operator.processing.Controller.cleanup(Controller.java:112)
	... 7 more