From 5fd47bf11596a912b6042c3623bab896f4ec611d Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Mon, 31 Aug 2020 19:31:12 +0000 Subject: [PATCH 01/13] changes for OWLS-83136 - Limit concurrent pod shutdowns during a cluster shrink --- docs/domains/Domain.json | 10 + docs/domains/Domain.md | 2 + kubernetes/crd/domain-crd.yaml | 18 + kubernetes/crd/domain-v1beta1-crd.yaml | 18 + .../operator/KubernetesConstants.java | 1 + .../operator/calls/AsyncRequestStep.java | 34 ++ .../helpers/AsyncRequestStepFactory.java | 2 + .../operator/helpers/CallBuilder.java | 39 ++ .../operator/helpers/DomainPresenceInfo.java | 103 ++++- .../operator/helpers/PodHelper.java | 28 +- .../operator/steps/DeleteDomainStep.java | 7 +- .../steps/ManagedServerUpIteratorStep.java | 24 +- .../operator/steps/ManagedServersUpStep.java | 58 ++- .../steps/ServerDownIteratorStep.java | 170 +++++++- .../operator/steps/ServerDownStep.java | 9 +- .../weblogic/domain/ClusterConfigurator.java | 2 + .../weblogic/domain/DomainConfigurator.java | 5 + .../domain/EffectiveConfigurationFactory.java | 2 + .../weblogic/domain/model/Cluster.java | 20 + .../weblogic/domain/model/Domain.java | 3 + .../model/DomainCommonConfigurator.java | 6 + .../weblogic/domain/model/DomainSpec.java | 39 +- .../helpers/KubernetesTestSupport.java | 2 + .../operator/helpers/PodHelperTestBase.java | 36 ++ .../steps/ManagedServersUpStepTest.java | 43 +- .../steps/ServerDownIteratorStepTest.java | 383 ++++++++++++++++++ .../weblogic/domain/DomainTestBase.java | 41 ++ 27 files changed, 1032 insertions(+), 73 deletions(-) create mode 100644 operator/src/test/java/oracle/kubernetes/operator/steps/ServerDownIteratorStepTest.java diff --git a/docs/domains/Domain.json b/docs/domains/Domain.json index 39f7e1d9ea7..afc70a1287b 100644 --- a/docs/domains/Domain.json +++ b/docs/domains/Domain.json @@ -117,6 +117,11 @@ "description": "Customization affecting Kubernetes Service generated for this WebLogic cluster.", "$ref": "#/definitions/KubernetesResource" }, + "maxConcurrentShutdown": { + "description": "The maximum number of Managed Servers instances that the operator will shutdown in parallel for this cluster in response to a change in the `replicas` count. If more Managed Server instances must be shutdown, the operator will wait until a Managed Server Pod is terminated before shutting down the next Managed Server instance. A value of 0 means all Managed Server instances will shutdown in parallel. Defaults to 1.", + "type": "number", + "minimum": 0 + }, "serverStartPolicy": { "description": "The strategy for deciding whether to start a WebLogic Server instance. Legal values are NEVER, or IF_NEEDED. Defaults to IF_NEEDED. More info: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#starting-and-stopping-servers.", "type": "string", @@ -354,6 +359,11 @@ "type": "number", "minimum": 0 }, + "maxClusterConcurrentShutdown": { + "description": "The maximum number of cluster member Managed Server instances that the operator will shutdown in parallel for a given cluster, if `maxConcurrentShutdown` is not specified for a specific cluster under the `clusters` field. A value of 0 means there is no configured limit. Defaults to 1.", + "type": "number", + "minimum": 0 + }, "domainHomeInImage": { "deprecated": "true", "description": "Deprecated. Use `domainHomeSourceType` instead. Ignored if `domainHomeSourceType` is specified. True indicates that the domain home file system is present in the container image specified by the image field. False indicates that the domain home file system is located on a persistent volume. Defaults to unset.", diff --git a/docs/domains/Domain.md b/docs/domains/Domain.md index ce19067c222..025e60d0a7e 100644 --- a/docs/domains/Domain.md +++ b/docs/domains/Domain.md @@ -34,6 +34,7 @@ The specification of the operation of the WebLogic domain. Required. | `logHome` | string | The directory in a server's container in which to store the domain, Node Manager, server logs, server *.out, introspector .out, and optionally HTTP access log files if `httpAccessLogInLogHome` is true. Ignored if `logHomeEnabled` is false. | | `logHomeEnabled` | Boolean | Specifies whether the log home folder is enabled. Defaults to true if `domainHomeSourceType` is PersistentVolume; false, otherwise. | | `managedServers` | array of [Managed Server](#managed-server) | Lifecycle options for individual Managed Servers, including Java options, environment variables, additional Pod content, and the ability to explicitly start, stop, or restart a named server instance. The `serverName` field of each entry must match a Managed Server that already exists in the WebLogic domain configuration or that matches a dynamic cluster member based on the server template. | +| `maxClusterConcurrentShutdown` | number | The maximum number of cluster member Managed Server instances that the operator will shutdown in parallel for a given cluster, if `maxConcurrentShutdown` is not specified for a specific cluster under the `clusters` field. A value of 0 means there is no configured limit. Defaults to 1. | | `maxClusterConcurrentStartup` | number | The maximum number of cluster member Managed Server instances that the operator will start in parallel for a given cluster, if `maxConcurrentStartup` is not specified for a specific cluster under the `clusters` field. A value of 0 means there is no configured limit. Defaults to 0. | | `replicas` | number | The default number of cluster member Managed Server instances to start for each WebLogic cluster in the domain configuration, unless `replicas` is specified for that cluster under the `clusters` field. For each cluster, the operator will sort cluster member Managed Server names from the WebLogic domain configuration by normalizing any numbers in the Managed Server name and then sorting alphabetically. This is done so that server names such as "managed-server10" come after "managed-server9". The operator will then start Managed Servers from the sorted list, up to the `replicas` count, unless specific Managed Servers are specified as starting in their entry under the `managedServers` field. In that case, the specified Managed Servers will be started and then additional cluster members will be started, up to the `replicas` count, by finding further cluster members in the sorted list that are not already started. If cluster members are started because of their entries under `managedServers`, then a cluster may have more cluster members running than its `replicas` count. Defaults to 0. | | `restartVersion` | string | Changes to this field cause the operator to restart WebLogic Server instances. More info: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#restarting-servers. | @@ -75,6 +76,7 @@ The current status of the operation of the WebLogic domain. Updated automaticall | `allowReplicasBelowMinDynClusterSize` | Boolean | Specifies whether the number of running cluster members is allowed to drop below the minimum dynamic cluster size configured in the WebLogic domain configuration. Otherwise, the operator will ensure that the number of running cluster members is not less than the minimum dynamic cluster setting. This setting applies to dynamic clusters only. Defaults to true. | | `clusterName` | string | The name of the cluster. This value must match the name of a WebLogic cluster already defined in the WebLogic domain configuration. Required. | | `clusterService` | [Kubernetes Resource](#kubernetes-resource) | Customization affecting Kubernetes Service generated for this WebLogic cluster. | +| `maxConcurrentShutdown` | number | The maximum number of Managed Servers instances that the operator will shutdown in parallel for this cluster in response to a change in the `replicas` count. If more Managed Server instances must be shutdown, the operator will wait until a Managed Server Pod is terminated before shutting down the next Managed Server instance. A value of 0 means all Managed Server instances will shutdown in parallel. Defaults to 1. | | `maxConcurrentStartup` | number | The maximum number of Managed Servers instances that the operator will start in parallel for this cluster in response to a change in the `replicas` count. If more Managed Server instances must be started, the operator will wait until a Managed Server Pod is in the `Ready` state before starting the next Managed Server instance. A value of 0 means all Managed Server instances will start in parallel. Defaults to 0. | | `maxUnavailable` | number | The maximum number of cluster members that can be temporarily unavailable. Defaults to 1. | | `replicas` | number | The number of cluster member Managed Server instances to start for this WebLogic cluster. The operator will sort cluster member Managed Server names from the WebLogic domain configuration by normalizing any numbers in the Managed Server name and then sorting alphabetically. This is done so that server names such as "managed-server10" come after "managed-server9". The operator will then start Managed Server instances from the sorted list, up to the `replicas` count, unless specific Managed Servers are specified as starting in their entry under the `managedServers` field. In that case, the specified Managed Server instances will be started and then additional cluster members will be started, up to the `replicas` count, by finding further cluster members in the sorted list that are not already started. If cluster members are started because of their related entries under `managedServers`, then this cluster may have more cluster members running than its `replicas` count. Defaults to 0. | diff --git a/kubernetes/crd/domain-crd.yaml b/kubernetes/crd/domain-crd.yaml index e1f008d5e18..a24a8ecf2d5 100644 --- a/kubernetes/crd/domain-crd.yaml +++ b/kubernetes/crd/domain-crd.yaml @@ -5127,6 +5127,16 @@ spec: additionalProperties: type: string type: object + maxConcurrentShutdown: + description: The maximum number of Managed Servers instances + that the operator will shutdown in parallel for this cluster + in response to a change in the `replicas` count. If more Managed + Server instances must be shutdown, the operator will wait + until a Managed Server Pod is terminated before shutting down + the next Managed Server instance. A value of 0 means all Managed + Server instances will shutdown in parallel. Defaults to 1. + type: number + minimum: 0.0 serverStartPolicy: description: 'The strategy for deciding whether to start a WebLogic Server instance. Legal values are NEVER, or IF_NEEDED. Defaults @@ -5200,6 +5210,14 @@ spec: a cluster may have more cluster members running than its `replicas` count. Defaults to 0. minimum: 0.0 + maxClusterConcurrentShutdown: + type: number + description: The maximum number of cluster member Managed Server instances + that the operator will shutdown in parallel for a given cluster, + if `maxConcurrentShutdown` is not specified for a specific cluster + under the `clusters` field. A value of 0 means there is no configured + limit. Defaults to 1. + minimum: 0.0 domainHomeInImage: type: boolean description: Deprecated. Use `domainHomeSourceType` instead. Ignored diff --git a/kubernetes/crd/domain-v1beta1-crd.yaml b/kubernetes/crd/domain-v1beta1-crd.yaml index 5ffe6ed0c52..64a18e1876f 100644 --- a/kubernetes/crd/domain-v1beta1-crd.yaml +++ b/kubernetes/crd/domain-v1beta1-crd.yaml @@ -5114,6 +5114,16 @@ spec: additionalProperties: type: string type: object + maxConcurrentShutdown: + description: The maximum number of Managed Servers instances that + the operator will shutdown in parallel for this cluster in response + to a change in the `replicas` count. If more Managed Server + instances must be shutdown, the operator will wait until a Managed + Server Pod is terminated before shutting down the next Managed + Server instance. A value of 0 means all Managed Server instances + will shutdown in parallel. Defaults to 1. + type: number + minimum: 0.0 serverStartPolicy: description: 'The strategy for deciding whether to start a WebLogic Server instance. Legal values are NEVER, or IF_NEEDED. Defaults @@ -5185,6 +5195,14 @@ spec: then a cluster may have more cluster members running than its `replicas` count. Defaults to 0. minimum: 0.0 + maxClusterConcurrentShutdown: + type: number + description: The maximum number of cluster member Managed Server instances + that the operator will shutdown in parallel for a given cluster, if + `maxConcurrentShutdown` is not specified for a specific cluster under + the `clusters` field. A value of 0 means there is no configured limit. + Defaults to 1. + minimum: 0.0 domainHomeInImage: type: boolean description: Deprecated. Use `domainHomeSourceType` instead. Ignored diff --git a/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java b/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java index 68a5870edf9..0d2b01ee51a 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/KubernetesConstants.java @@ -29,6 +29,7 @@ public interface KubernetesConstants { boolean DEFAULT_INCLUDE_SERVER_OUT_IN_POD_LOG = true; boolean DEFAULT_ALLOW_REPLICAS_BELOW_MIN_DYN_CLUSTER_SIZE = true; int DEFAULT_MAX_CLUSTER_CONCURRENT_START_UP = 0; + int DEFAULT_MAX_CLUSTER_CONCURRENT_SHUTDOWN = 1; String CONTAINER_NAME = "weblogic-server"; diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java b/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java index 3ea3a2ad0ec..9864d4e424c 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java @@ -51,6 +51,7 @@ public class AsyncRequestStep extends Step implements RetryStrategyListener { private final RequestParams requestParams; private final CallFactory factory; private final int maxRetryCount; + private final RetryStrategy customRetryStrategy; private final String fieldSelector; private final String labelSelector; private final String resourceVersion; @@ -79,10 +80,40 @@ public AsyncRequestStep( String fieldSelector, String labelSelector, String resourceVersion) { + this(next, requestParams,factory, null, helper, timeoutSeconds, maxRetryCount, + fieldSelector, labelSelector, resourceVersion); + } + + /** + * Construct async step. + * + * @param next Next + * @param requestParams Request parameters + * @param factory Factory + * @param customRetryStrategy Custom retry strategy + * @param helper Client pool + * @param timeoutSeconds Timeout + * @param maxRetryCount Max retry count + * @param fieldSelector Field selector + * @param labelSelector Label selector + * @param resourceVersion Resource version + */ + public AsyncRequestStep( + ResponseStep next, + RequestParams requestParams, + CallFactory factory, + RetryStrategy customRetryStrategy, + ClientPool helper, + int timeoutSeconds, + int maxRetryCount, + String fieldSelector, + String labelSelector, + String resourceVersion) { super(next); this.helper = helper; this.requestParams = requestParams; this.factory = factory; + this.customRetryStrategy = customRetryStrategy; this.timeoutSeconds = timeoutSeconds; this.maxRetryCount = maxRetryCount; this.fieldSelector = fieldSelector; @@ -240,6 +271,9 @@ public NextAction apply(Packet packet) { retry = oldResponse.getSpi(RetryStrategy.class); } + if ((retry == null) && (customRetryStrategy != null)) { + retry = customRetryStrategy; + } if (LOGGER.isFinerEnabled()) { logAsyncRequest(); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/AsyncRequestStepFactory.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/AsyncRequestStepFactory.java index 629be0e0b3a..bba4745d1f4 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/AsyncRequestStepFactory.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/AsyncRequestStepFactory.java @@ -5,6 +5,7 @@ import oracle.kubernetes.operator.calls.CallFactory; import oracle.kubernetes.operator.calls.RequestParams; +import oracle.kubernetes.operator.calls.RetryStrategy; import oracle.kubernetes.operator.work.Step; public interface AsyncRequestStepFactory { @@ -12,6 +13,7 @@ Step createRequestAsync( ResponseStep next, RequestParams requestParams, CallFactory factory, + RetryStrategy retryStrategy, ClientPool helper, int timeoutSeconds, int maxRetryCount, diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index 0bdade0d6ae..7721df0df95 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -48,6 +48,7 @@ import oracle.kubernetes.operator.calls.CallWrapper; import oracle.kubernetes.operator.calls.CancellableCall; import oracle.kubernetes.operator.calls.RequestParams; +import oracle.kubernetes.operator.calls.RetryStrategy; import oracle.kubernetes.operator.calls.SynchronousCallDispatcher; import oracle.kubernetes.operator.calls.SynchronousCallFactory; import oracle.kubernetes.operator.work.Step; @@ -1183,6 +1184,28 @@ public Step deletePodAsync( responseStep, new RequestParams("deletePod", namespace, name, deleteOptions, domainUid), deletePod); } + /** + * Asynchronous step for deleting pod. + * + * @param name Name + * @param namespace Namespace + * @param domainUid Identifier of the domain that the pod is associated with + * @param deleteOptions Delete options + * @param responseStep Response step for when call completes + * @return Asynchronous step + */ + public Step deletePodAsyncWithRetryStrategy( + String name, + String namespace, + String domainUid, + V1DeleteOptions deleteOptions, + ResponseStep responseStep, + RetryStrategy retryStrategy) { + return createRequestAsync( + responseStep, new RequestParams("deletePod", namespace, name, deleteOptions, domainUid), + deletePod, retryStrategy); + } + private Call patchPodAsync( ApiClient client, String name, String namespace, V1Patch patch, ApiCallback callback) throws ApiException { @@ -1835,6 +1858,7 @@ private Step createRequestAsync( next, requestParams, factory, + null, helper, timeoutSeconds, maxRetryCount, @@ -1843,6 +1867,21 @@ private Step createRequestAsync( resourceVersion); } + private Step createRequestAsync( + ResponseStep next, RequestParams requestParams, CallFactory factory, RetryStrategy retryStrategy) { + return STEP_FACTORY.createRequestAsync( + next, + requestParams, + factory, + retryStrategy, + helper, + timeoutSeconds, + maxRetryCount, + fieldSelector, + labelSelector, + resourceVersion); + } + private CancellableCall wrap(Call call) { return new CallWrapper(call); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java index ee4c2ca5b20..26161567094 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java @@ -46,6 +46,7 @@ public class DomainPresenceInfo { private final AtomicBoolean isPopulated = new AtomicBoolean(false); private final AtomicInteger retryCount = new AtomicInteger(0); private final AtomicReference> serverStartupInfo; + private final AtomicReference> serverShutdownInfo; private final ConcurrentMap servers = new ConcurrentHashMap<>(); private final ConcurrentMap clusters = new ConcurrentHashMap<>(); @@ -62,6 +63,7 @@ public DomainPresenceInfo(Domain domain) { this.namespace = domain.getMetadata().getNamespace(); this.domainUid = domain.getDomainUid(); this.serverStartupInfo = new AtomicReference<>(null); + this.serverShutdownInfo = new AtomicReference<>(null); } /** @@ -75,6 +77,7 @@ public DomainPresenceInfo(String namespace, String domainUid) { this.namespace = namespace; this.domainUid = domainUid; this.serverStartupInfo = new AtomicReference<>(null); + this.serverShutdownInfo = new AtomicReference<>(null); } private static boolean removeIfPresentAnd( @@ -462,6 +465,15 @@ public void setServerStartupInfo(Collection serverStartupInfo this.serverStartupInfo.set(serverStartupInfo); } + /** + * Sets server shutdown info. + * + * @param serverShutdownInfo Server shutdown info + */ + public void setServerShutdownInfo(Collection serverShutdownInfo) { + this.serverShutdownInfo.set(serverShutdownInfo); + } + @Override public String toString() { StringBuilder sb = new StringBuilder("DomainPresenceInfo{"); @@ -612,4 +624,93 @@ public int hashCode() { .toHashCode(); } } -} + + /** Details about a specific managed server that will be shutdown. */ + public static class ServerShutdownInfo { + public final WlsServerConfig serverConfig; + private final String clusterName; + private final ServerSpec serverSpec; + private final boolean isServiceOnly; + + /** + * Create server shutdown info. + * + * @param serverConfig Server config scan + * @param clusterName the name of the cluster + * @param serverSpec Server specifications + * @param isServiceOnly If service needs to be preserved + */ + public ServerShutdownInfo( + WlsServerConfig serverConfig, String clusterName, + ServerSpec serverSpec, boolean isServiceOnly) { + this.serverConfig = serverConfig; + this.clusterName = clusterName; + this.serverSpec = serverSpec; + this.isServiceOnly = isServiceOnly; + } + + public ServerShutdownInfo(String serverName, String clusterName) { + this(new WlsServerConfig(serverName, null, 0), clusterName, null, false); + } + + public String getName() { + return serverConfig.getName(); + } + + public String getServerName() { + return serverConfig.getName(); + } + + public String getClusterName() { + return clusterName; + } + + public boolean isServiceOnly() { + return isServiceOnly; + } + + public List getEnvironment() { + return serverSpec == null ? Collections.emptyList() : serverSpec.getEnvironmentVariables(); + } + + @Override + public String toString() { + return new ToStringBuilder(this) + .append("serverConfig", serverConfig) + .append("clusterName", clusterName) + .append("serverSpec", serverSpec) + .append("isServiceOnly", isServiceOnly) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + ServerShutdownInfo that = (ServerShutdownInfo) o; + + return new EqualsBuilder() + .append(serverConfig, that.serverConfig) + .append(clusterName, that.clusterName) + .append(serverSpec, that.serverSpec) + .append(isServiceOnly, that.isServiceOnly) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(serverConfig) + .append(clusterName) + .append(serverSpec) + .append(isServiceOnly) + .toHashCode(); + } + } +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 710448df026..801d0297207 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -22,6 +22,7 @@ import oracle.kubernetes.operator.ProcessingConstants; import oracle.kubernetes.operator.TuningParameters; import oracle.kubernetes.operator.calls.CallResponse; +import oracle.kubernetes.operator.calls.RetryStrategy; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; @@ -603,9 +604,32 @@ public NextAction onSuccess(Packet packet, CallResponse callResponse) { }); V1DeleteOptions deleteOptions = new V1DeleteOptions().gracePeriodSeconds(gracePeriodSeconds); + DeletePodRetryStrategy retryStrategy = new DeletePodRetryStrategy(next); return new CallBuilder() - .deletePodAsync( - name, namespace, domainUid, deleteOptions, new DefaultResponseStep<>(conflictStep, next)); + .deletePodAsyncWithRetryStrategy( + name, namespace, domainUid, deleteOptions, new DefaultResponseStep<>(conflictStep, next), retryStrategy); } } + + private static final class DeletePodRetryStrategy implements RetryStrategy { + private long retryCount = 0; + private final Step retryStep; + + DeletePodRetryStrategy(Step retryStep) { + this.retryStep = retryStep; + } + + @Override + public NextAction doPotentialRetry(Step conflictStep, Packet packet, int statusCode) { + NextAction na = new NextAction(); + na.invoke(retryStep, packet); + return na; + } + + @Override + public void reset() { + + } + } + } diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteDomainStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteDomainStep.java index 990a674aab1..8fa6fad8f26 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteDomainStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/DeleteDomainStep.java @@ -3,6 +3,8 @@ package oracle.kubernetes.operator.steps; +import java.util.ArrayList; +import java.util.List; import java.util.stream.Collectors; import io.kubernetes.client.openapi.models.V1ServiceList; @@ -44,9 +46,12 @@ public NextAction apply(Packet packet) { deleteServices(), ConfigMapHelper.deleteIntrospectorConfigMapStep(domainUid, namespace, getNext())); if (info != null) { + List ssi = new ArrayList<>(); + info.getServerPods().map(PodHelper::getPodServerName).collect(Collectors.toList()) + .forEach(s -> ssi.add(new DomainPresenceInfo.ServerShutdownInfo(s, null))); serverDownStep = new ServerDownIteratorStep( - info.getServerPods().map(PodHelper::getPodServerName).collect(Collectors.toList()), + ssi, serverDownStep); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java index ef2bbab27dc..ca5d161b368 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java @@ -9,7 +9,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; @@ -50,18 +49,6 @@ public ManagedServerUpIteratorStep(Collection startupInfos, S this.startupInfos = startupInfos; } - // pre-conditions: DomainPresenceInfo SPI - // "principal" - // "serverScan" - // "clusterScan" - // "envVars" - private static Step bringManagedServerUp(ServerStartupInfo ssi) { - return ssi.isServiceOnly() - ? ServiceHelper.createForServerStep( - true, new ServerDownStep(ssi.getServerName(), true, null)) - : ServiceHelper.createForServerStep(PodHelper.createManagedPodStep(null)); - } - @Override protected String getDetail() { List serversToStart = new ArrayList<>(); @@ -120,7 +107,8 @@ private List getServerNames(Collection startupInfos) } private StepAndPacket createManagedServerUpDetails(Packet packet, ServerStartupInfo ssi) { - return new StepAndPacket(bringManagedServerUp(ssi), createPacketForServer(packet, ssi)); + return new StepAndPacket(ServiceHelper.createForServerStep(PodHelper.createManagedPodStep(null)), + createPacketForServer(packet, ssi)); } private Packet createPacketForServer(Packet packet, ServerStartupInfo ssi) { @@ -177,9 +165,6 @@ public NextAction apply(Packet packet) { if (startDetailsQueue.isEmpty()) { return doNext(new ManagedServerUpAfterStep(getNext()), packet); - } else if (isServiceOnlyOrShuttingDown()) { - Collection servers = Collections.singletonList(startDetailsQueue.poll()); - return doForkJoin(this, packet, servers); } else if (serverAvailableToStart(packet.getSpi(DomainPresenceInfo.class))) { numStarted.getAndIncrement(); return doForkJoin(this, packet, Collections.singletonList(startDetailsQueue.poll())); @@ -188,11 +173,6 @@ public NextAction apply(Packet packet) { } } - private boolean isServiceOnlyOrShuttingDown() { - return Optional.ofNullable(startDetailsQueue.peek().step) - .map(step -> step.getNext() instanceof ServerDownStep).orElse(false); - } - private boolean serverAvailableToStart(DomainPresenceInfo info) { return ((numStarted.get() < PodHelper.getScheduledPods(info, clusterName).size()) && (canStartConcurrently(PodHelper.getReadyPods(info, clusterName).size()))); diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java index ad0923548b0..848ef7ad22d 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java @@ -18,6 +18,7 @@ import oracle.kubernetes.operator.DomainStatusUpdater; import oracle.kubernetes.operator.ProcessingConstants; import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerShutdownInfo; import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerStartupInfo; import oracle.kubernetes.operator.helpers.PodHelper; import oracle.kubernetes.operator.logging.LoggingFacade; @@ -37,8 +38,8 @@ public class ManagedServersUpStep extends Step { "Running servers for domain with UID: {0}, running list: {1}"; private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); private static NextStepFactory NEXT_STEP_FACTORY = - (info, config, servers, next) -> - scaleDownIfNecessary(info, config, servers, new ClusterServicesStep(next)); + (info, config, factory, next) -> + scaleDownIfNecessary(info, config, factory, new ClusterServicesStep(next)); public ManagedServersUpStep(Step next) { super(next); @@ -51,32 +52,36 @@ public static Collection getRunningServers(DomainPresenceInfo info) { private static Step scaleDownIfNecessary( DomainPresenceInfo info, WlsDomainConfig domainTopology, - Collection servers, + ServersUpStepFactory factory, Step next) { List steps = new ArrayList<>(Collections.singletonList(next)); - List serversToIgnore = new ArrayList<>(servers); + if (info.getDomain().isShuttingDown()) { + insert(steps, createAvailableHookStep()); + } + + List serversToIgnore = new ArrayList<>(factory.servers); if (info.getDomain().isShuttingDown()) { insert(steps, createAvailableHookStep()); } else { serversToIgnore.add(domainTopology.getAdminServerName()); } - List serversToStop = getServersToStop(info, serversToIgnore); + List serversToStop = getServersToStop(info, + factory.shutdownInfos); if (!serversToStop.isEmpty()) { - insert(steps, new ServerDownIteratorStep(serversToStop, null)); + insert(steps, new ServerDownIteratorStep(factory.shutdownInfos, null)); } return Step.chain(steps.toArray(new Step[0])); } - private static List getServersToStop( - DomainPresenceInfo info, List serversToIgnore) { - return info.getServerNames().stream() - .filter(n -> !serversToIgnore.contains(n)) - .collect(Collectors.toList()); + private static List getServersToStop( + DomainPresenceInfo info, List shutdownInfos) { + return shutdownInfos.stream() + .filter(ssi -> info.getServerNames().contains(ssi.getServerName())).collect(Collectors.toList()); } private static Step createAvailableHookStep() { @@ -103,11 +108,12 @@ public NextAction apply(Packet packet) { Optional.ofNullable(config).ifPresent(wlsDomainConfig -> addServersToFactory(factory, wlsDomainConfig)); info.setServerStartupInfo(factory.getStartupInfos()); + info.setServerShutdownInfo(factory.getShutdownInfos()); LOGGER.exiting(); return doNext( NEXT_STEP_FACTORY.createServerStep( - info, config, factory.servers, factory.createNextStep(getNext())), + info, config, factory, factory.createNextStep(getNext())), packet); } @@ -138,14 +144,16 @@ private void addClusteredServersToFactory(@Nonnull ServersUpStepFactory factory, // an interface to provide a hook for unit testing. interface NextStepFactory { Step createServerStep( - DomainPresenceInfo info, WlsDomainConfig config, Collection servers, Step next); + DomainPresenceInfo info, WlsDomainConfig config, ServersUpStepFactory factory, Step next); } - class ServersUpStepFactory { + static class ServersUpStepFactory { final WlsDomainConfig domainTopology; final Domain domain; Collection startupInfos; + List shutdownInfos = new ArrayList<>(); final Collection servers = new ArrayList<>(); + final Collection preCreateServers = new ArrayList<>(); final Map replicas = new HashMap<>(); ServersUpStepFactory(WlsDomainConfig domainTopology, Domain domain) { @@ -179,11 +187,16 @@ private void addServerIfNeeded(@Nonnull WlsServerConfig serverConfig, WlsCluster if (server.shouldStart(getReplicaCount(clusterName))) { servers.add(serverName); + if (shouldPrecreateServerService(server)) { + preCreateServers.add(serverName); + } addStartupInfo(new ServerStartupInfo(serverConfig, clusterName, server)); addToCluster(clusterName); } else if (shouldPrecreateServerService(server)) { - servers.add(serverName); - addStartupInfo(new ServerStartupInfo(serverConfig, clusterName, server, true)); + preCreateServers.add(serverName); + addShutdownInfo(new ServerShutdownInfo(serverConfig, clusterName, server, true)); + } else { + addShutdownInfo(new ServerShutdownInfo(serverConfig, clusterName, server, false)); } } @@ -198,7 +211,7 @@ boolean exceedsMaxConfiguredClusterSize(WlsClusterConfig clusterConfig) { return false; } - private Step createNextStep(Step next) { + private Step createNextStep(Step next) { if (servers.isEmpty()) { return next; } else { @@ -210,6 +223,10 @@ Collection getStartupInfos() { return startupInfos; } + Collection getShutdownInfos() { + return shutdownInfos; + } + private void addStartupInfo(ServerStartupInfo startupInfo) { if (startupInfos == null) { startupInfos = new ArrayList<>(); @@ -217,6 +234,13 @@ private void addStartupInfo(ServerStartupInfo startupInfo) { startupInfos.add(startupInfo); } + private void addShutdownInfo(DomainPresenceInfo.ServerShutdownInfo shutdownInfo) { + if (shutdownInfos == null) { + shutdownInfos = new ArrayList<>(); + } + shutdownInfos.add(shutdownInfo); + } + private void addToCluster(String clusterName) { if (clusterName != null) { replicas.put(clusterName, 1 + getReplicaCount(clusterName)); diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java index 66cbb300625..eba38d617d0 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java @@ -3,44 +3,180 @@ package oracle.kubernetes.operator.steps; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.stream.Collectors; +import java.util.stream.IntStream; +import oracle.kubernetes.operator.ProcessingConstants; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.helpers.ServiceHelper; import oracle.kubernetes.operator.work.NextAction; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.model.Domain; public class ServerDownIteratorStep extends Step { - private final List serverNames; + private final Collection serverNamesInfos; - ServerDownIteratorStep(List serverNames, Step next) { + ServerDownIteratorStep(List serverNamesInfos, Step next) { super(next); - this.serverNames = serverNames; + Collections.reverse(serverNamesInfos); + this.serverNamesInfos = serverNamesInfos; } List getServersToStop() { + List serverNames = new ArrayList<>(); + serverNamesInfos.forEach(s -> serverNames.add(s.getServerName())); return serverNames; } - @Override - protected String getDetail() { - return String.join(",", getServersToStop()); + Collection getServersInfoToStop() { + return serverNamesInfos; } @Override public NextAction apply(Packet packet) { - Collection startDetails = - getServersToStop().stream() - .map( - serverName -> - new StepAndPacket(new ServerDownStep(serverName, null), packet.clone())) - .collect(Collectors.toList()); - - if (startDetails.isEmpty()) { - return doNext(packet); + List shutdownDetails = + getServersInfoToStop().stream() + .filter(ssi -> !isServerInCluster(ssi)) + .map(ssi -> createManagedServerDownDetails(packet, ssi)).collect(Collectors.toList()); + + getShutdownClusteredServersStepFactories(getServersInfoToStop(), packet).values() + .forEach(factory -> shutdownDetails.addAll(factory.getServerShutdownStepAndPackets())); + + return doNext((new ShutdownManagedServersStep(shutdownDetails, getNext())), packet); + + } + + // pre-conditions: DomainPresenceInfo SPI + // "principal" + // "serverScan" + // "clusterScan" + // "envVars" + private static Step createServiceStep(DomainPresenceInfo.ServerShutdownInfo ssi) { + return ServiceHelper.createForServerStep( + true, new ServerDownStep(ssi.getServerName(), true, null)); + } + + private boolean isServerInCluster(DomainPresenceInfo.ServerShutdownInfo ssi) { + return ssi.getClusterName() != null; + } + + private int getMaxConcurrentShutdown(Domain domain, DomainPresenceInfo.ServerShutdownInfo ssi) { + return domain.getMaxConcurrentShutdown(ssi.getClusterName()); + } + + private int getReplicaCount(Domain domain, DomainPresenceInfo.ServerShutdownInfo ssi) { + return domain.getReplicaCount(ssi.getClusterName()); + } + + private StepAndPacket createManagedServerDownDetails(Packet packet, DomainPresenceInfo.ServerShutdownInfo ssi) { + if ((ssi.isServiceOnly()) && (ssi.serverConfig != null)) { + return new StepAndPacket(createServiceStep(ssi), + createPacketForServer(packet, ssi)); } else { - return doForkJoin(getNext(), packet, startDetails); + return new StepAndPacket(new ServerDownStep(ssi.getName(), null), packet.clone()); + } + } + + private Packet createPacketForServer(Packet packet, DomainPresenceInfo.ServerShutdownInfo ssi) { + Packet p = packet.clone(); + p.put(ProcessingConstants.CLUSTER_NAME, ssi.getClusterName()); + p.put(ProcessingConstants.SERVER_NAME, ssi.getName()); + p.put(ProcessingConstants.SERVER_SCAN, ssi.serverConfig); + p.put(ProcessingConstants.ENVVARS, ssi.getEnvironment()); + return p; + } + + private Map getShutdownClusteredServersStepFactories( + Collection shutdownInfos, Packet packet) { + DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class); + Domain domain = info.getDomain(); + + Map factories = new HashMap<>(); + shutdownInfos.stream() + .filter(this::isServerInCluster) + .forEach(ssi -> + factories.computeIfAbsent(ssi.getClusterName(), + k -> new ShutdownClusteredServersStepFactory(getMaxConcurrentShutdown(domain, ssi), + getReplicaCount(domain, ssi))) + .add(createManagedServerDownDetails(packet, ssi))); + return factories; + } + + private static class ShutdownClusteredServersStepFactory { + + private final Queue serversToShutdown = new ConcurrentLinkedQueue<>(); + private final int maxConcurrency; + private final int replicaCount; + + ShutdownClusteredServersStepFactory(int maxConcurrency, int replicaCount) { + this.maxConcurrency = maxConcurrency; + this.replicaCount = replicaCount; + } + + void add(StepAndPacket serverToShutdown) { + serversToShutdown.add(serverToShutdown); + } + + Collection getServerShutdownStepAndPackets() { + if ((maxConcurrency == 0) || (replicaCount == 0)) { + return serversToShutdown; + } + ArrayList steps = new ArrayList<>(maxConcurrency); + IntStream.range(0, maxConcurrency) + .forEach(i -> steps.add(ShutdownClusteredServersStep.createStepAndPacket(serversToShutdown))); + return steps; + } + } + + static class ShutdownManagedServersStep extends Step { + final Collection shutdownDetails; + + ShutdownManagedServersStep(Collection shutdownDetails, Step next) { + super(next); + this.shutdownDetails = shutdownDetails; + } + + @Override + public NextAction apply(Packet packet) { + if (shutdownDetails.isEmpty()) { + return doNext(getNext(), packet); + } else { + return doForkJoin(getNext(), packet, shutdownDetails); + } + } + } + + static class ShutdownClusteredServersStep extends Step { + + private final Queue serversToShutdown; + + static StepAndPacket createStepAndPacket(Queue serversToShutdown) { + return new StepAndPacket(new ShutdownClusteredServersStep(serversToShutdown), null); + } + + ShutdownClusteredServersStep(Queue serversToShutdown) { + super(null); + this.serversToShutdown = serversToShutdown; + } + + @Override + public NextAction apply(Packet packet) { + + if (serversToShutdown.isEmpty()) { + return doNext(packet); + } else { + Collection servers = Collections.singletonList(serversToShutdown.poll()); + return doForkJoin(this, packet, servers); + } } } -} +} \ No newline at end of file diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownStep.java index 4330c3ae733..4e98ea2350b 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownStep.java @@ -36,10 +36,11 @@ public NextAction apply(Packet packet) { next = getNext(); } else { next = ServiceHelper.deleteServicesStep(serverName, getNext()); - if (oldPod != null) { - PodAwaiterStepFactory pw = packet.getSpi(PodAwaiterStepFactory.class); - next = pw.waitForDelete(oldPod, next); - } + } + + if (oldPod != null) { + PodAwaiterStepFactory pw = packet.getSpi(PodAwaiterStepFactory.class); + next = pw.waitForDelete(oldPod, next); } return doNext(oldPod != null ? PodHelper.deletePodStep(serverName, next) : next, packet); diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/ClusterConfigurator.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/ClusterConfigurator.java index c8edd1d911c..6f2bea4c120 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/ClusterConfigurator.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/ClusterConfigurator.java @@ -132,4 +132,6 @@ ClusterConfigurator withLivenessProbeSettings( ClusterConfigurator withAllowReplicasBelowDynClusterSize(boolean allowReplicasBelowDynClusterSize); ClusterConfigurator withMaxConcurrentStartup(Integer maxConcurrentStartup); + + ClusterConfigurator withMaxConcurrentShutdown(Integer maxConcurrentShutdown); } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/DomainConfigurator.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/DomainConfigurator.java index 19742c719b7..67ef614d70c 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/DomainConfigurator.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/DomainConfigurator.java @@ -177,6 +177,11 @@ public DomainConfigurator withMaxConcurrentStartup(Integer maxConcurrentStartup) return this; } + public DomainConfigurator withMaxConcurrentShutdown(Integer maxConcurrentShutdown) { + getDomainSpec().setMaxClusterConcurrentShutdown(maxConcurrentShutdown); + return this; + } + /** * Sets the WebLogic configuration overrides configmap name for the domain. * diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/EffectiveConfigurationFactory.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/EffectiveConfigurationFactory.java index fe73e3d1aaf..055c42caed5 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/EffectiveConfigurationFactory.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/EffectiveConfigurationFactory.java @@ -34,4 +34,6 @@ public interface EffectiveConfigurationFactory { boolean isAllowReplicasBelowMinDynClusterSize(String clusterName); int getMaxConcurrentStartup(String clusterName); + + int getMaxConcurrentShutdown(String clusterName); } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Cluster.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Cluster.java index fc55dc78eea..089ffd6e30e 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Cluster.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Cluster.java @@ -88,6 +88,16 @@ public class Cluster extends BaseConfiguration implements Comparable { @Range(minimum = 0) private Integer maxConcurrentStartup; + @Description( + "The maximum number of Managed Servers instances that the operator will shutdown in parallel " + + "for this cluster in response to a change in the `replicas` count. " + + "If more Managed Server instances must be shutdown, the operator will wait until a Managed " + + "Server Pod is terminated before shutting down the next Managed Server instance. " + + "A value of 0 means all Managed Server instances will shutdown in parallel. Defaults to 1." + ) + @Range(minimum = 0) + private Integer maxConcurrentShutdown; + protected Cluster getConfiguration() { Cluster configuration = new Cluster(); configuration.fillInFrom(this); @@ -139,6 +149,14 @@ public void setMaxConcurrentStartup(Integer value) { maxConcurrentStartup = value; } + public Integer getMaxConcurrentShutdown() { + return maxConcurrentShutdown; + } + + public void setMaxConcurrentShutdown(Integer value) { + maxConcurrentShutdown = value; + } + @Nullable @Override public String getServerStartPolicy() { @@ -224,6 +242,7 @@ public boolean equals(Object o) { .append(maxUnavailable, cluster.maxUnavailable) .append(allowReplicasBelowMinDynClusterSize, cluster.allowReplicasBelowMinDynClusterSize) .append(maxConcurrentStartup, cluster.maxConcurrentStartup) + .append(maxConcurrentShutdown, cluster.maxConcurrentShutdown) .isEquals(); } @@ -238,6 +257,7 @@ public int hashCode() { .append(maxUnavailable) .append(allowReplicasBelowMinDynClusterSize) .append(maxConcurrentStartup) + .append(maxConcurrentShutdown) .toHashCode(); } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Domain.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Domain.java index 168b77ffe8c..bb11322f275 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Domain.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Domain.java @@ -302,6 +302,9 @@ public int getMaxConcurrentStartup(String clusterName) { return getEffectiveConfigurationFactory().getMaxConcurrentStartup(clusterName); } + public int getMaxConcurrentShutdown(String clusterName) { + return getEffectiveConfigurationFactory().getMaxConcurrentShutdown(clusterName); + } /** * DomainSpec is a description of a domain. diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainCommonConfigurator.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainCommonConfigurator.java index b650d0dd766..bd3e93b5797 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainCommonConfigurator.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainCommonConfigurator.java @@ -737,6 +737,12 @@ public ClusterConfigurator withMaxConcurrentStartup(Integer maxConcurrentStartup return this; } + @Override + public ClusterConfigurator withMaxConcurrentShutdown(Integer maxConcurrentShutdown) { + cluster.setMaxConcurrentShutdown(maxConcurrentShutdown); + return this; + } + @Override public ClusterConfigurator withSchedulerName(String schedulerName) { getDomainSpec().setSchedulerName(schedulerName); diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java index 5e24dee8892..7a6ca5138f2 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java @@ -34,6 +34,7 @@ import static oracle.kubernetes.operator.KubernetesConstants.ALWAYS_IMAGEPULLPOLICY; import static oracle.kubernetes.operator.KubernetesConstants.DEFAULT_ALLOW_REPLICAS_BELOW_MIN_DYN_CLUSTER_SIZE; import static oracle.kubernetes.operator.KubernetesConstants.DEFAULT_IMAGE; +import static oracle.kubernetes.operator.KubernetesConstants.DEFAULT_MAX_CLUSTER_CONCURRENT_SHUTDOWN; import static oracle.kubernetes.operator.KubernetesConstants.DEFAULT_MAX_CLUSTER_CONCURRENT_START_UP; import static oracle.kubernetes.operator.KubernetesConstants.IFNOTPRESENT_IMAGEPULLPOLICY; import static oracle.kubernetes.weblogic.domain.model.Model.DEFAULT_WDT_MODEL_HOME; @@ -208,6 +209,14 @@ public class DomainSpec extends BaseConfiguration { @Range(minimum = 0) private Integer maxClusterConcurrentStartup; + @Description( + "The maximum number of cluster member Managed Server instances that the operator will shutdown in parallel " + + "for a given cluster, if `maxConcurrentShutdown` is not specified for a specific cluster under the " + + "`clusters` field. A value of 0 means there is no configured limit. Defaults to 1." + ) + @Range(minimum = 0) + private Integer maxClusterConcurrentShutdown; + /** * Whether the domain home is part of the image. * @@ -682,6 +691,11 @@ public Integer getMaxClusterConcurrentStartup() { .orElse(DEFAULT_MAX_CLUSTER_CONCURRENT_START_UP); } + public Integer getMaxClusterConcurrentShutdown() { + return Optional.ofNullable(maxClusterConcurrentShutdown) + .orElse(DEFAULT_MAX_CLUSTER_CONCURRENT_SHUTDOWN); + } + @Nullable String getConfigOverrides() { return Optional.ofNullable(configuration).map(Configuration::getOverridesConfigMap).orElse(configOverrides); @@ -847,7 +861,8 @@ public int hashCode() { .append(configOverrides) .append(configOverrideSecrets) .append(allowReplicasBelowMinDynClusterSize) - .append(maxClusterConcurrentStartup); + .append(maxClusterConcurrentStartup) + .append(maxClusterConcurrentShutdown); return builder.toHashCode(); } @@ -886,7 +901,8 @@ public boolean equals(Object other) { .append(configOverrides, rhs.configOverrides) .append(configOverrideSecrets, rhs.configOverrideSecrets) .append(isAllowReplicasBelowMinDynClusterSize(), rhs.isAllowReplicasBelowMinDynClusterSize()) - .append(getMaxClusterConcurrentStartup(), rhs.getMaxClusterConcurrentStartup()); + .append(getMaxClusterConcurrentStartup(), rhs.getMaxClusterConcurrentStartup()) + .append(getMaxClusterConcurrentShutdown(), rhs.getMaxClusterConcurrentShutdown()); return builder.isEquals(); } @@ -958,6 +974,20 @@ public void setMaxClusterConcurrentStartup(Integer maxClusterConcurrentStartup) this.maxClusterConcurrentStartup = maxClusterConcurrentStartup; } + private int getMaxConcurrentShutdownFor(Cluster cluster) { + return hasMaxConcurrentShutdown(cluster) + ? cluster.getMaxConcurrentShutdown() + : getMaxClusterConcurrentShutdown(); + } + + private boolean hasMaxConcurrentShutdown(Cluster cluster) { + return cluster != null && cluster.getMaxConcurrentShutdown() != null; + } + + public void setMaxClusterConcurrentShutdown(Integer maxClusterConcurrentShutdown) { + this.maxClusterConcurrentShutdown = maxClusterConcurrentShutdown; + } + public AdminServer getAdminServer() { return adminServer; } @@ -1033,6 +1063,11 @@ public int getMaxConcurrentStartup(String clusterName) { return getMaxConcurrentStartupFor(getCluster(clusterName)); } + @Override + public int getMaxConcurrentShutdown(String clusterName) { + return getMaxConcurrentShutdownFor(getCluster(clusterName)); + } + private Cluster getOrCreateCluster(String clusterName) { Cluster cluster = getCluster(clusterName); if (cluster != null) { diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/KubernetesTestSupport.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/KubernetesTestSupport.java index 081ef74ac0c..1d9e4c2a3cb 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/KubernetesTestSupport.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/KubernetesTestSupport.java @@ -73,6 +73,7 @@ import oracle.kubernetes.operator.calls.CallFactory; import oracle.kubernetes.operator.calls.CallResponse; import oracle.kubernetes.operator.calls.RequestParams; +import oracle.kubernetes.operator.calls.RetryStrategy; import oracle.kubernetes.operator.calls.SynchronousCallDispatcher; import oracle.kubernetes.operator.calls.SynchronousCallFactory; import oracle.kubernetes.operator.work.Component; @@ -538,6 +539,7 @@ public Step createRequestAsync( ResponseStep next, RequestParams requestParams, CallFactory factory, + RetryStrategy retryStrategy, ClientPool helper, int timeoutSeconds, int maxRetryCount, diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java index f8f431b22c6..f78744722ef 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java @@ -11,6 +11,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.LogRecord; @@ -61,6 +62,7 @@ import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig; import oracle.kubernetes.operator.wlsconfig.WlsServerConfig; import oracle.kubernetes.operator.work.FiberTestSupport; +import oracle.kubernetes.operator.work.NextAction; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; import oracle.kubernetes.operator.work.TerminalStep; @@ -1456,4 +1458,38 @@ public Step waitForDelete(V1Pod pod, Step next) { return next; } } + + public static class DelayedPodAwaiterStepFactory implements PodAwaiterStepFactory { + private final int delaySeconds; + + public DelayedPodAwaiterStepFactory(int delaySeconds) { + this.delaySeconds = delaySeconds; + } + + @Override + public Step waitForReady(V1Pod pod, Step next) { + return new DelayStep(next, delaySeconds); + } + + @Override + public Step waitForDelete(V1Pod pod, Step next) { + return new DelayStep(next, delaySeconds); + } + } + + private static class DelayStep extends Step { + private final int delay; + private final Step next; + + DelayStep(Step next, int delay) { + this.delay = delay; + this.next = next; + } + + @Override + public NextAction apply(Packet packet) { + return doDelay(next, packet, delay, TimeUnit.SECONDS); + } + } + } diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java index 30d813b57ee..038df04a202 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java @@ -3,6 +3,7 @@ package oracle.kubernetes.operator.steps; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -10,6 +11,7 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.LogRecord; +import java.util.stream.Collectors; import com.meterware.simplestub.Memento; import com.meterware.simplestub.StaticStubSupport; @@ -21,6 +23,8 @@ import oracle.kubernetes.operator.helpers.DomainPresenceInfo; import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerStartupInfo; import oracle.kubernetes.operator.helpers.LegalNames; +import oracle.kubernetes.operator.helpers.PodHelper; +import oracle.kubernetes.operator.steps.ManagedServersUpStep.ServersUpStepFactory; import oracle.kubernetes.operator.utils.WlsDomainConfigSupport; import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig; import oracle.kubernetes.operator.wlsconfig.WlsServerConfig; @@ -41,6 +45,7 @@ import org.junit.Test; import static oracle.kubernetes.operator.steps.ManagedServersUpStep.SERVERS_UP_MSG; +import static oracle.kubernetes.operator.steps.ManagedServersUpStepTest.TestStepFactory.getPreCreateServers; import static oracle.kubernetes.operator.steps.ManagedServersUpStepTest.TestStepFactory.getServerStartupInfo; import static oracle.kubernetes.operator.steps.ManagedServersUpStepTest.TestStepFactory.getServers; import static oracle.kubernetes.utils.LogMatcher.containsFine; @@ -515,7 +520,7 @@ public void whenClusterStartupDefinedWithPreCreateServerService_addAllToServers( invokeStep(); - assertThat(getServers(), allOf(hasItem("ms1"), hasItem("ms2"))); + assertThat(TestStepFactory.getPreCreateServers(), allOf(hasItem("ms1"), hasItem("ms2"))); } @Test @@ -526,7 +531,7 @@ public void whenClusterStartupDefinedWithPreCreateServerService_adminServerDown_ invokeStep(); - assertThat(getServers(), allOf(hasItem("ms1"), hasItem("ms2"))); + assertThat(getPreCreateServers(), allOf(hasItem("ms1"), hasItem("ms2"))); } @Test @@ -536,7 +541,7 @@ public void whenClusterStartupDefinedWithPreCreateServerService_managedServerDow invokeStep(); - assertThat(getServers(), allOf(hasItem("ms1"), hasItem("ms2"))); + assertThat(getPreCreateServers(), allOf(hasItem("ms1"), hasItem("ms2"))); } @Test @@ -611,7 +616,26 @@ private Step createNextStep(List servers) { configSupport.setAdminServerName(ADMIN); WlsDomainConfig config = configSupport.createDomainConfig(); ManagedServersUpStep.NextStepFactory factory = factoryMemento.getOriginalValue(); - return factory.createServerStep(domainPresenceInfo, config, servers, nextStep); + ServersUpStepFactory serversUpStepFactory = new ServersUpStepFactory(config, domain); + List ssi = new ArrayList<>(); + domainPresenceInfo.getServerPods().map(PodHelper::getPodServerName).collect(Collectors.toList()) + .forEach(s -> addShutdownServerInfo(s, servers, ssi)); + serversUpStepFactory.shutdownInfos.addAll(ssi); + return factory.createServerStep(domainPresenceInfo, config, serversUpStepFactory, nextStep); + } + + private void addShutdownServerInfo(String serverName, List servers, + List ssi) { + if (isAdminAndNotShuttingDown(serverName)) { + return; + } else if (!servers.contains(serverName)) { + ssi.add(new DomainPresenceInfo.ServerShutdownInfo(serverName, null)); + } + } + + private boolean isAdminAndNotShuttingDown(String serverName) { + return (serverName.equals(configSupport.createDomainConfig().getAdminServerName())) + && (!domainPresenceInfo.getDomain().isShuttingDown()); } private void addWlsServer(String serverName) { @@ -705,6 +729,7 @@ static class TestStepFactory implements ManagedServersUpStep.NextStepFactory { private static WlsDomainConfig config; private static Collection servers; + private static Collection preCreateServers; private static Step next; private static TestStepFactory factory = new TestStepFactory(); @@ -717,22 +742,26 @@ static Collection getServers() { return servers; } + static Collection getPreCreateServers() { + return preCreateServers; + } + static ServerStartupInfo getServerStartupInfo(String serverName) { for (ServerStartupInfo startupInfo : info.getServerStartupInfo()) { if (startupInfo.serverConfig.getName().equals(serverName)) { return startupInfo; } } - return null; } @Override public Step createServerStep( - DomainPresenceInfo info, WlsDomainConfig config, Collection servers, Step next) { + DomainPresenceInfo info, WlsDomainConfig config, ServersUpStepFactory factory, Step next) { TestStepFactory.info = info; TestStepFactory.config = config; - TestStepFactory.servers = servers; + TestStepFactory.servers = factory.servers; + TestStepFactory.preCreateServers = factory.preCreateServers; TestStepFactory.next = next; return new TerminalStep(); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/ServerDownIteratorStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/ServerDownIteratorStepTest.java new file mode 100644 index 00000000000..be67514ec5c --- /dev/null +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/ServerDownIteratorStepTest.java @@ -0,0 +1,383 @@ +// Copyright (c) 2020, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +package oracle.kubernetes.operator.steps; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; +import javax.annotation.Nonnull; + +import com.meterware.simplestub.Memento; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.kubernetes.client.openapi.models.V1Pod; +import io.kubernetes.client.openapi.models.V1PodCondition; +import io.kubernetes.client.openapi.models.V1PodSpec; +import io.kubernetes.client.openapi.models.V1PodStatus; +import io.kubernetes.client.openapi.models.V1SecretReference; +import oracle.kubernetes.operator.KubernetesConstants; +import oracle.kubernetes.operator.LabelConstants; +import oracle.kubernetes.operator.PodAwaiterStepFactory; +import oracle.kubernetes.operator.ProcessingConstants; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerShutdownInfo; +import oracle.kubernetes.operator.helpers.KubernetesTestSupport; +import oracle.kubernetes.operator.helpers.LegalNames; +import oracle.kubernetes.operator.helpers.PodHelperTestBase; +import oracle.kubernetes.operator.helpers.TuningParametersStub; +import oracle.kubernetes.operator.utils.WlsDomainConfigSupport; +import oracle.kubernetes.operator.wlsconfig.WlsClusterConfig; +import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig; +import oracle.kubernetes.operator.wlsconfig.WlsServerConfig; +import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.operator.work.TerminalStep; +import oracle.kubernetes.utils.TestUtils; +import oracle.kubernetes.weblogic.domain.ClusterConfigurator; +import oracle.kubernetes.weblogic.domain.DomainConfigurator; +import oracle.kubernetes.weblogic.domain.DomainConfiguratorFactory; +import oracle.kubernetes.weblogic.domain.model.Domain; +import oracle.kubernetes.weblogic.domain.model.DomainSpec; +import org.hamcrest.junit.MatcherAssert; +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.junit.MatcherAssert.assertThat; + +public class ServerDownIteratorStepTest { + + protected static final String DOMAIN_NAME = "domain1"; + private static final String NS = "namespace"; + private static final String UID = "uid1"; + protected static final String KUBERNETES_UID = "12345"; + private static final String ADMIN = "asName"; + private static final String CLUSTER = "cluster1"; + private static final boolean INCLUDE_SERVER_OUT_IN_POD_LOG = true; + private static final String CREDENTIALS_SECRET_NAME = "webLogicCredentialsSecretName"; + private static final String LATEST_IMAGE = "image:latest"; + private static final String MS_PREFIX = "ms"; + private static final String MS1 = MS_PREFIX + "1"; + private static final String MS2 = MS_PREFIX + "2"; + private static final String MS3 = MS_PREFIX + "3"; + private static final String MS4 = MS_PREFIX + "4"; + private static final int MAX_SERVERS = 5; + private static final int PORT = 8001; + private static final String[] MANAGED_SERVER_NAMES = + IntStream.rangeClosed(1, MAX_SERVERS) + .mapToObj(ServerDownIteratorStepTest::getManagedServerName).toArray(String[]::new); + + @Nonnull + private static String getManagedServerName(int n) { + return MS_PREFIX + n; + } + + private final Domain domain = createDomain(); + private final DomainConfigurator configurator = DomainConfiguratorFactory.forDomain(domain); + private final WlsDomainConfigSupport configSupport = new WlsDomainConfigSupport(DOMAIN_NAME); + + private final Step nextStep = new TerminalStep(); + private final KubernetesTestSupport testSupport = new KubernetesTestSupport(); + private final List mementos = new ArrayList<>(); + private DomainPresenceInfo domainPresenceInfo = createDomainPresenceInfoWithServers(); + private final WlsDomainConfig domainConfig = createDomainConfig(); + + private static WlsDomainConfig createDomainConfig() { + WlsClusterConfig clusterConfig = new WlsClusterConfig(CLUSTER); + for (String serverName : MANAGED_SERVER_NAMES) { + clusterConfig.addServerConfig(new WlsServerConfig(serverName, "domain1-" + serverName, 8001)); + } + return new WlsDomainConfig("base_domain") + .withAdminServer(ADMIN, "domain1-admin-server", 7001) + .withCluster(clusterConfig); + } + + private DomainPresenceInfo createDomainPresenceInfoWithServers(String... serverNames) { + DomainPresenceInfo dpi = new DomainPresenceInfo(domain); + addServer(dpi, ADMIN); + Arrays.asList(serverNames).forEach(serverName -> addServer(dpi, serverName)); + return dpi; + } + + private Domain createDomain() { + return new Domain() + .withApiVersion(KubernetesConstants.DOMAIN_VERSION) + .withKind(KubernetesConstants.DOMAIN) + .withMetadata(new V1ObjectMeta().namespace(NS).name(DOMAIN_NAME).uid(KUBERNETES_UID)) + .withSpec(createDomainSpec()); + } + + private DomainSpec createDomainSpec() { + return new DomainSpec() + .withDomainUid(UID) + .withWebLogicCredentialsSecret(new V1SecretReference().name(CREDENTIALS_SECRET_NAME)) + .withIncludeServerOutInPodLog(INCLUDE_SERVER_OUT_IN_POD_LOG) + .withImage(LATEST_IMAGE); + } + + private static void addServer(DomainPresenceInfo domainPresenceInfo, String serverName) { + if (serverName.equals(ADMIN)) { + domainPresenceInfo.setServerPod(serverName, createReadyPod(serverName)); + } else { + domainPresenceInfo.setServerPod(serverName, createPod(serverName)); + } + } + + private static V1Pod createReadyPod(String serverName) { + return new V1Pod().metadata(withNames(new V1ObjectMeta().namespace(NS), serverName)) + .spec(new V1PodSpec().nodeName("Node1")) + .status(new V1PodStatus().phase("Running") + .addConditionsItem(new V1PodCondition().type("Ready").status("True"))); + } + + private static V1Pod createPod(String serverName) { + return new V1Pod().metadata(withNames(new V1ObjectMeta().namespace(NS), serverName)); + } + + private static V1ObjectMeta withNames(V1ObjectMeta objectMeta, String serverName) { + return objectMeta + .name(LegalNames.toPodName(UID, serverName)) + .putLabelsItem(LabelConstants.SERVERNAME_LABEL, serverName); + } + + /** + * Setup env for tests. + * @throws NoSuchFieldException if TestStepFactory fails to install + */ + @Before + public void setUp() throws NoSuchFieldException { + mementos.add(TestUtils.silenceOperatorLogger().ignoringLoggedExceptions(ApiException.class)); + mementos.add(TuningParametersStub.install()); + mementos.add(testSupport.install()); + + testSupport.defineResources(domain); + testSupport + .addToPacket(ProcessingConstants.DOMAIN_TOPOLOGY, domainConfig) + .addDomainPresenceInfo(domainPresenceInfo); + testSupport.addComponent( + ProcessingConstants.PODWATCHER_COMPONENT_NAME, + PodAwaiterStepFactory.class, + new PodHelperTestBase.DelayedPodAwaiterStepFactory(1)); + + } + + /** + * Cleanup env after tests. + * @throws Exception if test support failed + */ + @After + public void tearDown() throws Exception { + for (Memento memento : mementos) { + memento.revert(); + } + + testSupport.throwOnCompletionFailure(); + } + + @Test + public void withConcurrencyOf1_bothClusteredServersShutdownSequentially() { + configureCluster(CLUSTER).withMaxConcurrentShutdown(1).withReplicas(1); + addWlsCluster(CLUSTER, 8001, MS1, MS2); + domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2); + testSupport + .addDomainPresenceInfo(domainPresenceInfo); + + invokeStepWithServerShutdownInfos(createServerShutdownInfosForCluster(CLUSTER,MS1, MS2)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.FALSE)); + testSupport.setTime(10, TimeUnit.SECONDS); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.TRUE)); + } + + @Test + public void withConcurrencyOf2_bothClusteredServersShutdownConcurrently() { + configureCluster(CLUSTER).withMaxConcurrentShutdown(2).withReplicas(1); + addWlsCluster(CLUSTER, PORT, MS1, MS2); + domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2); + testSupport + .addDomainPresenceInfo(domainPresenceInfo); + + invokeStepWithServerShutdownInfos(createServerShutdownInfosForCluster(CLUSTER,MS1, MS2)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.TRUE)); + } + + @Test + public void withConcurrencyOf0_clusteredServersShutdownConcurrently() { + configureCluster(CLUSTER).withMaxConcurrentShutdown(0); + addWlsCluster(CLUSTER, PORT, MS1, MS2); + + domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2); + testSupport + .addDomainPresenceInfo(domainPresenceInfo); + + invokeStepWithServerShutdownInfos(createServerShutdownInfosForCluster(CLUSTER,MS1, MS2)); + + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.TRUE)); + } + + @Test + public void withReplicaCountOf0AndConcurrencyOf1_clusteredServersShutdownConcurrently() { + configureCluster(CLUSTER).withMaxConcurrentShutdown(1).withReplicas(0); + addWlsCluster(CLUSTER, PORT, MS1, MS2); + + domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2); + testSupport + .addDomainPresenceInfo(domainPresenceInfo); + + invokeStepWithServerShutdownInfos(createServerShutdownInfosForCluster(CLUSTER,MS1, MS2)); + + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.TRUE)); + } + + @Test + public void withConcurrencyOf2AndReplicaCount1_3clusteredServersShutdownIn2Threads() { + configureCluster(CLUSTER).withMaxConcurrentShutdown(2).withReplicas(1); + addWlsCluster(CLUSTER, PORT, MS1, MS2, MS3, MS4); + domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2,MS3,MS4); + testSupport + .addDomainPresenceInfo(domainPresenceInfo); + assertThat(MS2 + " pod", domainPresenceInfo.getServerPod(MS2), notNullValue()); + + invokeStepWithServerShutdownInfos(createServerShutdownInfosForCluster(CLUSTER, MS1, MS2, MS3, MS4)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS4), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS3), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.FALSE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.FALSE)); + testSupport.setTime(10, TimeUnit.SECONDS); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.FALSE)); + } + + @Test + public void withConcurrencyOf2AndReplicaCount0_4clusteredServersShutdownConcurrently() { + configureCluster(CLUSTER).withMaxConcurrentShutdown(2).withReplicas(0); + addWlsCluster(CLUSTER, PORT, MS1, MS2, MS3, MS4); + domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2,MS3,MS4); + testSupport + .addDomainPresenceInfo(domainPresenceInfo); + + invokeStepWithServerShutdownInfos(createServerShutdownInfosForCluster(CLUSTER, MS1, MS2, MS3, MS4)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS4), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS3), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); + } + + @Test + public void withMultipleClusters_differentClusterScheduleAndShutdownDifferently() { + final String CLUSTER2 = "cluster2"; + + configureCluster(CLUSTER).withMaxConcurrentShutdown(0).withReplicas(1); + configureCluster(CLUSTER2).withMaxConcurrentShutdown(1).withReplicas(1); + + addWlsCluster(CLUSTER, PORT, MS1, MS2); + addWlsCluster(CLUSTER2, PORT, MS3, MS4); + + domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2,MS3,MS4); + testSupport + .addDomainPresenceInfo(domainPresenceInfo); + + List serverShutdownInfos = + createServerShutdownInfosForCluster(CLUSTER, MS1, MS2); + serverShutdownInfos.addAll(createServerShutdownInfosForCluster(CLUSTER2, MS3, MS4)); + invokeStepWithServerShutdownInfos(serverShutdownInfos); + + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS4), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS3), is(Boolean.FALSE)); + } + + @Test + public void maxClusterConcurrentShutdown_doesNotApplyToNonClusteredServers() { + domain.getSpec().setMaxClusterConcurrentShutdown(1); + + addWlsServers(MS3, MS4); + domainPresenceInfo = createDomainPresenceInfoWithServers(MS3,MS4); + testSupport + .addDomainPresenceInfo(domainPresenceInfo); + + invokeStepWithServerShutdownInfos(createServerShutdownInfos(MS3, MS4)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS4), is(Boolean.TRUE)); + MatcherAssert.assertThat( + domainPresenceInfo.isServerPodBeingDeleted(MS3), is(Boolean.TRUE)); + } + + @NotNull + private List createServerShutdownInfosForCluster(String clusterName, String... servers) { + List serverShutdownInfos = new ArrayList<>(); + Arrays.stream(servers).forEach(server -> + serverShutdownInfos.add( + new ServerShutdownInfo(configSupport.getWlsServer(clusterName, server).getName(), + clusterName) + ) + ); + return serverShutdownInfos; + } + + @NotNull + private List createServerShutdownInfos(String... servers) { + List serverShutdownInfos = new ArrayList<>(); + Arrays.stream(servers).forEach(server -> + serverShutdownInfos.add( + new ServerShutdownInfo(configSupport.getWlsServer(server).getName(), null) + + ) + ); + return serverShutdownInfos; + } + + private void invokeStepWithServerShutdownInfos(List shutdownInfo) { + ServerDownIteratorStep step = new ServerDownIteratorStep(shutdownInfo, nextStep); + testSupport.runSteps(step); + } + + private ClusterConfigurator configureCluster(String clusterName) { + return configurator.configureCluster(clusterName); + } + + private void addWlsServers(String... serverNames) { + Arrays.asList(serverNames).forEach(this::addWlsServer); + } + + private void addWlsServer(String serverName) { + configSupport.addWlsServer(serverName, 8001); + } + + private void addWlsCluster(String clusterName, int port, String... serverNames) { + + configSupport.addWlsCluster(clusterName, port, serverNames); + } +} \ No newline at end of file diff --git a/operator/src/test/java/oracle/kubernetes/weblogic/domain/DomainTestBase.java b/operator/src/test/java/oracle/kubernetes/weblogic/domain/DomainTestBase.java index 22775fd4a51..251072dfa21 100644 --- a/operator/src/test/java/oracle/kubernetes/weblogic/domain/DomainTestBase.java +++ b/operator/src/test/java/oracle/kubernetes/weblogic/domain/DomainTestBase.java @@ -22,6 +22,7 @@ import static oracle.kubernetes.operator.KubernetesConstants.ALWAYS_IMAGEPULLPOLICY; import static oracle.kubernetes.operator.KubernetesConstants.DEFAULT_ALLOW_REPLICAS_BELOW_MIN_DYN_CLUSTER_SIZE; import static oracle.kubernetes.operator.KubernetesConstants.DEFAULT_IMAGE; +import static oracle.kubernetes.operator.KubernetesConstants.DEFAULT_MAX_CLUSTER_CONCURRENT_SHUTDOWN; import static oracle.kubernetes.operator.KubernetesConstants.DEFAULT_MAX_CLUSTER_CONCURRENT_START_UP; import static oracle.kubernetes.operator.KubernetesConstants.IFNOTPRESENT_IMAGEPULLPOLICY; import static oracle.kubernetes.operator.KubernetesConstants.LATEST_IMAGE_SUFFIX; @@ -400,6 +401,46 @@ public void whenBothClusterAndDomainSpecified_maxConcurrentStartupFromCluster() equalTo(1)); } + @Test + public void afterMaxConcurrentShutdownSetForCluster_canReadIt() { + configureCluster("cluster1").withMaxConcurrentShutdown(3); + + assertThat(domain.getMaxConcurrentShutdown("cluster1"), equalTo(3)); + } + + @Test + public void whenNotSpecified_maxConcurrentShutdownHasDefault() { + configureCluster("cluster1"); + configureDomain(domain).withMaxConcurrentShutdown(null); + + assertThat(domain.getMaxConcurrentShutdown("cluster1"), + equalTo(DEFAULT_MAX_CLUSTER_CONCURRENT_SHUTDOWN)); + } + + @Test + public void whenNotSpecified_maxConcurrentShutdownFromDomain() { + configureCluster("cluster1"); + configureDomain(domain).withMaxConcurrentShutdown(2); + + assertThat(domain.getMaxConcurrentShutdown("cluster1"), + equalTo(2)); + } + + @Test + public void whenNoClusterSpec_maxConcurrentShutdownHasDefault() { + assertThat(domain.getMaxConcurrentShutdown("cluster-with-no-spec"), + equalTo(DEFAULT_MAX_CLUSTER_CONCURRENT_SHUTDOWN)); + } + + @Test + public void whenBothClusterAndDomainSpecified_maxConcurrentShutdownFromCluster() { + configureCluster("cluster1").withMaxConcurrentShutdown(1); + configureDomain(domain).withMaxConcurrentShutdown(0); + + assertThat(domain.getMaxConcurrentShutdown("cluster1"), + equalTo(1)); + } + @Test public void whenBothClusterAndServerStateSpecified_managedServerUsesServerState() { configureServer(SERVER1).withDesiredState("STAND-BY"); From 91b582068e2de94e6e94339685d1ab22de42e066 Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Mon, 31 Aug 2020 21:15:23 +0000 Subject: [PATCH 02/13] Minor code cleanup for OWLS-83136 --- .../operator/calls/AsyncRequestStep.java | 2 +- .../operator/helpers/DomainPresenceInfo.java | 14 +++++++++---- .../operator/helpers/PodHelper.java | 2 +- .../operator/steps/ManagedServersUpStep.java | 7 +++---- .../steps/ServerDownIteratorStep.java | 20 +++++++++---------- .../steps/ManagedServersUpStepTest.java | 1 - 6 files changed, 25 insertions(+), 21 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java b/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java index 9864d4e424c..308d52f99bc 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/calls/AsyncRequestStep.java @@ -80,7 +80,7 @@ public AsyncRequestStep( String fieldSelector, String labelSelector, String resourceVersion) { - this(next, requestParams,factory, null, helper, timeoutSeconds, maxRetryCount, + this(next, requestParams, factory, null, helper, timeoutSeconds, maxRetryCount, fieldSelector, labelSelector, resourceVersion); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java index 26161567094..7357a0f1de5 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java @@ -632,6 +632,16 @@ public static class ServerShutdownInfo { private final ServerSpec serverSpec; private final boolean isServiceOnly; + /** + * Create server shutdown info. + * + * @param serverName the name of the server to shutdown + * @param clusterName the name of the cluster + */ + public ServerShutdownInfo(String serverName, String clusterName) { + this(new WlsServerConfig(serverName, null, 0), clusterName, null, false); + } + /** * Create server shutdown info. * @@ -649,10 +659,6 @@ public ServerShutdownInfo( this.isServiceOnly = isServiceOnly; } - public ServerShutdownInfo(String serverName, String clusterName) { - this(new WlsServerConfig(serverName, null, 0), clusterName, null, false); - } - public String getName() { return serverConfig.getName(); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 801d0297207..7509cb617e0 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -611,6 +611,7 @@ public NextAction onSuccess(Packet packet, CallResponse callResponse) { } } + /* Retry strategy for delete pod which will not perform any retries */ private static final class DeletePodRetryStrategy implements RetryStrategy { private long retryCount = 0; private final Step retryStep; @@ -628,7 +629,6 @@ public NextAction doPotentialRetry(Step conflictStep, Packet packet, int statusC @Override public void reset() { - } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java index 848ef7ad22d..a466af83525 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java @@ -57,19 +57,18 @@ private static Step scaleDownIfNecessary( List steps = new ArrayList<>(Collections.singletonList(next)); + List serversToIgnore = new ArrayList<>(factory.servers); if (info.getDomain().isShuttingDown()) { insert(steps, createAvailableHookStep()); } - List serversToIgnore = new ArrayList<>(factory.servers); if (info.getDomain().isShuttingDown()) { insert(steps, createAvailableHookStep()); } else { serversToIgnore.add(domainTopology.getAdminServerName()); } - List serversToStop = getServersToStop(info, - factory.shutdownInfos); + List serversToStop = getServersToStop(info, factory.shutdownInfos); if (!serversToStop.isEmpty()) { insert(steps, new ServerDownIteratorStep(factory.shutdownInfos, null)); @@ -211,7 +210,7 @@ boolean exceedsMaxConfiguredClusterSize(WlsClusterConfig clusterConfig) { return false; } - private Step createNextStep(Step next) { + private Step createNextStep(Step next) { if (servers.isEmpty()) { return next; } else { diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java index eba38d617d0..7ee0b896cab 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java @@ -23,32 +23,32 @@ import oracle.kubernetes.weblogic.domain.model.Domain; public class ServerDownIteratorStep extends Step { - private final Collection serverNamesInfos; + private final Collection serverShutdownInfos; - ServerDownIteratorStep(List serverNamesInfos, Step next) { + ServerDownIteratorStep(List serverShutdownInfos, Step next) { super(next); - Collections.reverse(serverNamesInfos); - this.serverNamesInfos = serverNamesInfos; + Collections.reverse(serverShutdownInfos); + this.serverShutdownInfos = serverShutdownInfos; } List getServersToStop() { List serverNames = new ArrayList<>(); - serverNamesInfos.forEach(s -> serverNames.add(s.getServerName())); + serverShutdownInfos.forEach(s -> serverNames.add(s.getServerName())); return serverNames; } - Collection getServersInfoToStop() { - return serverNamesInfos; + Collection getServerShutdownInfos() { + return serverShutdownInfos; } @Override public NextAction apply(Packet packet) { List shutdownDetails = - getServersInfoToStop().stream() + getServerShutdownInfos().stream() .filter(ssi -> !isServerInCluster(ssi)) .map(ssi -> createManagedServerDownDetails(packet, ssi)).collect(Collectors.toList()); - getShutdownClusteredServersStepFactories(getServersInfoToStop(), packet).values() + getShutdownClusteredServersStepFactories(getServerShutdownInfos(), packet).values() .forEach(factory -> shutdownDetails.addAll(factory.getServerShutdownStepAndPackets())); return doNext((new ShutdownManagedServersStep(shutdownDetails, getNext())), packet); @@ -78,7 +78,7 @@ private int getReplicaCount(Domain domain, DomainPresenceInfo.ServerShutdownInfo } private StepAndPacket createManagedServerDownDetails(Packet packet, DomainPresenceInfo.ServerShutdownInfo ssi) { - if ((ssi.isServiceOnly()) && (ssi.serverConfig != null)) { + if (ssi.isServiceOnly()) { return new StepAndPacket(createServiceStep(ssi), createPacketForServer(packet, ssi)); } else { diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java index 038df04a202..72382d769b4 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java @@ -3,7 +3,6 @@ package oracle.kubernetes.operator.steps; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; From e948c9293da8abf7d360af3a2cab3ae620d5f24a Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Mon, 31 Aug 2020 21:23:12 +0000 Subject: [PATCH 03/13] minor change to avoid duplicate step --- .../oracle/kubernetes/operator/steps/ManagedServersUpStep.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java index a466af83525..a4bb91752be 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java @@ -58,9 +58,6 @@ private static Step scaleDownIfNecessary( List steps = new ArrayList<>(Collections.singletonList(next)); List serversToIgnore = new ArrayList<>(factory.servers); - if (info.getDomain().isShuttingDown()) { - insert(steps, createAvailableHookStep()); - } if (info.getDomain().isShuttingDown()) { insert(steps, createAvailableHookStep()); From e7aea375215f94b3773d094e16342c360eaba040 Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Mon, 31 Aug 2020 21:59:27 +0000 Subject: [PATCH 04/13] fixed javadoc for deletePodAsyncWithRetryStrategy method --- .../java/oracle/kubernetes/operator/helpers/CallBuilder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index 7721df0df95..9647828c246 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -1185,13 +1185,14 @@ public Step deletePodAsync( } /** - * Asynchronous step for deleting pod. + * Asynchronous step for deleting pod with custom retry strategy. * * @param name Name * @param namespace Namespace * @param domainUid Identifier of the domain that the pod is associated with * @param deleteOptions Delete options * @param responseStep Response step for when call completes + * @param retryStrategy Custom retry strategy for async pod delete ! * @return Asynchronous step */ public Step deletePodAsyncWithRetryStrategy( From f9b5dcacdf3181a7aad2540b6b1c7b560c69dd4a Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Tue, 1 Sep 2020 17:46:46 +0000 Subject: [PATCH 05/13] fix for integration test and added maxConcurrentShutdown in index.html --- docs/domains/index.html | 10 ++++++++++ .../operator/steps/ManagedServersUpStep.java | 5 +---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/domains/index.html b/docs/domains/index.html index 563f33e508a..bbb5efed5b9 100644 --- a/docs/domains/index.html +++ b/docs/domains/index.html @@ -1051,6 +1051,11 @@ "type": "number", "minimum": 0.0 }, + "maxConcurrentShutdown": { + "description": "The maximum number of Managed Servers instances that the operator will shutdown in parallel for this cluster in response to a change in the `replicas` count. If more Managed Server instances must be shutdown, the operator will wait until a Managed Server Pod is terminated before shutting down the next Managed Server instance. A value of 0 means all Managed Server instances will shutdown in parallel. Defaults to 1.", + "type": "number", + "minimum": 0.0 + }, "restartVersion": { "description": "Changes to this field cause the operator to restart WebLogic Server instances. More info: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#restarting-servers.", "type": "string" @@ -1270,6 +1275,11 @@ "type": "number", "minimum": 0.0 }, + "maxClusterConcurrentShutdown": { + "description": "The maximum number of cluster member Managed Server instances that the operator will shutdown in parallel for a given cluster, if `maxConcurrentShutdown` is not specified for a specific cluster under the `clusters` field. A value of 0 means there is no configured limit. Defaults to 1.", + "type": "number", + "minimum": 0.0 + }, "replicas": { "description": "The default number of cluster member Managed Server instances to start for each WebLogic cluster in the domain configuration, unless `replicas` is specified for that cluster under the `clusters` field. For each cluster, the operator will sort cluster member Managed Server names from the WebLogic domain configuration by normalizing any numbers in the Managed Server name and then sorting alphabetically. This is done so that server names such as \"managed-server10\" come after \"managed-server9\". The operator will then start Managed Servers from the sorted list, up to the `replicas` count, unless specific Managed Servers are specified as starting in their entry under the `managedServers` field. In that case, the specified Managed Servers will be started and then additional cluster members will be started, up to the `replicas` count, by finding further cluster members in the sorted list that are not already started. If cluster members are started because of their entries under `managedServers`, then a cluster may have more cluster members running than its `replicas` count. Defaults to 0.", "type": "number", diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java index a4bb91752be..e56001fd528 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java @@ -57,12 +57,9 @@ private static Step scaleDownIfNecessary( List steps = new ArrayList<>(Collections.singletonList(next)); - List serversToIgnore = new ArrayList<>(factory.servers); - if (info.getDomain().isShuttingDown()) { insert(steps, createAvailableHookStep()); - } else { - serversToIgnore.add(domainTopology.getAdminServerName()); + factory.shutdownInfos.add(new ServerShutdownInfo(domainTopology.getAdminServerName(), null)); } List serversToStop = getServersToStop(info, factory.shutdownInfos); From 8597bca2836390659420e4603a74f6f363c45a82 Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Tue, 1 Sep 2020 19:36:44 +0000 Subject: [PATCH 06/13] fix unit test failure and shutdown servers concurrently when domain serverStartPolicy is NEVER --- .../kubernetes/operator/steps/ServerDownIteratorStep.java | 7 ++++--- .../operator/steps/ManagedServersUpStepTest.java | 7 +------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java index 7ee0b896cab..01c3eb4e1ef 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ServerDownIteratorStep.java @@ -43,13 +43,14 @@ Collection getServerShutdownInfos() { @Override public NextAction apply(Packet packet) { + DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class); List shutdownDetails = getServerShutdownInfos().stream() .filter(ssi -> !isServerInCluster(ssi)) .map(ssi -> createManagedServerDownDetails(packet, ssi)).collect(Collectors.toList()); getShutdownClusteredServersStepFactories(getServerShutdownInfos(), packet).values() - .forEach(factory -> shutdownDetails.addAll(factory.getServerShutdownStepAndPackets())); + .forEach(factory -> shutdownDetails.addAll(factory.getServerShutdownStepAndPackets(info))); return doNext((new ShutdownManagedServersStep(shutdownDetails, getNext())), packet); @@ -126,8 +127,8 @@ void add(StepAndPacket serverToShutdown) { serversToShutdown.add(serverToShutdown); } - Collection getServerShutdownStepAndPackets() { - if ((maxConcurrency == 0) || (replicaCount == 0)) { + Collection getServerShutdownStepAndPackets(DomainPresenceInfo info) { + if ((maxConcurrency == 0) || (replicaCount == 0) || info.getDomain().isShuttingDown()) { return serversToShutdown; } ArrayList steps = new ArrayList<>(maxConcurrency); diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java index 72382d769b4..532754718cf 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java @@ -625,18 +625,13 @@ private Step createNextStep(List servers) { private void addShutdownServerInfo(String serverName, List servers, List ssi) { - if (isAdminAndNotShuttingDown(serverName)) { + if (serverName.equals(configSupport.createDomainConfig().getAdminServerName())) { return; } else if (!servers.contains(serverName)) { ssi.add(new DomainPresenceInfo.ServerShutdownInfo(serverName, null)); } } - private boolean isAdminAndNotShuttingDown(String serverName) { - return (serverName.equals(configSupport.createDomainConfig().getAdminServerName())) - && (!domainPresenceInfo.getDomain().isShuttingDown()); - } - private void addWlsServer(String serverName) { configSupport.addWlsServer(serverName); } From b6fdd72d13cf7f40920b50b636fd5a8f3b1538cd Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Thu, 3 Sep 2020 00:05:55 +0000 Subject: [PATCH 07/13] Address PR review comments. --- .../operator/helpers/CallBuilder.java | 30 +-- .../operator/helpers/PodHelper.java | 5 +- .../weblogic/domain/model/Cluster.java | 9 +- .../weblogic/domain/model/DomainSpec.java | 7 +- .../steps/ServerDownIteratorStepTest.java | 223 +++++++----------- 5 files changed, 101 insertions(+), 173 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java index 9647828c246..24f28a3f00f 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CallBuilder.java @@ -230,6 +230,7 @@ public T execute( null, callback)); private String fieldSelector; + private RetryStrategy retryStrategy; /* Version */ private String labelSelector; @@ -516,6 +517,11 @@ public CallBuilder withFieldSelector(String fieldSelector) { return this; } + public CallBuilder withRetryStrategy(RetryStrategy retryStrategy) { + this.retryStrategy = retryStrategy; + return this; + } + private void tuning(int limit, int timeoutSeconds, int maxRetryCount) { this.limit = limit; this.timeoutSeconds = timeoutSeconds; @@ -1181,29 +1187,7 @@ public Step deletePodAsync( V1DeleteOptions deleteOptions, ResponseStep responseStep) { return createRequestAsync( - responseStep, new RequestParams("deletePod", namespace, name, deleteOptions, domainUid), deletePod); - } - - /** - * Asynchronous step for deleting pod with custom retry strategy. - * - * @param name Name - * @param namespace Namespace - * @param domainUid Identifier of the domain that the pod is associated with - * @param deleteOptions Delete options - * @param responseStep Response step for when call completes - * @param retryStrategy Custom retry strategy for async pod delete ! - * @return Asynchronous step - */ - public Step deletePodAsyncWithRetryStrategy( - String name, - String namespace, - String domainUid, - V1DeleteOptions deleteOptions, - ResponseStep responseStep, - RetryStrategy retryStrategy) { - return createRequestAsync( - responseStep, new RequestParams("deletePod", namespace, name, deleteOptions, domainUid), + responseStep, new RequestParams("deletePod", namespace, name, deleteOptions, domainUid), deletePod, retryStrategy); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 7509cb617e0..ba2d55c50b0 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -605,9 +605,8 @@ public NextAction onSuccess(Packet packet, CallResponse callResponse) { V1DeleteOptions deleteOptions = new V1DeleteOptions().gracePeriodSeconds(gracePeriodSeconds); DeletePodRetryStrategy retryStrategy = new DeletePodRetryStrategy(next); - return new CallBuilder() - .deletePodAsyncWithRetryStrategy( - name, namespace, domainUid, deleteOptions, new DefaultResponseStep<>(conflictStep, next), retryStrategy); + return new CallBuilder().withRetryStrategy(retryStrategy) + .deletePodAsync(name, namespace, domainUid, deleteOptions, new DefaultResponseStep<>(conflictStep, next)); } } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Cluster.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Cluster.java index 089ffd6e30e..b408d3dc6b2 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Cluster.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Cluster.java @@ -89,11 +89,10 @@ public class Cluster extends BaseConfiguration implements Comparable { private Integer maxConcurrentStartup; @Description( - "The maximum number of Managed Servers instances that the operator will shutdown in parallel " - + "for this cluster in response to a change in the `replicas` count. " - + "If more Managed Server instances must be shutdown, the operator will wait until a Managed " - + "Server Pod is terminated before shutting down the next Managed Server instance. " - + "A value of 0 means all Managed Server instances will shutdown in parallel. Defaults to 1." + "The maximum WebLogic Server instances that will shutdown in parallel " + + "for this cluster when it is being partially shutdown by lowering its replica count." + + "A value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown` " + + "(which in turn defaults to 1)." ) @Range(minimum = 0) private Integer maxConcurrentShutdown; diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java index 7a6ca5138f2..021214258b2 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java @@ -210,9 +210,10 @@ public class DomainSpec extends BaseConfiguration { private Integer maxClusterConcurrentStartup; @Description( - "The maximum number of cluster member Managed Server instances that the operator will shutdown in parallel " - + "for a given cluster, if `maxConcurrentShutdown` is not specified for a specific cluster under the " - + "`clusters` field. A value of 0 means there is no configured limit. Defaults to 1." + "The default maximum WebLogic Server instances that a cluster will shutdown in parallel when it is being " + + "partially shutdown by lowering its replica count. You can override this default on a per cluster " + + "basis by setting the cluster's `maxConcurrentShutdown` attribute. A value of 0 means there is " + + "no limit. Defaults to 1" ) @Range(minimum = 0) private Integer maxClusterConcurrentShutdown; diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/ServerDownIteratorStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/ServerDownIteratorStepTest.java index be67514ec5c..bf96cbfb95f 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/steps/ServerDownIteratorStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/ServerDownIteratorStepTest.java @@ -7,6 +7,7 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import java.util.stream.IntStream; import javax.annotation.Nonnull; @@ -17,8 +18,7 @@ import io.kubernetes.client.openapi.models.V1PodCondition; import io.kubernetes.client.openapi.models.V1PodSpec; import io.kubernetes.client.openapi.models.V1PodStatus; -import io.kubernetes.client.openapi.models.V1SecretReference; -import oracle.kubernetes.operator.KubernetesConstants; +import oracle.kubernetes.operator.DomainProcessorTestSetup; import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.PodAwaiterStepFactory; import oracle.kubernetes.operator.ProcessingConstants; @@ -39,15 +39,11 @@ import oracle.kubernetes.weblogic.domain.DomainConfigurator; import oracle.kubernetes.weblogic.domain.DomainConfiguratorFactory; import oracle.kubernetes.weblogic.domain.model.Domain; -import oracle.kubernetes.weblogic.domain.model.DomainSpec; -import org.hamcrest.junit.MatcherAssert; -import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Before; import org.junit.Test; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.junit.MatcherAssert.assertThat; public class ServerDownIteratorStepTest { @@ -55,12 +51,9 @@ public class ServerDownIteratorStepTest { protected static final String DOMAIN_NAME = "domain1"; private static final String NS = "namespace"; private static final String UID = "uid1"; - protected static final String KUBERNETES_UID = "12345"; private static final String ADMIN = "asName"; private static final String CLUSTER = "cluster1"; - private static final boolean INCLUDE_SERVER_OUT_IN_POD_LOG = true; - private static final String CREDENTIALS_SECRET_NAME = "webLogicCredentialsSecretName"; - private static final String LATEST_IMAGE = "image:latest"; + private static final String CLUSTER2 = "cluster2"; private static final String MS_PREFIX = "ms"; private static final String MS1 = MS_PREFIX + "1"; private static final String MS2 = MS_PREFIX + "2"; @@ -77,7 +70,7 @@ private static String getManagedServerName(int n) { return MS_PREFIX + n; } - private final Domain domain = createDomain(); + private final Domain domain = DomainProcessorTestSetup.createTestDomain(); private final DomainConfigurator configurator = DomainConfiguratorFactory.forDomain(domain); private final WlsDomainConfigSupport configSupport = new WlsDomainConfigSupport(DOMAIN_NAME); @@ -86,6 +79,7 @@ private static String getManagedServerName(int n) { private final List mementos = new ArrayList<>(); private DomainPresenceInfo domainPresenceInfo = createDomainPresenceInfoWithServers(); private final WlsDomainConfig domainConfig = createDomainConfig(); + private List serverShutdownInfos; private static WlsDomainConfig createDomainConfig() { WlsClusterConfig clusterConfig = new WlsClusterConfig(CLUSTER); @@ -104,22 +98,6 @@ private DomainPresenceInfo createDomainPresenceInfoWithServers(String... serverN return dpi; } - private Domain createDomain() { - return new Domain() - .withApiVersion(KubernetesConstants.DOMAIN_VERSION) - .withKind(KubernetesConstants.DOMAIN) - .withMetadata(new V1ObjectMeta().namespace(NS).name(DOMAIN_NAME).uid(KUBERNETES_UID)) - .withSpec(createDomainSpec()); - } - - private DomainSpec createDomainSpec() { - return new DomainSpec() - .withDomainUid(UID) - .withWebLogicCredentialsSecret(new V1SecretReference().name(CREDENTIALS_SECRET_NAME)) - .withIncludeServerOutInPodLog(INCLUDE_SERVER_OUT_IN_POD_LOG) - .withImage(LATEST_IMAGE); - } - private static void addServer(DomainPresenceInfo domainPresenceInfo, String serverName) { if (serverName.equals(ADMIN)) { domainPresenceInfo.setServerPod(serverName, createReadyPod(serverName)); @@ -163,7 +141,6 @@ public void setUp() throws NoSuchFieldException { ProcessingConstants.PODWATCHER_COMPONENT_NAME, PodAwaiterStepFactory.class, new PodHelperTestBase.DelayedPodAwaiterStepFactory(1)); - } /** @@ -184,17 +161,15 @@ public void withConcurrencyOf1_bothClusteredServersShutdownSequentially() { configureCluster(CLUSTER).withMaxConcurrentShutdown(1).withReplicas(1); addWlsCluster(CLUSTER, 8001, MS1, MS2); domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2); - testSupport - .addDomainPresenceInfo(domainPresenceInfo); + testSupport.addDomainPresenceInfo(domainPresenceInfo); + + createShutdownInfos() + .forClusteredServers(CLUSTER,MS1, MS2) + .shutdown(); - invokeStepWithServerShutdownInfos(createServerShutdownInfosForCluster(CLUSTER,MS1, MS2)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.FALSE)); + assertThat(serverPodsBeingDeleted(), containsInAnyOrder(MS2)); testSupport.setTime(10, TimeUnit.SECONDS); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.TRUE)); + assertThat(serverPodsBeingDeleted(), containsInAnyOrder(MS1, MS2)); } @Test @@ -202,73 +177,57 @@ public void withConcurrencyOf2_bothClusteredServersShutdownConcurrently() { configureCluster(CLUSTER).withMaxConcurrentShutdown(2).withReplicas(1); addWlsCluster(CLUSTER, PORT, MS1, MS2); domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2); - testSupport - .addDomainPresenceInfo(domainPresenceInfo); + testSupport.addDomainPresenceInfo(domainPresenceInfo); + + createShutdownInfos() + .forClusteredServers(CLUSTER,MS1, MS2) + .shutdown(); - invokeStepWithServerShutdownInfos(createServerShutdownInfosForCluster(CLUSTER,MS1, MS2)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.TRUE)); + assertThat(serverPodsBeingDeleted(), containsInAnyOrder(MS1, MS2)); } @Test public void withConcurrencyOf0_clusteredServersShutdownConcurrently() { configureCluster(CLUSTER).withMaxConcurrentShutdown(0); addWlsCluster(CLUSTER, PORT, MS1, MS2); - domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2); - testSupport - .addDomainPresenceInfo(domainPresenceInfo); + testSupport.addDomainPresenceInfo(domainPresenceInfo); - invokeStepWithServerShutdownInfos(createServerShutdownInfosForCluster(CLUSTER,MS1, MS2)); + createShutdownInfos() + .forClusteredServers(CLUSTER,MS1, MS2) + .shutdown(); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.TRUE)); + assertThat(serverPodsBeingDeleted(), containsInAnyOrder(MS1, MS2)); } @Test public void withReplicaCountOf0AndConcurrencyOf1_clusteredServersShutdownConcurrently() { configureCluster(CLUSTER).withMaxConcurrentShutdown(1).withReplicas(0); addWlsCluster(CLUSTER, PORT, MS1, MS2); - domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2); - testSupport - .addDomainPresenceInfo(domainPresenceInfo); + testSupport.addDomainPresenceInfo(domainPresenceInfo); - invokeStepWithServerShutdownInfos(createServerShutdownInfosForCluster(CLUSTER,MS1, MS2)); + createShutdownInfos() + .forClusteredServers(CLUSTER,MS1, MS2) + .shutdown(); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.TRUE)); + assertThat(serverPodsBeingDeleted(), containsInAnyOrder(MS1, MS2)); } @Test - public void withConcurrencyOf2AndReplicaCount1_3clusteredServersShutdownIn2Threads() { + public void withConcurrencyOf2AndReplicaCount1_3rdClusteredServerIsShutdownAfterPreviousPodTerminated() { configureCluster(CLUSTER).withMaxConcurrentShutdown(2).withReplicas(1); addWlsCluster(CLUSTER, PORT, MS1, MS2, MS3, MS4); domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2,MS3,MS4); - testSupport - .addDomainPresenceInfo(domainPresenceInfo); - assertThat(MS2 + " pod", domainPresenceInfo.getServerPod(MS2), notNullValue()); - - invokeStepWithServerShutdownInfos(createServerShutdownInfosForCluster(CLUSTER, MS1, MS2, MS3, MS4)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS4), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS3), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.FALSE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.FALSE)); + testSupport.addDomainPresenceInfo(domainPresenceInfo); + + createShutdownInfos() + .forClusteredServers(CLUSTER,MS1, MS2, MS3, MS4) + .shutdown(); + + assertThat(serverPodsBeingDeleted(), containsInAnyOrder(MS3, MS4)); testSupport.setTime(10, TimeUnit.SECONDS); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.FALSE)); + assertThat(serverPodsBeingDeleted(), containsInAnyOrder(MS2, MS3, MS4)); } @Test @@ -276,92 +235,78 @@ public void withConcurrencyOf2AndReplicaCount0_4clusteredServersShutdownConcurre configureCluster(CLUSTER).withMaxConcurrentShutdown(2).withReplicas(0); addWlsCluster(CLUSTER, PORT, MS1, MS2, MS3, MS4); domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2,MS3,MS4); - testSupport - .addDomainPresenceInfo(domainPresenceInfo); + testSupport.addDomainPresenceInfo(domainPresenceInfo); + + createShutdownInfos() + .forClusteredServers(CLUSTER,MS1, MS2, MS3, MS4) + .shutdown(); - invokeStepWithServerShutdownInfos(createServerShutdownInfosForCluster(CLUSTER, MS1, MS2, MS3, MS4)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS4), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS3), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); + assertThat(serverPodsBeingDeleted(), containsInAnyOrder(MS1, MS2, MS3, MS4)); } @Test public void withMultipleClusters_differentClusterScheduleAndShutdownDifferently() { - final String CLUSTER2 = "cluster2"; - configureCluster(CLUSTER).withMaxConcurrentShutdown(0).withReplicas(1); configureCluster(CLUSTER2).withMaxConcurrentShutdown(1).withReplicas(1); - addWlsCluster(CLUSTER, PORT, MS1, MS2); addWlsCluster(CLUSTER2, PORT, MS3, MS4); - domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2,MS3,MS4); - testSupport - .addDomainPresenceInfo(domainPresenceInfo); + testSupport.addDomainPresenceInfo(domainPresenceInfo); + + createShutdownInfos() + .forClusteredServers(CLUSTER,MS1, MS2) + .forClusteredServers(CLUSTER2, MS3, MS4) + .shutdown(); - List serverShutdownInfos = - createServerShutdownInfosForCluster(CLUSTER, MS1, MS2); - serverShutdownInfos.addAll(createServerShutdownInfosForCluster(CLUSTER2, MS3, MS4)); - invokeStepWithServerShutdownInfos(serverShutdownInfos); - - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS2), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS1), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS4), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS3), is(Boolean.FALSE)); + assertThat(serverPodsBeingDeleted(), containsInAnyOrder(MS1, MS2, MS4)); } @Test public void maxClusterConcurrentShutdown_doesNotApplyToNonClusteredServers() { domain.getSpec().setMaxClusterConcurrentShutdown(1); - addWlsServers(MS3, MS4); domainPresenceInfo = createDomainPresenceInfoWithServers(MS3,MS4); - testSupport - .addDomainPresenceInfo(domainPresenceInfo); + testSupport.addDomainPresenceInfo(domainPresenceInfo); + + createShutdownInfos() + .forServers(MS3, MS4) + .shutdown(); + + assertThat(serverPodsBeingDeleted(), containsInAnyOrder(MS3, MS4)); + } + + private List serverPodsBeingDeleted() { + return domainPresenceInfo.getServerNames().stream() + .filter(s -> domainPresenceInfo.isServerPodBeingDeleted(s)).collect(Collectors.toList()); + } - invokeStepWithServerShutdownInfos(createServerShutdownInfos(MS3, MS4)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS4), is(Boolean.TRUE)); - MatcherAssert.assertThat( - domainPresenceInfo.isServerPodBeingDeleted(MS3), is(Boolean.TRUE)); + private ServerDownIteratorStepTest createShutdownInfos() { + this.serverShutdownInfos = new ArrayList<>(); + return this; } - @NotNull - private List createServerShutdownInfosForCluster(String clusterName, String... servers) { - List serverShutdownInfos = new ArrayList<>(); - Arrays.stream(servers).forEach(server -> - serverShutdownInfos.add( - new ServerShutdownInfo(configSupport.getWlsServer(clusterName, server).getName(), - clusterName) - ) - ); - return serverShutdownInfos; + private ServerDownIteratorStepTest forServers(String... servers) { + this.serverShutdownInfos.addAll(Arrays.stream(servers).map(this::createShutdownInfo).collect(Collectors.toList())); + return this; } - @NotNull - private List createServerShutdownInfos(String... servers) { - List serverShutdownInfos = new ArrayList<>(); - Arrays.stream(servers).forEach(server -> - serverShutdownInfos.add( - new ServerShutdownInfo(configSupport.getWlsServer(server).getName(), null) + private ServerDownIteratorStepTest forClusteredServers(String clusterName, String... servers) { + this.serverShutdownInfos.addAll(Arrays.stream(servers).map(s -> createShutdownInfo(clusterName, s)) + .collect(Collectors.toList())); + return this; + } - ) - ); - return serverShutdownInfos; + private void shutdown() { + testSupport.runSteps(new ServerDownIteratorStep(this.serverShutdownInfos, nextStep)); } - private void invokeStepWithServerShutdownInfos(List shutdownInfo) { - ServerDownIteratorStep step = new ServerDownIteratorStep(shutdownInfo, nextStep); - testSupport.runSteps(step); + private ServerShutdownInfo createShutdownInfo(String clusterName, String serverName) { + return new ServerShutdownInfo(configSupport.getWlsServer(clusterName, serverName).getName(), clusterName); + } + + @Nonnull + private ServerShutdownInfo createShutdownInfo(String server) { + return new ServerShutdownInfo(configSupport.getWlsServer(server).getName(), null); } private ClusterConfigurator configureCluster(String clusterName) { From 147e5dd66aaba6663c4862a47f9734b1b2753123 Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Thu, 3 Sep 2020 19:23:26 +0000 Subject: [PATCH 08/13] Changes to address PR review comments. --- docs/domains/Domain.json | 4 ++-- docs/domains/Domain.md | 4 ++-- docs/domains/index.html | 18 +++++++-------- kubernetes/crd/domain-crd.yaml | 22 +++++++++---------- kubernetes/crd/domain-v1beta1-crd.yaml | 22 +++++++++---------- .../weblogic/domain/model/DomainSpec.java | 9 ++------ .../steps/ServerDownIteratorStepTest.java | 21 +++++++++++------- .../weblogic/domain/DomainTestBase.java | 1 - 8 files changed, 48 insertions(+), 53 deletions(-) diff --git a/docs/domains/Domain.json b/docs/domains/Domain.json index afc70a1287b..5a0f73aa8e6 100644 --- a/docs/domains/Domain.json +++ b/docs/domains/Domain.json @@ -118,7 +118,7 @@ "$ref": "#/definitions/KubernetesResource" }, "maxConcurrentShutdown": { - "description": "The maximum number of Managed Servers instances that the operator will shutdown in parallel for this cluster in response to a change in the `replicas` count. If more Managed Server instances must be shutdown, the operator will wait until a Managed Server Pod is terminated before shutting down the next Managed Server instance. A value of 0 means all Managed Server instances will shutdown in parallel. Defaults to 1.", + "description": "The maximum WebLogic Server instances that will shutdown in parallel for this cluster when it is being partially shutdown by lowering its replica count.A value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown` (which in turn defaults to 1).", "type": "number", "minimum": 0 }, @@ -360,7 +360,7 @@ "minimum": 0 }, "maxClusterConcurrentShutdown": { - "description": "The maximum number of cluster member Managed Server instances that the operator will shutdown in parallel for a given cluster, if `maxConcurrentShutdown` is not specified for a specific cluster under the `clusters` field. A value of 0 means there is no configured limit. Defaults to 1.", + "description": "The default maximum WebLogic Server instances that a cluster will shutdown in parallel when it is being partially shutdown by lowering its replica count. You can override this default on a per cluster basis by setting the cluster\u0027s `maxConcurrentShutdown` attribute. A value of 0 means there is no limit. Defaults to 1", "type": "number", "minimum": 0 }, diff --git a/docs/domains/Domain.md b/docs/domains/Domain.md index 025e60d0a7e..194a293083c 100644 --- a/docs/domains/Domain.md +++ b/docs/domains/Domain.md @@ -34,7 +34,7 @@ The specification of the operation of the WebLogic domain. Required. | `logHome` | string | The directory in a server's container in which to store the domain, Node Manager, server logs, server *.out, introspector .out, and optionally HTTP access log files if `httpAccessLogInLogHome` is true. Ignored if `logHomeEnabled` is false. | | `logHomeEnabled` | Boolean | Specifies whether the log home folder is enabled. Defaults to true if `domainHomeSourceType` is PersistentVolume; false, otherwise. | | `managedServers` | array of [Managed Server](#managed-server) | Lifecycle options for individual Managed Servers, including Java options, environment variables, additional Pod content, and the ability to explicitly start, stop, or restart a named server instance. The `serverName` field of each entry must match a Managed Server that already exists in the WebLogic domain configuration or that matches a dynamic cluster member based on the server template. | -| `maxClusterConcurrentShutdown` | number | The maximum number of cluster member Managed Server instances that the operator will shutdown in parallel for a given cluster, if `maxConcurrentShutdown` is not specified for a specific cluster under the `clusters` field. A value of 0 means there is no configured limit. Defaults to 1. | +| `maxClusterConcurrentShutdown` | number | The default maximum WebLogic Server instances that a cluster will shutdown in parallel when it is being partially shutdown by lowering its replica count. You can override this default on a per cluster basis by setting the cluster's `maxConcurrentShutdown` attribute. A value of 0 means there is no limit. Defaults to 1 | | `maxClusterConcurrentStartup` | number | The maximum number of cluster member Managed Server instances that the operator will start in parallel for a given cluster, if `maxConcurrentStartup` is not specified for a specific cluster under the `clusters` field. A value of 0 means there is no configured limit. Defaults to 0. | | `replicas` | number | The default number of cluster member Managed Server instances to start for each WebLogic cluster in the domain configuration, unless `replicas` is specified for that cluster under the `clusters` field. For each cluster, the operator will sort cluster member Managed Server names from the WebLogic domain configuration by normalizing any numbers in the Managed Server name and then sorting alphabetically. This is done so that server names such as "managed-server10" come after "managed-server9". The operator will then start Managed Servers from the sorted list, up to the `replicas` count, unless specific Managed Servers are specified as starting in their entry under the `managedServers` field. In that case, the specified Managed Servers will be started and then additional cluster members will be started, up to the `replicas` count, by finding further cluster members in the sorted list that are not already started. If cluster members are started because of their entries under `managedServers`, then a cluster may have more cluster members running than its `replicas` count. Defaults to 0. | | `restartVersion` | string | Changes to this field cause the operator to restart WebLogic Server instances. More info: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#restarting-servers. | @@ -76,7 +76,7 @@ The current status of the operation of the WebLogic domain. Updated automaticall | `allowReplicasBelowMinDynClusterSize` | Boolean | Specifies whether the number of running cluster members is allowed to drop below the minimum dynamic cluster size configured in the WebLogic domain configuration. Otherwise, the operator will ensure that the number of running cluster members is not less than the minimum dynamic cluster setting. This setting applies to dynamic clusters only. Defaults to true. | | `clusterName` | string | The name of the cluster. This value must match the name of a WebLogic cluster already defined in the WebLogic domain configuration. Required. | | `clusterService` | [Kubernetes Resource](#kubernetes-resource) | Customization affecting Kubernetes Service generated for this WebLogic cluster. | -| `maxConcurrentShutdown` | number | The maximum number of Managed Servers instances that the operator will shutdown in parallel for this cluster in response to a change in the `replicas` count. If more Managed Server instances must be shutdown, the operator will wait until a Managed Server Pod is terminated before shutting down the next Managed Server instance. A value of 0 means all Managed Server instances will shutdown in parallel. Defaults to 1. | +| `maxConcurrentShutdown` | number | The maximum WebLogic Server instances that will shutdown in parallel for this cluster when it is being partially shutdown by lowering its replica count.A value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown` (which in turn defaults to 1). | | `maxConcurrentStartup` | number | The maximum number of Managed Servers instances that the operator will start in parallel for this cluster in response to a change in the `replicas` count. If more Managed Server instances must be started, the operator will wait until a Managed Server Pod is in the `Ready` state before starting the next Managed Server instance. A value of 0 means all Managed Server instances will start in parallel. Defaults to 0. | | `maxUnavailable` | number | The maximum number of cluster members that can be temporarily unavailable. Defaults to 1. | | `replicas` | number | The number of cluster member Managed Server instances to start for this WebLogic cluster. The operator will sort cluster member Managed Server names from the WebLogic domain configuration by normalizing any numbers in the Managed Server name and then sorting alphabetically. This is done so that server names such as "managed-server10" come after "managed-server9". The operator will then start Managed Server instances from the sorted list, up to the `replicas` count, unless specific Managed Servers are specified as starting in their entry under the `managedServers` field. In that case, the specified Managed Server instances will be started and then additional cluster members will be started, up to the `replicas` count, by finding further cluster members in the sorted list that are not already started. If cluster members are started because of their related entries under `managedServers`, then this cluster may have more cluster members running than its `replicas` count. Defaults to 0. | diff --git a/docs/domains/index.html b/docs/domains/index.html index bbb5efed5b9..e5483f092a4 100644 --- a/docs/domains/index.html +++ b/docs/domains/index.html @@ -1038,6 +1038,11 @@ "description": "Customization affecting Kubernetes Service generated for this WebLogic cluster.", "$ref": "#/definitions/KubernetesResource" }, + "maxConcurrentShutdown": { + "description": "The maximum WebLogic Server instances that will shutdown in parallel for this cluster when it is being partially shutdown by lowering its replica count.A value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown` (which in turn defaults to 1).", + "type": "number", + "minimum": 0.0 + }, "serverStartPolicy": { "description": "The strategy for deciding whether to start a WebLogic Server instance. Legal values are NEVER, or IF_NEEDED. Defaults to IF_NEEDED. More info: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#starting-and-stopping-servers.", "type": "string", @@ -1051,11 +1056,6 @@ "type": "number", "minimum": 0.0 }, - "maxConcurrentShutdown": { - "description": "The maximum number of Managed Servers instances that the operator will shutdown in parallel for this cluster in response to a change in the `replicas` count. If more Managed Server instances must be shutdown, the operator will wait until a Managed Server Pod is terminated before shutting down the next Managed Server instance. A value of 0 means all Managed Server instances will shutdown in parallel. Defaults to 1.", - "type": "number", - "minimum": 0.0 - }, "restartVersion": { "description": "Changes to this field cause the operator to restart WebLogic Server instances. More info: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#restarting-servers.", "type": "string" @@ -1275,13 +1275,13 @@ "type": "number", "minimum": 0.0 }, - "maxClusterConcurrentShutdown": { - "description": "The maximum number of cluster member Managed Server instances that the operator will shutdown in parallel for a given cluster, if `maxConcurrentShutdown` is not specified for a specific cluster under the `clusters` field. A value of 0 means there is no configured limit. Defaults to 1.", + "replicas": { + "description": "The default number of cluster member Managed Server instances to start for each WebLogic cluster in the domain configuration, unless `replicas` is specified for that cluster under the `clusters` field. For each cluster, the operator will sort cluster member Managed Server names from the WebLogic domain configuration by normalizing any numbers in the Managed Server name and then sorting alphabetically. This is done so that server names such as \"managed-server10\" come after \"managed-server9\". The operator will then start Managed Servers from the sorted list, up to the `replicas` count, unless specific Managed Servers are specified as starting in their entry under the `managedServers` field. In that case, the specified Managed Servers will be started and then additional cluster members will be started, up to the `replicas` count, by finding further cluster members in the sorted list that are not already started. If cluster members are started because of their entries under `managedServers`, then a cluster may have more cluster members running than its `replicas` count. Defaults to 0.", "type": "number", "minimum": 0.0 }, - "replicas": { - "description": "The default number of cluster member Managed Server instances to start for each WebLogic cluster in the domain configuration, unless `replicas` is specified for that cluster under the `clusters` field. For each cluster, the operator will sort cluster member Managed Server names from the WebLogic domain configuration by normalizing any numbers in the Managed Server name and then sorting alphabetically. This is done so that server names such as \"managed-server10\" come after \"managed-server9\". The operator will then start Managed Servers from the sorted list, up to the `replicas` count, unless specific Managed Servers are specified as starting in their entry under the `managedServers` field. In that case, the specified Managed Servers will be started and then additional cluster members will be started, up to the `replicas` count, by finding further cluster members in the sorted list that are not already started. If cluster members are started because of their entries under `managedServers`, then a cluster may have more cluster members running than its `replicas` count. Defaults to 0.", + "maxClusterConcurrentShutdown": { + "description": "The default maximum WebLogic Server instances that a cluster will shutdown in parallel when it is being partially shutdown by lowering its replica count. You can override this default on a per cluster basis by setting the cluster\u0027s `maxConcurrentShutdown` attribute. A value of 0 means there is no limit. Defaults to 1", "type": "number", "minimum": 0.0 }, diff --git a/kubernetes/crd/domain-crd.yaml b/kubernetes/crd/domain-crd.yaml index a24a8ecf2d5..9c3a8066a6d 100644 --- a/kubernetes/crd/domain-crd.yaml +++ b/kubernetes/crd/domain-crd.yaml @@ -5128,13 +5128,11 @@ spec: type: string type: object maxConcurrentShutdown: - description: The maximum number of Managed Servers instances - that the operator will shutdown in parallel for this cluster - in response to a change in the `replicas` count. If more Managed - Server instances must be shutdown, the operator will wait - until a Managed Server Pod is terminated before shutting down - the next Managed Server instance. A value of 0 means all Managed - Server instances will shutdown in parallel. Defaults to 1. + description: The maximum WebLogic Server instances that will + shutdown in parallel for this cluster when it is being partially + shutdown by lowering its replica count.A value of 0 means + there is no limit. Defaults to `spec.maxClusterConcurrentShutdown` + (which in turn defaults to 1). type: number minimum: 0.0 serverStartPolicy: @@ -5212,11 +5210,11 @@ spec: minimum: 0.0 maxClusterConcurrentShutdown: type: number - description: The maximum number of cluster member Managed Server instances - that the operator will shutdown in parallel for a given cluster, - if `maxConcurrentShutdown` is not specified for a specific cluster - under the `clusters` field. A value of 0 means there is no configured - limit. Defaults to 1. + description: The default maximum WebLogic Server instances that a + cluster will shutdown in parallel when it is being partially shutdown + by lowering its replica count. You can override this default on + a per cluster basis by setting the cluster's `maxConcurrentShutdown` + attribute. A value of 0 means there is no limit. Defaults to 1 minimum: 0.0 domainHomeInImage: type: boolean diff --git a/kubernetes/crd/domain-v1beta1-crd.yaml b/kubernetes/crd/domain-v1beta1-crd.yaml index 64a18e1876f..ac1f4c6f872 100644 --- a/kubernetes/crd/domain-v1beta1-crd.yaml +++ b/kubernetes/crd/domain-v1beta1-crd.yaml @@ -5115,13 +5115,11 @@ spec: type: string type: object maxConcurrentShutdown: - description: The maximum number of Managed Servers instances that - the operator will shutdown in parallel for this cluster in response - to a change in the `replicas` count. If more Managed Server - instances must be shutdown, the operator will wait until a Managed - Server Pod is terminated before shutting down the next Managed - Server instance. A value of 0 means all Managed Server instances - will shutdown in parallel. Defaults to 1. + description: The maximum WebLogic Server instances that will shutdown + in parallel for this cluster when it is being partially shutdown + by lowering its replica count.A value of 0 means there is no + limit. Defaults to `spec.maxClusterConcurrentShutdown` (which + in turn defaults to 1). type: number minimum: 0.0 serverStartPolicy: @@ -5197,11 +5195,11 @@ spec: minimum: 0.0 maxClusterConcurrentShutdown: type: number - description: The maximum number of cluster member Managed Server instances - that the operator will shutdown in parallel for a given cluster, if - `maxConcurrentShutdown` is not specified for a specific cluster under - the `clusters` field. A value of 0 means there is no configured limit. - Defaults to 1. + description: The default maximum WebLogic Server instances that a cluster + will shutdown in parallel when it is being partially shutdown by lowering + its replica count. You can override this default on a per cluster + basis by setting the cluster's `maxConcurrentShutdown` attribute. + A value of 0 means there is no limit. Defaults to 1 minimum: 0.0 domainHomeInImage: type: boolean diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java index 021214258b2..b1fdf1d070f 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java @@ -976,13 +976,8 @@ public void setMaxClusterConcurrentStartup(Integer maxClusterConcurrentStartup) } private int getMaxConcurrentShutdownFor(Cluster cluster) { - return hasMaxConcurrentShutdown(cluster) - ? cluster.getMaxConcurrentShutdown() - : getMaxClusterConcurrentShutdown(); - } - - private boolean hasMaxConcurrentShutdown(Cluster cluster) { - return cluster != null && cluster.getMaxConcurrentShutdown() != null; + return Optional.ofNullable(cluster).map(Cluster::getMaxConcurrentShutdown) + .orElse(getMaxClusterConcurrentShutdown()); } public void setMaxClusterConcurrentShutdown(Integer maxClusterConcurrentShutdown) { diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/ServerDownIteratorStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/ServerDownIteratorStepTest.java index bf96cbfb95f..45881ae9e85 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/steps/ServerDownIteratorStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/ServerDownIteratorStepTest.java @@ -59,6 +59,8 @@ public class ServerDownIteratorStepTest { private static final String MS2 = MS_PREFIX + "2"; private static final String MS3 = MS_PREFIX + "3"; private static final String MS4 = MS_PREFIX + "4"; + private static final String MS5 = MS_PREFIX + "5"; + private static final String MS6 = MS_PREFIX + "6"; private static final int MAX_SERVERS = 5; private static final int PORT = 8001; private static final String[] MANAGED_SERVER_NAMES = @@ -201,7 +203,7 @@ public void withConcurrencyOf0_clusteredServersShutdownConcurrently() { } @Test - public void withReplicaCountOf0AndConcurrencyOf1_clusteredServersShutdownConcurrently() { + public void whenClusterShutdown_concurrencySettingIsIgnored() { configureCluster(CLUSTER).withMaxConcurrentShutdown(1).withReplicas(0); addWlsCluster(CLUSTER, PORT, MS1, MS2); domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2); @@ -215,7 +217,7 @@ public void withReplicaCountOf0AndConcurrencyOf1_clusteredServersShutdownConcurr } @Test - public void withConcurrencyOf2AndReplicaCount1_3rdClusteredServerIsShutdownAfterPreviousPodTerminated() { + public void whenMaxConcurrentShutdownSet_limitNumberOfServersShuttingDownAtOnce() { configureCluster(CLUSTER).withMaxConcurrentShutdown(2).withReplicas(1); addWlsCluster(CLUSTER, PORT, MS1, MS2, MS3, MS4); domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2,MS3,MS4); @@ -231,17 +233,20 @@ public void withConcurrencyOf2AndReplicaCount1_3rdClusteredServerIsShutdownAfter } @Test - public void withConcurrencyOf2AndReplicaCount0_4clusteredServersShutdownConcurrently() { - configureCluster(CLUSTER).withMaxConcurrentShutdown(2).withReplicas(0); - addWlsCluster(CLUSTER, PORT, MS1, MS2, MS3, MS4); - domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2,MS3,MS4); + public void withMultipleClusters_concurrencySettingIsIgnoredForShuttingDownClusterAndHonoredForShrinkingCluster() { + configureCluster(CLUSTER).withMaxConcurrentShutdown(1).withReplicas(0); + configureCluster(CLUSTER2).withMaxConcurrentShutdown(1).withReplicas(1); + addWlsCluster(CLUSTER, PORT, MS1, MS2, MS3); + addWlsCluster(CLUSTER2, PORT, MS4, MS5, MS6); + domainPresenceInfo = createDomainPresenceInfoWithServers(MS1, MS2,MS3,MS4,MS5,MS6); testSupport.addDomainPresenceInfo(domainPresenceInfo); createShutdownInfos() - .forClusteredServers(CLUSTER,MS1, MS2, MS3, MS4) + .forClusteredServers(CLUSTER,MS1, MS2, MS3) + .forClusteredServers(CLUSTER2,MS4, MS5, MS6) .shutdown(); - assertThat(serverPodsBeingDeleted(), containsInAnyOrder(MS1, MS2, MS3, MS4)); + assertThat(serverPodsBeingDeleted(), containsInAnyOrder(MS1, MS2, MS3, MS6)); } @Test diff --git a/operator/src/test/java/oracle/kubernetes/weblogic/domain/DomainTestBase.java b/operator/src/test/java/oracle/kubernetes/weblogic/domain/DomainTestBase.java index 251072dfa21..c43eb3d40fb 100644 --- a/operator/src/test/java/oracle/kubernetes/weblogic/domain/DomainTestBase.java +++ b/operator/src/test/java/oracle/kubernetes/weblogic/domain/DomainTestBase.java @@ -419,7 +419,6 @@ public void whenNotSpecified_maxConcurrentShutdownHasDefault() { @Test public void whenNotSpecified_maxConcurrentShutdownFromDomain() { - configureCluster("cluster1"); configureDomain(domain).withMaxConcurrentShutdown(2); assertThat(domain.getMaxConcurrentShutdown("cluster1"), From fa9a802c9a4a8cfd08ab497c151b0f26c149f33f Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Fri, 4 Sep 2020 20:47:07 +0000 Subject: [PATCH 09/13] Changes to address PR review comments. --- docs/domains/Domain.json | 6 +++--- docs/domains/Domain.md | 6 +++--- .../kubernetes/weblogic/domain/model/Cluster.java | 10 ++++++---- .../kubernetes/weblogic/domain/model/DomainSpec.java | 12 +++++++----- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/domains/Domain.json b/docs/domains/Domain.json index 5a0f73aa8e6..edd6c3d10fc 100644 --- a/docs/domains/Domain.json +++ b/docs/domains/Domain.json @@ -118,7 +118,7 @@ "$ref": "#/definitions/KubernetesResource" }, "maxConcurrentShutdown": { - "description": "The maximum WebLogic Server instances that will shutdown in parallel for this cluster when it is being partially shutdown by lowering its replica count.A value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown` (which in turn defaults to 1).", + "description": "The maximum number of WebLogic Server instances that will shut down in parallel for this cluster when it is being partially shut down by lowering its replica count. A value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown`, which defaults to 1.", "type": "number", "minimum": 0 }, @@ -360,7 +360,7 @@ "minimum": 0 }, "maxClusterConcurrentShutdown": { - "description": "The default maximum WebLogic Server instances that a cluster will shutdown in parallel when it is being partially shutdown by lowering its replica count. You can override this default on a per cluster basis by setting the cluster\u0027s `maxConcurrentShutdown` attribute. A value of 0 means there is no limit. Defaults to 1", + "description": "The default maximum number of WebLogic Server instances that a cluster will shut down in parallel when it is being partially shut down by lowering its replica count. You can override this default on a per cluster basis by setting the cluster\u0027s `maxConcurrentShutdown` field. A value of 0 means there is no limit. Defaults to 1.", "type": "number", "minimum": 0 }, @@ -841,4 +841,4 @@ "metadata", "spec" ] -} \ No newline at end of file +} diff --git a/docs/domains/Domain.md b/docs/domains/Domain.md index 194a293083c..ac6a83bfc80 100644 --- a/docs/domains/Domain.md +++ b/docs/domains/Domain.md @@ -34,7 +34,7 @@ The specification of the operation of the WebLogic domain. Required. | `logHome` | string | The directory in a server's container in which to store the domain, Node Manager, server logs, server *.out, introspector .out, and optionally HTTP access log files if `httpAccessLogInLogHome` is true. Ignored if `logHomeEnabled` is false. | | `logHomeEnabled` | Boolean | Specifies whether the log home folder is enabled. Defaults to true if `domainHomeSourceType` is PersistentVolume; false, otherwise. | | `managedServers` | array of [Managed Server](#managed-server) | Lifecycle options for individual Managed Servers, including Java options, environment variables, additional Pod content, and the ability to explicitly start, stop, or restart a named server instance. The `serverName` field of each entry must match a Managed Server that already exists in the WebLogic domain configuration or that matches a dynamic cluster member based on the server template. | -| `maxClusterConcurrentShutdown` | number | The default maximum WebLogic Server instances that a cluster will shutdown in parallel when it is being partially shutdown by lowering its replica count. You can override this default on a per cluster basis by setting the cluster's `maxConcurrentShutdown` attribute. A value of 0 means there is no limit. Defaults to 1 | +| `maxClusterConcurrentShutdown` | number | The default maximum number of WebLogic Server instances that a cluster will shut down in parallel when it is being partially shut down by lowering its replica count. You can override this default on a per cluster basis by setting the cluster's `maxConcurrentShutdown` field. A value of 0 means there is no limit. Defaults to 1. | | `maxClusterConcurrentStartup` | number | The maximum number of cluster member Managed Server instances that the operator will start in parallel for a given cluster, if `maxConcurrentStartup` is not specified for a specific cluster under the `clusters` field. A value of 0 means there is no configured limit. Defaults to 0. | | `replicas` | number | The default number of cluster member Managed Server instances to start for each WebLogic cluster in the domain configuration, unless `replicas` is specified for that cluster under the `clusters` field. For each cluster, the operator will sort cluster member Managed Server names from the WebLogic domain configuration by normalizing any numbers in the Managed Server name and then sorting alphabetically. This is done so that server names such as "managed-server10" come after "managed-server9". The operator will then start Managed Servers from the sorted list, up to the `replicas` count, unless specific Managed Servers are specified as starting in their entry under the `managedServers` field. In that case, the specified Managed Servers will be started and then additional cluster members will be started, up to the `replicas` count, by finding further cluster members in the sorted list that are not already started. If cluster members are started because of their entries under `managedServers`, then a cluster may have more cluster members running than its `replicas` count. Defaults to 0. | | `restartVersion` | string | Changes to this field cause the operator to restart WebLogic Server instances. More info: https://oracle.github.io/weblogic-kubernetes-operator/userguide/managing-domains/domain-lifecycle/startup/#restarting-servers. | @@ -76,7 +76,7 @@ The current status of the operation of the WebLogic domain. Updated automaticall | `allowReplicasBelowMinDynClusterSize` | Boolean | Specifies whether the number of running cluster members is allowed to drop below the minimum dynamic cluster size configured in the WebLogic domain configuration. Otherwise, the operator will ensure that the number of running cluster members is not less than the minimum dynamic cluster setting. This setting applies to dynamic clusters only. Defaults to true. | | `clusterName` | string | The name of the cluster. This value must match the name of a WebLogic cluster already defined in the WebLogic domain configuration. Required. | | `clusterService` | [Kubernetes Resource](#kubernetes-resource) | Customization affecting Kubernetes Service generated for this WebLogic cluster. | -| `maxConcurrentShutdown` | number | The maximum WebLogic Server instances that will shutdown in parallel for this cluster when it is being partially shutdown by lowering its replica count.A value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown` (which in turn defaults to 1). | +| `maxConcurrentShutdown` | number | The maximum number of WebLogic Server instances that will shut down in parallel for this cluster when it is being partially shut down by lowering its replica count. A value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown`, which defaults to 1. | | `maxConcurrentStartup` | number | The maximum number of Managed Servers instances that the operator will start in parallel for this cluster in response to a change in the `replicas` count. If more Managed Server instances must be started, the operator will wait until a Managed Server Pod is in the `Ready` state before starting the next Managed Server instance. A value of 0 means all Managed Server instances will start in parallel. Defaults to 0. | | `maxUnavailable` | number | The maximum number of cluster members that can be temporarily unavailable. Defaults to 1. | | `replicas` | number | The number of cluster member Managed Server instances to start for this WebLogic cluster. The operator will sort cluster member Managed Server names from the WebLogic domain configuration by normalizing any numbers in the Managed Server name and then sorting alphabetically. This is done so that server names such as "managed-server10" come after "managed-server9". The operator will then start Managed Server instances from the sorted list, up to the `replicas` count, unless specific Managed Servers are specified as starting in their entry under the `managedServers` field. In that case, the specified Managed Server instances will be started and then additional cluster members will be started, up to the `replicas` count, by finding further cluster members in the sorted list that are not already started. If cluster members are started because of their related entries under `managedServers`, then this cluster may have more cluster members running than its `replicas` count. Defaults to 0. | @@ -253,4 +253,4 @@ The current status of the operation of the WebLogic domain. Updated automaticall | --- | --- | --- | | `health` | string | Server health of this WebLogic Server instance. | | `subsystemName` | string | Name of subsystem providing symptom information. | -| `symptoms` | array of string | Symptoms provided by the reporting subsystem. | \ No newline at end of file +| `symptoms` | array of string | Symptoms provided by the reporting subsystem. | diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Cluster.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Cluster.java index b408d3dc6b2..4f8e10e5f54 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Cluster.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/Cluster.java @@ -89,10 +89,10 @@ public class Cluster extends BaseConfiguration implements Comparable { private Integer maxConcurrentStartup; @Description( - "The maximum WebLogic Server instances that will shutdown in parallel " - + "for this cluster when it is being partially shutdown by lowering its replica count." - + "A value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown` " - + "(which in turn defaults to 1)." + "The maximum number of WebLogic Server instances that will shut down in parallel " + + "for this cluster when it is being partially shut down by lowering its replica count. " + + "A value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown`, " + + "which defaults to 1." ) @Range(minimum = 0) private Integer maxConcurrentShutdown; @@ -217,6 +217,8 @@ public String toString() { .append("clusterService", clusterService) .append("maxUnavailable", maxUnavailable) .append("allowReplicasBelowMinDynClusterSize", allowReplicasBelowMinDynClusterSize) + .append("maxConcurrentStartup", maxConcurrentStartup) + .append("maxConcurrentShutdown", maxConcurrentShutdown) .toString(); } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java index b1fdf1d070f..edf6324664a 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java @@ -210,10 +210,10 @@ public class DomainSpec extends BaseConfiguration { private Integer maxClusterConcurrentStartup; @Description( - "The default maximum WebLogic Server instances that a cluster will shutdown in parallel when it is being " - + "partially shutdown by lowering its replica count. You can override this default on a per cluster " - + "basis by setting the cluster's `maxConcurrentShutdown` attribute. A value of 0 means there is " - + "no limit. Defaults to 1" + "The default maximum number of WebLogic Server instances that a cluster will shut down in parallel when it " + + "is being partially shut down by lowering its replica count. You can override this default on a per " + + "cluster basis by setting the cluster's `maxConcurrentShutdown` field. A value of 0 means " + + "there is no limit. Defaults to 1." ) @Range(minimum = 0) private Integer maxClusterConcurrentShutdown; @@ -831,7 +831,9 @@ public String toString() { .append("logHomeEnabled", logHomeEnabled) .append("includeServerOutInPodLog", includeServerOutInPodLog) .append("configOverrides", configOverrides) - .append("configOverrideSecrets", configOverrideSecrets); + .append("configOverrideSecrets", configOverrideSecrets) + .append("maxClusterConcurrentStartup",maxClusterConcurrentStartup) + .append("maxClusterConcurrentShutdown",maxClusterConcurrentShutdown); return builder.toString(); } From 8f71eaf45a2b2d71b9782a1bba75c4dc27297087 Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Fri, 4 Sep 2020 20:57:36 +0000 Subject: [PATCH 10/13] Changes to address PR review comments --- docs/domains/Domain.json | 2 +- docs/domains/Domain.md | 2 +- docs/domains/index.html | 4 ++-- kubernetes/crd/domain-crd.yaml | 20 +++++++++---------- kubernetes/crd/domain-v1beta1-crd.yaml | 20 +++++++++---------- .../weblogic/domain/model/DomainSpec.java | 4 ++-- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/domains/Domain.json b/docs/domains/Domain.json index edd6c3d10fc..459d6356d78 100644 --- a/docs/domains/Domain.json +++ b/docs/domains/Domain.json @@ -841,4 +841,4 @@ "metadata", "spec" ] -} +} \ No newline at end of file diff --git a/docs/domains/Domain.md b/docs/domains/Domain.md index ac6a83bfc80..3189de22c46 100644 --- a/docs/domains/Domain.md +++ b/docs/domains/Domain.md @@ -253,4 +253,4 @@ The current status of the operation of the WebLogic domain. Updated automaticall | --- | --- | --- | | `health` | string | Server health of this WebLogic Server instance. | | `subsystemName` | string | Name of subsystem providing symptom information. | -| `symptoms` | array of string | Symptoms provided by the reporting subsystem. | +| `symptoms` | array of string | Symptoms provided by the reporting subsystem. | \ No newline at end of file diff --git a/docs/domains/index.html b/docs/domains/index.html index e5483f092a4..568f95c0c2f 100644 --- a/docs/domains/index.html +++ b/docs/domains/index.html @@ -1039,7 +1039,7 @@ "$ref": "#/definitions/KubernetesResource" }, "maxConcurrentShutdown": { - "description": "The maximum WebLogic Server instances that will shutdown in parallel for this cluster when it is being partially shutdown by lowering its replica count.A value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown` (which in turn defaults to 1).", + "description": "The maximum number of WebLogic Server instances that will shut down in parallel for this cluster when it is being partially shut down by lowering its replica count. A value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown`, which defaults to 1.", "type": "number", "minimum": 0.0 }, @@ -1281,7 +1281,7 @@ "minimum": 0.0 }, "maxClusterConcurrentShutdown": { - "description": "The default maximum WebLogic Server instances that a cluster will shutdown in parallel when it is being partially shutdown by lowering its replica count. You can override this default on a per cluster basis by setting the cluster\u0027s `maxConcurrentShutdown` attribute. A value of 0 means there is no limit. Defaults to 1", + "description": "The default maximum number of WebLogic Server instances that a cluster will shut down in parallel when it is being partially shut down by lowering its replica count. You can override this default on a per cluster basis by setting the cluster\u0027s `maxConcurrentShutdown` field. A value of 0 means there is no limit. Defaults to 1.", "type": "number", "minimum": 0.0 }, diff --git a/kubernetes/crd/domain-crd.yaml b/kubernetes/crd/domain-crd.yaml index 9c3a8066a6d..6ad595800e8 100644 --- a/kubernetes/crd/domain-crd.yaml +++ b/kubernetes/crd/domain-crd.yaml @@ -5128,11 +5128,11 @@ spec: type: string type: object maxConcurrentShutdown: - description: The maximum WebLogic Server instances that will - shutdown in parallel for this cluster when it is being partially - shutdown by lowering its replica count.A value of 0 means - there is no limit. Defaults to `spec.maxClusterConcurrentShutdown` - (which in turn defaults to 1). + description: The maximum number of WebLogic Server instances + that will shut down in parallel for this cluster when it is + being partially shut down by lowering its replica count. A + value of 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown`, + which defaults to 1. type: number minimum: 0.0 serverStartPolicy: @@ -5210,11 +5210,11 @@ spec: minimum: 0.0 maxClusterConcurrentShutdown: type: number - description: The default maximum WebLogic Server instances that a - cluster will shutdown in parallel when it is being partially shutdown - by lowering its replica count. You can override this default on - a per cluster basis by setting the cluster's `maxConcurrentShutdown` - attribute. A value of 0 means there is no limit. Defaults to 1 + description: The default maximum number of WebLogic Server instances + that a cluster will shut down in parallel when it is being partially + shut down by lowering its replica count. You can override this default + on a per cluster basis by setting the cluster's `maxConcurrentShutdown` + field. A value of 0 means there is no limit. Defaults to 1. minimum: 0.0 domainHomeInImage: type: boolean diff --git a/kubernetes/crd/domain-v1beta1-crd.yaml b/kubernetes/crd/domain-v1beta1-crd.yaml index ac1f4c6f872..7afb27c58f2 100644 --- a/kubernetes/crd/domain-v1beta1-crd.yaml +++ b/kubernetes/crd/domain-v1beta1-crd.yaml @@ -5115,11 +5115,11 @@ spec: type: string type: object maxConcurrentShutdown: - description: The maximum WebLogic Server instances that will shutdown - in parallel for this cluster when it is being partially shutdown - by lowering its replica count.A value of 0 means there is no - limit. Defaults to `spec.maxClusterConcurrentShutdown` (which - in turn defaults to 1). + description: The maximum number of WebLogic Server instances that + will shut down in parallel for this cluster when it is being + partially shut down by lowering its replica count. A value of + 0 means there is no limit. Defaults to `spec.maxClusterConcurrentShutdown`, + which defaults to 1. type: number minimum: 0.0 serverStartPolicy: @@ -5195,11 +5195,11 @@ spec: minimum: 0.0 maxClusterConcurrentShutdown: type: number - description: The default maximum WebLogic Server instances that a cluster - will shutdown in parallel when it is being partially shutdown by lowering - its replica count. You can override this default on a per cluster - basis by setting the cluster's `maxConcurrentShutdown` attribute. - A value of 0 means there is no limit. Defaults to 1 + description: The default maximum number of WebLogic Server instances + that a cluster will shut down in parallel when it is being partially + shut down by lowering its replica count. You can override this default + on a per cluster basis by setting the cluster's `maxConcurrentShutdown` + field. A value of 0 means there is no limit. Defaults to 1. minimum: 0.0 domainHomeInImage: type: boolean diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java index edf6324664a..8cdd7c0e3ee 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/DomainSpec.java @@ -211,8 +211,8 @@ public class DomainSpec extends BaseConfiguration { @Description( "The default maximum number of WebLogic Server instances that a cluster will shut down in parallel when it " - + "is being partially shut down by lowering its replica count. You can override this default on a per " - + "cluster basis by setting the cluster's `maxConcurrentShutdown` field. A value of 0 means " + + "is being partially shut down by lowering its replica count. You can override this default on a " + + "per cluster basis by setting the cluster's `maxConcurrentShutdown` field. A value of 0 means " + "there is no limit. Defaults to 1." ) @Range(minimum = 0) From cfb2a3258acef8d01c5a8f3240f7928197f988a5 Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Fri, 4 Sep 2020 22:27:16 +0000 Subject: [PATCH 11/13] Resolve merge conflict --- .../kubernetes/operator/steps/ManagedServersUpStep.java | 7 ++++++- .../operator/steps/ManagedServersUpStepTest.java | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java index e56001fd528..8b8240db96a 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java @@ -33,6 +33,9 @@ import oracle.kubernetes.weblogic.domain.model.Domain; import oracle.kubernetes.weblogic.domain.model.ServerSpec; +import static oracle.kubernetes.operator.DomainStatusUpdater.MANAGED_SERVERS_STARTING_PROGRESS_REASON; +import static oracle.kubernetes.operator.DomainStatusUpdater.createProgressingStep; + public class ManagedServersUpStep extends Step { static final String SERVERS_UP_MSG = "Running servers for domain with UID: {0}, running list: {1}"; @@ -65,7 +68,9 @@ private static Step scaleDownIfNecessary( List serversToStop = getServersToStop(info, factory.shutdownInfos); if (!serversToStop.isEmpty()) { - insert(steps, new ServerDownIteratorStep(factory.shutdownInfos, null)); + insert(steps, + Step.chain(createProgressingStep(info, MANAGED_SERVERS_STARTING_PROGRESS_REASON, true, + null), new ServerDownIteratorStep(factory.shutdownInfos, null))); } return Step.chain(steps.toArray(new Step[0])); diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java index 532754718cf..b07ce52fdbe 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java @@ -487,7 +487,7 @@ public void whenNotShuttingDown_dontInsertCreateAvailableStep() { public void whenShuttingDownAtLeastOneServer_prependServerDownIteratorStep() { addServer(domainPresenceInfo, "server1"); - assertThat(createNextStep(), instanceOf(ServerDownIteratorStep.class)); + assertThat(createNextStep().getNext(), instanceOf(ServerDownIteratorStep.class)); } @Test @@ -600,7 +600,7 @@ public void whenDomainToplogyIsMissing_noExceptionAndDontStartServers() { } private void assertStoppingServers(Step step, String... servers) { - assertThat(((ServerDownIteratorStep) step).getServersToStop(), containsInAnyOrder(servers)); + assertThat(((ServerDownIteratorStep) step.getNext()).getServersToStop(), containsInAnyOrder(servers)); } private Step createNextStepWithout(String... serverNames) { From 1da2459d657f3d8e9806dbdf325f19cf75e03aed Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Fri, 4 Sep 2020 22:33:38 +0000 Subject: [PATCH 12/13] Enable unit tests --- .../kubernetes/operator/helpers/IntrospectionStatusTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectionStatusTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectionStatusTest.java index bbfe5aef481..936b1d9b64d 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectionStatusTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/IntrospectionStatusTest.java @@ -28,7 +28,6 @@ import org.apache.commons.lang.RandomStringUtils; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import static oracle.kubernetes.operator.DomainProcessorTestSetup.NS; @@ -100,7 +99,6 @@ public void whenNewIntrospectorJobPodStatusNull_ignoreIt() { } @Test - @Ignore("Anil! - you need to fix this one") public void whenNewIntrospectorJobPodCreatedWithErrImagePullStatus_patchDomain() { processor.dispatchPodWatch( WatchEvent.createAddedEvent( @@ -125,7 +123,6 @@ public void whenNewIntrospectorJobPodCreatedWithNullMessage_ignoreIt() { } @Test - @Ignore("Anil! - you need to fix this one") public void whenNewIntrospectorJobPodCreatedWithImagePullBackupStatus_patchDomain() { processor.dispatchPodWatch( WatchEvent.createAddedEvent( From 7bc236c8dccf0bd9dfbf50c8a17b476ef71cdce0 Mon Sep 17 00:00:00 2001 From: "ANIL.KEDIA@ORACLE.COM" Date: Fri, 4 Sep 2020 22:53:36 +0000 Subject: [PATCH 13/13] Use Dongbo's change in unit test --- .../operator/DomainStatusUpdater.java | 2 +- .../steps/ManagedServersUpStepTest.java | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java b/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java index c3599e41d1f..a3ce154a9a5 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java +++ b/operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java @@ -591,7 +591,7 @@ private Integer getClusterSizeGoal(String clusterName) { } } - private static class ProgressingStep extends DomainStatusUpdaterStep { + public static class ProgressingStep extends DomainStatusUpdaterStep { private final String reason; private final boolean isPreserveAvailable; diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java index b07ce52fdbe..3dcb43b13af 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServersUpStepTest.java @@ -17,6 +17,7 @@ import io.kubernetes.client.openapi.models.V1EnvVar; import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.openapi.models.V1Pod; +import oracle.kubernetes.operator.DomainStatusUpdater; import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.ProcessingConstants; import oracle.kubernetes.operator.helpers.DomainPresenceInfo; @@ -487,7 +488,7 @@ public void whenNotShuttingDown_dontInsertCreateAvailableStep() { public void whenShuttingDownAtLeastOneServer_prependServerDownIteratorStep() { addServer(domainPresenceInfo, "server1"); - assertThat(createNextStep().getNext(), instanceOf(ServerDownIteratorStep.class)); + assertThat(skipProgressingStep(createNextStep()), instanceOf(ServerDownIteratorStep.class)); } @Test @@ -497,7 +498,8 @@ public void whenExclusionsSpecified_doNotAddToListOfServers() { addServer(domainPresenceInfo, "server3"); addServer(domainPresenceInfo, ADMIN); - assertStoppingServers(createNextStepWithout("server2"), "server1", "server3"); + assertStoppingServers(skipProgressingStep(createNextStepWithout("server2")), + "server1", "server3"); } @Test @@ -509,7 +511,8 @@ public void whenShuttingDown_allowAdminServerNameInListOfServers() { addServer(domainPresenceInfo, "server3"); addServer(domainPresenceInfo, ADMIN); - assertStoppingServers(createNextStepWithout("server2"), "server1", "server3", ADMIN); + assertStoppingServers(skipProgressingStep(createNextStepWithout("server2")), "server1", + "server3", ADMIN); } @Test @@ -599,8 +602,15 @@ public void whenDomainToplogyIsMissing_noExceptionAndDontStartServers() { assertServersWillNotBeStarted(); } + private static Step skipProgressingStep(Step step) { + if (step instanceof DomainStatusUpdater.ProgressingStep) { + return step.getNext(); + } + return step; + } + private void assertStoppingServers(Step step, String... servers) { - assertThat(((ServerDownIteratorStep) step.getNext()).getServersToStop(), containsInAnyOrder(servers)); + assertThat(((ServerDownIteratorStep) step).getServersToStop(), containsInAnyOrder(servers)); } private Step createNextStepWithout(String... serverNames) {