diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 7442b5bccb9..9423df6011d 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -34,6 +34,7 @@ import oracle.kubernetes.weblogic.domain.model.ServerSpec; import oracle.kubernetes.weblogic.domain.model.Shutdown; +import static oracle.kubernetes.operator.LabelConstants.CLUSTERNAME_LABEL; import static oracle.kubernetes.operator.ProcessingConstants.SERVERS_TO_ROLL; public class PodHelper { @@ -85,6 +86,63 @@ public static boolean isReady(V1Pod pod) { return ready; } + /** + * Get list of scheduled pods for a particular cluster or non-clustered servers. + * @param info Domain presence info + * @param clusterName cluster name of the pod server + * @return list containing scheduled pods + */ + public static List getScheduledPods(DomainPresenceInfo info, String clusterName) { + // These are presently scheduled servers + List scheduledServers = new ArrayList<>(); + for (Map.Entry entry : info.getServers().entrySet()) { + V1Pod pod = entry.getValue().getPod().get(); + if (pod != null && !PodHelper.isDeleting(pod) && PodHelper.getScheduledStatus(pod)) { + String wlsClusterName = pod.getMetadata().getLabels().get(CLUSTERNAME_LABEL); + if ((wlsClusterName == null) || (wlsClusterName.contains(clusterName))) { + scheduledServers.add(entry.getKey()); + } + } + } + return scheduledServers; + } + + /** + * Get list of ready pods for a particular cluster or non-clustered servers. + * @param info Domain presence info + * @param clusterName cluster name of the pod server + * @return list containing ready pods + */ + public static List getReadyPods(DomainPresenceInfo info, String clusterName) { + // These are presently Ready servers + List readyServers = new ArrayList<>(); + for (Map.Entry entry : info.getServers().entrySet()) { + V1Pod pod = entry.getValue().getPod().get(); + if (pod != null && !PodHelper.isDeleting(pod) && PodHelper.getReadyStatus(pod)) { + String wlsClusterName = pod.getMetadata().getLabels().get(CLUSTERNAME_LABEL); + if ((wlsClusterName == null) || (wlsClusterName.contains(clusterName))) { + readyServers.add(entry.getKey()); + } + } + } + return readyServers; + } + + /** + * get if pod is in scheduled state. + * @param pod pod + * @return true, if pod is scheduled + */ + public static boolean getScheduledStatus(V1Pod pod) { + V1PodSpec spec = pod.getSpec(); + if (spec != null) { + if (spec.getNodeName() != null) { + return true; + } + } + return false; + } + /** * get if pod is in ready state. * @param pod pod @@ -444,7 +502,7 @@ protected String getPodReplacedMessageKey() { protected V1ObjectMeta createMetadata() { V1ObjectMeta metadata = super.createMetadata(); if (getClusterName() != null) { - metadata.putLabelsItem(LabelConstants.CLUSTERNAME_LABEL, getClusterName()); + metadata.putLabelsItem(CLUSTERNAME_LABEL, getClusterName()); } return metadata; } @@ -502,7 +560,7 @@ public NextAction apply(Packet packet) { if (oldPod != null) { Map labels = oldPod.getMetadata().getLabels(); if (labels != null) { - clusterName = labels.get(LabelConstants.CLUSTERNAME_LABEL); + clusterName = labels.get(CLUSTERNAME_LABEL); } ServerSpec serverSpec = info.getDomain().getServer(serverName, clusterName); diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java index cabacd161f7..ef2bbab27dc 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java @@ -4,16 +4,18 @@ package oracle.kubernetes.operator.steps; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import java.util.stream.IntStream; import oracle.kubernetes.operator.DomainStatusUpdater; import oracle.kubernetes.operator.ProcessingConstants; @@ -43,9 +45,6 @@ public class ManagedServerUpIteratorStep extends Step { private final Collection startupInfos; - private static NextStepFactory NEXT_STEP_FACTORY = - (next) -> DomainStatusUpdater.createStatusUpdateStep(next); - public ManagedServerUpIteratorStep(Collection startupInfos, Step next) { super(next); this.startupInfos = startupInfos; @@ -89,12 +88,26 @@ public NextAction apply(Packet packet) { .filter(ssi -> !isServerInCluster(ssi)) .map(ssi -> createManagedServerUpDetails(packet, ssi)).collect(Collectors.toList()); - getStartClusteredServersStepFactories(startupInfos, packet).values() - .forEach(factory -> startDetails.addAll(factory.getServerStartsStepAndPackets())); + Collection work = new ArrayList<>(); + if (!startDetails.isEmpty()) { + work.add( + new StepAndPacket( + new StartManagedServersStep(null, 0, startDetails, null), packet)); + } + + for (Map.Entry entry + : getStartClusteredServersStepFactories(startupInfos, packet).entrySet()) { + work.add( + new StepAndPacket( + new StartManagedServersStep(entry.getKey(), entry.getValue().getMaxConcurrency(), + entry.getValue().getServerStartsStepAndPackets(), null), packet.clone())); + } + + if (!work.isEmpty()) { + return doForkJoin(DomainStatusUpdater.createStatusUpdateStep(getNext()), packet, work); + } - return doNext( - NEXT_STEP_FACTORY.createStatusUpdateStep(new StartManagedServersStep(startDetails, getNext())), - packet); + return doNext(DomainStatusUpdater.createStatusUpdateStep(getNext()), packet); } @@ -142,19 +155,52 @@ private boolean isServerInCluster(ServerStartupInfo ssi) { static class StartManagedServersStep extends Step { final Collection startDetails; + final Queue startDetailsQueue = new ConcurrentLinkedQueue<>(); + final String clusterName; + final int maxConcurrency; + final AtomicInteger numStarted = new AtomicInteger(0); - StartManagedServersStep(Collection startDetails, Step next) { + StartManagedServersStep(String clusterName, int maxConcurrency, Collection startDetails, Step next) { super(next); + this.clusterName = clusterName; this.startDetails = startDetails; + this.maxConcurrency = maxConcurrency; + startDetails.forEach(this::add); } - Collection getStartDetails() { - return startDetails; + void add(StepAndPacket serverToStart) { + startDetailsQueue.add(new StepAndPacket(serverToStart.step, serverToStart.packet)); } @Override public NextAction apply(Packet packet) { - return doForkJoin(new ManagedServerUpAfterStep(getNext()), packet, startDetails); + + if (startDetailsQueue.isEmpty()) { + return doNext(new ManagedServerUpAfterStep(getNext()), packet); + } else if (isServiceOnlyOrShuttingDown()) { + Collection servers = Collections.singletonList(startDetailsQueue.poll()); + return doForkJoin(this, packet, servers); + } else if (serverAvailableToStart(packet.getSpi(DomainPresenceInfo.class))) { + numStarted.getAndIncrement(); + return doForkJoin(this, packet, Collections.singletonList(startDetailsQueue.poll())); + } else { + return doDelay(this, packet, 100, TimeUnit.MILLISECONDS); + } + } + + private boolean isServiceOnlyOrShuttingDown() { + return Optional.ofNullable(startDetailsQueue.peek().step) + .map(step -> step.getNext() instanceof ServerDownStep).orElse(false); + } + + private boolean serverAvailableToStart(DomainPresenceInfo info) { + return ((numStarted.get() < PodHelper.getScheduledPods(info, clusterName).size()) + && (canStartConcurrently(PodHelper.getReadyPods(info, clusterName).size()))); + } + + private boolean canStartConcurrently(int numReady) { + return ((this.maxConcurrency > 0) && (numStarted.get() < (this.maxConcurrency + numReady - 1))) + || (this.maxConcurrency == 0); } } @@ -171,58 +217,17 @@ private static class StartClusteredServersStepFactory { this.maxConcurrency = maxConcurrency; } + public int getMaxConcurrency() { + return this.maxConcurrency; + } + void add(StepAndPacket serverToStart) { serversToStart.add(serverToStart); } Collection getServerStartsStepAndPackets() { - if (maxConcurrency == 0 || serversToStart.size() <= maxConcurrency) { - return serversToStart; - } - ArrayList steps = new ArrayList<>(maxConcurrency); - IntStream.range(0, maxConcurrency) - .forEach(i -> steps.add(StartClusteredServersStep.createStepAndPacket(serversToStart))); - return steps; - } - - } - - static class StartClusteredServersStep extends Step { - - private final Queue serversToStart; - - static StepAndPacket createStepAndPacket(Queue serversToStart) { - return new StepAndPacket(new StartClusteredServersStep(serversToStart), null); - } - - StartClusteredServersStep(Queue serversToStart) { - super(null); - this.serversToStart = serversToStart; - serversToStart.forEach(stepAndPacket -> setupSequentialStartPacket(stepAndPacket.packet)); - } - - Collection getServersToStart() { return serversToStart; } - - private void setupSequentialStartPacket(Packet packet) { - packet.put(ProcessingConstants.WAIT_FOR_POD_READY, true); - } - - @Override - public NextAction apply(Packet packet) { - Collection servers = Arrays.asList(serversToStart.poll()); - if (servers.isEmpty()) { - return doNext(packet); - } else { - return doForkJoin(this, packet, servers); - } - } - } - - // an interface to provide a hook for unit testing. - interface NextStepFactory { - Step createStatusUpdateStep(Step next); } } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/BaseConfiguration.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/BaseConfiguration.java index 94b8f2ca9ce..fce9fa872c5 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/BaseConfiguration.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/BaseConfiguration.java @@ -217,7 +217,7 @@ public String getServiceAccountName() { return serverPod.getServiceAccountName(); } - void setNodeName(String nodeName) { + public void setNodeName(String nodeName) { serverPod.setNodeName(nodeName); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTestSetup.java b/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTestSetup.java index 9a00221c80b..1605a2912a8 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTestSetup.java +++ b/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTestSetup.java @@ -32,6 +32,7 @@ public class DomainProcessorTestSetup { public static final String NS = "namespace"; public static final String SECRET_NAME = "secret-name"; public static final String KUBERNETES_UID = "12345"; + public static final String NODE_NAME = "Node1"; private static final String INTROSPECTION_JOB = LegalNames.toJobIntrospectorName(UID); private static final String INTROSPECT_RESULT = @@ -88,13 +89,14 @@ private static V1ObjectMeta withTimestamps(V1ObjectMeta meta) { * @return a domain */ public static Domain createTestDomain() { + DomainSpec ds = new DomainSpec() + .withWebLogicCredentialsSecret(new V1SecretReference().name(SECRET_NAME).namespace(NS)); + ds.setNodeName(NODE_NAME); return new Domain() .withApiVersion(KubernetesConstants.DOMAIN_GROUP + "/" + KubernetesConstants.DOMAIN_VERSION) .withKind(KubernetesConstants.DOMAIN) .withMetadata(withTimestamps(new V1ObjectMeta().name(UID).namespace(NS).uid(KUBERNETES_UID))) - .withSpec( - new DomainSpec() - .withWebLogicCredentialsSecret(new V1SecretReference().name(SECRET_NAME).namespace(NS))); + .withSpec(ds); } /** diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStepTest.java index 04deb857520..7c6fc54d324 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStepTest.java @@ -6,22 +6,33 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; +import javax.annotation.Nonnull; import com.meterware.simplestub.Memento; -import com.meterware.simplestub.StaticStubSupport; +import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.kubernetes.client.openapi.models.V1Pod; +import io.kubernetes.client.openapi.models.V1PodCondition; +import io.kubernetes.client.openapi.models.V1PodSpec; +import io.kubernetes.client.openapi.models.V1PodStatus; +import io.kubernetes.client.openapi.models.V1SecretReference; +import oracle.kubernetes.operator.KubernetesConstants; +import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.ProcessingConstants; import oracle.kubernetes.operator.helpers.DomainPresenceInfo; import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerStartupInfo; -import oracle.kubernetes.operator.steps.ManagedServerUpIteratorStep.StartClusteredServersStep; -import oracle.kubernetes.operator.steps.ManagedServerUpIteratorStep.StartManagedServersStep; +import oracle.kubernetes.operator.helpers.KubernetesTestSupport; +import oracle.kubernetes.operator.helpers.LegalNames; +import oracle.kubernetes.operator.helpers.TuningParametersStub; import oracle.kubernetes.operator.utils.WlsDomainConfigSupport; -import oracle.kubernetes.operator.work.FiberTestSupport; +import oracle.kubernetes.operator.wlsconfig.WlsClusterConfig; +import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig; +import oracle.kubernetes.operator.wlsconfig.WlsServerConfig; import oracle.kubernetes.operator.work.Step; -import oracle.kubernetes.operator.work.Step.StepAndPacket; import oracle.kubernetes.operator.work.TerminalStep; import oracle.kubernetes.utils.TestUtils; import oracle.kubernetes.weblogic.domain.ClusterConfigurator; @@ -34,43 +45,103 @@ import org.junit.Before; import org.junit.Test; -import static oracle.kubernetes.operator.steps.ManagedServerUpIteratorStepTest.TestStepFactory.getServers; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.junit.MatcherAssert.assertThat; public class ManagedServerUpIteratorStepTest { - private static final String DOMAIN = "domain"; + protected static final String DOMAIN_NAME = "domain1"; private static final String NS = "namespace"; private static final String UID = "uid1"; + protected static final String KUBERNETES_UID = "12345"; private static final String ADMIN = "asName"; private static final String CLUSTER = "cluster1"; + private static final boolean INCLUDE_SERVER_OUT_IN_POD_LOG = true; + private static final String CREDENTIALS_SECRET_NAME = "webLogicCredentialsSecretName"; + private static final String LATEST_IMAGE = "image:latest"; + private static final String MS_PREFIX = "ms"; + private static final String MS1 = MS_PREFIX + "1"; + private static final String MS2 = MS_PREFIX + "2"; + private static final String MS3 = MS_PREFIX + "3"; + private static final String MS4 = MS_PREFIX + "4"; + private static final int MAX_SERVERS = 5; + private static final int PORT = 8001; + private static final String[] MANAGED_SERVER_NAMES = + IntStream.rangeClosed(1, MAX_SERVERS) + .mapToObj(ManagedServerUpIteratorStepTest::getManagedServerName).toArray(String[]::new); + + @Nonnull + private static String getManagedServerName(int n) { + return MS_PREFIX + n; + } + private final Domain domain = createDomain(); private final DomainConfigurator configurator = DomainConfiguratorFactory.forDomain(domain); - private WlsDomainConfigSupport configSupport = new WlsDomainConfigSupport(DOMAIN); - - private Step nextStep = new TerminalStep(); - private FiberTestSupport testSupport = new FiberTestSupport(); - private List mementos = new ArrayList<>(); - private DomainPresenceInfo domainPresenceInfo = createDomainPresenceInfo(); - private TestUtils.ConsoleHandlerMemento consoleHandlerMemento; + private final WlsDomainConfigSupport configSupport = new WlsDomainConfigSupport(DOMAIN_NAME); + + private final Step nextStep = new TerminalStep(); + private final KubernetesTestSupport testSupport = new KubernetesTestSupport(); + private final List mementos = new ArrayList<>(); + private final DomainPresenceInfo domainPresenceInfo = createDomainPresenceInfoWithServers(); + private final WlsDomainConfig domainConfig = createDomainConfig(); + + private static WlsDomainConfig createDomainConfig() { + WlsClusterConfig clusterConfig = new WlsClusterConfig(CLUSTER); + for (String serverName : MANAGED_SERVER_NAMES) { + clusterConfig.addServerConfig(new WlsServerConfig(serverName, "domain1-" + serverName, 8001)); + } + return new WlsDomainConfig("base_domain") + .withAdminServer(ADMIN, "domain1-admin-server", 7001) + .withCluster(clusterConfig); + } - private DomainPresenceInfo createDomainPresenceInfo() { - return new DomainPresenceInfo(domain); + private DomainPresenceInfo createDomainPresenceInfoWithServers(String... serverNames) { + DomainPresenceInfo dpi = new DomainPresenceInfo(domain); + addServer(dpi, ADMIN); + Arrays.asList(serverNames).forEach(serverName -> addServer(dpi, serverName)); + return dpi; } private Domain createDomain() { - return new Domain().withMetadata(createMetaData()).withSpec(createDomainSpec()); + return new Domain() + .withApiVersion(KubernetesConstants.DOMAIN_VERSION) + .withKind(KubernetesConstants.DOMAIN) + .withMetadata(new V1ObjectMeta().namespace(NS).name(DOMAIN_NAME).uid(KUBERNETES_UID)) + .withSpec(createDomainSpec()); } - private V1ObjectMeta createMetaData() { - return new V1ObjectMeta().namespace(NS); + private DomainSpec createDomainSpec() { + return new DomainSpec() + .withDomainUid(UID) + .withWebLogicCredentialsSecret(new V1SecretReference().name(CREDENTIALS_SECRET_NAME)) + .withIncludeServerOutInPodLog(INCLUDE_SERVER_OUT_IN_POD_LOG) + .withImage(LATEST_IMAGE); } - private DomainSpec createDomainSpec() { - return new DomainSpec().withDomainUid(UID).withReplicas(1); + private static void addServer(DomainPresenceInfo domainPresenceInfo, String serverName) { + if (serverName.equals(ADMIN)) { + domainPresenceInfo.setServerPod(serverName, createReadyPod(serverName)); + } else { + domainPresenceInfo.setServerPod(serverName, createPod(serverName)); + } + } + + private static V1Pod createReadyPod(String serverName) { + return new V1Pod().metadata(withNames(new V1ObjectMeta().namespace(NS), serverName)) + .spec(new V1PodSpec().nodeName("Node1")) + .status(new V1PodStatus().phase("Running") + .addConditionsItem(new V1PodCondition().type("Ready").status("True"))); + } + + private static V1Pod createPod(String serverName) { + return new V1Pod().metadata(withNames(new V1ObjectMeta().namespace(NS), serverName)); + } + + private static V1ObjectMeta withNames(V1ObjectMeta objectMeta, String serverName) { + return objectMeta + .name(LegalNames.toPodName(UID, serverName)) + .putLabelsItem(LabelConstants.SERVERNAME_LABEL, serverName); } /** @@ -79,9 +150,14 @@ private DomainSpec createDomainSpec() { */ @Before public void setUp() throws NoSuchFieldException { - mementos.add(consoleHandlerMemento = TestUtils.silenceOperatorLogger()); - mementos.add(TestStepFactory.install()); - testSupport.addDomainPresenceInfo(domainPresenceInfo); + mementos.add(TestUtils.silenceOperatorLogger().ignoringLoggedExceptions(ApiException.class)); + mementos.add(TuningParametersStub.install()); + mementos.add(testSupport.install()); + + testSupport.defineResources(domain); + testSupport + .addToPacket(ProcessingConstants.DOMAIN_TOPOLOGY, domainConfig) + .addDomainPresenceInfo(domainPresenceInfo); } /** @@ -97,80 +173,130 @@ public void tearDown() throws Exception { testSupport.throwOnCompletionFailure(); } - @Test - public void withConcurrencyOf1_bothClusteredServersStartSequentially() { - configureCluster(CLUSTER).withMaxConcurrentStartup(1); - addWlsCluster(CLUSTER, "ms1", "ms2"); + private void makePodReady(String serverName) { + domainPresenceInfo.getServerPod(serverName).status(new V1PodStatus().phase("Running")); + Objects.requireNonNull(domainPresenceInfo.getServerPod(serverName).getStatus()) + .addConditionsItem(new V1PodCondition().status("True").type("Ready")); + } - invokeStepWithServerStartupInfos(createServerStartupInfosForCluster(CLUSTER,"ms1", "ms2")); + private void schedulePod(String serverName, String nodeName) { + Objects.requireNonNull(domainPresenceInfo.getServerPod(serverName).getSpec()).setNodeName(nodeName); + } - assertThat(getServers(), hasItem(Arrays.asList("ms1", "ms2"))); - assertThat(getServers().size(), equalTo(1)); + @Test + public void withConcurrencyOf1_bothClusteredServersScheduleAndStartSequentially() { + configureCluster(CLUSTER).withMaxConcurrentStartup(1); + //addWlsCluster(CLUSTER, 8001, MS1, MS2); + addWlsCluster(CLUSTER, 8001, MS1, MS2); + + invokeStepWithServerStartupInfos(createServerStartupInfosForCluster(CLUSTER,MS1, MS2)); + + assertThat(MS1 + " pod", domainPresenceInfo.getServerPod(MS1), notNullValue()); + schedulePod(MS1, "Node1"); + testSupport.setTime(100, TimeUnit.MILLISECONDS); + assertThat(MS2 + " pod", domainPresenceInfo.getServerPod(MS2), nullValue()); + makePodReady(MS1); + testSupport.setTime(10, TimeUnit.SECONDS); + assertThat(MS2 + " pod", domainPresenceInfo.getServerPod(MS2), notNullValue()); } @Test - public void withConcurrencyOf0_bothClusteredServersStartConcurrently() { + public void withConcurrencyOf0_clusteredServersScheduleSequentiallyAndStartConcurrently() { configureCluster(CLUSTER).withMaxConcurrentStartup(0); - addWlsCluster(CLUSTER, "ms1", "ms2"); + addWlsCluster(CLUSTER, PORT, MS1, MS2); - invokeStepWithServerStartupInfos(createServerStartupInfosForCluster(CLUSTER,"ms1", "ms2")); + invokeStepWithServerStartupInfos(createServerStartupInfosForCluster(CLUSTER,MS1, MS2)); - assertThat(getServers(), allOf(hasItem("ms1"), hasItem("ms2"))); + assertThat(MS1 + " pod", domainPresenceInfo.getServerPod(MS1), notNullValue()); + assertThat(MS2 + " pod", domainPresenceInfo.getServerPod(MS2), nullValue()); + schedulePod(MS1, "Node1"); + testSupport.setTime(100, TimeUnit.MILLISECONDS); + assertThat(MS2 + " pod", domainPresenceInfo.getServerPod(MS2), notNullValue()); } @Test - public void withConcurrencyOf2_bothClusteredServersStartConcurrently() { + public void withConcurrencyOf2_clusteredServersScheduleSequentiallyAndStartConcurrently() { configureCluster(CLUSTER).withMaxConcurrentStartup(2); - addWlsCluster(CLUSTER, "ms1", "ms2"); + addWlsCluster(CLUSTER, PORT, MS1, MS2); - invokeStepWithServerStartupInfos(createServerStartupInfosForCluster(CLUSTER, "ms1", "ms2")); + invokeStepWithServerStartupInfos(createServerStartupInfosForCluster(CLUSTER, MS1, MS2)); - assertThat(getServers(), allOf(hasItem("ms1"), hasItem("ms2"))); + assertThat(MS1 + " pod", domainPresenceInfo.getServerPod(MS1), notNullValue()); + assertThat(MS2 + " pod", domainPresenceInfo.getServerPod(MS2), nullValue()); + schedulePod(MS1, "Node1"); + testSupport.setTime(100, TimeUnit.MILLISECONDS); + assertThat(MS2 + " pod", domainPresenceInfo.getServerPod(MS2), notNullValue()); } @Test - public void withConcurrencyOf2_4clusteredServersStartIn2Threads() { + public void withConcurrencyOf2_4clusteredServersScheduleSequentiallyAndStartIn2Threads() { configureCluster(CLUSTER).withMaxConcurrentStartup(2); - addWlsCluster(CLUSTER, "ms1", "ms2", "ms3", "ms4"); - - invokeStepWithServerStartupInfos(createServerStartupInfosForCluster(CLUSTER, "ms1", "ms2", "ms3", "ms4")); - - assertThat(getServers(), hasItem(Arrays.asList("ms1", "ms2", "ms3", "ms4"))); - assertThat(getServers().size(), equalTo(2)); + addWlsCluster(CLUSTER, PORT, MS1, MS2, MS3, MS4); + + invokeStepWithServerStartupInfos(createServerStartupInfosForCluster(CLUSTER, MS1, MS2, MS3, MS4)); + assertThat(MS1 + " pod", domainPresenceInfo.getServerPod(MS1), notNullValue()); + assertThat(MS2 + " pod", domainPresenceInfo.getServerPod(MS2), nullValue()); + schedulePod(MS1, "Node1"); + testSupport.setTime(100, TimeUnit.MILLISECONDS); + assertThat(MS2 + " pod", domainPresenceInfo.getServerPod(MS2), notNullValue()); + assertThat(MS3 + " pod", domainPresenceInfo.getServerPod(MS3), nullValue()); + schedulePod(MS2, "Node2"); + testSupport.setTime(100, TimeUnit.MILLISECONDS); + assertThat(MS3 + " pod", domainPresenceInfo.getServerPod(MS3), nullValue()); + makePodReady(MS1); + testSupport.setTime(10, TimeUnit.SECONDS); + assertThat(MS3 + " pod", domainPresenceInfo.getServerPod(MS3), notNullValue()); + assertThat(MS4 + " pod", domainPresenceInfo.getServerPod(MS4), nullValue()); + makePodReady(MS2); + schedulePod(MS3, "Node3"); + testSupport.setTime(10, TimeUnit.SECONDS); + assertThat(MS4 + " pod", domainPresenceInfo.getServerPod(MS4), notNullValue()); } @Test - public void withMultipleClusters_differentClusterStartDifferently() { + public void withMultipleClusters_differentClusterScheduleAndStartDifferently() { final String CLUSTER2 = "cluster2"; - configureCluster(CLUSTER).withMaxConcurrentStartup(1); - configureCluster(CLUSTER2).withMaxConcurrentStartup(0); - addWlsCluster(CLUSTER, "ms1", "ms2"); - addWlsCluster(CLUSTER2, "ms3", "ms4"); + configureCluster(CLUSTER).withMaxConcurrentStartup(0); + configureCluster(CLUSTER2).withMaxConcurrentStartup(1); - Collection serverStartupInfos = createServerStartupInfosForCluster(CLUSTER, "ms1", "ms2"); - serverStartupInfos.addAll(createServerStartupInfosForCluster(CLUSTER2, "ms3", "ms4")); + addWlsCluster(CLUSTER, PORT, MS1, MS2); + addWlsCluster(CLUSTER2, PORT, MS3, MS4); + + Collection serverStartupInfos = createServerStartupInfosForCluster(CLUSTER, MS1, MS2); + serverStartupInfos.addAll(createServerStartupInfosForCluster(CLUSTER2, MS3, MS4)); invokeStepWithServerStartupInfos(serverStartupInfos); - assertThat(getServers(), hasItem(Arrays.asList("ms1", "ms2"))); - assertThat(getServers(), allOf(hasItem("ms3"), hasItem("ms4"))); + assertThat(MS1 + " pod", domainPresenceInfo.getServerPod(MS1), notNullValue()); + assertThat(MS3 + " pod", domainPresenceInfo.getServerPod(MS3), notNullValue()); + schedulePod(MS1, "Node1"); + schedulePod(MS3, "Node2"); + testSupport.setTime(100, TimeUnit.MILLISECONDS); + assertThat(MS2 + " pod", domainPresenceInfo.getServerPod(MS2), notNullValue()); + assertThat(MS4 + " pod", domainPresenceInfo.getServerPod(MS4), nullValue()); + //makePodReady(MS3); + //k8sTestSupport.setTime(10, TimeUnit.SECONDS); + //assertThat(MS4 + " pod", domainPresenceInfo.getServerPod(MS4), notNullValue()); } @Test public void maxClusterConcurrentStartup_doesNotApplyToNonClusteredServers() { domain.getSpec().setMaxClusterConcurrentStartup(1); - addWlsServers("ms3", "ms4"); + addWlsServers(MS3, MS4); - invokeStepWithServerStartupInfos(createServerStartupInfos("ms3", "ms4")); + invokeStepWithServerStartupInfos(createServerStartupInfos(MS3, MS4)); - assertThat(getServers(), allOf(hasItem("ms3"), hasItem("ms4"))); + assertThat(MS3 + " pod", domainPresenceInfo.getServerPod(MS3), notNullValue()); + schedulePod(MS3, "Node2"); + testSupport.setTime(200, TimeUnit.MILLISECONDS); + assertThat(MS3 + " pod", domainPresenceInfo.getServerPod(MS3), notNullValue()); } @NotNull private Collection createServerStartupInfosForCluster(String clusterName, String... servers) { Collection serverStartupInfos = new ArrayList<>(); - Arrays.asList(servers).stream().forEach(server -> + Arrays.stream(servers).forEach(server -> serverStartupInfos.add( new ServerStartupInfo(configSupport.getWlsServer(clusterName, server), clusterName, @@ -183,7 +309,7 @@ private Collection createServerStartupInfosForCluster(String @NotNull private Collection createServerStartupInfos(String... servers) { Collection serverStartupInfos = new ArrayList<>(); - Arrays.asList(servers).stream().forEach(server -> + Arrays.stream(servers).forEach(server -> serverStartupInfos.add( new ServerStartupInfo(configSupport.getWlsServer(server), null, @@ -195,10 +321,6 @@ private Collection createServerStartupInfos(String... servers private void invokeStepWithServerStartupInfos(Collection startupInfos) { ManagedServerUpIteratorStep step = new ManagedServerUpIteratorStep(startupInfos, nextStep); - // configSupport.setAdminServerName(ADMIN); - - testSupport.addToPacket( - ProcessingConstants.DOMAIN_TOPOLOGY, configSupport.createDomainConfig()); testSupport.runSteps(step); } @@ -207,49 +329,15 @@ private ClusterConfigurator configureCluster(String clusterName) { } private void addWlsServers(String... serverNames) { - Arrays.asList(serverNames).forEach(serverName -> addWlsServer(serverName)); + Arrays.asList(serverNames).forEach(this::addWlsServer); } private void addWlsServer(String serverName) { - configSupport.addWlsServer(serverName); - } - - private void addWlsCluster(String clusterName, String... serverNames) { - configSupport.addWlsCluster(clusterName, serverNames); + configSupport.addWlsServer(serverName, 8001); } - static class TestStepFactory implements ManagedServerUpIteratorStep.NextStepFactory { + private void addWlsCluster(String clusterName, int port, String... serverNames) { - private static Step next; - private static TestStepFactory factory = new TestStepFactory(); - - private static Memento install() throws NoSuchFieldException { - return StaticStubSupport.install(ManagedServerUpIteratorStep.class, "NEXT_STEP_FACTORY", factory); - } - - static Collection getServers() { - if (next instanceof StartManagedServersStep) { - return ((StartManagedServersStep)next).getStartDetails() - .stream() - .map(serverToStart -> getServerFromStepAndPacket(serverToStart)).collect(Collectors.toList()); - } - return Collections.emptyList(); - } - - static Object getServerFromStepAndPacket(StepAndPacket startDetail) { - if (startDetail.step instanceof StartClusteredServersStep) { - Collection serversToStart = ((StartClusteredServersStep)startDetail.step).getServersToStart(); - return serversToStart.stream().map(serverToStart -> getServerFromStepAndPacket(serverToStart)) - .collect(Collectors.toList()); - } - return startDetail.packet.get(ProcessingConstants.SERVER_NAME); - } - - @Override - public Step createStatusUpdateStep(Step next) { - TestStepFactory.next = next; - return new TerminalStep(); - } + configSupport.addWlsCluster(clusterName, port, serverNames); } - -} +} \ No newline at end of file diff --git a/operator/src/test/java/oracle/kubernetes/operator/utils/WlsDomainConfigSupport.java b/operator/src/test/java/oracle/kubernetes/operator/utils/WlsDomainConfigSupport.java index 5c3171b3951..2c502e9c28e 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/utils/WlsDomainConfigSupport.java +++ b/operator/src/test/java/oracle/kubernetes/operator/utils/WlsDomainConfigSupport.java @@ -118,13 +118,25 @@ public WlsServerConfig getWlsServer(String clusterName, String serverName) { * @param serverNames the names of the servers */ public void addWlsCluster(String clusterName, String... serverNames) { + addWlsCluster(clusterName, null, serverNames); + } + + /** + * Adds a WLS cluster to the configuration, including its member servers. + * + * @param clusterName the name of the cluster + * @param port - the port of the servers + * @param serverNames the names of the servers + */ + public void addWlsCluster(String clusterName, Integer port, String... serverNames) { ClusterConfigBuilder builder = new ClusterConfigBuilder(clusterName); for (String serverName : serverNames) { - builder.addServer(serverName); + builder.addServer(serverName, port); } wlsClusters.put(clusterName, builder.build()); } + /** * Returns the configuration for the named cluster, if any has been defined. *