diff --git a/integration-tests/src/test/java/oracle/weblogic/domain/ProbeTuning.java b/integration-tests/src/test/java/oracle/weblogic/domain/ProbeTuning.java index be1227798b4..887d85ba28e 100644 --- a/integration-tests/src/test/java/oracle/weblogic/domain/ProbeTuning.java +++ b/integration-tests/src/test/java/oracle/weblogic/domain/ProbeTuning.java @@ -19,6 +19,12 @@ public class ProbeTuning { @ApiModelProperty("The number of seconds with no response that indicates a failure.") private Integer timeoutSeconds; + @ApiModelProperty("Number of times the check is performed before giving up.") + private Integer failureThreshold = 1; + + @ApiModelProperty("Minimum number of times the check needs to pass for the probe to be considered successful") + private Integer successThreshold = 1; + public ProbeTuning initialDelaySeconds(Integer initialDelaySeconds) { this.initialDelaySeconds = initialDelaySeconds; return this; @@ -70,12 +76,32 @@ public void setTimeoutSeconds(Integer timeoutSeconds) { this.timeoutSeconds = timeoutSeconds; } + public Integer getFailureThreshold() { + return failureThreshold; + } + + public ProbeTuning failureThreshold(Integer failureThreshold) { + this.failureThreshold = failureThreshold; + return this; + } + + public Integer getSuccessThreshold() { + return successThreshold; + } + + public ProbeTuning successThreshold(Integer successThreshold) { + this.successThreshold = successThreshold; + return this; + } + @Override public String toString() { return new ToStringBuilder(this) .append("initialDelaySeconds", initialDelaySeconds) .append("periodSeconds", periodSeconds) .append("timeoutSeconds", timeoutSeconds) + .append("failureThreshold", failureThreshold) + .append("successThreshold", successThreshold) .toString(); } @@ -93,6 +119,8 @@ public boolean equals(Object other) { .append(initialDelaySeconds, rhs.initialDelaySeconds) .append(periodSeconds, rhs.periodSeconds) .append(timeoutSeconds, rhs.timeoutSeconds) + .append(failureThreshold, rhs.failureThreshold) + .append(successThreshold, rhs.successThreshold) .isEquals(); } @@ -102,6 +130,8 @@ public int hashCode() { .append(initialDelaySeconds) .append(periodSeconds) .append(timeoutSeconds) + .append(failureThreshold) + .append(successThreshold) .toHashCode(); } } diff --git a/integration-tests/src/test/java/oracle/weblogic/domain/ServerPod.java b/integration-tests/src/test/java/oracle/weblogic/domain/ServerPod.java index 560a101e9c6..6b3a9ae573b 100644 --- a/integration-tests/src/test/java/oracle/weblogic/domain/ServerPod.java +++ b/integration-tests/src/test/java/oracle/weblogic/domain/ServerPod.java @@ -225,7 +225,7 @@ public void setAnnotations(Map annotations) { this.annotations = annotations; } - public ServerPod livenessProve(ProbeTuning livenessProbe) { + public ServerPod livenessProbe(ProbeTuning livenessProbe) { this.livenessProbe = livenessProbe; return this; } diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLivenessProbeCustomization.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLivenessProbeCustomization.java index 9cbcc2fda3d..b7ae3ddbdda 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLivenessProbeCustomization.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLivenessProbeCustomization.java @@ -7,10 +7,13 @@ import java.io.FileWriter; import java.io.IOException; import java.nio.file.Paths; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; +import io.kubernetes.client.custom.V1Patch; import io.kubernetes.client.openapi.models.V1EnvVar; import io.kubernetes.client.openapi.models.V1LocalObjectReference; import io.kubernetes.client.openapi.models.V1ObjectMeta; @@ -22,6 +25,7 @@ import oracle.weblogic.domain.Domain; import oracle.weblogic.domain.DomainSpec; import oracle.weblogic.domain.Model; +import oracle.weblogic.domain.ProbeTuning; import oracle.weblogic.domain.ServerPod; import oracle.weblogic.kubernetes.annotations.IntegrationTest; import oracle.weblogic.kubernetes.annotations.Namespaces; @@ -34,15 +38,20 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; +import static io.kubernetes.client.custom.V1Patch.PATCH_FORMAT_JSON_PATCH; import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE; import static oracle.weblogic.kubernetes.TestConstants.DOMAIN_API_VERSION; import static oracle.weblogic.kubernetes.TestConstants.MANAGED_SERVER_NAME_BASE; import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_APP_NAME; import static oracle.weblogic.kubernetes.TestConstants.OCIR_SECRET_NAME; +import static oracle.weblogic.kubernetes.TestConstants.OPERATOR_RELEASE_NAME; import static oracle.weblogic.kubernetes.TestConstants.WLS_DOMAIN_TYPE; import static oracle.weblogic.kubernetes.actions.ActionConstants.RESOURCE_DIR; import static oracle.weblogic.kubernetes.actions.TestActions.getContainerRestartCount; import static oracle.weblogic.kubernetes.actions.TestActions.getDomainCustomResource; +import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorPodName; +import static oracle.weblogic.kubernetes.actions.TestActions.getPodCreationTimestamp; +import static oracle.weblogic.kubernetes.actions.impl.Domain.patchDomainCustomResource; import static oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes.copyFileToPod; import static oracle.weblogic.kubernetes.assertions.TestAssertions.appAccessibleInPod; import static oracle.weblogic.kubernetes.assertions.TestAssertions.appNotAccessibleInPod; @@ -52,7 +61,9 @@ import static oracle.weblogic.kubernetes.utils.ImageUtils.createMiiImageAndVerify; import static oracle.weblogic.kubernetes.utils.ImageUtils.createOcirRepoSecret; import static oracle.weblogic.kubernetes.utils.ImageUtils.dockerLoginAndPushImageToRegistry; +import static oracle.weblogic.kubernetes.utils.LoggingUtil.checkPodLogContainsString; import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator; +import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodRestarted; import static oracle.weblogic.kubernetes.utils.PodUtils.setPodAntiAffinity; import static oracle.weblogic.kubernetes.utils.SecretUtils.createSecretWithUsernamePassword; import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; @@ -71,8 +82,6 @@ @Tag("okdenv") class ItLivenessProbeCustomization { - private static String domainNamespace = null; - // domain constants private static final String domainUid = "liveprobecustom"; private static final String MII_IMAGE_NAME = "liveprobecustom-mii"; @@ -85,7 +94,10 @@ class ItLivenessProbeCustomization { private static final String internalPort = "8001"; private static final String appPath = "sample-war/index.jsp"; + private static String domainNamespace = null; + private static String opNamespace = null; private static LoggingFacade logger = null; + private static File tempFile = null; /** * Get namespaces for operator and WebLogic domain. @@ -99,7 +111,7 @@ public static void initAll(@Namespaces(2) List namespaces) { // get a unique operator namespace logger.info("Getting a unique namespace for operator"); assertNotNull(namespaces.get(0), "Namespace list is null"); - final String opNamespace = namespaces.get(0); + opNamespace = namespaces.get(0); // get a unique domain namespace logger.info("Getting a unique namespace for WebLogic domain"); @@ -114,6 +126,11 @@ public static void initAll(@Namespaces(2) List namespaces) { // create a basic model in image domain createAndVerifyMiiDomain(imageName); + + // create temp file to be copied to managed server pod which will be used in customLivenessProbe.sh + String fileName = "tempFile"; + tempFile = assertDoesNotThrow(() -> createTempfile(fileName), "Failed to create temp file"); + logger.info("File created {0}", tempFile); } /** @@ -134,12 +151,6 @@ void testCustomLivenessProbe() { domainUid, domainNamespace)); assertNotNull(domain1, "Got null domain resource"); - // create temp file - String fileName = "tempFile"; - File tempFile = assertDoesNotThrow(() -> createTempfile(fileName), - "Failed to create temp file"); - logger.info("File created {0}", tempFile); - for (int i = 1; i <= NUMBER_OF_CLUSTERS_MIIDOMAIN; i++) { for (int j = 1; j <= replicaCount; j++) { String managedServerPodName = @@ -231,6 +242,323 @@ void testCustomLivenessProbeNotTrigged() { } } + /** + * Patch the domain with custom livenessProbe failureThreshold and successThreshold value in serverPod. + * Verify the domain is restarted and the failureThreshold and successThreshold is updated. + * Also verify the failureThreshold runtime behavior is corrected. + */ + @Test + @DisplayName("Test custom livenessProbe failureThreshold and successThreshold in serverPod") + void testCustomLivenessProbeFailureThresholdSuccessThreshold() { + Domain domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace), + String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s", + domainUid, domainNamespace)); + assertNotNull(domain1, "Got null domain resource"); + assertNotNull(domain1.getSpec(), "domain1.getSpec() is null"); + assertNotNull(domain1.getSpec().getServerPod(), "domain1.getSpec().getServerPod() is null"); + assertNotNull(domain1.getSpec().getServerPod().getLivenessProbe(), + "domain1.getSpec().getServerPod().getLivenessProbe() is null"); + + // get the original failureThreshold of livenessProbe + Integer failureThreshold = domain1.getSpec().getServerPod().getLivenessProbe().getFailureThreshold(); + logger.info("Original livenessProbe failureThreshold is: {0}", failureThreshold); + assertTrue(failureThreshold.intValue() == 1, "The original livenessProbe failureThreshold is not 1"); + + // get the original successThreshold of livenessProbe + Integer successThreshold = domain1.getSpec().getServerPod().getLivenessProbe().getSuccessThreshold(); + logger.info("Original livenessProbe successThreshold is: {0}", successThreshold); + assertTrue(successThreshold.intValue() == 1, "The original livenessProbe successThreshold is not 1"); + + // get the original admin server pod and managed server pods creation time + OffsetDateTime adminPodCreationTime = + assertDoesNotThrow(() -> getPodCreationTimestamp(domainNamespace, "", adminServerPodName), + String.format("Failed to get creationTimestamp for pod %s", adminServerPodName)); + assertNotNull(adminPodCreationTime, "creationTimestamp of the admin server pod is null"); + + Map managedServerPodsCreationTime = new HashMap<>(); + for (int i = 1; i <= NUMBER_OF_CLUSTERS_MIIDOMAIN; i++) { + for (int j = 1; j <= replicaCount; j++) { + String managedServerPodName = + domainUid + "-" + CLUSTER_NAME_PREFIX + i + "-" + MANAGED_SERVER_NAME_BASE + j; + OffsetDateTime managedServerPodCreationTime = + assertDoesNotThrow(() -> getPodCreationTimestamp(domainNamespace, "", managedServerPodName), + String.format("Failed to get creationTimestamp for pod %s", managedServerPodName)); + assertNotNull(managedServerPodCreationTime, "creationTimestamp of the managed server pod is null"); + managedServerPodsCreationTime.put(managedServerPodName, managedServerPodCreationTime); + } + } + + // patch the domain with custom failureThreshold + String patchStr + = "[{\"op\": \"replace\", \"path\": \"/spec/serverPod/livenessProbe/failureThreshold\", \"value\": 3}," + + "{\"op\": \"add\", \"path\": \"/spec/serverPod/livenessProbe/periodSeconds\", \"value\": 30}]"; + logger.info("Updating domain configuration using patch string: {0}", patchStr); + assertTrue(patchDomainCustomResource(domainUid, domainNamespace, new V1Patch(patchStr), PATCH_FORMAT_JSON_PATCH), + String.format("failed to patch domain %s in namespace %s", domainUid, domainNamespace)); + + // check the domain get restarted + checkPodRestarted(domainUid, domainNamespace, adminServerPodName, adminPodCreationTime); + + for (int i = 1; i <= NUMBER_OF_CLUSTERS_MIIDOMAIN; i++) { + for (int j = 1; j <= replicaCount; j++) { + String managedServerPodName = + domainUid + "-" + CLUSTER_NAME_PREFIX + i + "-" + MANAGED_SERVER_NAME_BASE + j; + checkPodRestarted(domainUid, domainNamespace, managedServerPodName, + managedServerPodsCreationTime.get(managedServerPodName)); + } + } + + // check the livenessProbe failureThreshold and successThreshold after the domain got patched + domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace), + String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s", + domainUid, domainNamespace)); + assertNotNull(domain1, "Got null domain resource"); + assertNotNull(domain1.getSpec(), "domain1.getSpec() is null"); + assertNotNull(domain1.getSpec().getServerPod(), "domain1.getSpec().getServerPod() is null"); + assertNotNull(domain1.getSpec().getServerPod().getLivenessProbe(), + "domain1.getSpec().getServerPod().getLivenessProbe() is null"); + + // get the failureThreshold of livenessProbe after patch + failureThreshold = domain1.getSpec().getServerPod().getLivenessProbe().getFailureThreshold(); + logger.info("livenessProbe failureThreshold after patch is: {0}", failureThreshold); + assertTrue(failureThreshold.intValue() == 3, "The livenessProbe failureThreshold after patch is not 3"); + + // verify the failureThreshold behavior of livenessProbe + // copy temp file to pod and verify the restartCount only happens after 1m 30 second + for (int i = 1; i <= NUMBER_OF_CLUSTERS_MIIDOMAIN; i++) { + for (int j = 1; j <= replicaCount; j++) { + String managedServerPodName = + domainUid + "-" + CLUSTER_NAME_PREFIX + i + "-" + MANAGED_SERVER_NAME_BASE + j; + + // get the restart count of the container in pod before liveness probe restarts + int beforeRestartCount = + assertDoesNotThrow(() -> getContainerRestartCount(domainNamespace, null, + managedServerPodName, null), + String.format("Failed to get the restart count of the container from pod {0} in namespace {1}", + managedServerPodName, domainNamespace)); + logger.info("For server {0} restart count before liveness probe is: {1}", + managedServerPodName, beforeRestartCount); + + String destLocation = "/u01/tempFile.txt"; + assertDoesNotThrow(() -> copyFileToPod(domainNamespace, managedServerPodName, "weblogic-server", + tempFile.toPath(), Paths.get(destLocation)), + String.format("Failed to copy file %s to pod %s in namespace %s", + tempFile, managedServerPodName, domainNamespace)); + logger.info("File copied to Pod {0} in namespace {1}", managedServerPodName, domainNamespace); + + // check the pod should be restarted after 1m since the livenessProbe periodSeconds is changed to 30s. + // sleep for 45s + try { + Thread.sleep(45000); + } catch (InterruptedException ie) { + // ignore + } + int afterDelayRestartCount = + assertDoesNotThrow(() -> getContainerRestartCount(domainNamespace, null, + managedServerPodName, null), + String.format("Failed to get the restart count of the container from pod {0} in namespace {1} after 1m", + managedServerPodName, domainNamespace)); + logger.info("checking after 45s the restartCount is not changed."); + assertTrue(beforeRestartCount == afterDelayRestartCount, "The pod was restarted after 45s, " + + "it should restart after that"); + + String expectedStr = "Hello World, you have reached server " + + CLUSTER_NAME_PREFIX + i + "-" + MANAGED_SERVER_NAME_BASE + j; + checkAppNotRunning(domainNamespace, managedServerPodName, expectedStr); + checkAppIsRunning(domainNamespace, managedServerPodName, expectedStr); + + // get the restart count of the container in pod after liveness probe restarts + int afterRestartCount = assertDoesNotThrow(() -> + getContainerRestartCount(domainNamespace, null, managedServerPodName, null), + String.format("Failed to get the restart count of the container from pod {0} in namespace {1}", + managedServerPodName, domainNamespace)); + logger.info("Restart count after liveness probe {0}", afterRestartCount); + assertTrue(afterRestartCount - beforeRestartCount == 1, + String.format("Liveness probe did not start the container in pod %s in namespace %s", + managedServerPodName, domainNamespace)); + } + } + + // patch the domain with custom livenessProbe successThreshold + patchStr = "[{\"op\": \"replace\", \"path\": \"/spec/serverPod/livenessProbe/successThreshold\", \"value\": 2}]"; + logger.info("Updating domain configuration using patch string: {0}", patchStr); + assertTrue(patchDomainCustomResource(domainUid, domainNamespace, new V1Patch(patchStr), PATCH_FORMAT_JSON_PATCH), + String.format("failed to patch domain %s in namespace %s", domainUid, domainNamespace)); + + // check the operator log contains expected error msg + String expectedErrorMsg = "Invalid value '2' for the liveness probe success threshold under " + + "'spec.adminServer.serverPod.livenessProbe.successThreshold'. " + + "The liveness probe successThreshold value must be 1."; + String operatorPodName = assertDoesNotThrow(() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace)); + checkPodLogContainsString(opNamespace, operatorPodName, expectedErrorMsg); + + // patch the domain back to the original state + // get the original admin server pod and managed server pods creation time + adminPodCreationTime = + assertDoesNotThrow(() -> getPodCreationTimestamp(domainNamespace, "", adminServerPodName), + String.format("Failed to get creationTimestamp for pod %s", adminServerPodName)); + assertNotNull(adminPodCreationTime, "creationTimestamp of the admin server pod is null"); + + managedServerPodsCreationTime = new HashMap<>(); + for (int i = 1; i <= NUMBER_OF_CLUSTERS_MIIDOMAIN; i++) { + for (int j = 1; j <= replicaCount; j++) { + String managedServerPodName = + domainUid + "-" + CLUSTER_NAME_PREFIX + i + "-" + MANAGED_SERVER_NAME_BASE + j; + OffsetDateTime managedServerPodCreationTime = + assertDoesNotThrow(() -> getPodCreationTimestamp(domainNamespace, "", managedServerPodName), + String.format("Failed to get creationTimestamp for pod %s", managedServerPodName)); + assertNotNull(managedServerPodCreationTime, "creationTimestamp of the managed server pod is null"); + managedServerPodsCreationTime.put(managedServerPodName, managedServerPodCreationTime); + } + } + + patchStr + = "[{\"op\": \"replace\", \"path\": \"/spec/serverPod/livenessProbe/failureThreshold\", \"value\": 1}, " + + "{\"op\": \"replace\", \"path\": \"/spec/serverPod/livenessProbe/successThreshold\", \"value\": 1}, " + + "{\"op\": \"replace\", \"path\": \"/spec/serverPod/livenessProbe/periodSeconds\", \"value\": 45}]"; + + logger.info("Updating domain configuration using patch string: {0}", patchStr); + assertTrue(patchDomainCustomResource(domainUid, domainNamespace, new V1Patch(patchStr), PATCH_FORMAT_JSON_PATCH), + String.format("failed to patch domain %s in namespace %s", domainUid, domainNamespace)); + + // check the domain get restarted + checkPodRestarted(domainUid, domainNamespace, adminServerPodName, adminPodCreationTime); + + for (int i = 1; i <= NUMBER_OF_CLUSTERS_MIIDOMAIN; i++) { + for (int j = 1; j <= replicaCount; j++) { + String managedServerPodName = + domainUid + "-" + CLUSTER_NAME_PREFIX + i + "-" + MANAGED_SERVER_NAME_BASE + j; + checkPodRestarted(domainUid, domainNamespace, managedServerPodName, + managedServerPodsCreationTime.get(managedServerPodName)); + } + } + } + + /** + * Patch the domain with custom readinessProbe failureThreshold and successThreshold value in serverPod. + * Verify the domain is restarted and the failureThreshold and sucessThreshold are updated. + */ + @Test + @DisplayName("Test custom readinessProbe failureThreshold and successThreshold") + void testCustomReadinessProbeFailureThresholdSuccessThreshold() { + Domain domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace), + String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s", + domainUid, domainNamespace)); + assertNotNull(domain1, "Got null domain resource"); + assertNotNull(domain1.getSpec(), "domain1.getSpec() is null"); + assertNotNull(domain1.getSpec().getServerPod(), "domain1.getSpec().getServerPod() is null"); + assertNotNull(domain1.getSpec().getServerPod().getLivenessProbe(), + "domain1.getSpec().getServerPod().getLivenessProbe() is null"); + + // get the original failureThreshold of readinessProbe + Integer failureThreshold = domain1.getSpec().getServerPod().getReadinessProbe().getFailureThreshold(); + logger.info("Original readinessProbe failureThreshold is: {0}", failureThreshold); + assertTrue(failureThreshold.intValue() == 1, "The original readinessProbe failureThreshold is not 1"); + + // get the original successThreshold of readinessProbe + Integer successThreshold = domain1.getSpec().getServerPod().getReadinessProbe().getSuccessThreshold(); + logger.info("Original readinessProbe successThreshold is: {0}", successThreshold); + assertTrue(successThreshold.intValue() == 1, "The original readinessProbe successThreshold is not 1"); + + // get the original admin server pod and managed server pods creation time + OffsetDateTime adminPodCreationTime = + assertDoesNotThrow(() -> getPodCreationTimestamp(domainNamespace, "", adminServerPodName), + String.format("Failed to get creationTimestamp for pod %s", adminServerPodName)); + assertNotNull(adminPodCreationTime, "creationTimestamp of the admin server pod is null"); + + Map managedServerPodsCreationTime = new HashMap<>(); + for (int i = 1; i <= NUMBER_OF_CLUSTERS_MIIDOMAIN; i++) { + for (int j = 1; j <= replicaCount; j++) { + String managedServerPodName = + domainUid + "-" + CLUSTER_NAME_PREFIX + i + "-" + MANAGED_SERVER_NAME_BASE + j; + OffsetDateTime managedServerPodCreationTime = + assertDoesNotThrow(() -> getPodCreationTimestamp(domainNamespace, "", managedServerPodName), + String.format("Failed to get creationTimestamp for pod %s", managedServerPodName)); + assertNotNull(managedServerPodCreationTime, "creationTimestamp of the managed server pod is null"); + managedServerPodsCreationTime.put(managedServerPodName, managedServerPodCreationTime); + } + } + + // patch the domain with custom readinessProbe failureThreshold and successThreshold in serverPod + String patchStr + = "[{\"op\": \"replace\", \"path\": \"/spec/serverPod/readinessProbe/failureThreshold\", \"value\": 3}, " + + "{\"op\": \"replace\", \"path\": \"/spec/serverPod/readinessProbe/successThreshold\", \"value\": 3}]"; + logger.info("Updating domain configuration using patch string: {0}", patchStr); + assertTrue(patchDomainCustomResource(domainUid, domainNamespace, new V1Patch(patchStr), PATCH_FORMAT_JSON_PATCH), + String.format("failed to patch domain %s in namespace %s", domainUid, domainNamespace)); + + // check the domain get restarted + checkPodRestarted(domainUid, domainNamespace, adminServerPodName, adminPodCreationTime); + + for (int i = 1; i <= NUMBER_OF_CLUSTERS_MIIDOMAIN; i++) { + for (int j = 1; j <= replicaCount; j++) { + String managedServerPodName = + domainUid + "-" + CLUSTER_NAME_PREFIX + i + "-" + MANAGED_SERVER_NAME_BASE + j; + checkPodRestarted(domainUid, domainNamespace, managedServerPodName, + managedServerPodsCreationTime.get(managedServerPodName)); + } + } + + // check the readinessProbe failureThreshold and successThreshold after the domain got patched + domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, domainNamespace), + String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s", + domainUid, domainNamespace)); + assertNotNull(domain1, "Got null domain resource"); + assertNotNull(domain1.getSpec(), "domain1.getSpec() is null"); + assertNotNull(domain1.getSpec().getServerPod(), "domain1.getSpec().getServerPod() is null"); + assertNotNull(domain1.getSpec().getServerPod().getReadinessProbe(), + "domain1.getSpec().getServerPod().getReadinessProbe() is null"); + + // get the failureThreshold of readinessProbe after patch + failureThreshold = domain1.getSpec().getServerPod().getReadinessProbe().getFailureThreshold(); + logger.info("The readinessProbe failureThreshold after patch is: {0}", failureThreshold); + assertTrue(failureThreshold.intValue() == 3, "The readinessProbe failureThreshold after patch is not 3"); + + // get the successThreshold of readinessProbe + successThreshold = domain1.getSpec().getServerPod().getReadinessProbe().getSuccessThreshold(); + logger.info("readinessProbe successThreshold after patch is: {0}", successThreshold); + assertTrue(successThreshold.intValue() == 3, "The readinessProbe successThreshold after patch is not 3"); + + // patch the domain to the original state + // get the original admin server pod and managed server pods creation time + adminPodCreationTime = + assertDoesNotThrow(() -> getPodCreationTimestamp(domainNamespace, "", adminServerPodName), + String.format("Failed to get creationTimestamp for pod %s", adminServerPodName)); + assertNotNull(adminPodCreationTime, "creationTimestamp of the admin server pod is null"); + + managedServerPodsCreationTime = new HashMap<>(); + for (int i = 1; i <= NUMBER_OF_CLUSTERS_MIIDOMAIN; i++) { + for (int j = 1; j <= replicaCount; j++) { + String managedServerPodName = + domainUid + "-" + CLUSTER_NAME_PREFIX + i + "-" + MANAGED_SERVER_NAME_BASE + j; + OffsetDateTime managedServerPodCreationTime = + assertDoesNotThrow(() -> getPodCreationTimestamp(domainNamespace, "", managedServerPodName), + String.format("Failed to get creationTimestamp for pod %s", managedServerPodName)); + assertNotNull(managedServerPodCreationTime, "creationTimestamp of the managed server pod is null"); + managedServerPodsCreationTime.put(managedServerPodName, managedServerPodCreationTime); + } + } + + // patch the domain with original readinessProbe failureThreshold and successThreshold in serverPod + patchStr + = "[{\"op\": \"replace\", \"path\": \"/spec/serverPod/readinessProbe/failureThreshold\", \"value\": 1}, " + + "{\"op\": \"replace\", \"path\": \"/spec/serverPod/readinessProbe/successThreshold\", \"value\": 1}]"; + logger.info("Updating domain configuration using patch string: {0}", patchStr); + assertTrue(patchDomainCustomResource(domainUid, domainNamespace, new V1Patch(patchStr), PATCH_FORMAT_JSON_PATCH), + String.format("failed to patch domain %s in namespace %s", domainUid, domainNamespace)); + + // check the domain get restarted + checkPodRestarted(domainUid, domainNamespace, adminServerPodName, adminPodCreationTime); + + for (int i = 1; i <= NUMBER_OF_CLUSTERS_MIIDOMAIN; i++) { + for (int j = 1; j <= replicaCount; j++) { + String managedServerPodName = + domainUid + "-" + CLUSTER_NAME_PREFIX + i + "-" + MANAGED_SERVER_NAME_BASE + j; + checkPodRestarted(domainUid, domainNamespace, managedServerPodName, + managedServerPodsCreationTime.get(managedServerPodName)); + } + } + } /** * Create a model in image domain and verify the server pods are ready. @@ -288,6 +616,12 @@ private static void createAndVerifyMiiDomain(String miiImage) { .addEnvItem(new V1EnvVar() .name("LIVENESS_PROBE_CUSTOM_SCRIPT") .value("/u01/customLivenessProbe.sh")) + .livenessProbe(new ProbeTuning() + .failureThreshold(1) + .successThreshold(1)) + .readinessProbe(new ProbeTuning() + .failureThreshold(1) + .successThreshold(1)) .resources(new V1ResourceRequirements() .limits(new HashMap<>()) .requests(new HashMap<>()))) @@ -299,6 +633,7 @@ private static void createAndVerifyMiiDomain(String miiImage) { .domainType(WLS_DOMAIN_TYPE) .runtimeEncryptionSecret(encryptionSecretName)))); setPodAntiAffinity(domain); + // create model in image domain logger.info("Creating model in image domain {0} in namespace {1} using docker image {2}", domainUid, domainNamespace, miiImage); @@ -413,7 +748,7 @@ private static String createAndVerifyDomainImage() { return miiImage; } - private File createTempfile(String filename) throws IOException { + private static File createTempfile(String filename) throws IOException { File tempFile = File.createTempFile(filename, ".txt"); //deletes the file when VM terminates tempFile.deleteOnExit(); diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java index 20fc7f1864b..9c952ab28d7 100644 --- a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java +++ b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java @@ -21,6 +21,8 @@ import io.kubernetes.client.openapi.ApiException; import oracle.weblogic.domain.Cluster; +import oracle.weblogic.domain.Domain; +import oracle.weblogic.domain.DomainCondition; import oracle.weblogic.kubernetes.actions.TestActions; import oracle.weblogic.kubernetes.actions.impl.primitive.Command; import oracle.weblogic.kubernetes.actions.impl.primitive.CommandParams; @@ -37,6 +39,7 @@ import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT; import static oracle.weblogic.kubernetes.TestConstants.K8S_NODEPORT_HOST; import static oracle.weblogic.kubernetes.TestConstants.OKD; +import static oracle.weblogic.kubernetes.actions.TestActions.getDomainCustomResource; import static oracle.weblogic.kubernetes.actions.TestActions.getPodCreationTimestamp; import static oracle.weblogic.kubernetes.actions.TestActions.scaleCluster; import static oracle.weblogic.kubernetes.actions.TestActions.scaleClusterWithRestApi; @@ -910,4 +913,42 @@ public static void verifyCommandResultContainsMsg(String command, String expecte "Waiting until command result contains expected message \"{0}\"", expectedMsg); } + + /** + * Verify domain status conditions contains the given condition type and message. + * + * @param domainUid uid of the domain + * @param domainNamespace namespace of the domain + * @param conditionType condition type + * @param conditionMsg messsage in condition + * @return true if the condition matches + */ + public static boolean verifyDomainStatusCondition(String domainUid, + String domainNamespace, + String conditionType, + String conditionMsg) { + withLongRetryPolicy + .conditionEvaluationListener( + condition -> getLogger().info("Waiting for domain status condition message contains the expected msg " + + "\"{0}\", (elapsed time {1}ms, remaining time {2}ms)", + conditionMsg, + condition.getElapsedTimeInMS(), + condition.getRemainingTimeInMS())) + .until(() -> { + Domain domain = getDomainCustomResource(domainUid, domainNamespace); + if ((domain != null) && (domain.getStatus() != null)) { + for (DomainCondition domainCondition : domain.getStatus().getConditions()) { + getLogger().info("Condition Type =" + domainCondition.getType() + + " Condition Msg =" + domainCondition.getMessage()); + if ((domainCondition.getType() != null && domainCondition.getType().equalsIgnoreCase(conditionType)) + && (domainCondition.getMessage() != null && domainCondition.getMessage().contains(conditionMsg))) { + return true; + } + } + } + return false; + }); + return false; + } + }