From 8d97105b74c05b4b283cbd673ea4df07ed5c3a95 Mon Sep 17 00:00:00 2001 From: asm582 Date: Fri, 27 Jan 2023 15:03:33 -0500 Subject: [PATCH 01/14] test for no generic item --- test/e2e/queue.go | 15 +++++++++++++++ test/e2e/util.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/test/e2e/queue.go b/test/e2e/queue.go index a1db58aa6..637325eae 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -313,6 +313,21 @@ var _ = Describe("AppWrapper E2E Test", func() { }) + It("Create AppWrapper - Bad Generic Item Only", func() { + fmt.Fprintf(os.Stdout, "[e2e] Create AppWrapper - Bad Generic Item Only - Started.\n") + context := initTestContext() + var appwrappers []*arbv1.AppWrapper + appwrappersPtr := &appwrappers + defer cleanupTestObjectsPtr(context, appwrappersPtr) + + aw := createBadGenericItemAW(context, "aw-bad-generic-item-1") + appwrappers = append(appwrappers, aw) + + err := waitAWPodsReady(context, aw) + Expect(err).To(HaveOccurred()) + + }) + It("Create AppWrapper - Namespace Only - 0 Pods", func() { fmt.Fprintf(os.Stdout, "[e2e] Create AppWrapper - Namespace Only - 0 Pods - Started.\n") context := initTestContext() diff --git a/test/e2e/util.go b/test/e2e/util.go index a88979214..0907ce302 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -3003,6 +3003,42 @@ func createBadGenericPodAW(context *context, name string) *arbv1.AppWrapper { return appwrapper } + +func createBadGenericItemAW(context *context, name string) *arbv1.AppWrapper { + rb := []byte(` `) + var schedSpecMin int = 1 + + aw := &arbv1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: context.namespace, + }, + Spec: arbv1.AppWrapperSpec{ + SchedSpec: arbv1.SchedulingSpecTemplate{ + MinAvailable: schedSpecMin, + }, + AggrResources: arbv1.AppWrapperResourceList{ + GenericItems: []arbv1.AppWrapperGenericResource{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", name, "item"), + Namespace: context.namespace, + }, + GenericTemplate: runtime.RawExtension{ + Raw: rb, + }, + }, + }, + }, + }, + } + + appwrapper, err := context.karclient.ArbV1().AppWrappers(context.namespace).Create(aw) + Expect(err).NotTo(HaveOccurred()) + + return appwrapper +} + func createBadGenericPodTemplateAW(context *context, name string) (*arbv1.AppWrapper, error) { rb := []byte(`{"metadata": { From 7db36993ae95c17c1252d1548766bcb8f12b6224 Mon Sep 17 00:00:00 2001 From: asm582 Date: Mon, 30 Jan 2023 16:28:50 -0500 Subject: [PATCH 02/14] add test case for malformed AW --- test/e2e/queue.go | 64 ++++++++++++++- test/e2e/util.go | 196 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 255 insertions(+), 5 deletions(-) diff --git a/test/e2e/queue.go b/test/e2e/queue.go index 637325eae..4bc8f8678 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -36,7 +36,6 @@ package e2e import ( "fmt" "os" - "time" . "github.com/onsi/ginkgo" @@ -502,6 +501,69 @@ var _ = Describe("AppWrapper E2E Test", func() { }) + It("MCAD Multi-Item Job Completion Test", func() { + fmt.Fprintf(os.Stdout, "[e2e] MCAD Multi-Item Job Completion Test - Started.\n") + context := initTestContext() + var appwrappers []*arbv1.AppWrapper + appwrappersPtr := &appwrappers + defer cleanupTestObjectsPtr(context, appwrappersPtr) + + aw := createGenericJobAWWithMultipleStatus(context, "aw-test-job-with-comp-2") + err1 := waitAWPodsReady(context, aw) + Expect(err1).NotTo(HaveOccurred()) + time.Sleep(1 * time.Minute) + aw1, err := context.karclient.ArbV1().AppWrappers(aw.Namespace).Get(aw.Name, metav1.GetOptions{}) + if err != nil { + fmt.Fprintf(os.Stdout, "Error getting status") + } + pass := false + fmt.Fprintf(os.Stdout, "[e2e] status of AW %v.\n", aw1.Status.State) + if aw1.Status.State == arbv1.AppWrapperStateCompleted { + pass = true + } + Expect(pass).To(BeTrue()) + appwrappers = append(appwrappers, aw) + fmt.Fprintf(os.Stdout, "[e2e] MCAD Job Completion Test - Completed.\n") + + }) + + It("MCAD Multi-Item Job Completion Test", func() { + fmt.Fprintf(os.Stdout, "[e2e] MCAD Multi-Item Job Completion Test - Started.\n") + context := initTestContext() + var appwrappers []*arbv1.AppWrapper + appwrappersPtr := &appwrappers + defer cleanupTestObjectsPtr(context, appwrappersPtr) + + aw := createGenericJobAWWithMultipleStatus(context, "aw-test-job-with-comp-2") + err1 := waitAWPodsReady(context, aw) + Expect(err1).NotTo(HaveOccurred()) + time.Sleep(1 * time.Minute) + pass := false + aw1, err := context.karclient.ArbV1().AppWrappers(aw.Namespace).Get(aw.Name, metav1.GetOptions{}) + if err != nil { + fmt.Fprintf(os.Stdout, "Error getting status") + } + if aw1.Status.State == arbv1.AppWrapperStateCompleted { + pass = true + } + Expect(pass).To(BeTrue()) + + }) + + FIt("MCAD GenericItem Without Status Test", func() { + fmt.Fprintf(os.Stdout, "[e2e] MCAD GenericItem Without Status Test - Started.\n") + context := initTestContext() + var appwrappers []*arbv1.AppWrapper + appwrappersPtr := &appwrappers + defer cleanupTestObjectsPtr(context, appwrappersPtr) + + aw := createAWGenericItemWithoutStatus(context, "aw-test-job-with-comp-44") + err1 := waitAWPodsReady(context, aw) + fmt.Fprintf(os.Stdout, "The error is: %v", err1) + Expect(err1).To(HaveOccurred()) + + }) + It("MCAD Job Completion No-requeue Test", func() { fmt.Fprintf(os.Stdout, "[e2e] MCAD Job Completion No-requeue Test - Started.\n") context := initTestContext() diff --git a/test/e2e/util.go b/test/e2e/util.go index 0907ce302..4ae2823c9 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1627,6 +1627,194 @@ func createGenericJobAWWithStatus(context *context, name string) *arbv1.AppWrapp return appwrapper } +func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1.AppWrapper { + rb := []byte(`{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "aw-test-job-with-comp-2", + "namespace": "test" + }, + "spec": { + "completions": 1, + "parallelism": 1, + "template": { + "metadata": { + "labels": { + "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-2" + } + }, + "spec": { + "containers": [ + { + "args": [ + "sleep 5" + ], + "command": [ + "/bin/bash", + "-c", + "--" + ], + "image": "ubuntu:latest", + "imagePullPolicy": "IfNotPresent", + "name": "aw-test-job-with-comp-21", + "resources": { + "limits": { + "cpu": "100m", + "memory": "256M" + }, + "requests": { + "cpu": "100m", + "memory": "256M" + } + } + } + ], + "restartPolicy": "Never" + } + } + } + }`) + + rb2 := []byte(`{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "aw-test-job-with-comp-3", + "namespace": "test" + }, + "spec": { + "completions": 1, + "parallelism": 1, + "template": { + "metadata": { + "labels": { + "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-2" + } + }, + "spec": { + "containers": [ + { + "args": [ + "sleep 5" + ], + "command": [ + "/bin/bash", + "-c", + "--" + ], + "image": "ubuntu:latest", + "imagePullPolicy": "IfNotPresent", + "name": "aw-test-job-with-comp-22", + "resources": { + "limits": { + "cpu": "100m", + "memory": "256M" + }, + "requests": { + "cpu": "100m", + "memory": "256M" + } + } + } + ], + "restartPolicy": "Never" + } + } + } + }`) + //var schedSpecMin int = 1 + + aw := &arbv1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test", + }, + Spec: arbv1.AppWrapperSpec{ + SchedSpec: arbv1.SchedulingSpecTemplate{ + //MinAvailable: schedSpecMin, + }, + AggrResources: arbv1.AppWrapperResourceList{ + GenericItems: []arbv1.AppWrapperGenericResource{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-2"), + Namespace: "test", + }, + DesiredAvailable: 1, + GenericTemplate: runtime.RawExtension{ + Raw: rb, + }, + CompletionStatus: "Complete", + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-3"), + Namespace: "test", + }, + DesiredAvailable: 1, + GenericTemplate: runtime.RawExtension{ + Raw: rb2, + }, + CompletionStatus: "Complete", + }, + }, + }, + }, + } + + appwrapper, err := context.karclient.ArbV1().AppWrappers(context.namespace).Create(aw) + Expect(err).NotTo(HaveOccurred()) + + return appwrapper +} + +func createAWGenericItemWithoutStatus(context *context, name string) *arbv1.AppWrapper { + rb := []byte(`{ + "apiVersion": "scheduling.sigs.k8s.io/v1alpha1", + "kind": "PodGroup", + "metadata": { + "name": "aw-schd-spec-with-timeout-1", + "namespace": "default" + }, + "spec": { + "minMember": 1 + } + }`) + var schedSpecMin int = 1 + aw := &arbv1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test", + }, + Spec: arbv1.AppWrapperSpec{ + SchedSpec: arbv1.SchedulingSpecTemplate{ + MinAvailable: schedSpecMin, + }, + AggrResources: arbv1.AppWrapperResourceList{ + GenericItems: []arbv1.AppWrapperGenericResource{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-44"), + Namespace: "test", + }, + DesiredAvailable: 1, + GenericTemplate: runtime.RawExtension{ + Raw: rb, + }, + //CompletionStatus: "Complete", + }, + }, + }, + }, + } + + appwrapper, err := context.karclient.ArbV1().AppWrappers(context.namespace).Create(aw) + Expect(err).NotTo(HaveOccurred()) + + return appwrapper +} + func createGenericJobAWWithScheduleSpec(context *context, name string) *arbv1.AppWrapper { rb := []byte(`{ "apiVersion": "batch/v1", @@ -3005,7 +3193,7 @@ func createBadGenericPodAW(context *context, name string) *arbv1.AppWrapper { } func createBadGenericItemAW(context *context, name string) *arbv1.AppWrapper { - rb := []byte(` `) + //rb := []byte(`""`) var schedSpecMin int = 1 aw := &arbv1.AppWrapper{ @@ -3024,9 +3212,9 @@ func createBadGenericItemAW(context *context, name string) *arbv1.AppWrapper { Name: fmt.Sprintf("%s-%s", name, "item"), Namespace: context.namespace, }, - GenericTemplate: runtime.RawExtension{ - Raw: rb, - }, + // GenericTemplate: runtime.RawExtension{ + // Raw: rb, + // }, }, }, }, From 2fa4a1dfbcf9927383be5305b1680085ef45449d Mon Sep 17 00:00:00 2001 From: asm582 Date: Tue, 31 Jan 2023 10:03:27 -0500 Subject: [PATCH 03/14] move to helm3 --- hack/run-e2e-kind.sh | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index 93ff4eb7c..fcec49f48 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -280,26 +280,33 @@ function kube-test-env-up { # Install Helm Client - echo "---" - echo "Installing Helm Client..." - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > install-helm.sh - chmod u+x install-helm.sh - ./install-helm.sh --version v2.17.0 + # echo "---" + # echo "Installing Helm Client..." + # curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > install-helm.sh + # chmod u+x install-helm.sh + # ./install-helm.sh --version v2.17.0 # Start Helm Server - echo "Installing Helm Server..." - kubectl -n kube-system create serviceaccount tiller - kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller + # echo "Installing Helm Server..." + # kubectl -n kube-system create serviceaccount tiller + # kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller + + # echo "Initialize Helm Server..." + # helm init --service-account tiller + # echo "Wait for Helm Server to complete startup..." + # sleep 25 - echo "Initialize Helm Server..." - helm init --service-account tiller - echo "Wait for Helm Server to complete startup..." - sleep 25 + # echo "Getting Helm Server info..." + # tiller_pod=$(kubectl get pods --namespace kube-system | grep tiller | awk '{print $1}') - echo "Getting Helm Server info..." - tiller_pod=$(kubectl get pods --namespace kube-system | grep tiller | awk '{print $1}') + # kubectl describe pod ${tiller_pod} -n kube-system - kubectl describe pod ${tiller_pod} -n kube-system + # Installing helm3 + + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh + sleep 10 helm version @@ -308,7 +315,7 @@ function kube-test-env-up { # start mcad controller echo "Starting MCAD Controller..." echo "helm install mcad-controller namespace kube-system wait set loglevel=2 set resources.requests.cpu=1000m set resources.requests.memory=1024Mi set resources.limits.cpu=4000m set resources.limits.memory=4096Mi set image.repository=$IMAGE_REPOSITORY_MCAD set image.tag=$IMAGE_TAG_MCAD set image.pullPolicy=$MCAD_IMAGE_PULL_POLICY" - helm install mcad-controller --namespace kube-system --wait --set loglevel=2 --set resources.requests.cpu=1000m --set resources.requests.memory=1024Mi --set resources.limits.cpu=4000m --set resources.limits.memory=4096Mi --set image.repository=$IMAGE_REPOSITORY_MCAD --set image.tag=$IMAGE_TAG_MCAD --set image.pullPolicy=$MCAD_IMAGE_PULL_POLICY + helm upgrade --install mcad-controller . --namespace kube-system --wait --set loglevel=2 --set resources.requests.cpu=1000m --set resources.requests.memory=1024Mi --set resources.limits.cpu=4000m --set resources.limits.memory=4096Mi --set image.repository=$IMAGE_REPOSITORY_MCAD --set image.tag=$IMAGE_TAG_MCAD --set image.pullPolicy=$MCAD_IMAGE_PULL_POLICY sleep 10 echo "Listing MCAD Controller Helm Chart and Pod YAML..." From 495eafe31cbca676836fc734045c0bcdb1afac8d Mon Sep 17 00:00:00 2001 From: asm582 Date: Tue, 31 Jan 2023 10:13:53 -0500 Subject: [PATCH 04/14] change dir to helm chart --- hack/run-e2e-kind.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index fcec49f48..bdd7f5c79 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -310,7 +310,7 @@ function kube-test-env-up { helm version - cd deployment + cd deployment/mcad-controller # start mcad controller echo "Starting MCAD Controller..." From 135ed2a0407a8eeb1fc904189de89024ce87e184 Mon Sep 17 00:00:00 2001 From: asm582 Date: Tue, 31 Jan 2023 10:26:38 -0500 Subject: [PATCH 05/14] run all tests --- test/e2e/queue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/queue.go b/test/e2e/queue.go index 4bc8f8678..69169de02 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -550,7 +550,7 @@ var _ = Describe("AppWrapper E2E Test", func() { }) - FIt("MCAD GenericItem Without Status Test", func() { + It("MCAD GenericItem Without Status Test", func() { fmt.Fprintf(os.Stdout, "[e2e] MCAD GenericItem Without Status Test - Started.\n") context := initTestContext() var appwrappers []*arbv1.AppWrapper From bc5624c784c0f50f05ed5c4170bd31d1c3099d7c Mon Sep 17 00:00:00 2001 From: asm582 Date: Tue, 31 Jan 2023 13:37:01 -0500 Subject: [PATCH 06/14] add podgroup CRD --- hack/run-e2e-kind.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index bdd7f5c79..4d29d09d6 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -310,6 +310,10 @@ function kube-test-env-up { helm version + echo "Installing Podgroup CRD" + + kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/scheduler-plugins/277b6bdec18f8a9e9ccd1bfeaf4b66495bfc6f92/config/crd/bases/scheduling.sigs.k8s.io_podgroups.yaml + cd deployment/mcad-controller # start mcad controller From 159a2cbb791273dc58f0ed64f85f13c410f12d4b Mon Sep 17 00:00:00 2001 From: asm582 Date: Tue, 31 Jan 2023 16:34:34 -0500 Subject: [PATCH 07/14] add podgroup rbac --- hack/run-e2e-kind.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index 4d29d09d6..2534b8319 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -319,7 +319,7 @@ function kube-test-env-up { # start mcad controller echo "Starting MCAD Controller..." echo "helm install mcad-controller namespace kube-system wait set loglevel=2 set resources.requests.cpu=1000m set resources.requests.memory=1024Mi set resources.limits.cpu=4000m set resources.limits.memory=4096Mi set image.repository=$IMAGE_REPOSITORY_MCAD set image.tag=$IMAGE_TAG_MCAD set image.pullPolicy=$MCAD_IMAGE_PULL_POLICY" - helm upgrade --install mcad-controller . --namespace kube-system --wait --set loglevel=2 --set resources.requests.cpu=1000m --set resources.requests.memory=1024Mi --set resources.limits.cpu=4000m --set resources.limits.memory=4096Mi --set image.repository=$IMAGE_REPOSITORY_MCAD --set image.tag=$IMAGE_TAG_MCAD --set image.pullPolicy=$MCAD_IMAGE_PULL_POLICY + helm upgrade --install mcad-controller . --namespace kube-system --wait --set loglevel=2 --set resources.requests.cpu=1000m --set resources.requests.memory=1024Mi --set resources.limits.cpu=4000m --set resources.limits.memory=4096Mi --set coscheduler.rbac.resource=podgroups --set image.repository=$IMAGE_REPOSITORY_MCAD --set image.tag=$IMAGE_TAG_MCAD --set image.pullPolicy=$MCAD_IMAGE_PULL_POLICY sleep 10 echo "Listing MCAD Controller Helm Chart and Pod YAML..." From 3990f6464de5d5e2eccc469d6679255e5fa81f4e Mon Sep 17 00:00:00 2001 From: asm582 Date: Wed, 1 Feb 2023 08:32:13 -0500 Subject: [PATCH 08/14] fix helm command --- hack/run-e2e-kind.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index 2534b8319..06b563031 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -319,7 +319,7 @@ function kube-test-env-up { # start mcad controller echo "Starting MCAD Controller..." echo "helm install mcad-controller namespace kube-system wait set loglevel=2 set resources.requests.cpu=1000m set resources.requests.memory=1024Mi set resources.limits.cpu=4000m set resources.limits.memory=4096Mi set image.repository=$IMAGE_REPOSITORY_MCAD set image.tag=$IMAGE_TAG_MCAD set image.pullPolicy=$MCAD_IMAGE_PULL_POLICY" - helm upgrade --install mcad-controller . --namespace kube-system --wait --set loglevel=2 --set resources.requests.cpu=1000m --set resources.requests.memory=1024Mi --set resources.limits.cpu=4000m --set resources.limits.memory=4096Mi --set coscheduler.rbac.resource=podgroups --set image.repository=$IMAGE_REPOSITORY_MCAD --set image.tag=$IMAGE_TAG_MCAD --set image.pullPolicy=$MCAD_IMAGE_PULL_POLICY + helm upgrade --install mcad-controller . --namespace kube-system --wait --set loglevel=2 --set resources.requests.cpu=1000m --set resources.requests.memory=1024Mi --set resources.limits.cpu=4000m --set resources.limits.memory=4096Mi --set configMap.name=mcad-controller-configmap --set configMap.podCreationTimeout='"120000"' --set configMap.quotaEnabled='"false"' --set coscheduler.rbac.apiGroup=scheduling.sigs.k8s.io --set coscheduler.rbac.resource=podgroups --set image.repository=$IMAGE_REPOSITORY_MCAD --set image.tag=$IMAGE_TAG_MCAD --set image.pullPolicy=$MCAD_IMAGE_PULL_POLICY sleep 10 echo "Listing MCAD Controller Helm Chart and Pod YAML..." From 7b90ae7fd42d6c58e8d609f0857fb1b6b8cf9ced Mon Sep 17 00:00:00 2001 From: asm582 Date: Wed, 1 Feb 2023 12:08:37 -0500 Subject: [PATCH 09/14] make aw name unique --- test/e2e/queue.go | 2 +- test/e2e/util.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/e2e/queue.go b/test/e2e/queue.go index 69169de02..aabb91d9e 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -508,7 +508,7 @@ var _ = Describe("AppWrapper E2E Test", func() { appwrappersPtr := &appwrappers defer cleanupTestObjectsPtr(context, appwrappersPtr) - aw := createGenericJobAWWithMultipleStatus(context, "aw-test-job-with-comp-2") + aw := createGenericJobAWWithMultipleStatus(context, "aw-test-job-with-comp-ms-2") err1 := waitAWPodsReady(context, aw) Expect(err1).NotTo(HaveOccurred()) time.Sleep(1 * time.Minute) diff --git a/test/e2e/util.go b/test/e2e/util.go index 4ae2823c9..187a76225 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1632,7 +1632,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. "apiVersion": "batch/v1", "kind": "Job", "metadata": { - "name": "aw-test-job-with-comp-2", + "name": "aw-test-job-with-comp-ms-21", "namespace": "test" }, "spec": { @@ -1641,7 +1641,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. "template": { "metadata": { "labels": { - "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-2" + "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-ms-2" } }, "spec": { @@ -1680,7 +1680,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. "apiVersion": "batch/v1", "kind": "Job", "metadata": { - "name": "aw-test-job-with-comp-3", + "name": "aw-test-job-with-comp-ms-3", "namespace": "test" }, "spec": { @@ -1689,7 +1689,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. "template": { "metadata": { "labels": { - "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-2" + "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-ms-2" } }, "spec": { @@ -1738,7 +1738,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. GenericItems: []arbv1.AppWrapperGenericResource{ { ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-2"), + Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-21"), Namespace: "test", }, DesiredAvailable: 1, @@ -1749,7 +1749,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. }, { ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-3"), + Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-22"), Namespace: "test", }, DesiredAvailable: 1, From fe56c6049ad6bd61f9241589bc5aac8270064065 Mon Sep 17 00:00:00 2001 From: asm582 Date: Wed, 1 Feb 2023 13:16:47 -0500 Subject: [PATCH 10/14] add correct aw name --- test/e2e/queue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/queue.go b/test/e2e/queue.go index aabb91d9e..9e309329f 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -534,7 +534,7 @@ var _ = Describe("AppWrapper E2E Test", func() { appwrappersPtr := &appwrappers defer cleanupTestObjectsPtr(context, appwrappersPtr) - aw := createGenericJobAWWithMultipleStatus(context, "aw-test-job-with-comp-2") + aw := createGenericJobAWWithMultipleStatus(context, "aw-test-job-with-comp-ms-2") err1 := waitAWPodsReady(context, aw) Expect(err1).NotTo(HaveOccurred()) time.Sleep(1 * time.Minute) From b252b5fa0933eaf95cf831a200c1584d4114d2eb Mon Sep 17 00:00:00 2001 From: asm582 Date: Wed, 1 Feb 2023 14:54:26 -0500 Subject: [PATCH 11/14] remove duplicate test --- test/e2e/queue.go | 24 +----------------------- test/e2e/util.go | 6 +++--- 2 files changed, 4 insertions(+), 26 deletions(-) diff --git a/test/e2e/queue.go b/test/e2e/queue.go index 9e309329f..1892373a5 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -527,29 +527,6 @@ var _ = Describe("AppWrapper E2E Test", func() { }) - It("MCAD Multi-Item Job Completion Test", func() { - fmt.Fprintf(os.Stdout, "[e2e] MCAD Multi-Item Job Completion Test - Started.\n") - context := initTestContext() - var appwrappers []*arbv1.AppWrapper - appwrappersPtr := &appwrappers - defer cleanupTestObjectsPtr(context, appwrappersPtr) - - aw := createGenericJobAWWithMultipleStatus(context, "aw-test-job-with-comp-ms-2") - err1 := waitAWPodsReady(context, aw) - Expect(err1).NotTo(HaveOccurred()) - time.Sleep(1 * time.Minute) - pass := false - aw1, err := context.karclient.ArbV1().AppWrappers(aw.Namespace).Get(aw.Name, metav1.GetOptions{}) - if err != nil { - fmt.Fprintf(os.Stdout, "Error getting status") - } - if aw1.Status.State == arbv1.AppWrapperStateCompleted { - pass = true - } - Expect(pass).To(BeTrue()) - - }) - It("MCAD GenericItem Without Status Test", func() { fmt.Fprintf(os.Stdout, "[e2e] MCAD GenericItem Without Status Test - Started.\n") context := initTestContext() @@ -561,6 +538,7 @@ var _ = Describe("AppWrapper E2E Test", func() { err1 := waitAWPodsReady(context, aw) fmt.Fprintf(os.Stdout, "The error is: %v", err1) Expect(err1).To(HaveOccurred()) + fmt.Fprintf(os.Stdout, "[e2e] MCAD GenericItem Without Status Test - Completed.\n") }) diff --git a/test/e2e/util.go b/test/e2e/util.go index 187a76225..d3eac6727 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1657,7 +1657,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. ], "image": "ubuntu:latest", "imagePullPolicy": "IfNotPresent", - "name": "aw-test-job-with-comp-21", + "name": "aw-test-job-with-comp-ms-21", "resources": { "limits": { "cpu": "100m", @@ -1680,7 +1680,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. "apiVersion": "batch/v1", "kind": "Job", "metadata": { - "name": "aw-test-job-with-comp-ms-3", + "name": "aw-test-job-with-comp-ms-22", "namespace": "test" }, "spec": { @@ -1705,7 +1705,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. ], "image": "ubuntu:latest", "imagePullPolicy": "IfNotPresent", - "name": "aw-test-job-with-comp-22", + "name": "aw-test-job-with-comp-ms-22", "resources": { "limits": { "cpu": "100m", From 23f0c1893d5ed21ff53aaf019cf9e9dd7561524a Mon Sep 17 00:00:00 2001 From: asm582 Date: Thu, 23 Feb 2023 12:25:47 -0500 Subject: [PATCH 12/14] address review --- hack/run-e2e-kind.sh | 23 ----------------------- test/e2e/queue.go | 2 +- test/e2e/util.go | 25 +++++++++++-------------- 3 files changed, 12 insertions(+), 38 deletions(-) diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index 06b563031..6d3196fa6 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -278,29 +278,6 @@ function kube-test-env-up { cat $HOME/.kube/config fi - # Install Helm Client - - # echo "---" - # echo "Installing Helm Client..." - # curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > install-helm.sh - # chmod u+x install-helm.sh - # ./install-helm.sh --version v2.17.0 - - # Start Helm Server - # echo "Installing Helm Server..." - # kubectl -n kube-system create serviceaccount tiller - # kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller - - # echo "Initialize Helm Server..." - # helm init --service-account tiller - # echo "Wait for Helm Server to complete startup..." - # sleep 25 - - # echo "Getting Helm Server info..." - # tiller_pod=$(kubectl get pods --namespace kube-system | grep tiller | awk '{print $1}') - - # kubectl describe pod ${tiller_pod} -n kube-system - # Installing helm3 curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 diff --git a/test/e2e/queue.go b/test/e2e/queue.go index 1892373a5..449589139 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -508,7 +508,7 @@ var _ = Describe("AppWrapper E2E Test", func() { appwrappersPtr := &appwrappers defer cleanupTestObjectsPtr(context, appwrappersPtr) - aw := createGenericJobAWWithMultipleStatus(context, "aw-test-job-with-comp-ms-2") + aw := createGenericJobAWWithMultipleStatus(context, "aw-test-job-with-comp-ms-21") err1 := waitAWPodsReady(context, aw) Expect(err1).NotTo(HaveOccurred()) time.Sleep(1 * time.Minute) diff --git a/test/e2e/util.go b/test/e2e/util.go index d3eac6727..a141d1a0f 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1632,7 +1632,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. "apiVersion": "batch/v1", "kind": "Job", "metadata": { - "name": "aw-test-job-with-comp-ms-21", + "name": "aw-test-job-with-comp-ms-21-1", "namespace": "test" }, "spec": { @@ -1641,7 +1641,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. "template": { "metadata": { "labels": { - "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-ms-2" + "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-ms-21" } }, "spec": { @@ -1657,7 +1657,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. ], "image": "ubuntu:latest", "imagePullPolicy": "IfNotPresent", - "name": "aw-test-job-with-comp-ms-21", + "name": "aw-test-job-with-comp-ms-21-1", "resources": { "limits": { "cpu": "100m", @@ -1680,7 +1680,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. "apiVersion": "batch/v1", "kind": "Job", "metadata": { - "name": "aw-test-job-with-comp-ms-22", + "name": "aw-test-job-with-comp-ms-21-2", "namespace": "test" }, "spec": { @@ -1689,7 +1689,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. "template": { "metadata": { "labels": { - "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-ms-2" + "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-ms-21" } }, "spec": { @@ -1705,7 +1705,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. ], "image": "ubuntu:latest", "imagePullPolicy": "IfNotPresent", - "name": "aw-test-job-with-comp-ms-22", + "name": "aw-test-job-with-comp-ms-21-2", "resources": { "limits": { "cpu": "100m", @@ -1723,7 +1723,6 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. } } }`) - //var schedSpecMin int = 1 aw := &arbv1.AppWrapper{ ObjectMeta: metav1.ObjectMeta{ @@ -1731,14 +1730,11 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. Namespace: "test", }, Spec: arbv1.AppWrapperSpec{ - SchedSpec: arbv1.SchedulingSpecTemplate{ - //MinAvailable: schedSpecMin, - }, AggrResources: arbv1.AppWrapperResourceList{ GenericItems: []arbv1.AppWrapperGenericResource{ { ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-21"), + Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-ms-21-1"), Namespace: "test", }, DesiredAvailable: 1, @@ -1749,7 +1745,7 @@ func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1. }, { ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-22"), + Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-ms-21-2"), Namespace: "test", }, DesiredAvailable: 1, @@ -1776,6 +1772,8 @@ func createAWGenericItemWithoutStatus(context *context, name string) *arbv1.AppW "metadata": { "name": "aw-schd-spec-with-timeout-1", "namespace": "default" + labels: + appwrapper.mcad.ibm.com: "aw-test-job-with-comp-44" }, "spec": { "minMember": 1 @@ -1795,14 +1793,13 @@ func createAWGenericItemWithoutStatus(context *context, name string) *arbv1.AppW GenericItems: []arbv1.AppWrapperGenericResource{ { ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-44"), + Name: name, Namespace: "test", }, DesiredAvailable: 1, GenericTemplate: runtime.RawExtension{ Raw: rb, }, - //CompletionStatus: "Complete", }, }, }, From 1ccb1c004daef30ef0c9964a8bc1b94fb209d96b Mon Sep 17 00:00:00 2001 From: asm582 Date: Thu, 23 Feb 2023 13:32:19 -0500 Subject: [PATCH 13/14] fix yaml format --- test/e2e/util.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/e2e/util.go b/test/e2e/util.go index a141d1a0f..1f47338e5 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1771,9 +1771,10 @@ func createAWGenericItemWithoutStatus(context *context, name string) *arbv1.AppW "kind": "PodGroup", "metadata": { "name": "aw-schd-spec-with-timeout-1", - "namespace": "default" - labels: - appwrapper.mcad.ibm.com: "aw-test-job-with-comp-44" + "namespace": "default", + "labels":{ + appwrapper.mcad.ibm.com: "aw-test-job-with-comp-44" + } }, "spec": { "minMember": 1 @@ -1793,7 +1794,7 @@ func createAWGenericItemWithoutStatus(context *context, name string) *arbv1.AppW GenericItems: []arbv1.AppWrapperGenericResource{ { ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: "aw-test-job-with-comp-44", Namespace: "test", }, DesiredAvailable: 1, From 119ac560fd95603e7a7c80a8e2a90596088cbff2 Mon Sep 17 00:00:00 2001 From: asm582 Date: Thu, 23 Feb 2023 16:45:34 -0500 Subject: [PATCH 14/14] fix key format --- test/e2e/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/util.go b/test/e2e/util.go index 1f47338e5..144e67e1a 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1773,7 +1773,7 @@ func createAWGenericItemWithoutStatus(context *context, name string) *arbv1.AppW "name": "aw-schd-spec-with-timeout-1", "namespace": "default", "labels":{ - appwrapper.mcad.ibm.com: "aw-test-job-with-comp-44" + "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-44" } }, "spec": { @@ -1794,7 +1794,7 @@ func createAWGenericItemWithoutStatus(context *context, name string) *arbv1.AppW GenericItems: []arbv1.AppWrapperGenericResource{ { ObjectMeta: metav1.ObjectMeta{ - Name: "aw-test-job-with-comp-44", + Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-44"), Namespace: "test", }, DesiredAvailable: 1,