diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index 93ff4eb7c..6d3196fa6 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -278,37 +278,25 @@ function kube-test-env-up { cat $HOME/.kube/config fi - # Install Helm Client + # Installing helm3 - echo "---" - echo "Installing Helm Client..." - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > install-helm.sh - chmod u+x install-helm.sh - ./install-helm.sh --version v2.17.0 - - # Start Helm Server - echo "Installing Helm Server..." - kubectl -n kube-system create serviceaccount tiller - kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller - - echo "Initialize Helm Server..." - helm init --service-account tiller - echo "Wait for Helm Server to complete startup..." - sleep 25 + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh + sleep 10 - echo "Getting Helm Server info..." - tiller_pod=$(kubectl get pods --namespace kube-system | grep tiller | awk '{print $1}') + helm version - kubectl describe pod ${tiller_pod} -n kube-system + echo "Installing Podgroup CRD" - helm version + kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/scheduler-plugins/277b6bdec18f8a9e9ccd1bfeaf4b66495bfc6f92/config/crd/bases/scheduling.sigs.k8s.io_podgroups.yaml - cd deployment + cd deployment/mcad-controller # start mcad controller echo "Starting MCAD Controller..." echo "helm install mcad-controller namespace kube-system wait set loglevel=2 set resources.requests.cpu=1000m set resources.requests.memory=1024Mi set resources.limits.cpu=4000m set resources.limits.memory=4096Mi set image.repository=$IMAGE_REPOSITORY_MCAD set image.tag=$IMAGE_TAG_MCAD set image.pullPolicy=$MCAD_IMAGE_PULL_POLICY" - helm install mcad-controller --namespace kube-system --wait --set loglevel=2 --set resources.requests.cpu=1000m --set resources.requests.memory=1024Mi --set resources.limits.cpu=4000m --set resources.limits.memory=4096Mi --set image.repository=$IMAGE_REPOSITORY_MCAD --set image.tag=$IMAGE_TAG_MCAD --set image.pullPolicy=$MCAD_IMAGE_PULL_POLICY + helm upgrade --install mcad-controller . --namespace kube-system --wait --set loglevel=2 --set resources.requests.cpu=1000m --set resources.requests.memory=1024Mi --set resources.limits.cpu=4000m --set resources.limits.memory=4096Mi --set configMap.name=mcad-controller-configmap --set configMap.podCreationTimeout='"120000"' --set configMap.quotaEnabled='"false"' --set coscheduler.rbac.apiGroup=scheduling.sigs.k8s.io --set coscheduler.rbac.resource=podgroups --set image.repository=$IMAGE_REPOSITORY_MCAD --set image.tag=$IMAGE_TAG_MCAD --set image.pullPolicy=$MCAD_IMAGE_PULL_POLICY sleep 10 echo "Listing MCAD Controller Helm Chart and Pod YAML..." diff --git a/test/e2e/queue.go b/test/e2e/queue.go index a1db58aa6..449589139 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -36,7 +36,6 @@ package e2e import ( "fmt" "os" - "time" . "github.com/onsi/ginkgo" @@ -313,6 +312,21 @@ var _ = Describe("AppWrapper E2E Test", func() { }) + It("Create AppWrapper - Bad Generic Item Only", func() { + fmt.Fprintf(os.Stdout, "[e2e] Create AppWrapper - Bad Generic Item Only - Started.\n") + context := initTestContext() + var appwrappers []*arbv1.AppWrapper + appwrappersPtr := &appwrappers + defer cleanupTestObjectsPtr(context, appwrappersPtr) + + aw := createBadGenericItemAW(context, "aw-bad-generic-item-1") + appwrappers = append(appwrappers, aw) + + err := waitAWPodsReady(context, aw) + Expect(err).To(HaveOccurred()) + + }) + It("Create AppWrapper - Namespace Only - 0 Pods", func() { fmt.Fprintf(os.Stdout, "[e2e] Create AppWrapper - Namespace Only - 0 Pods - Started.\n") context := initTestContext() @@ -487,6 +501,47 @@ var _ = Describe("AppWrapper E2E Test", func() { }) + It("MCAD Multi-Item Job Completion Test", func() { + fmt.Fprintf(os.Stdout, "[e2e] MCAD Multi-Item Job Completion Test - Started.\n") + context := initTestContext() + var appwrappers []*arbv1.AppWrapper + appwrappersPtr := &appwrappers + defer cleanupTestObjectsPtr(context, appwrappersPtr) + + aw := createGenericJobAWWithMultipleStatus(context, "aw-test-job-with-comp-ms-21") + err1 := waitAWPodsReady(context, aw) + Expect(err1).NotTo(HaveOccurred()) + time.Sleep(1 * time.Minute) + aw1, err := context.karclient.ArbV1().AppWrappers(aw.Namespace).Get(aw.Name, metav1.GetOptions{}) + if err != nil { + fmt.Fprintf(os.Stdout, "Error getting status") + } + pass := false + fmt.Fprintf(os.Stdout, "[e2e] status of AW %v.\n", aw1.Status.State) + if aw1.Status.State == arbv1.AppWrapperStateCompleted { + pass = true + } + Expect(pass).To(BeTrue()) + appwrappers = append(appwrappers, aw) + fmt.Fprintf(os.Stdout, "[e2e] MCAD Job Completion Test - Completed.\n") + + }) + + It("MCAD GenericItem Without Status Test", func() { + fmt.Fprintf(os.Stdout, "[e2e] MCAD GenericItem Without Status Test - Started.\n") + context := initTestContext() + var appwrappers []*arbv1.AppWrapper + appwrappersPtr := &appwrappers + defer cleanupTestObjectsPtr(context, appwrappersPtr) + + aw := createAWGenericItemWithoutStatus(context, "aw-test-job-with-comp-44") + err1 := waitAWPodsReady(context, aw) + fmt.Fprintf(os.Stdout, "The error is: %v", err1) + Expect(err1).To(HaveOccurred()) + fmt.Fprintf(os.Stdout, "[e2e] MCAD GenericItem Without Status Test - Completed.\n") + + }) + It("MCAD Job Completion No-requeue Test", func() { fmt.Fprintf(os.Stdout, "[e2e] MCAD Job Completion No-requeue Test - Started.\n") context := initTestContext() diff --git a/test/e2e/util.go b/test/e2e/util.go index a88979214..144e67e1a 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1627,6 +1627,192 @@ func createGenericJobAWWithStatus(context *context, name string) *arbv1.AppWrapp return appwrapper } +func createGenericJobAWWithMultipleStatus(context *context, name string) *arbv1.AppWrapper { + rb := []byte(`{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "aw-test-job-with-comp-ms-21-1", + "namespace": "test" + }, + "spec": { + "completions": 1, + "parallelism": 1, + "template": { + "metadata": { + "labels": { + "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-ms-21" + } + }, + "spec": { + "containers": [ + { + "args": [ + "sleep 5" + ], + "command": [ + "/bin/bash", + "-c", + "--" + ], + "image": "ubuntu:latest", + "imagePullPolicy": "IfNotPresent", + "name": "aw-test-job-with-comp-ms-21-1", + "resources": { + "limits": { + "cpu": "100m", + "memory": "256M" + }, + "requests": { + "cpu": "100m", + "memory": "256M" + } + } + } + ], + "restartPolicy": "Never" + } + } + } + }`) + + rb2 := []byte(`{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "aw-test-job-with-comp-ms-21-2", + "namespace": "test" + }, + "spec": { + "completions": 1, + "parallelism": 1, + "template": { + "metadata": { + "labels": { + "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-ms-21" + } + }, + "spec": { + "containers": [ + { + "args": [ + "sleep 5" + ], + "command": [ + "/bin/bash", + "-c", + "--" + ], + "image": "ubuntu:latest", + "imagePullPolicy": "IfNotPresent", + "name": "aw-test-job-with-comp-ms-21-2", + "resources": { + "limits": { + "cpu": "100m", + "memory": "256M" + }, + "requests": { + "cpu": "100m", + "memory": "256M" + } + } + } + ], + "restartPolicy": "Never" + } + } + } + }`) + + aw := &arbv1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test", + }, + Spec: arbv1.AppWrapperSpec{ + AggrResources: arbv1.AppWrapperResourceList{ + GenericItems: []arbv1.AppWrapperGenericResource{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-ms-21-1"), + Namespace: "test", + }, + DesiredAvailable: 1, + GenericTemplate: runtime.RawExtension{ + Raw: rb, + }, + CompletionStatus: "Complete", + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-ms-21-2"), + Namespace: "test", + }, + DesiredAvailable: 1, + GenericTemplate: runtime.RawExtension{ + Raw: rb2, + }, + CompletionStatus: "Complete", + }, + }, + }, + }, + } + + appwrapper, err := context.karclient.ArbV1().AppWrappers(context.namespace).Create(aw) + Expect(err).NotTo(HaveOccurred()) + + return appwrapper +} + +func createAWGenericItemWithoutStatus(context *context, name string) *arbv1.AppWrapper { + rb := []byte(`{ + "apiVersion": "scheduling.sigs.k8s.io/v1alpha1", + "kind": "PodGroup", + "metadata": { + "name": "aw-schd-spec-with-timeout-1", + "namespace": "default", + "labels":{ + "appwrapper.mcad.ibm.com": "aw-test-job-with-comp-44" + } + }, + "spec": { + "minMember": 1 + } + }`) + var schedSpecMin int = 1 + aw := &arbv1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test", + }, + Spec: arbv1.AppWrapperSpec{ + SchedSpec: arbv1.SchedulingSpecTemplate{ + MinAvailable: schedSpecMin, + }, + AggrResources: arbv1.AppWrapperResourceList{ + GenericItems: []arbv1.AppWrapperGenericResource{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", name, "aw-test-job-with-comp-44"), + Namespace: "test", + }, + DesiredAvailable: 1, + GenericTemplate: runtime.RawExtension{ + Raw: rb, + }, + }, + }, + }, + }, + } + + appwrapper, err := context.karclient.ArbV1().AppWrappers(context.namespace).Create(aw) + Expect(err).NotTo(HaveOccurred()) + + return appwrapper +} + func createGenericJobAWWithScheduleSpec(context *context, name string) *arbv1.AppWrapper { rb := []byte(`{ "apiVersion": "batch/v1", @@ -3003,6 +3189,42 @@ func createBadGenericPodAW(context *context, name string) *arbv1.AppWrapper { return appwrapper } + +func createBadGenericItemAW(context *context, name string) *arbv1.AppWrapper { + //rb := []byte(`""`) + var schedSpecMin int = 1 + + aw := &arbv1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: context.namespace, + }, + Spec: arbv1.AppWrapperSpec{ + SchedSpec: arbv1.SchedulingSpecTemplate{ + MinAvailable: schedSpecMin, + }, + AggrResources: arbv1.AppWrapperResourceList{ + GenericItems: []arbv1.AppWrapperGenericResource{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", name, "item"), + Namespace: context.namespace, + }, + // GenericTemplate: runtime.RawExtension{ + // Raw: rb, + // }, + }, + }, + }, + }, + } + + appwrapper, err := context.karclient.ArbV1().AppWrappers(context.namespace).Create(aw) + Expect(err).NotTo(HaveOccurred()) + + return appwrapper +} + func createBadGenericPodTemplateAW(context *context, name string) (*arbv1.AppWrapper, error) { rb := []byte(`{"metadata": {