Create e2e test to emulate deployment use cases for MCAD

hack/run-e2e-kind.sh

@@ -44,9 +44,8 @@ export MCAD_IMAGE_PULL_POLICY="${3-Always}"
 export IMAGE_MCAD="${IMAGE_REPOSITORY_MCAD}:${IMAGE_TAG_MCAD}"
 CLUSTER_STARTED="false"
 export KUTTL_VERSION=0.15.0
-export KUTTL_TEST_OPT="--config ${ROOT_DIR}/kuttl-test.yaml"
-# FOR DEBUGGING
-#export KUTTL_TEST_OPT="--config ${ROOT_DIR}/kuttl-test.yaml --skip-delete"
+export KUTTL_OPTIONS=${TEST_KUTTL_OPTIONS}
+export KUTTL_TEST_SUITES=("${ROOT_DIR}/test/kuttl-test.yaml" "${ROOT_DIR}/test/kuttl-test-deployment-03.yaml" "${ROOT_DIR}/test/kuttl-test-deployment-02.yaml" "${ROOT_DIR}/test/kuttl-test-deployment-01.yaml") 
 
 function update_test_host {
 
@@ -310,15 +309,14 @@ function cleanup {
     fi 
 }
 
-function mcad-quota-management-down {
-
+function undeploy_mcad_helm {
     # Helm chart install name
     local helm_chart_name=$(helm list -n kube-system --short | grep mcad-controller)
 
     # start mcad controller
     echo "Stopping MCAD Controller for Quota Management Testing..."
     echo "helm delete ${helm_chart_name}"
-    helm delete -n kube-system ${helm_chart_name}
+    helm delete -n kube-system ${helm_chart_name} --wait
     if [ $? -ne 0 ]
     then
       echo "Failed to undeploy controller"
@@ -372,16 +370,24 @@ function setup-mcad-env {
 }
 
 function kuttl-tests {
-  echo "kubectl kuttl test ${KUTTL_TEST_OPT}"
-  kubectl kuttl test ${KUTTL_TEST_OPT}
-  if [ $? -ne 0 ]
-  then
-    echo "quota management kuttl e2e tests failure, exiting."
-    exit 1
-  else
-    # Takes a bit of time for namespace created in kuttl testing to completely delete.
-    sleep 40
-  fi
+  for kuttl_test in ${KUTTL_TEST_SUITES[@]}; do
+    echo "kubectl kuttl test --config ${kuttl_test}"
+    kubectl kuttl test --config ${kuttl_test}
+    if [ $? -ne 0 ]
+    then
+      echo "kuttl e2e test '${kuttl_test}' failure, exiting."
+      exit 1
+    fi
+    #clean up after sucessfull execution of a test by removing all quota subtrees
+    #and undeploying mcad helm chart.
+    kubectl delete quotasubtrees -n kube-system --all --wait
+    if [ $? -ne 0 ]
+    then
+      echo "Failed to delete quotasubtrees for test: '${kuttl_test}'"
+      exit 1
+    fi  
+    undeploy_mcad_helm
+  done
   rm -f kubeconfig
 }
 
@@ -392,6 +398,5 @@ kind-up-cluster
 setup-mcad-env
 # MCAD with quotamanagement options is started by kuttl-tests
 kuttl-tests
-mcad-quota-management-down
 mcad-up
 go test ./test/e2e -v -timeout 130m -count=1

pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr.go

@@ -128,7 +128,7 @@ func NewQuotaManager(dispatchedAWDemands map[string]*clusterstateapi.Resource, d
 
 	// Set the name of the forest in the backend
 	qm.quotaManagerBackend.AddForest(QuotaManagerForestName)
-	klog.V(10).Infof("[NewQuotaManager] Before initialization QuotaSubtree informer - %s", qm.quotaManagerBackend.String())
+	klog.V(4).Infof("[NewQuotaManager] Before initialization QuotaSubtree informer - %s", qm.quotaManagerBackend.String())
 
 	// Create a resource plan manager
 	qm.quotaSubtreeManager, err = qstmanager.NewQuotaSubtreeManager(config, qm.quotaManagerBackend)
@@ -171,19 +171,47 @@ func (qm *QuotaManager) loadDispatchedAWs(dispatchedAWDemands map[string]*cluste
 		klog.V(4).Infof("[loadDispatchedAWs] No dispatched AppWrappers found to preload.")
 		return nil
 	}
+	allTrees := qm.GetValidQuotaLabels()
+	klog.V(4).Infof("[loadDispatchedAWs] valid quota labels:%v", allTrees)
+	if len(allTrees) == 0 && len(dispatchedAWs) > 0 {
+		klog.Warning("[loadDispatchedAWs] No quota trees are defined in the cluster.")
+		klog.Warning("[loadDispatchedAWs] The resources for the following app wrappers will not be counted in the quota tree:")
+		for k := range dispatchedAWDemands {
+			aw := getDispatchedAppWrapper(dispatchedAWs, k)
+			if aw != nil {
+				klog.Warningf("[loadDispatchedAWs] app wrapper %s/%s not counted. AW labels: %v", aw.Namespace, aw.Name, aw.GetLabels())
+			}
+		}
+		return nil
+	}
 
 	// Process list of AppWrappers that are already dispatched
 	var result *multierror.Error
 
 	for k, v := range dispatchedAWDemands {
 		aw := getDispatchedAppWrapper(dispatchedAWs, k)
 		if aw != nil {
+			klog.V(4).Infof("[loadDispatchedAWs] Dispatched AppWrappers %s/%s found to preload.", aw.Namespace, aw.Name)
+			newLabels := make(map[string]string)
+			for key, value := range aw.Labels {
+				newLabels[key] = value
+			}
+			for _, treeName := range allTrees {
+				if _, quotaSetForAW := newLabels[treeName]; !quotaSetForAW {
+					newLabels[treeName] = "default"
+					klog.V(4).Infof("[loadDispatchedAWs] Dispatched AppWrappers %s/%s adding default quota labels.", aw.Namespace, aw.Name)
+				}
+
+			}
+			aw.SetLabels(newLabels)
+
 			doesFit, preemptionIds, errorMessage := qm.Fits(aw, v, nil)
 			if !doesFit {
 				klog.Errorf("[loadDispatchedAWs] Loading of AppWrapper %s/%s failed.",
 					aw.Namespace, aw.Name)
 				result = multierror.Append(result, fmt.Errorf("loading of AppWrapper %s/%s failed, %s",
 					aw.Namespace, aw.Name, errorMessage))
+				qm.Release(aw)
 			}
 
 			if len(preemptionIds) > 0 {
@@ -192,7 +220,6 @@ func (qm *QuotaManager) loadDispatchedAWs(dispatchedAWDemands map[string]*cluste
 				result = multierror.Append(result, fmt.Errorf("loading of AppWrapper %s/%s caused invalid preemptions: %v.  Quota Manager is in inconsistent state",
 					aw.Namespace, aw.Name, preemptionIds))
 			}
-			klog.V(4).Infof("[loadDispatchedAWs] Dispatched AppWrappers %s/%s found to preload.", aw.Namespace, aw.Name)
 		} else {
 			klog.Warningf("[loadDispatchedAWs] Unable to obtain AppWrapper from key: %s.  Loading of AppWrapper will be skipped.", k)
 		}

pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr/quotasubtmgr/event_handlers.go

@@ -33,7 +33,7 @@ func (qstm *QuotaSubtreeManager) addQST(obj interface{}) {
 	qstm.qstMap[qst.Namespace+"/"+qst.Name] = qst
 	qstm.setQuotasubtreeChanged()
 	qstm.qstMutex.Unlock()
-	klog.V(10).Infof("[addQST] Add complete for: %s/%s", qst.Name, qst.Namespace)
+	klog.V(4).Infof("[addQST] Add complete for: %s/%s", qst.Name, qst.Namespace)
 }
 
 func (qstm *QuotaSubtreeManager) updateQST(oldObj, newObj interface{}) {
@@ -65,7 +65,7 @@ func (qstm *QuotaSubtreeManager) updateQST(oldObj, newObj interface{}) {
 		qstm.setQuotasubtreeChanged()
 		qstm.qstMutex.Unlock()
 	}
-	klog.V(10).Infof("[updateQST] Update complete for: %s/%s", newQST.Name, newQST.Namespace)
+	klog.V(4).Infof("[updateQST] Update complete for: %s/%s", newQST.Name, newQST.Namespace)
 }
 
 func (qstm *QuotaSubtreeManager) deleteQST(obj interface{}) {
@@ -79,5 +79,5 @@ func (qstm *QuotaSubtreeManager) deleteQST(obj interface{}) {
 
 	delete(qstm.qstMap, string(qst.UID))
 	delete(qstm.qstMap, qst.Namespace+"/"+qst.Name)
-	klog.V(10).Infof("[deleteQST] Delete complete for: %s/%s", qst.Name, qst.Namespace)
+	klog.V(4).Infof("[deleteQST] Delete complete for: %s/%s", qst.Name, qst.Namespace)
 }

pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr/quotasubtmgr/quota_subtree_manager.go

@@ -61,6 +61,7 @@ func newQuotaSubtreeManager(config *rest.Config, quotaManagerBackend *qmlib.Mana
 	qstm := &QuotaSubtreeManager{
 		quotaManagerBackend: quotaManagerBackend,
 		qstMap:              make(map[string]*qstv1.QuotaSubtree),
+		qstChanged:          true,
 	}
 	// QuotaSubtree informer setup
 	qstClient, err := qst.NewForConfigOrDie(config)
@@ -83,19 +84,19 @@ func newQuotaSubtreeManager(config *rest.Config, quotaManagerBackend *qmlib.Mana
 
 	// Start resource plan informers
 	neverStop := make(chan struct{})
-	klog.V(10).Infof("[newQuotaSubtreeManager] Starting QuotaSubtree Informer.")
+	klog.V(4).Infof("[newQuotaSubtreeManager] Starting QuotaSubtree Informer.")
 	go qstm.quotaSubtreeInformer.Informer().Run(neverStop)
 
 	// Wait for cache sync
-	klog.V(10).Infof("[newQuotaSubtreeManager] Waiting for QuotaSubtree informer cache sync. to complete.")
+	klog.V(4).Infof("[newQuotaSubtreeManager] Waiting for QuotaSubtree informer cache sync. to complete.")
 	qstm.qstSynced = qstm.quotaSubtreeInformer.Informer().HasSynced
 	if !cache.WaitForCacheSync(neverStop, qstm.qstSynced) {
 		return nil, errors.New("failed to wait for the quota sub tree informer to synch")
 	}
 
 	// Initialize Quota Trees
 	qstm.initializeQuotaTreeBackend()
-	klog.V(10).Infof("[newQuotaSubtreeManager] QuotaSubtree Manager initialization complete.")
+	klog.V(4).Infof("[newQuotaSubtreeManager] QuotaSubtree Manager initialization complete.")
 	return qstm, nil
 }
 
@@ -129,7 +130,7 @@ func (qstm *QuotaSubtreeManager) clearQuotasubtreeChanged() {
 func (qstm *QuotaSubtreeManager) IsQuotasubtreeChanged() bool {
 	qstm.qstMutex.RLock()
 	defer qstm.qstMutex.RUnlock()
-
+	klog.V(4).Infof("[IsQuotasubtreeChanged] QuotaSubtree Manager changed %t.", qstm.qstChanged)
 	return qstm.qstChanged
 }
 
@@ -178,7 +179,7 @@ func (qstm *QuotaSubtreeManager) createTreeNodesFromQST(qst *qstv1.QuotaSubtree)
 			Quota:  quota,
 			Hard:   strconv.FormatBool(qstChild.Quotas.HardLimit),
 		}
-		klog.V(10).Infof("[createTreeNodesFromQST] Created node: %s=%#v for QuotaSubtree  %s completed.",
+		klog.V(4).Infof("[createTreeNodesFromQST] Created node: %s=%#v for QuotaSubtree  %s completed.",
 			child_key, *node, qst.Name)
 
 		//Add to the list of nodes from this quotasubtree

test/e2e-kuttl-deployment-01/steps/00-assert.yaml

@@ -0,0 +1,19 @@
+# Verify CRDs existence
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: appwrappers.mcad.ibm.com
+status:
+  acceptedNames:
+    kind: AppWrapper
+    listKind: AppWrapperList
+    plural: appwrappers
+    singular: appwrapper
+  storedVersions:
+    - v1beta1
+---
+# Verify test namespace existence
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: start-up

test/e2e-kuttl-deployment-01/steps/00-install.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: start-up

test/e2e-kuttl-deployment-01/steps/01-assert.yaml

@@ -0,0 +1,24 @@
+# Verify AppWrapper was dispatched and pod was created
+apiVersion: mcad.ibm.com/v1beta1
+kind: AppWrapper
+metadata:
+  name: no-quota-deployment-01
+  namespace: start-up
+status:
+  state: Running
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: no-quota-deployment-01
+  namespace: start-up
+  labels:
+      app:  no-quota-deployment-01
+      appwrapper.mcad.ibm.com: no-quota-deployment-01
+      resourceName: no-quota-deployment-01
+status:
+    availableReplicas: 1
+    observedGeneration: 1
+    readyReplicas: 1
+    replicas: 1
+    updatedReplicas: 1

test/e2e-kuttl-deployment-01/steps/01-install.yaml

@@ -0,0 +1,39 @@
+apiVersion: mcad.ibm.com/v1beta1
+kind: AppWrapper
+metadata:
+  name: no-quota-deployment-01
+  namespace: start-up
+spec:
+  resources:
+    GenericItems:
+    - replicas: 1
+      generictemplate:
+        apiVersion: apps/v1
+        kind: Deployment
+        metadata:
+          name: no-quota-deployment-01
+          namespace: start-up
+          labels:
+            app: no-quota-deployment-01
+        spec:
+          selector:
+            matchLabels:
+              app: no-quota-deployment-01
+          replicas: 1
+          template:
+            metadata:
+              labels:
+                app: deployment-echoserver-01
+            spec:
+              containers:
+               - name:  no-quota-deployment-01
+                 image: kicbase/echo-server:1.0
+                 ports:
+                 - containerPort: 80
+                 resources:
+                   requests:
+                     cpu: 300m
+                     memory: 32Mi
+                   limits:
+                     cpu: 300m
+                     memory: 32Mi

test/e2e-kuttl-deployment-01/steps/02-assert.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: mcad.ibm.com/v1beta1
+kind: AppWrapper
+metadata:
+    name: no-quota-job-02
+    namespace: start-up
+status:
+    state: Completed

test/e2e-kuttl-deployment-01/steps/02-install.yaml

@@ -0,0 +1,60 @@
+apiVersion: mcad.ibm.com/v1beta1
+kind: AppWrapper
+metadata:
+    name: no-quota-job-02
+    namespace: start-up
+spec:
+    schedulingSpec:
+        minAvailable: 1
+    resources:
+        GenericItems:
+            - replicas: 1
+              completionstatus: Complete
+              custompodresources:
+                  - replicas: 1
+                    requests:
+                        cpu: 500m
+                        nvidia.com/gpu: 0
+                        memory: 300Mi
+                    limits:
+                        cpu: 500m
+                        nvidia.com/gpu: 0
+                        memory: 300Mi
+              generictemplate:
+                  apiVersion: batch/v1
+                  kind: Job
+                  metadata:
+                      name: no-quota-job-02
+                      namespace: start-up
+                      labels:
+                          appwrapper.mcad.ibm.com: no-quota-job-02
+                  spec:
+                      parallelism: 1
+                      completions: 1
+                      template:
+                          metadata:
+                              name: no-quota-job-1
+                              namespace: start-up
+                              labels:
+                                  appwrapper.mcad.ibm.com: no-quota-job-02
+                          spec:
+                              terminationGracePeriodSeconds: 1
+                              restartPolicy: Never
+                              containers:
+                                  - name: ubuntu
+                                    image: ubuntu:latest
+                                    imagePullPolicy: IfNotPresent
+                                    command:
+                                        - sh
+                                        - -c
+                                        - |
+                                          sleep 30
+                                    resources:
+                                        requests:
+                                            cpu: 500m
+                                            nvidia.com/gpu: 0
+                                            memory: 300Mi
+                                        limits:
+                                            cpu: 500m
+                                            nvidia.com/gpu: 0
+                                            memory: 300Mi