From 69163dfc68762df9ce9217eaff844f2197ce6ad0 Mon Sep 17 00:00:00 2001 From: "Pedro D. Bello-Maldonado" Date: Mon, 15 May 2023 17:26:44 -0400 Subject: [PATCH 1/2] Quota release when AppWrapper is completed --- pkg/controller/queuejob/queuejob_controller_ex.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/queuejob/queuejob_controller_ex.go b/pkg/controller/queuejob/queuejob_controller_ex.go index 03f9b55dd..348af6593 100644 --- a/pkg/controller/queuejob/queuejob_controller_ex.go +++ b/pkg/controller/queuejob/queuejob_controller_ex.go @@ -2105,6 +2105,7 @@ func (cc *XController) manageQueueJob(qj *arbv1.AppWrapper, podPhaseChanges bool updateQj = qj.DeepCopy() } cc.updateEtcd(updateQj, "[syncQueueJob] setCompleted") + cc.quotaManager.Release(updateQj) } // Bugfix to eliminate performance problem of overloading the event queue. From 5ef78b68fe7c889e5142783e651375d524406eac Mon Sep 17 00:00:00 2001 From: "Pedro D. Bello-Maldonado" Date: Mon, 29 May 2023 12:13:19 -0400 Subject: [PATCH 2/2] Quota release testing. Fix in the Fits function to allocate consumer. --- .../qm_lib_backend_with_quotasubt_mgr.go | 2 +- test/e2e-kuttl/quota-forest/09-assert.yaml | 21 +++ test/e2e-kuttl/quota-forest/09-install.yaml | 133 ++++++++++++++++++ 3 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 test/e2e-kuttl/quota-forest/09-assert.yaml create mode 100644 test/e2e-kuttl/quota-forest/09-install.yaml diff --git a/pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr.go b/pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr.go index 1143186ee..9fab86d26 100644 --- a/pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr.go +++ b/pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr.go @@ -523,7 +523,7 @@ func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi _, err = qm.quotaManagerBackend.AddConsumer(consumerInfo) if err != nil { - return false, nil, err.Error() + klog.Errorf("[Fits] Consumer '%s' already exists.", consumerInfo.GetID()) } consumerID := consumerInfo.GetID() diff --git a/test/e2e-kuttl/quota-forest/09-assert.yaml b/test/e2e-kuttl/quota-forest/09-assert.yaml new file mode 100644 index 000000000..c4ec24c96 --- /dev/null +++ b/test/e2e-kuttl/quota-forest/09-assert.yaml @@ -0,0 +1,21 @@ +# Verify AppWrappers finished successfully +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 120 +--- +apiVersion: mcad.ibm.com/v1beta1 +kind: AppWrapper +metadata: + name: my-job-1 + namespace: test +status: + state: Completed +--- +apiVersion: mcad.ibm.com/v1beta1 +kind: AppWrapper +metadata: + name: my-job-2 + namespace: test +status: + state: Completed diff --git a/test/e2e-kuttl/quota-forest/09-install.yaml b/test/e2e-kuttl/quota-forest/09-install.yaml new file mode 100644 index 000000000..a8f437be2 --- /dev/null +++ b/test/e2e-kuttl/quota-forest/09-install.yaml @@ -0,0 +1,133 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: kubectl -n test delete appwrappers,jobs --all +--- +apiVersion: mcad.ibm.com/v1beta1 +kind: AppWrapper +metadata: + name: my-job-1 + namespace: test + labels: + quota_context: bronze + quota_service: service-root +spec: + schedulingSpec: + minAvailable: 1 + resources: + GenericItems: + - replicas: 1 + completionstatus: Complete + custompodresources: + - replicas: 1 + requests: + cpu: 900m + nvidia.com/gpu: 0 + memory: 300Mi + limits: + cpu: 900m + nvidia.com/gpu: 0 + memory: 300Mi + generictemplate: + apiVersion: batch/v1 + kind: Job + metadata: + name: my-job-1 + namespace: test + labels: + appwrapper.mcad.ibm.com: my-job-1 + spec: + parallelism: 1 + completions: 1 + template: + metadata: + name: my-job-1 + namespace: test + labels: + appwrapper.mcad.ibm.com: my-job-1 + spec: + terminationGracePeriodSeconds: 1 + restartPolicy: Never + containers: + - name: ubuntu + image: ubuntu:latest + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + sleep 30 + resources: + requests: + cpu: 900m + nvidia.com/gpu: 0 + memory: 300Mi + limits: + cpu: 900m + nvidia.com/gpu: 0 + memory: 300Mi +--- +apiVersion: mcad.ibm.com/v1beta1 +kind: AppWrapper +metadata: + name: my-job-2 + namespace: test + labels: + quota_context: bronze + quota_service: service-root +spec: + schedulingSpec: + minAvailable: 1 + resources: + GenericItems: + - replicas: 1 + completionstatus: Complete + custompodresources: + - replicas: 1 + requests: + cpu: 900m + nvidia.com/gpu: 0 + memory: 300Mi + limits: + cpu: 900m + nvidia.com/gpu: 0 + memory: 300Mi + generictemplate: + apiVersion: batch/v1 + kind: Job + metadata: + name: my-job-2 + namespace: test + labels: + appwrapper.mcad.ibm.com: my-job-2 + spec: + parallelism: 1 + completions: 1 + template: + metadata: + name: my-job-2 + namespace: test + labels: + appwrapper.mcad.ibm.com: my-job-2 + spec: + terminationGracePeriodSeconds: 1 + restartPolicy: Never + containers: + - name: ubuntu + image: ubuntu:latest + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + sleep 30 + resources: + requests: + cpu: 900m + nvidia.com/gpu: 0 + memory: 300Mi + limits: + cpu: 900m + nvidia.com/gpu: 0 + memory: 300Mi