Quota check preceding resource check

hack/run-e2e-kind.sh

@@ -45,7 +45,7 @@ export IMAGE_MCAD="${IMAGE_REPOSITORY_MCAD}:${IMAGE_TAG_MCAD}"
 CLUSTER_STARTED="false"
 export KUTTL_VERSION=0.15.0
 export KUTTL_OPTIONS=${TEST_KUTTL_OPTIONS}
-export KUTTL_TEST_SUITES=("${ROOT_DIR}/test/kuttl-test.yaml" "${ROOT_DIR}/test/kuttl-test-deployment-03.yaml" "${ROOT_DIR}/test/kuttl-test-deployment-02.yaml" "${ROOT_DIR}/test/kuttl-test-deployment-01.yaml")
+export KUTTL_TEST_SUITES=("${ROOT_DIR}/test/kuttl-test.yaml" "${ROOT_DIR}/test/kuttl-test-borrowing.yaml" "${ROOT_DIR}/test/kuttl-test-deployment-03.yaml" "${ROOT_DIR}/test/kuttl-test-deployment-02.yaml" "${ROOT_DIR}/test/kuttl-test-deployment-01.yaml")
 DUMP_LOGS="true"
 
 function update_test_host {

pkg/controller/clusterstate/api/node_info.go

@@ -43,8 +43,10 @@ type NodeInfo struct {
 
 	// The releasing resource on that node
 	Releasing *Resource
+
 	// The idle resource on that node
 	Idle *Resource
+
 	// The used resource on that node, including running and terminating
 	// pods
 	Used *Resource
@@ -104,7 +106,6 @@ func (ni *NodeInfo) Clone() *NodeInfo {
 func (ni *NodeInfo) SetNode(node *v1.Node) {
 	if ni.Node == nil {
 		ni.Idle = NewResource(node.Status.Allocatable)
-
 	}
 
 	ni.Name = node.Name
@@ -121,5 +122,4 @@ func (ni NodeInfo) String() string {
 
 	return fmt.Sprintf("Node (%s): idle <%v>, used <%v>, releasing <%v>%s",
 		ni.Name, ni.Idle, ni.Used, ni.Releasing, res)
-
 }

pkg/controller/quota/quota_manager_interface.go

@@ -21,7 +21,7 @@ import (
 )
 
 type QuotaManagerInterface interface {
-	Fits(aw *arbv1.AppWrapper, resources *clusterstateapi.Resource, proposedPremptions []*arbv1.AppWrapper) (bool, []*arbv1.AppWrapper, string)
+	Fits(aw *arbv1.AppWrapper, requestedResources *clusterstateapi.Resource, clusterResources *clusterstateapi.Resource, proposedPremptions []*arbv1.AppWrapper) (bool, []*arbv1.AppWrapper, string)
 	Release(aw *arbv1.AppWrapper) bool
 	GetValidQuotaLabels() []string
 }

pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr.go

@@ -32,6 +32,7 @@ import (
 	qmbackend "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/quotaplugins/quota-forest/quota-manager/quota"
 	qmbackendutils "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/quotaplugins/quota-forest/quota-manager/quota/utils"
 	"github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/quotaplugins/util"
+	"github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejobresources/genericresource"
 	"k8s.io/client-go/rest"
 
 	"math"
@@ -205,7 +206,7 @@ func (qm *QuotaManager) loadDispatchedAWs(dispatchedAWDemands map[string]*cluste
 			}
 			aw.SetLabels(newLabels)
 
-			doesFit, preemptionIds, errorMessage := qm.Fits(aw, v, nil)
+			doesFit, preemptionIds, errorMessage := qm.Fits(aw, v, nil, nil)
 			if !doesFit {
 				klog.Errorf("[loadDispatchedAWs] Loading of AppWrapper %s/%s failed.",
 					aw.Namespace, aw.Name)
@@ -509,7 +510,7 @@ func (qm *QuotaManager) buildRequest(aw *arbv1.AppWrapper,
 
 	return consumerInfo, err
 }
-func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi.Resource,
+func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi.Resource, clusterResources *clusterstateapi.Resource,
 	proposedPreemptions []*arbv1.AppWrapper) (bool, []*arbv1.AppWrapper, string) {
 
 	// If a Quota Manager Backend instance does not exists then assume quota failed
@@ -555,7 +556,7 @@ func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi
 
 	consumerID := consumerInfo.GetID()
 	klog.V(4).Infof("[Fits] Sending quota allocation request: %#v ", consumerInfo)
-	allocResponse, err := qm.quotaManagerBackend.AllocateForest(QuotaManagerForestName, consumerID)
+	allocResponse, err := qm.quotaManagerBackend.TryAllocateForest(QuotaManagerForestName, consumerID)
 	if err != nil {
 		qm.removeConsumer(consumerID)
 		klog.Errorf("[Fits] Error allocating consumer: %s/%s, err=%#v.", aw.Namespace, aw.Name, err)
@@ -569,9 +570,47 @@ func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi
 		return doesFit, preemptIds, strings.TrimSpace(allocResponse.GetMessage())
 	}
 	preemptIds = qm.getAppWrappers(allocResponse.GetPreemptedIds())
+
+	// Update cluster resources in the even that preemption happens
+	// TODO: Potentially move this resource updated out to the calling function (would need to comeback again to undo the allocation
+	// if the resources are not enough after preemption)
+	if clusterResources != nil {
+		updatedResources := clusterResources
+
+		for _, appWrapper := range preemptIds {
+			updatedResources.Add(qm.getAggregatedResources(appWrapper))
+		}
+
+		// Check if job fits with the update resources after preempted AppWrappers are removed
+		if clusterResources != nil && !awResDemands.LessEqual(updatedResources) {
+			qm.quotaManagerBackend.UndoAllocateForest(QuotaManagerForestName, consumerID)
+			qm.removeConsumer(consumerID)
+			return false, preemptIds, fmt.Sprintf("[Fits] AppWrapper '%s/%s' does not fit in the cluster, even after borrowed quota is freed", aw.Namespace, aw.Name)
+		}
+	}
+
 	return doesFit, preemptIds, strings.TrimSpace(allocResponse.GetMessage())
 }
 
+func (qm *QuotaManager) getAggregatedResources(appWrapper *arbv1.AppWrapper) *clusterstateapi.Resource {
+	// After quota evaluation, a set of AppWrappers is returned for preemption. Before deciding to delete them,
+	// we need to make sure enough resources are free for the new AppWrapper after the preemptable list is deleted.
+	// For this we need to add back the requests consumed by the preemptable AppWrappers to the available resources
+	// in order to perform a correct resource check with updated values.
+	allocated := clusterstateapi.EmptyResource()
+
+	for _, genericItem := range appWrapper.Spec.AggrResources.GenericItems {
+		resources, err := genericresource.GetResources(&genericItem)
+		if err != nil {
+			klog.V(8).Infof("[GetAggregatedResources] Failure aggregating resources for %s/%s, err=%#v, genericItem=%#v",
+				appWrapper.Namespace, appWrapper.Name, err, genericItem)
+		}
+		allocated = allocated.Add(resources)
+	}
+
+	return allocated
+}
+
 func (qm *QuotaManager) getAppWrappers(preemptIds []string) []*arbv1.AppWrapper {
 	var aws []*arbv1.AppWrapper
 	if len(preemptIds) <= 0 {

pkg/quotaplugins/quota-forest/quota-manager/quota/core/quotanode.go

@@ -138,7 +138,7 @@ func (qn *QuotaNode) SlideDown() {
 func (qn *QuotaNode) SlideUp(c *Consumer, applyPriority bool, allocationRecovery *AllocationRecovery,
 	preemptedConsumers *[]string) bool {
 
-	if qn.isHard {
+	if qn.isHard && !qn.IsRoot() {
 		return false
 	}
 

pkg/quotaplugins/quota-forest/quota-manager/quota/quotamanager.go

@@ -494,12 +494,14 @@ func (m *Manager) AllocateForest(forestName string, consumerID string) (response
 
 // TryAllocateForest : allocate a consumer on a forest
 func (m *Manager) TryAllocateForest(forestName string, consumerID string) (response *core.AllocationResponse, err error) {
+	if m.mode != Normal {
+		response, err = m.AllocateForest(forestName, consumerID)
+		return response, err
+	}
+
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	if m.mode != Normal {
-		return nil, fmt.Errorf("manager is not in normal mode")
-	}
 	forestController, forestConsumer, err := m.preAllocateForest(forestName, consumerID)
 	if err == nil && forestController.IsConsumerAllocated(consumerID) {
 		err = fmt.Errorf("consumer %s already allocated on forest %s", consumerID, forestName)

pkg/quotaplugins/quota-forest/quota-manager/quota/utils/defaults.go

@@ -21,7 +21,7 @@ var (
 	DefaultTreeName string = "default"
 
 	// DefaultResourceNames : the default resource names
-	DefaultResourceNames []string = []string{"cpu", "memory"}
+	DefaultResourceNames []string = []string{"cpu", "memory", "nvidia.com/gpu"}
 
 	// DefaultTreeKind : the default kind attribute of the tree
 	DefaultTreeKind string = "QuotaTree"

pkg/quotaplugins/quota-simple-rest/quota_rest_manager.go

@@ -257,7 +257,7 @@ func (qm *QuotaManager) getQuotaDesignation(aw *arbv1.AppWrapper) []QuotaGroup {
 	return groups
 }
 
-func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi.Resource,
+func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi.Resource, clusterResources *clusterstateapi.Resource,
 	proposedPreemptions []*arbv1.AppWrapper) (bool, []*arbv1.AppWrapper, string) {
 
 	// Handle uninitialized quota manager

test/e2e-kuttl-borrowing/install-quota-subtree.yaml

@@ -0,0 +1,95 @@
+---
+apiVersion: ibm.com/v1
+kind: QuotaSubtree
+metadata:
+  name: context-root
+  namespace: kube-system
+  labels:
+    tree: quota_context
+spec:
+  children:
+    - name: context-root
+      quotas:
+        requests:
+          cpu: 1075m
+          memory: 1045Mi
+          nvidia.com/gpu: 16
+---
+apiVersion: ibm.com/v1
+kind: QuotaSubtree
+metadata:
+  name: service-root
+  namespace: kube-system
+  labels:
+    tree: quota_service
+spec:
+  children:
+    - name: service-root
+      quotas:
+        requests:
+          cpu: 1075m
+          memory: 1045Mi
+          nvidia.com/gpu: 16
+---
+apiVersion: ibm.com/v1
+kind: QuotaSubtree
+metadata:
+  name: context-root-children
+  namespace: kube-system
+  labels:
+    tree: quota_context
+spec:
+  parent: context-root
+  children:
+    - name: gold
+      quotas:
+        hardLimit: false
+        requests:
+          cpu: 1075m
+          memory: 450Mi
+          nvidia.com/gpu: 8
+    - name: silver
+      quotas:
+        hardLimit: false
+        requests:
+          cpu: 1075m
+          memory: 400Mi
+          nvidia.com/gpu: 8
+    - name: bronze
+      quotas:
+        hardLimit: true
+        requests:
+          cpu: 900m
+          memory: 300Mi
+          nvidia.com/gpu: 8
+    - name: default
+      quotas:
+        hardLimit: false
+        requests:
+          cpu: 0m
+          memory: 0Mi
+          nvidia.com/gpu: 0
+---
+apiVersion: ibm.com/v1
+kind: QuotaSubtree
+metadata:
+  name: service-root-children
+  namespace: kube-system
+  labels:
+    tree: quota_service
+spec:
+  parent: service-root
+  children:
+    - name: gold
+      quotas:
+        requests:
+          cpu: 1075m
+          memory: 1045Mi
+          nvidia.com/gpu: 16
+    - name: default
+      quotas:
+        hardLimit: false
+        requests:
+          cpu: 0m
+          memory: 0Mi
+          nvidia.com/gpu: 0

test/e2e-kuttl-borrowing/steps/00-assert.yaml

@@ -0,0 +1,25 @@
+# Verify CRDs existence
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: appwrappers.mcad.ibm.com
+status:
+  acceptedNames:
+    kind: AppWrapper
+    listKind: AppWrapperList
+    plural: appwrappers
+    singular: appwrapper
+  storedVersions:
+    - v1beta1
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: quotasubtrees.ibm.com
+status:
+  acceptedNames:
+    kind: QuotaSubtree
+    singular: quotasubtree
+    plural: quotasubtrees
+  storedVersions:
+    - v1

test/e2e-kuttl-borrowing/steps/01-assert.yaml

@@ -0,0 +1,32 @@
+# Verify subtree creations
+apiVersion: ibm.com/v1
+kind: QuotaSubtree
+metadata:
+  name: context-root
+  namespace: kube-system
+  labels:
+    tree: quota_context
+---
+apiVersion: ibm.com/v1
+kind: QuotaSubtree
+metadata:
+  name: service-root
+  namespace: kube-system
+  labels:
+    tree: quota_service
+---
+apiVersion: ibm.com/v1
+kind: QuotaSubtree
+metadata:
+  name: context-root-children
+  namespace: kube-system
+  labels:
+    tree: quota_context
+---
+apiVersion: ibm.com/v1
+kind: QuotaSubtree
+metadata:
+  name: service-root-children
+  namespace: kube-system
+  labels:
+    tree: quota_service

test/e2e-kuttl-borrowing/steps/02-assert.yaml

@@ -0,0 +1,5 @@
+# Verify test namespace existence
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test

test/e2e-kuttl-borrowing/steps/02-install.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test

test/e2e-kuttl-borrowing/steps/03-assert.yaml

@@ -0,0 +1,9 @@
+# Verify job is running
+---
+apiVersion: mcad.ibm.com/v1beta1
+kind: AppWrapper
+metadata:
+    name: my-job-1
+    namespace: test
+status:
+    state: Running