[quota management] Incorrect condition message generated for quota overflow

[asm582]

Incorrect status:

Status:
  Conditions:
    Last Transition Micro Time:  2023-05-16T23:36:20.541399Z
    Last Update Micro Time:      2023-05-16T23:36:20.541399Z
    Status:                      True
    Type:                        Init
    Last Transition Micro Time:  2023-05-16T23:36:20.541521Z
    Last Update Micro Time:      2023-05-16T23:36:20.541521Z
    Reason:                      AwaitingHeadOfLine
    Status:                      True
    Type:                        Queueing
    Last Transition Micro Time:  2023-05-16T23:36:20.547527Z
    Last Update Micro Time:      2023-05-16T23:36:20.547527Z
    Reason:                      FrontOfQueue.
    Status:                      True
    Type:                        HeadOfLine
    Last Transition Micro Time:  2023-05-16T23:36:20.552796Z
    Last Update Micro Time:      2023-05-16T23:36:20.552796Z
    Message:                     Insufficient resources to dispatch AppWrapper.
    Reason:                      AppWrapperNotRunnable.
    Status:                      True
    Type:                        Backoff
  Controllerfirsttimestamp:      2023-05-16T23:36:20.541261Z
  Filterignore:                  true
  Queuejobstate:                 HeadOfLine
  Sender:                        before ScheduleNext - setHOL
  State:                         Pending
  Systempriority:                100
Events:                          <none>

Setup quota tree as below:

apiVersion: ibm.com/v1
kind: QuotaSubtree
metadata:
  name: context-root
  namespace: kube-system
  labels:
    tree: quota_context
spec:
  children:
    - name: context-root
      quotas:
        requests:
          cpu: 24000
          memory: 32Gi
          nvidia.com/gpu: 2
---
apiVersion: ibm.com/v1
kind: QuotaSubtree
metadata:
  name: context-root-children
  namespace: kube-system
  labels:
    tree: quota_context
spec:
  parent: context-root
  children:
    - name: namespace-1
      quotas:
        hardLimit: true
        requests:
          cpu: 3000
          memory: 3Gi
    - name: namespace-2
      quotas:
        hardLimit: false
        requests:
          cpu: 3000
          memory: 6Gi
    - name: namespace-3
      quotas:
        hardLimit: false
        requests:
          cpu: 3000
          memory: 6Gi

fire in the below order:

apiVersion: mcad.ibm.com/v1beta1
kind: AppWrapper
metadata:
  name: batch-job-3
  namespace: namespace-1
  labels:
    quota_context: "namespace-2"
spec:
  service:
    spec: {}
  priority: 100
  resources:
    metadata: {}
    GenericItems:
    - replicas: 1
      completionstatus: Complete
      custompodresources:
      - replicas: 1
        requests:
          cpu: 3000m
          memory: 6Gi
          nvidia.com/gpu: 0
        limits:
          cpu: 3000m
          memory: 6Gi
          nvidia.com/gpu: 0
      generictemplate:
        apiVersion: batch/v1
        kind: Job
        metadata:
          name: batch-job-3
          labels:
              appwrapper.mcad.ibm.com: batch-job-3
          namespace: namespace-1
        spec:
          parallelism: 1
          completions: 1
          template:
            metadata:
              labels:
                appwrapper.mcad.ibm.com: batch-job-3
              namespace: namespace-1
            spec:
              containers:
              - name: batch-job-3
                image: ubi8-minimal:latest
                command: [ "/bin/bash", "-c", "--" ]
                args: [ "sleep infinity" ]
                resources:
                  requests:
                    memory: 6Gi
                    cpu: "3000m"
                  limits:
                    memory: 6Gi
                    cpu: "3000m"
              restartPolicy: Never

apiVersion: mcad.ibm.com/v1beta1
kind: AppWrapper
metadata:
  name: batch-job-2
  namespace: namespace-1
  labels:
    quota_context: "namespace-2"
spec:
  service:
    spec: {}
  priority: 100
  resources:
    metadata: {}
    GenericItems:
    - replicas: 1
      completionstatus: Complete
      custompodresources:
      - replicas: 1
        requests:
          cpu: 2000m
          memory: 4Gi
          nvidia.com/gpu: 0
        limits:
          cpu: 2000m
          memory: 4Gi
          nvidia.com/gpu: 0
      generictemplate:
        apiVersion: batch/v1
        kind: Job
        metadata:
          name: batch-job-2
          labels:
              appwrapper.mcad.ibm.com: batch-job-2
          namespace: namespace-1
        spec:
          parallelism: 1
          completions: 1
          template:
            metadata:
              labels:
                appwrapper.mcad.ibm.com: batch-job-2
              namespace: namespace-1
            spec:
              containers:
              - name: batch-job-2
                image: ubi8-minimal:latest
                command: [ "/bin/bash", "-c", "--" ]
                args: [ "sleep infinity" ]
                resources:
                  requests:
                    memory: 4Gi
                    cpu: "2000m"
                  limits:
                    memory: 4Gi
                    cpu: "2000m"
              restartPolicy: Never

apiVersion: mcad.ibm.com/v1beta1
kind: AppWrapper
metadata:
  name: batch-job-1
  namespace: namespace-1
  labels:
    quota_context: "namespace-1"
spec:
  service:
    spec: {}
  priority: 100
  resources:
    metadata: {}
    GenericItems:
    - replicas: 1
      completionstatus: Complete
      custompodresources:
      - replicas: 1
        requests:
          cpu: 24000m
          memory: 32Gi
          nvidia.com/gpu: 0
        limits:
          cpu: 24000m
          memory: 32Gi
          nvidia.com/gpu: 0
      generictemplate:
        apiVersion: batch/v1
        kind: Job
        metadata:
          name: batch-job-1
          labels:
              appwrapper.mcad.ibm.com: batch-job-1
          namespace: namespace-1
        spec:
          parallelism: 1
          completions: 1
          template:
            metadata:
              labels:
                appwrapper.mcad.ibm.com: batch-job-1
              namespace: namespace-1
            spec:
              containers:
              - name: batch-job-1
                image: ubi8-minimal:latest
                command: [ "/bin/bash", "-c", "--" ]
                args: [ "sleep infinity" ]
                resources:
                  requests:
                    memory: 32Gi
                    cpu: "24000m"
                  limits:
                    memory: 32Gi
                    cpu: "24000m"
              restartPolicy: Never