Updated testing for new requeuing mechanism

Author: metalcycling

test/e2e/queue.go

@@ -36,7 +36,6 @@ package e2e
 import (
 	"fmt"
 	"os"
-
 	"time"
 
 	. "github.com/onsi/ginkgo"
@@ -140,7 +139,7 @@ var _ = Describe("AppWrapper E2E Test", func() {
 		Expect(err).NotTo(HaveOccurred())
 	})
 
-	It("MCAD CPU Requeuing Test", func() {
+	It("MCAD CPU Requeuing - Completion After enough Requeuing Times Test", func() {
 		fmt.Fprintf(os.Stdout, "[e2e] MCAD CPU Requeuing Test - Started.\n")
 
 		context := initTestContext()
@@ -153,13 +152,31 @@ var _ = Describe("AppWrapper E2E Test", func() {
 		// The job should be requeued 3 times before it finishes since the wait time is doubled each time the job is requeued (i.e., initially it waits
 		// for 1 minutes before requeuing, then 2 minutes, and then 4 minutes). Since the init containers take 3 minutes
 		// and 20 seconds to finish, a 4 minute wait should be long enough to finish the job successfully
-		aw := createJobAWWithInitContainer(context, "aw-job-3-init-container")
+		aw := createJobAWWithInitContainer(context, "aw-job-3-init-container", 60, "exponential", 0)
 		appwrappers = append(appwrappers, aw)
 
-		err := waitAWPodsCompleted(context, aw) // This test waits for 10 minutes to make sure all PODs complete
+		err := waitAWPodsCompleted(context, aw, 600 * time.Second) // This test waits for 10 minutes to make sure all PODs complete
 		Expect(err).NotTo(HaveOccurred())
 	})
 
+	It("MCAD CPU Requeuing - Deletion After Maximum Requeuing Times Test", func() {
+		fmt.Fprintf(os.Stdout, "[e2e] MCAD CPU Requeuing Test - Started.\n")
+
+		context := initTestContext()
+		var appwrappers []*arbv1.AppWrapper
+		appwrappersPtr := &appwrappers
+		defer cleanupTestObjectsPtr(context, appwrappersPtr)
+
+		// Create a job with init containers that need 200 seconds to be ready before the container starts.
+		// The requeuing mechanism is set to fire after 1 second (plus the 60 seconds time interval of the background thread)
+		// Within 5 minutes, the AppWrapper will be requeued up to 3 times at which point it will be deleted
+		aw := createJobAWWithInitContainer(context, "aw-job-3-init-container", 1, "none", 3)
+		appwrappers = append(appwrappers, aw)
+
+		err := waitAWPodsCompleted(context, aw, 300 * time.Second)
+		Expect(err).To(HaveOccurred())
+	})
+
 	It("Create AppWrapper - StatefulSet Only - 2 Pods", func() {
 		fmt.Fprintf(os.Stdout, "[e2e] Create AppWrapper - StatefulSet Only - 2 Pods - Started.\n")
 
@@ -371,7 +388,7 @@ var _ = Describe("AppWrapper E2E Test", func() {
 
 		// This should fit on cluster with customPodResources matching deployment resource demands so AW pods are created
 		aw := createGenericDeploymentCustomPodResourcesWithCPUAW(
-			context, "aw-deployment-2-550-vs-550-cpu", "550m", "550m", 2, 1)
+			context, "aw-deployment-2-550-vs-550-cpu", "550m", "550m", 2, 60)
 
 		appwrappers = append(appwrappers, aw)
 
@@ -399,7 +416,7 @@ var _ = Describe("AppWrapper E2E Test", func() {
 
 		// This should not fit on any node but should dispatch because there is enough aggregated resources.
 		aw2 := createGenericDeploymentCustomPodResourcesWithCPUAW(
-			context, "aw-ff-deployment-1-700-cpu", "700m", "700m", 1, 1)
+			context, "aw-ff-deployment-1-700-cpu", "700m", "700m", 1, 60)
 
 		appwrappers = append(appwrappers, aw2)
 
@@ -412,7 +429,7 @@ var _ = Describe("AppWrapper E2E Test", func() {
 		// This should fit on cluster after AW aw-deployment-1-700-cpu above is automatically preempted on
 		// scheduling failure
 		aw3 := createGenericDeploymentCustomPodResourcesWithCPUAW(
-			context, "aw-ff-deployment-2-340-cpu", "340m", "340m", 2, 1)
+			context, "aw-ff-deployment-2-340-cpu", "340m", "340m", 2, 60)
 
 		appwrappers = append(appwrappers, aw3)
 
@@ -445,7 +462,7 @@ var _ = Describe("AppWrapper E2E Test", func() {
 
 		// This should not fit on cluster but customPodResources is incorrect so AW pods are created
 		aw2 := createGenericDeploymentCustomPodResourcesWithCPUAW(
-			context, "aw-deployment-2-425-vs-426-cpu", "425m", "426m", 2, 1)
+			context, "aw-deployment-2-425-vs-426-cpu", "425m", "426m", 2, 60)
 
 		appwrappers = append(appwrappers, aw2)
 
@@ -472,7 +489,7 @@ var _ = Describe("AppWrapper E2E Test", func() {
 
 		// This should fit on cluster but customPodResources is incorrect so AW pods are not created
 		aw2 := createGenericDeploymentCustomPodResourcesWithCPUAW(
-			context, "aw-deployment-2-426-vs-425-cpu", "426m", "425m", 2, 1)
+			context, "aw-deployment-2-426-vs-425-cpu", "426m", "425m", 2, 60)
 
 		appwrappers = append(appwrappers, aw2)
 
@@ -517,7 +534,7 @@ var _ = Describe("AppWrapper E2E Test", func() {
 		aw := createGenericJobAWWithScheduleSpec(context, "aw-test-job-with-scheduling-spec")
 		err1 := waitAWPodsReady(context, aw)
 		Expect(err1).NotTo(HaveOccurred())
-		err2 := waitAWPodsCompleted(context, aw)
+		err2 := waitAWPodsCompleted(context, aw, 90 * time.Second)
 		Expect(err2).NotTo(HaveOccurred())
 
 		// Once pods are completed, we wait for them to see if they change their status to anything BUT "Completed"

test/e2e/util.go

@@ -612,8 +612,8 @@ func waitAWPodsReady(ctx *context, aw *arbv1.AppWrapper) error {
 	return waitAWPodsReadyEx(ctx, aw, int(aw.Spec.SchedSpec.MinAvailable), false)
 }
 
-func waitAWPodsCompleted(ctx *context, aw *arbv1.AppWrapper) error {
-	return waitAWPodsCompletedEx(ctx, aw, int(aw.Spec.SchedSpec.MinAvailable), false)
+func waitAWPodsCompleted(ctx *context, aw *arbv1.AppWrapper, timeout time.Duration) error {
+	return waitAWPodsCompletedEx(ctx, aw, int(aw.Spec.SchedSpec.MinAvailable), false, timeout)
 }
 
 func waitAWPodsNotCompleted(ctx *context, aw *arbv1.AppWrapper) error {
@@ -654,8 +654,8 @@ func waitAWPodsReadyEx(ctx *context, aw *arbv1.AppWrapper, taskNum int, quite bo
 		[]v1.PodPhase{v1.PodRunning, v1.PodSucceeded}, taskNum, quite))
 }
 
-func waitAWPodsCompletedEx(ctx *context, aw *arbv1.AppWrapper, taskNum int, quite bool) error {
-	return wait.Poll(100*time.Millisecond, tenMinutes, awPodPhase(ctx, aw,
+func waitAWPodsCompletedEx(ctx *context, aw *arbv1.AppWrapper, taskNum int, quite bool, timeout time.Duration ) error {
+	return wait.Poll(100*time.Millisecond, timeout, awPodPhase(ctx, aw,
 		[]v1.PodPhase{v1.PodSucceeded}, taskNum, quite))
 }
 
@@ -745,7 +745,7 @@ func createReplicaSet(context *context, name string, rep int32, img string, req
 	return deployment
 }
 
-func createJobAWWithInitContainer(context *context, name string) *arbv1.AppWrapper {
+func createJobAWWithInitContainer(context *context, name string, requeuingTimeInSeconds int, requeuingGrowthType string, requeuingMaxNumRequeuings int ) *arbv1.AppWrapper {
 	rb := []byte(`{"apiVersion": "batch/v1",
 		"kind": "Job",
 	"metadata": {
@@ -798,7 +798,6 @@ func createJobAWWithInitContainer(context *context, name string) *arbv1.AppWrapp
 	}} `)
 
 	var minAvailable int = 3
-	var requeuingTimeMinutes int = 1
 
 	aw := &arbv1.AppWrapper{
 		ObjectMeta: metav1.ObjectMeta{
@@ -808,7 +807,11 @@ func createJobAWWithInitContainer(context *context, name string) *arbv1.AppWrapp
 		Spec: arbv1.AppWrapperSpec{
 			SchedSpec: arbv1.SchedulingSpecTemplate{
 				MinAvailable: minAvailable,
-				RequeuingTimeMinutes: requeuingTimeMinutes,
+				Requeuing: arbv1.RequeuingTemplate{
+					TimeInSeconds: requeuingTimeInSeconds,
+					GrowthType: requeuingGrowthType,
+					MaxNumRequeuings: requeuingMaxNumRequeuings,
+				},
 			},
 			AggrResources: arbv1.AppWrapperResourceList{
 				GenericItems: []arbv1.AppWrapperGenericResource{
@@ -2291,7 +2294,7 @@ func createGenericDeploymentWithCPUAW(context *context, name string, cpuDemand s
 	return appwrapper
 }
 
-func createGenericDeploymentCustomPodResourcesWithCPUAW(context *context, name string, customPodCpuDemand string, cpuDemand string, replicas int, requeuingTimeMinutes int) *arbv1.AppWrapper {
+func createGenericDeploymentCustomPodResourcesWithCPUAW(context *context, name string, customPodCpuDemand string, cpuDemand string, replicas int, requeuingTimeInSeconds int) *arbv1.AppWrapper {
 	rb := []byte(fmt.Sprintf(`{
 	"apiVersion": "apps/v1",
 	"kind": "Deployment", 
@@ -2350,7 +2353,9 @@ func createGenericDeploymentCustomPodResourcesWithCPUAW(context *context, name s
 		Spec: arbv1.AppWrapperSpec{
 			SchedSpec: arbv1.SchedulingSpecTemplate{
 				MinAvailable: schedSpecMin,
-				RequeuingTimeMinutes: requeuingTimeMinutes,
+				Requeuing: arbv1.RequeuingTemplate{
+					TimeInSeconds: requeuingTimeInSeconds,
+				},
 			},
 			AggrResources: arbv1.AppWrapperResourceList{
 				GenericItems: []arbv1.AppWrapperGenericResource{