From 83ad5a60e18c408bb57146c2329dd931490b558b Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Wed, 24 Jan 2024 23:02:10 +0000 Subject: [PATCH 1/2] Added e2e test for nodepools --- test/e2e/instascale_app_wrapper.go | 1 + test/e2e/instascale_nodepool_test.go | 66 ++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 test/e2e/instascale_nodepool_test.go diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go index e030817f3..45305ddab 100644 --- a/test/e2e/instascale_app_wrapper.go +++ b/test/e2e/instascale_app_wrapper.go @@ -48,6 +48,7 @@ func instaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *cor Image: GetPyTorchImage(), Env: []corev1.EnvVar{ {Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, Args: []string{"$PYTHONUSERBASE"}, diff --git a/test/e2e/instascale_nodepool_test.go b/test/e2e/instascale_nodepool_test.go new file mode 100644 index 000000000..17945040c --- /dev/null +++ b/test/e2e/instascale_nodepool_test.go @@ -0,0 +1,66 @@ +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestInstascaleNodepool(t *testing.T) { + + test := With(t) + test.T().Parallel() + + clusterType := GetClusterType(test) + if clusterType != HypershiftCluster { + test.T().Skipf("Skipping test as not running on an Hypershift cluster, resolved cluster type: %s", clusterType) + } + + namespace := test.NewTestNamespace() + + // Test configuration + cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }) + + //create OCM connection + connection := CreateOCMConnection(test) + defer connection.Close() + + // check existing cluster resources + // look for a node pool with a label key equal to aw name - expect NOT to find it + test.Expect(GetNodePools(test, connection)). + ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + + // Setup batch job and AppWrapper + aw := instaScaleJobAppWrapper(test, namespace, cm) + + // apply AppWrapper to cluster + _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) + + // assert that AppWrapper goes to "Running" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + // look for a node pool with a label key equal to aw name - expect to find it + test.Eventually(NodePools(test, connection), TestTimeoutLong). + Should(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + + // assert that the AppWrapper goes to "Completed" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) + + // look for a node pool with a label key equal to aw name - expect NOT to find it + test.Eventually(NodePools(test, connection), TestTimeoutLong). + ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + +} From 95ee58abaaf42c17a6019dba0864c0b17f0bbf0d Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Wed, 31 Jan 2024 13:38:35 +0000 Subject: [PATCH 2/2] address feedback --- test/e2e/instascale_nodepool_test.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/test/e2e/instascale_nodepool_test.go b/test/e2e/instascale_nodepool_test.go index 17945040c..8721663d0 100644 --- a/test/e2e/instascale_nodepool_test.go +++ b/test/e2e/instascale_nodepool_test.go @@ -1,6 +1,7 @@ package e2e import ( + "fmt" "testing" . "github.com/onsi/gomega" @@ -34,14 +35,15 @@ func TestInstascaleNodepool(t *testing.T) { connection := CreateOCMConnection(test) defer connection.Close() - // check existing cluster resources - // look for a node pool with a label key equal to aw name - expect NOT to find it - test.Expect(GetNodePools(test, connection)). - ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) - // Setup batch job and AppWrapper aw := instaScaleJobAppWrapper(test, namespace, cm) + expectedLabel := fmt.Sprintf("%s-%s", aw.Name, aw.Namespace) + // check existing cluster resources + // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect NOT to find it + test.Expect(GetNodePools(test, connection)). + ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) + // apply AppWrapper to cluster _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) @@ -51,16 +53,16 @@ func TestInstascaleNodepool(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - // look for a node pool with a label key equal to aw name - expect to find it + // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect to find it test.Eventually(NodePools(test, connection), TestTimeoutLong). - Should(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + Should(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) // assert that the AppWrapper goes to "Completed" state test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - // look for a node pool with a label key equal to aw name - expect NOT to find it + // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect NOT to find it test.Eventually(NodePools(test, connection), TestTimeoutLong). - ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) }