Enable TestMNISTRayClusterSDK test

Author: sutaakar

.github/actions/kind/action.yml

@@ -1,6 +1,12 @@
 name: "Set up KinD"
 description: "Step to start and configure KinD cluster"
 
+inputs:
+  kind-node-hostname:
+    description: "Hostname of the main kind node"
+    required: false
+    default: kind
+
 runs:
   using: "composite"
   steps:
@@ -56,3 +62,13 @@ runs:
         curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/"${VERSION}"/deploy/static/provider/kind/deploy.yaml | sed "s/--publish-status-address=localhost/--report-node-internal-ip-address\\n        - --status-update-interval=10/g" | kubectl apply -f -
         kubectl annotate ingressclass nginx "ingressclass.kubernetes.io/is-default-class=true"
         kubectl -n ingress-nginx wait --timeout=300s --for=condition=Available deployments --all
+
+    - name: Add ${{ inputs.kind-node-hostname }} host to machine hosts
+      shell: bash
+      run: echo "127.0.0.1 ${{ inputs.kind-node-hostname }}" | sudo tee -a /etc/hosts
+
+    - name: Set env variables for tests to properly leverage KinD cluster
+      shell: bash
+      run: |
+        echo "CLUSTER_TYPE=KIND" >> $GITHUB_ENV
+        echo "CLUSTER_HOSTNAME=kind" >> $GITHUB_ENV

.github/workflows/e2e_tests.yaml

@@ -71,7 +71,7 @@ jobs:
           echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
 
           set -euo pipefail
-          go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
+          go test -timeout 30m -run TestMNISTRayClusterSDK -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
 
       - name: Print CodeFlare operator logs
         if: always() && steps.deploy.outcome == 'success'
@@ -85,6 +85,13 @@ jobs:
           echo "Printing KubeRay operator logs"
           kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
 
+      - name: Print Ingress
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing Ingress"
+          kubectl describe ingress -A
+          kubectl get ingress -A -o yaml
+
       - name: Upload logs
         uses: actions/upload-artifact@v3
         if: always() && steps.deploy.outcome == 'success'

.github/workflows/olm_tests.yaml

@@ -21,6 +21,8 @@ require (
 
 replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3
 
+replace github.com/project-codeflare/codeflare-common => github.com/sutaakar/codeflare-common v0.0.0-20231110120152-dbaedcac175e
+
 require (
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect

go.mod

@@ -391,8 +391,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 h1:TRMLDP6IYt0CAd3+BkvY/r2lkpjI3sOsxf3tnQojZ9k=
-github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk=
 github.com/project-codeflare/instascale v0.3.0 h1:PSlwbqqUsFTkTQ5KUhMFRebfokySnEZwav97xZixLQs=
 github.com/project-codeflare/instascale v0.3.0/go.mod h1:IU1Wl+zqTpMpZ49BOcr6U+A6gF3AjcmFdKo9ZwP3TDI=
 github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 h1:dU2Ev0SijdNm30Y9mjdKJL1Fp6l07rnRBKhSbx1kX9g=
@@ -463,6 +461,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/sutaakar/codeflare-common v0.0.0-20231110120152-dbaedcac175e h1:xWNK1SCWHkmlpnelVjnmAYdoCTJ466B21Ssa1GGUbk4=
+github.com/sutaakar/codeflare-common v0.0.0-20231110120152-dbaedcac175e/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=

go.sum

@@ -30,6 +30,11 @@ nodes:
         nodeRegistration:
           kubeletExtraArgs:
             node-labels: "ingress-ready=true"
+    extraPortMappings:
+      - containerPort: 80
+        hostPort: 80
+        listenAddress: "0.0.0.0"
+        protocol: tcp
 EOF
 
 echo "Deploying Ingress controller into KinD cluster"

test/e2e/kind.sh

@@ -1,4 +1,5 @@
 import sys
+import os
 
 from time import sleep
 
@@ -9,16 +10,36 @@
 
 namespace = sys.argv[1]
 
+ingress_options = {}
+
+host = os.getenv('CLUSTER_HOSTNAME')
+if host is not None:
+    ingress_options = {
+        "ingresses": [
+            {
+                "ingressName": "ray-dashboard",
+                "port": 8265,
+                "pathType": "Prefix",
+                "path": "/",
+                "host": host,
+            },
+        ]
+    }
+
+
 cluster = Cluster(ClusterConfiguration(
     name='mnist',
     namespace=namespace,
     num_workers=1,
+    head_cpus='500m',
+    head_memory=2,
     min_cpus='500m',
     max_cpus=1,
     min_memory=0.5,
-    max_memory=1,
+    max_memory=2,
     num_gpus=0,
     instascale=False,
+    ingress_options = ingress_options
 ))
 
 cluster.up()

test/e2e/mnist_raycluster_sdk.py

@@ -40,9 +40,6 @@ func TestMNISTRayClusterSDK(t *testing.T) {
 	test := With(t)
 	test.T().Parallel()
 
-	// Currently blocked by https://github.com/project-codeflare/codeflare-sdk/pull/251 , remove the skip once SDK with the PR is released
-	test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/251")
-
 	// Create a namespace
 	namespace := test.NewTestNamespace()
 
@@ -109,6 +106,11 @@ func TestMNISTRayClusterSDK(t *testing.T) {
 				APIGroups: []string{"route.openshift.io"},
 				Resources: []string{"routes"},
 			},
+			{
+				Verbs:     []string{"get", "list"},
+				APIGroups: []string{"networking.k8s.io"},
+				Resources: []string{"ingresses"},
+			},
 		},
 	}
 	role, err = test.Client().Core().RbacV1().Roles(namespace.Name).Create(test.Ctx(), role, metav1.CreateOptions{})
@@ -163,7 +165,7 @@ func TestMNISTRayClusterSDK(t *testing.T) {
 							Env: []corev1.EnvVar{
 								corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"},
 							},
-							Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name},
+							Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==0.11.0 && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name},
 							VolumeMounts: []corev1.VolumeMount{
 								{
 									Name:      "test",
@@ -211,6 +213,24 @@ func TestMNISTRayClusterSDK(t *testing.T) {
 			},
 		},
 	}
+	if GetClusterType(test) == KindCluster {
+		// Take first KinD node and redirect pod hostname requests there
+		node := GetNodes(test)[0]
+		hostname := GetClusterHostname(test)
+		IP := GetNodeInternalIP(test, node)
+
+		test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP)
+		job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{
+			{
+				IP:        IP,
+				Hostnames: []string{hostname},
+			},
+		}
+
+		hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname}
+		job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar)
+	}
+
 	job, err = test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{})
 	test.Expect(err).NotTo(HaveOccurred())
 	test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name)