diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 413c8c5eb..1a1aab2c7 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -18,7 +18,6 @@ package e2e import ( "encoding/base64" - "net/url" "testing" . "github.com/onsi/gomega" @@ -26,12 +25,8 @@ import ( rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - routev1 "github.com/openshift/api/route/v1" . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -257,98 +252,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) - var rayDashboardURL url.URL - if IsOpenShift(test) { - // Create a route to expose the Ray cluster API - route := &routev1.Route{ - TypeMeta: metav1.TypeMeta{ - APIVersion: routev1.GroupVersion.String(), - Kind: "Route", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace.Name, - Name: "ray-dashboard", - }, - Spec: routev1.RouteSpec{ - To: routev1.RouteTargetReference{ - Name: "raycluster-head-svc", - }, - Port: &routev1.RoutePort{ - TargetPort: intstr.FromString("dashboard"), - }, - }, - } - - _, err := test.Client().Route().RouteV1().Routes(namespace.Name).Create(test.Ctx(), route, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Route %s/%s successfully", route.Namespace, route.Name) - - test.T().Logf("Waiting for Route %s/%s to be admitted", route.Namespace, route.Name) - test.Eventually(Route(test, route.Namespace, route.Name), TestTimeoutMedium). - Should(WithTransform(ConditionStatus(routev1.RouteAdmitted), Equal(corev1.ConditionTrue))) - - route = GetRoute(test, route.Namespace, route.Name) - - rayDashboardURL = url.URL{ - Scheme: "http", - Host: route.Status.Ingress[0].Host, - } - } else { - ingress := &networkingv1.Ingress{ - TypeMeta: metav1.TypeMeta{ - APIVersion: networkingv1.SchemeGroupVersion.String(), - Kind: "Ingress", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace.Name, - Name: "ray-dashboard", - Annotations: map[string]string{ - "nginx.ingress.kubernetes.io/use-regex": "true", - "nginx.ingress.kubernetes.io/rewrite-target": "/$2", - }, - }, - Spec: networkingv1.IngressSpec{ - Rules: []networkingv1.IngressRule{ - { - IngressRuleValue: networkingv1.IngressRuleValue{ - HTTP: &networkingv1.HTTPIngressRuleValue{ - Paths: []networkingv1.HTTPIngressPath{ - { - Path: "/ray-dashboard(/|$)(.*)", - PathType: Ptr(networkingv1.PathTypePrefix), - Backend: networkingv1.IngressBackend{ - Service: &networkingv1.IngressServiceBackend{ - Name: "raycluster-head-svc", - Port: networkingv1.ServiceBackendPort{ - Name: "dashboard", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - _, err := test.Client().Core().NetworkingV1().Ingresses(ingress.Namespace).Create(test.Ctx(), ingress, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Ingress %s/%s successfully", ingress.Namespace, ingress.Name) - - test.T().Logf("Waiting for Ingress %s/%s to be admitted", ingress.Namespace, ingress.Name) - test.Eventually(Ingress(test, ingress.Namespace, ingress.Name), TestTimeoutMedium). - Should(WithTransform(LoadBalancerIngresses, HaveLen(1))) - - ingress = GetIngress(test, ingress.Namespace, ingress.Name) - - rayDashboardURL = url.URL{ - Scheme: "http", - Host: ingress.Status.LoadBalancer.Ingress[0].IP, - Path: "ray-dashboard", - } - } + rayDashboardURL := ExposeService(test, "ray-dashboard", namespace.Name, "raycluster-head-svc", "dashboard") test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) rayClient := NewRayClusterClient(rayDashboardURL) diff --git a/test/support/ingress.go b/test/support/ingress.go index 667abe6e4..d834f3b15 100644 --- a/test/support/ingress.go +++ b/test/support/ingress.go @@ -17,6 +17,8 @@ limitations under the License. package support import ( + "net/url" + "github.com/onsi/gomega" networkingv1 "k8s.io/api/networking/v1" @@ -39,3 +41,61 @@ func GetIngress(t Test, namespace, name string) *networkingv1.Ingress { func LoadBalancerIngresses(ingress *networkingv1.Ingress) []networkingv1.IngressLoadBalancerIngress { return ingress.Status.LoadBalancer.Ingress } + +func ExposeServiceByIngress(t Test, name string, namespace string, serviceName string, servicePort string) url.URL { + ingress := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: networkingv1.SchemeGroupVersion.String(), + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "nginx.ingress.kubernetes.io/use-regex": "true", + "nginx.ingress.kubernetes.io/rewrite-target": "/$2", + }, + }, + Spec: networkingv1.IngressSpec{ + Rules: []networkingv1.IngressRule{ + networkingv1.IngressRule{ + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + networkingv1.HTTPIngressPath{ + Path: "/" + name + "(/|$)(.*)", + PathType: Ptr(networkingv1.PathTypePrefix), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: serviceName, + Port: networkingv1.ServiceBackendPort{ + Name: servicePort, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + _, err := t.Client().Core().NetworkingV1().Ingresses(ingress.Namespace).Create(t.Ctx(), ingress, metav1.CreateOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + t.T().Logf("Created Ingress %s/%s successfully", ingress.Namespace, ingress.Name) + + t.T().Logf("Waiting for Ingress %s/%s to be admitted", ingress.Namespace, ingress.Name) + t.Eventually(Ingress(t, ingress.Namespace, ingress.Name), TestTimeoutMedium). + Should(gomega.WithTransform(LoadBalancerIngresses, gomega.HaveLen(1))) + + ingress = GetIngress(t, ingress.Namespace, ingress.Name) + + ingressURL := url.URL{ + Scheme: "http", + Host: ingress.Status.LoadBalancer.Ingress[0].IP, + Path: name, + } + return ingressURL +} diff --git a/test/support/route.go b/test/support/route.go index 8a74ca12f..df1608909 100644 --- a/test/support/route.go +++ b/test/support/route.go @@ -17,9 +17,14 @@ limitations under the License. package support import ( + "net/http" + "net/url" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" routev1 "github.com/openshift/api/route/v1" ) @@ -36,3 +41,51 @@ func GetRoute(t Test, namespace, name string) *routev1.Route { t.T().Helper() return Route(t, namespace, name)(t) } + +func ExposeServiceByRoute(t Test, name string, namespace string, serviceName string, servicePort string) url.URL { + r := &routev1.Route{ + TypeMeta: metav1.TypeMeta{ + APIVersion: routev1.SchemeGroupVersion.String(), + Kind: "Route", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: routev1.RouteSpec{ + To: routev1.RouteTargetReference{ + Name: serviceName, + }, + Port: &routev1.RoutePort{ + TargetPort: intstr.FromString(servicePort), + }, + }, + } + + _, err := t.Client().Route().RouteV1().Routes(r.Namespace).Create(t.Ctx(), r, metav1.CreateOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + t.T().Logf("Created Route %s/%s successfully", r.Namespace, r.Name) + + t.T().Logf("Waiting for Route %s/%s to be available", r.Namespace, r.Name) + t.Eventually(Route(t, r.Namespace, r.Name), TestTimeoutLong). + Should(gomega.WithTransform(ConditionStatus(routev1.RouteAdmitted), gomega.Equal(corev1.ConditionTrue))) + + // Retrieve hostname + r, err = t.Client().Route().RouteV1().Routes(r.Namespace).Get(t.Ctx(), r.Name, metav1.GetOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + hostname := r.Status.Ingress[0].Host + + // Wait for expected HTTP code + t.Eventually(func() int { + resp, _ := http.Get("http://" + hostname) + return resp.StatusCode + }, TestTimeoutLong).Should(gomega.Not(gomega.Equal(503))) + + r = GetRoute(t, r.Namespace, r.Name) + routeURL := url.URL{ + Scheme: "http", + Host: r.Status.Ingress[0].Host, + } + + return routeURL +} diff --git a/test/support/service.go b/test/support/service.go new file mode 100644 index 000000000..d59ea3ec9 --- /dev/null +++ b/test/support/service.go @@ -0,0 +1,27 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import "net/url" + +func ExposeService(t Test, name string, namespace string, serviceName string, servicePort string) url.URL { + if IsOpenShift(t) { + return ExposeServiceByRoute(t, name, namespace, serviceName, servicePort) + } else { + return ExposeServiceByIngress(t, name, namespace, serviceName, servicePort) + } +} diff --git a/test/support/support.go b/test/support/support.go index 782e65e7a..1255baa8c 100644 --- a/test/support/support.go +++ b/test/support/support.go @@ -28,11 +28,11 @@ import ( ) var ( + ApplyOptions = metav1.ApplyOptions{FieldManager: "codeflare-test", Force: true} + TestTimeoutShort = 1 * time.Minute TestTimeoutMedium = 2 * time.Minute TestTimeoutLong = 5 * time.Minute - - ApplyOptions = metav1.ApplyOptions{FieldManager: "codeflare-test", Force: true} ) func init() {