From 653fd1516f73f768b7aee9b4e144252b39043215 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 29 Mar 2024 17:52:30 -0400 Subject: [PATCH] raise error if raycluster crd not available Signed-off-by: Kevin --- src/codeflare_sdk/cluster/cluster.py | 25 +++++++++++++++++++++++++ tests/unit_test.py | 24 ++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index 127c6fade..fa9dded80 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -48,6 +48,7 @@ import requests from kubernetes import config +from kubernetes.client.rest import ApiException class Cluster: @@ -225,6 +226,10 @@ def up(self): Applies the AppWrapper yaml, pushing the resource request onto the MCAD queue. """ + + # check if RayCluster CustomResourceDefinition exists if not throw RuntimeError + self._throw_for_no_raycluster() + namespace = self.config.namespace try: @@ -255,12 +260,32 @@ def up(self): except Exception as e: # pragma: no cover return _kube_api_error_handling(e) + def _throw_for_no_raycluster(self): + api_instance = client.CustomObjectsApi(api_config_handler()) + try: + api_instance.list_namespaced_custom_object( + group="ray.io", + version="v1", + namespace=self.config.namespace, + plural="rayclusters", + ) + except ApiException as e: + if e.status == 404: + raise RuntimeError( + "RayCluster CustomResourceDefinition unavailable contact your administrator." + ) + else: + raise RuntimeError( + "Failed to get RayCluster CustomResourceDefinition: " + str(e) + ) + def down(self): """ Deletes the AppWrapper yaml, scaling-down and deleting all resources associated with the cluster. """ namespace = self.config.namespace + self._throw_for_no_raycluster() try: config_check() api_instance = client.CustomObjectsApi(api_config_handler()) diff --git a/tests/unit_test.py b/tests/unit_test.py index 9a28d1c1a..145bb45d1 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -474,6 +474,7 @@ def arg_check_del_effect(group, version, namespace, plural, name, *args): def test_cluster_up_down(mocker): mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") + mocker.patch("codeflare_sdk.cluster.cluster.Cluster._throw_for_no_raycluster") mocker.patch( "kubernetes.client.CustomObjectsApi.get_cluster_custom_object", return_value={"spec": {"domain": ""}}, @@ -496,6 +497,7 @@ def test_cluster_up_down(mocker): def test_cluster_up_down_no_mcad(mocker): + mocker.patch("codeflare_sdk.cluster.cluster.Cluster._throw_for_no_raycluster") mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch("kubernetes.config.load_kube_config", return_value="ignore") mocker.patch( @@ -3172,6 +3174,28 @@ def test_gen_app_wrapper_with_oauth(mocker: MockerFixture): ) +def test_cluster_throw_for_no_raycluster(mocker: MockerFixture): + mocker.patch("kubernetes.client.ApisApi.get_api_versions") + mocker.patch( + "codeflare_sdk.cluster.cluster.get_current_namespace", + return_value="opendatahub", + ) + mocker.patch( + "kubernetes.client.CustomObjectsApi.list_namespaced_custom_object", + side_effect=client.ApiException(status=404), + ) + cluster = Cluster( + ClusterConfiguration( + "test_cluster", + image="quay.io/project-codeflare/ray:latest-py39-cu118", + ingress_domain="apps.cluster.awsroute.org", + write_to_file=False, + ) + ) + with pytest.raises(RuntimeError): + cluster.up() + + """ Ray Jobs tests """