diff --git a/custom-nb-image/imagestream.yaml b/custom-nb-image/imagestream.yaml index e44c2f091..bd17076ff 100644 --- a/custom-nb-image/imagestream.yaml +++ b/custom-nb-image/imagestream.yaml @@ -21,7 +21,7 @@ metadata: annotations: opendatahub.io/notebook-image-name: "CodeFlare Notebook" - opendatahub.io/notebook-image-desc: "Custom Jupyter notebook image with CodeFlare SDK, Python 3.8, Ray 2.1.0 and PyTorch 1.12.1" + opendatahub.io/notebook-image-desc: "Custom Jupyter notebook image with CodeFlare SDK, Python 3.8, Ray 2.5.0 and PyTorch 1.12.1" spec: lookupPolicy: local: true diff --git a/custom-nb-image/requirements.txt b/custom-nb-image/requirements.txt index 17822fabf..35b5d5591 100644 --- a/custom-nb-image/requirements.txt +++ b/custom-nb-image/requirements.txt @@ -157,7 +157,7 @@ python-json-logger==2.0.4; python_version >= '3.5' pytz==2022.2.1 pyyaml==6.0; python_version >= '3.6' pyzmq==24.0.1; python_version >= '3.6' -ray[default]==2.1.0 +ray[default]==2.5.0 requests-oauthlib==1.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' requests==2.28.1; python_version >= '3.7' and python_version < '4' rsa==4.9; python_version >= '3.6' diff --git a/demo-notebooks/batch-job/batch_mnist_ray.ipynb b/demo-notebooks/batch-job/batch_mnist_ray.ipynb index 88b050701..b725bfc55 100644 --- a/demo-notebooks/batch-job/batch_mnist_ray.ipynb +++ b/demo-notebooks/batch-job/batch_mnist_ray.ipynb @@ -63,6 +63,7 @@ " min_memory=16,\n", " max_memory=16,\n", " gpu=4,\n", + " image=\"quay.io/project-codeflare/ray:2.5.0-py38-cu116\",\n", " instascale=True, # Can be set to false if scaling not needed\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"] # Can be removed if above is false\n", "))" @@ -88,7 +89,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "657ebdfb", "metadata": {}, @@ -222,7 +222,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "b3a55fe4", "metadata": {}, @@ -5224,7 +5223,7 @@ ], "metadata": { "kernelspec": { - "display_name": "sdktest", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/demo-notebooks/guided-demos/0_basic_ray.ipynb b/demo-notebooks/guided-demos/0_basic_ray.ipynb index 7daf4b88b..338f5730b 100644 --- a/demo-notebooks/guided-demos/0_basic_ray.ipynb +++ b/demo-notebooks/guided-demos/0_basic_ray.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "id": "8d4a42f6", "metadata": {}, @@ -65,6 +64,7 @@ " max_cpus=1,\n", " min_memory=4,\n", " max_memory=4,\n", + " image=\"quay.io/project-codeflare/ray:2.5.0-py38-cu116\",\n", " gpu=0,\n", " instascale=False\n", "))" @@ -90,7 +90,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "657ebdfb", "metadata": {}, @@ -129,7 +128,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "b3a55fe4", "metadata": {}, @@ -178,7 +176,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -192,7 +190,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.8.13" }, "vscode": { "interpreter": { diff --git a/demo-notebooks/guided-demos/1_basic_instascale.ipynb b/demo-notebooks/guided-demos/1_basic_instascale.ipynb index 8bb86e125..cc5c8224b 100644 --- a/demo-notebooks/guided-demos/1_basic_instascale.ipynb +++ b/demo-notebooks/guided-demos/1_basic_instascale.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "id": "9865ee8c", "metadata": {}, @@ -38,7 +37,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "bc27f84c", "metadata": {}, @@ -64,13 +62,13 @@ " min_memory=8,\n", " max_memory=8,\n", " gpu=1,\n", + " image=\"quay.io/project-codeflare/ray:2.5.0-py38-cu116\",\n", " instascale=True, # InstaScale now enabled, will scale OCP cluster to guarantee resource request\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"] # Head, worker AWS machine types desired\n", "))" ] }, { - "attachments": {}, "cell_type": "markdown", "id": "12eef53c", "metadata": {}, @@ -91,7 +89,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "6abfe904", "metadata": {}, @@ -130,7 +127,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "c883caea", "metadata": {}, @@ -151,7 +147,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -165,7 +161,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.8.13" }, "vscode": { "interpreter": { diff --git a/demo-notebooks/guided-demos/2_basic_jobs.ipynb b/demo-notebooks/guided-demos/2_basic_jobs.ipynb index 37560b172..3fe46a8a3 100644 --- a/demo-notebooks/guided-demos/2_basic_jobs.ipynb +++ b/demo-notebooks/guided-demos/2_basic_jobs.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "id": "464af595", "metadata": {}, @@ -38,7 +37,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "bc27f84c", "metadata": {}, @@ -64,6 +62,7 @@ " min_memory=4,\n", " max_memory=4,\n", " gpu=0,\n", + " image=\"quay.io/project-codeflare/ray:2.5.0-py38-cu116\",\n", " instascale=False\n", "))" ] @@ -91,7 +90,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "33663f47", "metadata": {}, @@ -110,7 +108,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "83d77b74", "metadata": {}, @@ -134,7 +131,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "5b9ae53a", "metadata": {}, @@ -163,7 +159,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "5af8cd32", "metadata": {}, @@ -182,7 +177,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "31096641", "metadata": {}, @@ -211,7 +205,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "0837e43b", "metadata": {}, @@ -240,7 +233,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "aebf376a", "metadata": {}, @@ -271,7 +263,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -285,7 +277,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.8.13" }, "vscode": { "interpreter": { diff --git a/demo-notebooks/guided-demos/3_basic_interactive.ipynb b/demo-notebooks/guided-demos/3_basic_interactive.ipynb index 4a1b0f6f7..aad498500 100644 --- a/demo-notebooks/guided-demos/3_basic_interactive.ipynb +++ b/demo-notebooks/guided-demos/3_basic_interactive.ipynb @@ -62,6 +62,7 @@ " min_memory=8,\n", " max_memory=8,\n", " gpu=1,\n", + " image=\"quay.io/project-codeflare/ray:2.5.0-py38-cu116\",\n", " instascale=True,\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"]\n", " \n", @@ -275,7 +276,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -289,7 +290,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.8.13" }, "vscode": { "interpreter": { diff --git a/demo-notebooks/guided-demos/4_gpt.ipynb b/demo-notebooks/guided-demos/4_gpt.ipynb index 0132bd4a8..e489c9bcb 100644 --- a/demo-notebooks/guided-demos/4_gpt.ipynb +++ b/demo-notebooks/guided-demos/4_gpt.ipynb @@ -45,6 +45,7 @@ " min_memory=8,\n", " max_memory=8,\n", " gpu=1,\n", + " image=\"quay.io/project-codeflare/ray:2.5.0-py38-cu116\",\n", " instascale=True,\n", " machine_types=[\"m5.xlarge\", \"g4dn.xlarge\"],\n", "))" diff --git a/demo-notebooks/interactive/hf_interactive.ipynb b/demo-notebooks/interactive/hf_interactive.ipynb index ed7748216..64ad6ef7f 100644 --- a/demo-notebooks/interactive/hf_interactive.ipynb +++ b/demo-notebooks/interactive/hf_interactive.ipynb @@ -86,7 +86,17 @@ ], "source": [ "# Create our cluster and submit appwrapper\n", - "cluster = Cluster(ClusterConfiguration(name='hfgputest', min_worker=1, max_worker=1, min_cpus=8, max_cpus=8, min_memory=16, max_memory=16, gpu=4, instascale=True, machine_types=[\"m5.xlarge\", \"p3.8xlarge\"]))" + "cluster = Cluster(ClusterConfiguration(name='hfgputest', \n", + " namespace=\"default\",\n", + " min_worker=1, \n", + " max_worker=1, \n", + " min_cpus=8, \n", + " max_cpus=8, \n", + " min_memory=16, \n", + " max_memory=16, \n", + " gpu=4,\n", + " image=\"quay.io/project-codeflare/ray:2.5.0-py38-cu116\",\n", + " instascale=True, machine_types=[\"m5.xlarge\", \"p3.8xlarge\"]))" ] }, { @@ -108,7 +118,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "657ebdfb", "metadata": {}, @@ -189,7 +198,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "477ac246", "metadata": {}, @@ -309,7 +317,7 @@ "# establish connection to ray cluster\n", "\n", "#install additionall libraries that will be required for this training\n", - "runtime_env = {\"pip\": [\"transformers\", \"datasets\", \"evaluate\"]}\n", + "runtime_env = {\"pip\": [\"transformers\", \"datasets\", \"evaluate\", \"pyarrow<7.0.0\", \"accelerate\"]}\n", "\n", "ray.init(address=f'{ray_cluster_uri}', runtime_env=runtime_env)\n", "\n", @@ -1443,7 +1451,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.9.7 64-bit", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -1457,7 +1465,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.8.13" }, "vscode": { "interpreter": { diff --git a/pyproject.toml b/pyproject.toml index 79b90cb37..ee90ff08d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,8 +23,8 @@ keywords = ['codeflare', 'python', 'sdk', 'client', 'batch', 'scale'] python = "^3.7" openshift-client = "1.0.18" rich = "^12.5" -ray = {version = "2.1.0", extras = ["default"]} -kubernetes = ">= 25.3.0, < 27" +ray = {version = "2.5.0", extras = ["default"]} +kubernetes = "25.3.0" codeflare-torchx = "0.6.0.dev0" cryptography = "40.0.2" diff --git a/requirements.txt b/requirements.txt index e529bc39f..6c3c59d4a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ openshift-client==1.0.18 rich==12.5.1 -ray[default]==2.1.0 +ray[default]==2.5.0 kubernetes>=25.3.0,<27 codeflare-torchx==0.6.0.dev0 diff --git a/src/codeflare_sdk/cluster/config.py b/src/codeflare_sdk/cluster/config.py index a00d128a2..f24425635 100644 --- a/src/codeflare_sdk/cluster/config.py +++ b/src/codeflare_sdk/cluster/config.py @@ -47,6 +47,6 @@ class ClusterConfiguration: template: str = f"{dir}/templates/base-template.yaml" instascale: bool = False envs: dict = field(default_factory=dict) - image: str = "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103" + image: str = "quay.io/project-codeflare/ray:2.5.0-py38-cu116" local_interactive: bool = False image_pull_secrets: list = field(default_factory=list) diff --git a/tests/test-case-bad.yaml b/tests/test-case-bad.yaml index 1d273ca60..5518cb48e 100644 --- a/tests/test-case-bad.yaml +++ b/tests/test-case-bad.yaml @@ -73,7 +73,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103 + image: quay.io/project-codeflare/ray:2.5.0-py38-cu116 imagePullPolicy: Always lifecycle: preStop: @@ -130,7 +130,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103 + image: quay.io/project-codeflare/ray:2.5.0-py38-cu116 lifecycle: preStop: exec: diff --git a/tests/test-case.yaml b/tests/test-case.yaml index da6058698..167bb61b3 100644 --- a/tests/test-case.yaml +++ b/tests/test-case.yaml @@ -81,7 +81,7 @@ spec: value: /home/ray/workspace/tls/server.key - name: RAY_TLS_CA_CERT value: /home/ray/workspace/tls/ca.crt - image: ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103 + image: quay.io/project-codeflare/ray:2.5.0-py38-cu116 imagePullPolicy: Always lifecycle: preStop: @@ -148,7 +148,7 @@ spec: value: /home/ray/workspace/tls/server.key - name: RAY_TLS_CA_CERT value: /home/ray/workspace/tls/ca.crt - image: ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103 + image: quay.io/project-codeflare/ray:2.5.0-py38-cu116 lifecycle: preStop: exec: diff --git a/tests/unit_test.py b/tests/unit_test.py index d6e6c6ef6..d1ea3e75e 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -228,10 +228,7 @@ def test_config_creation(): assert config.min_cpus == 3 and config.max_cpus == 4 assert config.min_memory == 5 and config.max_memory == 6 assert config.gpu == 7 - assert ( - config.image - == "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103" - ) + assert config.image == "quay.io/project-codeflare/ray:2.5.0-py38-cu116" assert config.template == f"{parent}/src/codeflare_sdk/templates/base-template.yaml" assert config.instascale assert config.machine_types == ["cpu.small", "gpu.large"] @@ -668,7 +665,7 @@ def get_ray_obj(cls=None): "spec": { "containers": [ { - "image": "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103", + "image": "quay.io/project-codeflare/ray:2.5.0-py38-cu116", "imagePullPolicy": "Always", "lifecycle": { "preStop": { @@ -738,7 +735,7 @@ def get_ray_obj(cls=None): }, } ], - "image": "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103", + "image": "quay.io/project-codeflare/ray:2.5.0-py38-cu116", "lifecycle": { "preStop": { "exec": { @@ -802,7 +799,7 @@ def get_aw_obj(): "kind": "AppWrapper", "metadata": { "annotations": { - "kubectl.kubernetes.io/last-applied-configuration": '{"apiVersion":"mcad.ibm.com/v1beta1","kind":"AppWrapper","metadata":{"annotations":{},"name":"quicktest1","namespace":"ns"},"spec":{"priority":9,"resources":{"GenericItems":[{"custompodresources":[{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"replicas":1,"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}},{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"replicas":1,"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}],"generictemplate":{"apiVersion":"ray.io/v1alpha1","kind":"RayCluster","metadata":{"labels":{"appwrapper.mcad.ibm.com":"quicktest1","controller-tools.k8s.io":"1.0"},"name":"quicktest1","namespace":"ns"},"spec":{"autoscalerOptions":{"idleTimeoutSeconds":60,"imagePullPolicy":"Always","resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"500m","memory":"512Mi"}},"upscalingMode":"Default"},"enableInTreeAutoscaling":false,"headGroupSpec":{"rayStartParams":{"block":"true","dashboard-host":"0.0.0.0","num-gpus":"0"},"serviceType":"ClusterIP","template":{"spec":{"containers":[{"image":"ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103","imagePullPolicy":"Always","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"ray-head","ports":[{"containerPort":6379,"name":"gcs"},{"containerPort":8265,"name":"dashboard"},{"containerPort":10001,"name":"client"}],"resources":{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}}}]}}},"rayVersion":"1.12.0","workerGroupSpecs":[{"groupName":"small-group-quicktest","maxReplicas":1,"minReplicas":1,"rayStartParams":{"block":"true","num-gpus":"0"},"replicas":1,"template":{"metadata":{"annotations":{"key":"value"},"labels":{"key":"value"}},"spec":{"containers":[{"env":[{"name":"MY_POD_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}}],"image":"ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"machine-learning","resources":{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}}],"initContainers":[{"command":["sh","-c","until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"],"image":"busybox:1.28","name":"init-myservice"}]}}}]}},"replicas":1},{"generictemplate":{"apiVersion":"route.openshift.io/v1","kind":"Route","metadata":{"labels":{"odh-ray-cluster-service":"quicktest-head-svc"},"name":"ray-dashboard-quicktest","namespace":"default"},"spec":{"port":{"targetPort":"dashboard"},"to":{"kind":"Service","name":"quicktest-head-svc"}}},"replica":1}],"Items":[]}}}\n' + "kubectl.kubernetes.io/last-applied-configuration": '{"apiVersion":"mcad.ibm.com/v1beta1","kind":"AppWrapper","metadata":{"annotations":{},"name":"quicktest1","namespace":"ns"},"spec":{"priority":9,"resources":{"GenericItems":[{"custompodresources":[{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"replicas":1,"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}},{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"replicas":1,"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}],"generictemplate":{"apiVersion":"ray.io/v1alpha1","kind":"RayCluster","metadata":{"labels":{"appwrapper.mcad.ibm.com":"quicktest1","controller-tools.k8s.io":"1.0"},"name":"quicktest1","namespace":"ns"},"spec":{"autoscalerOptions":{"idleTimeoutSeconds":60,"imagePullPolicy":"Always","resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"500m","memory":"512Mi"}},"upscalingMode":"Default"},"enableInTreeAutoscaling":false,"headGroupSpec":{"rayStartParams":{"block":"true","dashboard-host":"0.0.0.0","num-gpus":"0"},"serviceType":"ClusterIP","template":{"spec":{"containers":[{"image":"quay.io/project-codeflare/ray:2.5.0-py38-cu116","imagePullPolicy":"Always","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"ray-head","ports":[{"containerPort":6379,"name":"gcs"},{"containerPort":8265,"name":"dashboard"},{"containerPort":10001,"name":"client"}],"resources":{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}}}]}}},"rayVersion":"1.12.0","workerGroupSpecs":[{"groupName":"small-group-quicktest","maxReplicas":1,"minReplicas":1,"rayStartParams":{"block":"true","num-gpus":"0"},"replicas":1,"template":{"metadata":{"annotations":{"key":"value"},"labels":{"key":"value"}},"spec":{"containers":[{"env":[{"name":"MY_POD_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}}],"image":"quay.io/project-codeflare/ray:2.5.0-py38-cu116","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"machine-learning","resources":{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}}],"initContainers":[{"command":["sh","-c","until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"],"image":"busybox:1.28","name":"init-myservice"}]}}}]}},"replicas":1},{"generictemplate":{"apiVersion":"route.openshift.io/v1","kind":"Route","metadata":{"labels":{"odh-ray-cluster-service":"quicktest-head-svc"},"name":"ray-dashboard-quicktest","namespace":"default"},"spec":{"port":{"targetPort":"dashboard"},"to":{"kind":"Service","name":"quicktest-head-svc"}}},"replica":1}],"Items":[]}}}\n' }, "creationTimestamp": "2023-02-22T16:26:07Z", "generation": 4, @@ -931,7 +928,7 @@ def get_aw_obj(): "spec": { "containers": [ { - "image": "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103", + "image": "quay.io/project-codeflare/ray:2.5.0-py38-cu116", "imagePullPolicy": "Always", "lifecycle": { "preStop": { @@ -1005,7 +1002,7 @@ def get_aw_obj(): }, } ], - "image": "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103", + "image": "quay.io/project-codeflare/ray:2.5.0-py38-cu116", "lifecycle": { "preStop": { "exec": { @@ -1124,7 +1121,7 @@ def get_aw_obj(): "kind": "AppWrapper", "metadata": { "annotations": { - "kubectl.kubernetes.io/last-applied-configuration": '{"apiVersion":"mcad.ibm.com/v1beta1","kind":"AppWrapper","metadata":{"annotations":{},"name":"quicktest2","namespace":"ns"},"spec":{"priority":9,"resources":{"GenericItems":[{"custompodresources":[{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"replicas":1,"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}},{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"replicas":1,"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}],"generictemplate":{"apiVersion":"ray.io/v1alpha1","kind":"RayCluster","metadata":{"labels":{"appwrapper.mcad.ibm.com":"quicktest2","controller-tools.k8s.io":"1.0"},"name":"quicktest2","namespace":"ns"},"spec":{"autoscalerOptions":{"idleTimeoutSeconds":60,"imagePullPolicy":"Always","resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"500m","memory":"512Mi"}},"upscalingMode":"Default"},"enableInTreeAutoscaling":false,"headGroupSpec":{"rayStartParams":{"block":"true","dashboard-host":"0.0.0.0","num-gpus":"0"},"serviceType":"ClusterIP","template":{"spec":{"containers":[{"image":"ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103","imagePullPolicy":"Always","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"ray-head","ports":[{"containerPort":6379,"name":"gcs"},{"containerPort":8265,"name":"dashboard"},{"containerPort":10001,"name":"client"}],"resources":{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}}}]}}},"rayVersion":"1.12.0","workerGroupSpecs":[{"groupName":"small-group-quicktest","maxReplicas":1,"minReplicas":1,"rayStartParams":{"block":"true","num-gpus":"0"},"replicas":1,"template":{"metadata":{"annotations":{"key":"value"},"labels":{"key":"value"}},"spec":{"containers":[{"env":[{"name":"MY_POD_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}}],"image":"ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"machine-learning","resources":{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}}],"initContainers":[{"command":["sh","-c","until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"],"image":"busybox:1.28","name":"init-myservice"}]}}}]}},"replicas":1},{"generictemplate":{"apiVersion":"route.openshift.io/v1","kind":"Route","metadata":{"labels":{"odh-ray-cluster-service":"quicktest-head-svc"},"name":"ray-dashboard-quicktest","namespace":"default"},"spec":{"port":{"targetPort":"dashboard"},"to":{"kind":"Service","name":"quicktest-head-svc"}}},"replica":1}],"Items":[]}}}\n' + "kubectl.kubernetes.io/last-applied-configuration": '{"apiVersion":"mcad.ibm.com/v1beta1","kind":"AppWrapper","metadata":{"annotations":{},"name":"quicktest2","namespace":"ns"},"spec":{"priority":9,"resources":{"GenericItems":[{"custompodresources":[{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"replicas":1,"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}},{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"replicas":1,"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}],"generictemplate":{"apiVersion":"ray.io/v1alpha1","kind":"RayCluster","metadata":{"labels":{"appwrapper.mcad.ibm.com":"quicktest2","controller-tools.k8s.io":"1.0"},"name":"quicktest2","namespace":"ns"},"spec":{"autoscalerOptions":{"idleTimeoutSeconds":60,"imagePullPolicy":"Always","resources":{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"500m","memory":"512Mi"}},"upscalingMode":"Default"},"enableInTreeAutoscaling":false,"headGroupSpec":{"rayStartParams":{"block":"true","dashboard-host":"0.0.0.0","num-gpus":"0"},"serviceType":"ClusterIP","template":{"spec":{"containers":[{"image":"quay.io/project-codeflare/ray:2.5.0-py38-cu116","imagePullPolicy":"Always","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"ray-head","ports":[{"containerPort":6379,"name":"gcs"},{"containerPort":8265,"name":"dashboard"},{"containerPort":10001,"name":"client"}],"resources":{"limits":{"cpu":2,"memory":"8G","nvidia.com/gpu":0},"requests":{"cpu":2,"memory":"8G","nvidia.com/gpu":0}}}]}}},"rayVersion":"1.12.0","workerGroupSpecs":[{"groupName":"small-group-quicktest","maxReplicas":1,"minReplicas":1,"rayStartParams":{"block":"true","num-gpus":"0"},"replicas":1,"template":{"metadata":{"annotations":{"key":"value"},"labels":{"key":"value"}},"spec":{"containers":[{"env":[{"name":"MY_POD_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}}],"image":"quay.io/project-codeflare/ray:2.5.0-py38-cu116","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","ray stop"]}}},"name":"machine-learning","resources":{"limits":{"cpu":1,"memory":"2G","nvidia.com/gpu":0},"requests":{"cpu":1,"memory":"2G","nvidia.com/gpu":0}}}],"initContainers":[{"command":["sh","-c","until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"],"image":"busybox:1.28","name":"init-myservice"}]}}}]}},"replicas":1},{"generictemplate":{"apiVersion":"route.openshift.io/v1","kind":"Route","metadata":{"labels":{"odh-ray-cluster-service":"quicktest-head-svc"},"name":"ray-dashboard-quicktest","namespace":"default"},"spec":{"port":{"targetPort":"dashboard"},"to":{"kind":"Service","name":"quicktest-head-svc"}}},"replica":1}],"Items":[]}}}\n' }, "creationTimestamp": "2023-02-22T16:26:07Z", "generation": 4, @@ -1253,7 +1250,7 @@ def get_aw_obj(): "spec": { "containers": [ { - "image": "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103", + "image": "quay.io/project-codeflare/ray:2.5.0-py38-cu116", "imagePullPolicy": "Always", "lifecycle": { "preStop": { @@ -1327,7 +1324,7 @@ def get_aw_obj(): }, } ], - "image": "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103", + "image": "quay.io/project-codeflare/ray:2.5.0-py38-cu116", "lifecycle": { "preStop": { "exec": {