From 240e336f20b9684962f25f613666e0dea72f4e3d Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 19 Jul 2024 12:54:53 +0200 Subject: [PATCH] Remove ODH integration tests from CFO repo --- test/odh/environment.go | 49 ------ test/odh/notebook.go | 95 ------------ test/odh/ray_test.go | 129 --------------- test/odh/resources/custom-nb-small.yaml | 165 -------------------- test/odh/resources/mnist.py | 190 ----------------------- test/odh/resources/mnist_mcad_mini.ipynb | 95 ------------ test/odh/resources/mnist_ray_mini.ipynb | 146 ----------------- test/odh/resources/requirements.txt | 6 - test/odh/support.go | 34 ---- test/odh/template.go | 40 ----- 10 files changed, 949 deletions(-) delete mode 100644 test/odh/environment.go delete mode 100644 test/odh/notebook.go delete mode 100644 test/odh/ray_test.go delete mode 100644 test/odh/resources/custom-nb-small.yaml delete mode 100644 test/odh/resources/mnist.py delete mode 100644 test/odh/resources/mnist_mcad_mini.ipynb delete mode 100644 test/odh/resources/mnist_ray_mini.ipynb delete mode 100644 test/odh/resources/requirements.txt delete mode 100644 test/odh/support.go delete mode 100644 test/odh/template.go diff --git a/test/odh/environment.go b/test/odh/environment.go deleted file mode 100644 index 0a087c9d5..000000000 --- a/test/odh/environment.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package odh - -import ( - "os" - - . "github.com/project-codeflare/codeflare-common/support" -) - -const ( - // The environment variable for namespace where ODH is installed to. - odhNamespaceEnvVar = "ODH_NAMESPACE" - // The environment variable for ODH Notebook ImageStream name - notebookImageStreamName = "NOTEBOOK_IMAGE_STREAM_NAME" -) - -func GetOpenDataHubNamespace() string { - return lookupEnvOrDefault(odhNamespaceEnvVar, "opendatahub") -} - -func GetNotebookImageStreamName(t Test) string { - isName, ok := os.LookupEnv(notebookImageStreamName) - if !ok { - t.T().Fatalf("Expected environment variable %s not found, please use this environment variable to specify what ImageStream to use for Notebook.", notebookImageStreamName) - } - return isName -} - -func lookupEnvOrDefault(key, value string) string { - if v, ok := os.LookupEnv(key); ok { - return v - } - return value -} diff --git a/test/odh/notebook.go b/test/odh/notebook.go deleted file mode 100644 index 70bdda82c..000000000 --- a/test/odh/notebook.go +++ /dev/null @@ -1,95 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package odh - -import ( - "bytes" - - gomega "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/yaml" - - imagev1 "github.com/openshift/api/image/v1" -) - -const recommendedTagAnnotation = "opendatahub.io/workbench-image-recommended" - -var notebookResource = schema.GroupVersionResource{Group: "kubeflow.org", Version: "v1", Resource: "notebooks"} - -type NotebookProps struct { - IngressDomain string - OpenShiftApiUrl string - KubernetesBearerToken string - Namespace string - OpenDataHubNamespace string - ImageStreamName string - ImageStreamTag string - RayImage string - NotebookConfigMapName string - NotebookConfigMapFileName string - NotebookPVC string -} - -func createNotebook(test Test, namespace *corev1.Namespace, notebookToken, jupyterNotebookConfigMapName, jupyterNotebookConfigMapFileName string) { - // Create PVC for Notebook - notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "10Gi", corev1.ReadWriteOnce) - - // Retrieve ImageStream tag for - is := GetImageStream(test, GetOpenDataHubNamespace(), GetNotebookImageStreamName(test)) - recommendedTagName := getRecommendedImageStreamTag(test, is) - - // Read the Notebook CR from resources and perform replacements for custom values using go template - notebookProps := NotebookProps{ - IngressDomain: GetOpenShiftIngressDomain(test), - OpenShiftApiUrl: GetOpenShiftApiUrl(test), - KubernetesBearerToken: notebookToken, - Namespace: namespace.Name, - OpenDataHubNamespace: GetOpenDataHubNamespace(), - ImageStreamName: GetNotebookImageStreamName(test), - ImageStreamTag: recommendedTagName, - RayImage: GetRayImage(), - NotebookConfigMapName: jupyterNotebookConfigMapName, - NotebookConfigMapFileName: jupyterNotebookConfigMapFileName, - NotebookPVC: notebookPVC.Name, - } - notebookTemplate, err := files.ReadFile("resources/custom-nb-small.yaml") - test.Expect(err).NotTo(gomega.HaveOccurred()) - - parsedNotebookTemplate := ParseTemplate(test, notebookTemplate, notebookProps) - - // Create Notebook CR - notebookCR := &unstructured.Unstructured{} - err = yaml.NewYAMLOrJSONDecoder(bytes.NewBuffer(parsedNotebookTemplate), 8192).Decode(notebookCR) - test.Expect(err).NotTo(gomega.HaveOccurred()) - _, err = test.Client().Dynamic().Resource(notebookResource).Namespace(namespace.Name).Create(test.Ctx(), notebookCR, metav1.CreateOptions{}) - test.Expect(err).NotTo(gomega.HaveOccurred()) -} - -func getRecommendedImageStreamTag(test Test, is *imagev1.ImageStream) (tagName string) { - for _, tag := range is.Spec.Tags { - if tag.Annotations[recommendedTagAnnotation] == "true" { - return tag.Name - } - } - test.T().Fatalf("tag with annotation '%s' not found in ImageStream %s", recommendedTagAnnotation, is.Name) - return -} diff --git a/test/odh/ray_test.go b/test/odh/ray_test.go deleted file mode 100644 index 14856919d..000000000 --- a/test/odh/ray_test.go +++ /dev/null @@ -1,129 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package odh - -import ( - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" - - rbacv1 "k8s.io/api/rbac/v1" -) - -func TestRay(t *testing.T) { - test := With(t) - - // Create a namespace - namespace := test.NewTestNamespace() - - // Test configuration - jupyterNotebookConfigMapFileName := "mnist_ray_mini.ipynb" - config := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // MNIST Ray Notebook - jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_ray_mini.ipynb"), - "mnist.py": readMnistPy(test), - "requirements.txt": readRequirementsTxt(test), - }) - - // Create RBAC, retrieve token for user with limited rights - policyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1.GroupVersion.Group}, - Resources: []string{"rayclusters", "rayclusters/status"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"route.openshift.io"}, - Resources: []string{"routes"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"ingresses"}, - }, - } - - // Create cluster wide RBAC, required for SDK OpenShift check - // TODO reevaluate once SDK change OpenShift detection logic - clusterPolicyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"config.openshift.io"}, - Resources: []string{"ingresses"}, - ResourceNames: []string{"cluster"}, - }, - } - - sa := CreateServiceAccount(test, namespace.Name) - role := CreateRole(test, namespace.Name, policyRules) - CreateRoleBinding(test, namespace.Name, sa, role) - clusterRole := CreateClusterRole(test, clusterPolicyRules) - CreateClusterRoleBinding(test, sa, clusterRole) - token := CreateToken(test, namespace.Name, sa) - - // Create Notebook CR - createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) - - // Make sure the RayCluster is created and running - test.Eventually(RayClusters(test, namespace.Name), TestTimeoutLong). - Should( - And( - HaveLen(1), - ContainElement(WithTransform(RayClusterState, Equal(rayv1.Ready))), - ), - ) - - // Make sure the RayCluster finishes and is deleted - test.Eventually(RayClusters(test, namespace.Name), TestTimeoutLong). - Should(HaveLen(0)) -} - -func readRequirementsTxt(test Test) []byte { - // Read the requirements.txt from resources and perform replacements for custom values using go template - props := struct { - PipIndexUrl string - PipTrustedHost string - }{ - PipIndexUrl: "--index " + GetPipIndexURL(), - } - - // Provide trusted host only if defined - if len(GetPipTrustedHost()) > 0 { - props.PipTrustedHost = "--trusted-host " + GetPipTrustedHost() - } - - template, err := files.ReadFile("resources/requirements.txt") - test.Expect(err).NotTo(HaveOccurred()) - - return ParseTemplate(test, template, props) -} - -func readMnistPy(test Test) []byte { - // Read the mnist.py from resources and perform replacements for custom values using go template - props := struct { - MnistDatasetURL string - }{ - MnistDatasetURL: GetMnistDatasetURL(), - } - template, err := files.ReadFile("resources/mnist.py") - test.Expect(err).NotTo(HaveOccurred()) - - return ParseTemplate(test, template, props) -} diff --git a/test/odh/resources/custom-nb-small.yaml b/test/odh/resources/custom-nb-small.yaml deleted file mode 100644 index 791a2d98a..000000000 --- a/test/odh/resources/custom-nb-small.yaml +++ /dev/null @@ -1,165 +0,0 @@ -# This template maybe used to spin up a custom notebook image -# i.e.: sed s/{{.IngressDomain}}/$(oc get ingresses.config/cluster -o jsonpath={.spec.domain})/g tests/resources/custom-nb.template | oc apply -f - -# resources generated: -# pod/jupyter-nb-kube-3aadmin-0 -# service/jupyter-nb-kube-3aadmin -# route.route.openshift.io/jupyter-nb-kube-3aadmin (jupyter-nb-kube-3aadmin-opendatahub.apps.tedbig412.cp.fyre.ibm.com) -# service/jupyter-nb-kube-3aadmin-tls -apiVersion: kubeflow.org/v1 -kind: Notebook -metadata: - annotations: - notebooks.opendatahub.io/inject-oauth: "true" - notebooks.opendatahub.io/last-image-selection: codeflare-notebook:latest - notebooks.opendatahub.io/last-size-selection: Small - notebooks.opendatahub.io/oauth-logout-url: https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}/notebookController/kube-3aadmin/home - opendatahub.io/link: https://jupyter-nb-kube-3aadmin-{{.Namespace}}.{{.IngressDomain}}/notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin - opendatahub.io/username: kube:admin - generation: 1 - labels: - app: jupyter-nb-kube-3aadmin - opendatahub.io/dashboard: "true" - opendatahub.io/odh-managed: "true" - opendatahub.io/user: kube-3aadmin - name: jupyter-nb-kube-3aadmin - namespace: {{.Namespace}} -spec: - template: - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: nvidia.com/gpu.present - operator: NotIn - values: - - "true" - weight: 1 - containers: - - env: - - name: NOTEBOOK_ARGS - value: |- - --ServerApp.port=8888 - --ServerApp.token='' - --ServerApp.password='' - --ServerApp.base_url=/notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin - --ServerApp.quit_button=False - --ServerApp.tornado_settings={"user":"kube-3aadmin","hub_host":"https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}","hub_prefix":"/notebookController/kube-3aadmin"} - - name: JUPYTER_IMAGE - value: image-registry.openshift-image-registry.svc:5000/{{.OpenDataHubNamespace}}/{{.ImageStreamName}}:{{.ImageStreamTag}} - - name: JUPYTER_NOTEBOOK_PORT - value: "8888" - - name: OCP_SERVER - value: {{.OpenShiftApiUrl}} - - name: OCP_TOKEN - value: {{.KubernetesBearerToken}} - image: image-registry.openshift-image-registry.svc:5000/{{.OpenDataHubNamespace}}/{{.ImageStreamName}}:{{.ImageStreamTag}} - command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{.NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{.Namespace}} -p ray_image {{.RayImage}} && sleep infinity"] - # args: ["pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/mcad.ipynb /opt/app-root/src/mcad-out.ipynb" ] - imagePullPolicy: Always - # livenessProbe: - # failureThreshold: 3 - # httpGet: - # path: /notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin/api - # port: notebook-port - # scheme: HTTP - # initialDelaySeconds: 10 - # periodSeconds: 5 - # successThreshold: 1 - # timeoutSeconds: 1 - name: jupyter-nb-kube-3aadmin - ports: - - containerPort: 8888 - name: notebook-port - protocol: TCP - resources: - limits: - cpu: "2" - memory: 3Gi - requests: - cpu: "1" - memory: 3Gi - volumeMounts: - - mountPath: /opt/app-root/src - name: jupyterhub-nb-kube-3aadmin-pvc - - mountPath: /opt/app-root/notebooks - name: {{.NotebookConfigMapName}} - workingDir: /opt/app-root/src - - args: - - --provider=openshift - - --https-address=:8443 - - --http-address= - - --openshift-service-account=jupyter-nb-kube-3aadmin - - --cookie-secret-file=/etc/oauth/config/cookie_secret - - --cookie-expire=24h0m0s - - --tls-cert=/etc/tls/private/tls.crt - - --tls-key=/etc/tls/private/tls.key - - --upstream=http://localhost:8888 - - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - - --skip-auth-regex=^(?:/notebook/$(NAMESPACE)/jupyter-nb-kube-3aadmin)?/api$ - - --email-domain=* - - --skip-provider-button - - --openshift-sar={"verb":"get","resource":"notebooks","resourceAPIGroup":"kubeflow.org","resourceName":"jupyter-nb-kube-3aadmin","namespace":"$(NAMESPACE)"} - - --logout-url=https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}/notebookController/kube-3aadmin/home - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.redhat.io/openshift4/ose-oauth-proxy:v4.10 - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - httpGet: - path: /oauth/healthz - port: oauth-proxy - scheme: HTTPS - initialDelaySeconds: 30 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 1 - name: oauth-proxy - ports: - - containerPort: 8443 - name: oauth-proxy - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /oauth/healthz - port: oauth-proxy - scheme: HTTPS - initialDelaySeconds: 5 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 1 - resources: - limits: - cpu: 100m - memory: 64Mi - requests: - cpu: 100m - memory: 64Mi - volumeMounts: - - mountPath: /etc/oauth/config - name: oauth-config - - mountPath: /etc/tls/private - name: tls-certificates - enableServiceLinks: false - serviceAccountName: jupyter-nb-kube-3aadmin - volumes: - - name: jupyterhub-nb-kube-3aadmin-pvc - persistentVolumeClaim: - claimName: {{.NotebookPVC}} - - name: oauth-config - secret: - defaultMode: 420 - secretName: jupyter-nb-kube-3aadmin-oauth-config - - name: tls-certificates - secret: - defaultMode: 420 - secretName: jupyter-nb-kube-3aadmin-tls - - name: {{.NotebookConfigMapName}} - configMap: - name: {{.NotebookConfigMapName}} diff --git a/test/odh/resources/mnist.py b/test/odh/resources/mnist.py deleted file mode 100644 index 85d420f48..000000000 --- a/test/odh/resources/mnist.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2022 IBM, Red Hat -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import torch -import requests -from pytorch_lightning import LightningModule, Trainer -from pytorch_lightning.callbacks.progress import TQDMProgressBar -from torch import nn -from torch.nn import functional as F -from torch.utils.data import DataLoader, random_split, RandomSampler -from torchmetrics import Accuracy -from torchvision import transforms -from torchvision.datasets import MNIST - -PATH_DATASETS = os.environ.get("PATH_DATASETS", ".") -BATCH_SIZE = 256 if torch.cuda.is_available() else 64 -# %% - -print("prior to running the trainer") -print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) -print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) - -MNIST_DATASET_URL = "{{.MnistDatasetURL}}" -print("MNIST_DATASET_URL: is ", MNIST_DATASET_URL) - -class LitMNIST(LightningModule): - def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4): - - super().__init__() - - # Set our init args as class attributes - self.data_dir = data_dir - self.hidden_size = hidden_size - self.learning_rate = learning_rate - - # Hardcode some dataset specific attributes - self.num_classes = 10 - self.dims = (1, 28, 28) - channels, width, height = self.dims - self.transform = transforms.Compose( - [ - transforms.ToTensor(), - transforms.Normalize((0.1307,), (0.3081,)), - ] - ) - - # Define PyTorch model - self.model = nn.Sequential( - nn.Flatten(), - nn.Linear(channels * width * height, hidden_size), - nn.ReLU(), - nn.Dropout(0.1), - nn.Linear(hidden_size, hidden_size), - nn.ReLU(), - nn.Dropout(0.1), - nn.Linear(hidden_size, self.num_classes), - ) - - self.val_accuracy = Accuracy() - self.test_accuracy = Accuracy() - - def forward(self, x): - x = self.model(x) - return F.log_softmax(x, dim=1) - - def training_step(self, batch, batch_idx): - x, y = batch - logits = self(x) - loss = F.nll_loss(logits, y) - return loss - - def validation_step(self, batch, batch_idx): - x, y = batch - logits = self(x) - loss = F.nll_loss(logits, y) - preds = torch.argmax(logits, dim=1) - self.val_accuracy.update(preds, y) - - # Calling self.log will surface up scalars for you in TensorBoard - self.log("val_loss", loss, prog_bar=True) - self.log("val_acc", self.val_accuracy, prog_bar=True) - - def test_step(self, batch, batch_idx): - x, y = batch - logits = self(x) - loss = F.nll_loss(logits, y) - preds = torch.argmax(logits, dim=1) - self.test_accuracy.update(preds, y) - - # Calling self.log will surface up scalars for you in TensorBoard - self.log("test_loss", loss, prog_bar=True) - self.log("test_acc", self.test_accuracy, prog_bar=True) - - def configure_optimizers(self): - optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) - return optimizer - - #################### - # DATA RELATED HOOKS - #################### - - def prepare_data(self): - datasetFiles = [ - "t10k-images-idx3-ubyte", - "t10k-labels-idx1-ubyte", - "train-images-idx3-ubyte", - "train-labels-idx1-ubyte" - ] - - # Create required folder structure - downloadLocation = os.path.join(PATH_DATASETS, "MNIST", "raw") - os.makedirs(downloadLocation, exist_ok=True) - print(f"{downloadLocation} folder_path created!") - - for file in datasetFiles: - print(f"Downloading MNIST dataset {file}... to path : {downloadLocation}") - response = requests.get(f"{MNIST_DATASET_URL}{file}", stream=True) - filePath = os.path.join(downloadLocation, file) - - #to download dataset file - try: - if response.status_code == 200: - open(filePath, 'wb').write(response.content) - print(f"{file}: Downloaded and saved zipped file to path - {filePath}") - else: - print(f"Failed to download file {file}") - except Exception as e: - print(e) - print(f"Downloaded MNIST dataset to... {downloadLocation}") - - MNIST(self.data_dir, train=True, download=True) - MNIST(self.data_dir, train=False, download=True) - - def setup(self, stage=None): - - # Assign train/val datasets for use in dataloaders - if stage == "fit" or stage is None: - mnist_full = MNIST(self.data_dir, train=True, transform=self.transform) - self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000]) - - # Assign test dataset for use in dataloader(s) - if stage == "test" or stage is None: - self.mnist_test = MNIST( - self.data_dir, train=False, transform=self.transform - ) - - def train_dataloader(self): - return DataLoader(self.mnist_train, batch_size=BATCH_SIZE, sampler=RandomSampler(self.mnist_train, num_samples=1000)) - - def val_dataloader(self): - return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) - - def test_dataloader(self): - return DataLoader(self.mnist_test, batch_size=BATCH_SIZE) - - -# Init DataLoader from MNIST Dataset - -model = LitMNIST() - -print("GROUP: ", int(os.environ.get("GROUP_WORLD_SIZE", 1))) -print("LOCAL: ", int(os.environ.get("LOCAL_WORLD_SIZE", 1))) - -# Initialize a trainer -trainer = Trainer( - accelerator="auto", - # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs - max_epochs=3, - callbacks=[TQDMProgressBar(refresh_rate=20)], - num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), - devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), - replace_sampler_ddp=False, - strategy="ddp", -) - -# Train the model ⚡ -trainer.fit(model) diff --git a/test/odh/resources/mnist_mcad_mini.ipynb b/test/odh/resources/mnist_mcad_mini.ipynb deleted file mode 100644 index 341eb12ce..000000000 --- a/test/odh/resources/mnist_mcad_mini.ipynb +++ /dev/null @@ -1,95 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Import pieces from codeflare-sdk\n", - "from codeflare_sdk.job.jobs import DDPJobDefinition\n", - "from time import sleep" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "47ca5c15", - "metadata": { - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "#parameters\n", - "namespace = \"default\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26b21373", - "metadata": {}, - "outputs": [], - "source": [ - "job = DDPJobDefinition(name=\"mnistjob\", script=\"/test/mnist.py\", scheduler_args={\"namespace\": namespace}, j=\"1x1\", gpu=0, cpu=1, memMB=2000, image=\"quay.io/project-codeflare/demo-images:pytorch-mnist-v0.0.1\").submit()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d24e9f95", - "metadata": {}, - "outputs": [], - "source": [ - "finished = False\n", - "while not finished:\n", - " sleep(1)\n", - " try:\n", - " finished = (\"Epoch 2: 100%\" in job.logs())\n", - " except:\n", - " finished = False" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f078b7cd", - "metadata": {}, - "outputs": [], - "source": [ - "job.cancel()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - }, - "vscode": { - "interpreter": { - "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/test/odh/resources/mnist_ray_mini.ipynb b/test/odh/resources/mnist_ray_mini.ipynb deleted file mode 100644 index 0d8fcc53a..000000000 --- a/test/odh/resources/mnist_ray_mini.ipynb +++ /dev/null @@ -1,146 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Import pieces from codeflare-sdk\n", - "from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration\n", - "from codeflare_sdk.job.jobs import DDPJobDefinition\n", - "from time import sleep" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "30888aed", - "metadata": { - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "#parameters\n", - "namespace = \"default\"\n", - "ray_image = \"has to be specified\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f4bc870-091f-4e11-9642-cba145710159", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Create our cluster and submit appwrapper\n", - "cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', head_cpus=1, head_memory=2, num_workers=1, min_cpus=1, max_cpus=1, min_memory=1, max_memory=2, num_gpus=0, instascale=False, image=ray_image))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f0884bbc-c224-4ca0-98a0-02dfa09c2200", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Bring up the cluster\n", - "cluster.up()\n", - "cluster.wait_ready()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "df71c1ed", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "cluster.status()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7fd45bc5-03c0-4ae5-9ec5-dd1c30f1a084", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "cluster.details()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "47ca5c15", - "metadata": {}, - "outputs": [], - "source": [ - "job = DDPJobDefinition(name=\"mnisttest\", script=\"mnist.py\", workspace=\"file:///opt/app-root/notebooks/..data\", scheduler_args={\"requirements\": \"/opt/app-root/notebooks/requirements.txt\"}).submit(cluster)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f63a178a", - "metadata": {}, - "outputs": [], - "source": [ - "finished = False\n", - "while not finished:\n", - " sleep(1)\n", - " status = job.status()\n", - " finished = (str(status.state) == \"SUCCEEDED\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6b099777", - "metadata": {}, - "outputs": [], - "source": [ - "cluster.down()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - }, - "vscode": { - "interpreter": { - "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/test/odh/resources/requirements.txt b/test/odh/resources/requirements.txt deleted file mode 100644 index a58df2950..000000000 --- a/test/odh/resources/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -{{.PipIndexUrl}} -{{.PipTrustedHost}} -pytorch_lightning==1.9.5 -ray_lightning -torchmetrics==0.9.1 -torchvision==0.12.0 diff --git a/test/odh/support.go b/test/odh/support.go deleted file mode 100644 index d828ed950..000000000 --- a/test/odh/support.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package odh - -import ( - "embed" - - "github.com/onsi/gomega" - "github.com/project-codeflare/codeflare-common/support" -) - -//go:embed resources/* -var files embed.FS - -func ReadFile(t support.Test, fileName string) []byte { - t.T().Helper() - file, err := files.ReadFile(fileName) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return file -} diff --git a/test/odh/template.go b/test/odh/template.go deleted file mode 100644 index 3ff4da17f..000000000 --- a/test/odh/template.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package odh - -import ( - "bytes" - "html/template" - - "github.com/onsi/gomega" - "github.com/project-codeflare/codeflare-common/support" -) - -func ParseTemplate(t support.Test, inputTemplate []byte, props interface{}) []byte { - t.T().Helper() - - // Parse input template - parsedTemplate, err := template.New("template").Parse(string(inputTemplate)) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - // Filter template and store results to the buffer - buffer := new(bytes.Buffer) - err = parsedTemplate.Execute(buffer, props) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - return buffer.Bytes() -}