diff --git a/.github/workflows/coverage-badge.yaml b/.github/workflows/coverage-badge.yaml new file mode 100644 index 000000000..7dde84490 --- /dev/null +++ b/.github/workflows/coverage-badge.yaml @@ -0,0 +1,53 @@ +# This workflow will generate and push an updated coverage badge + +name: Coverage Badge + +on: + push: + branches: [ main ] + +jobs: + report: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest==6.2.4 + pip install pytest-mock==3.6.1 + pip install coverage + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Generate coverage report + run: | + coverage run -m --source=src pytest -v tests/unit_test.py + + - name: Coverage Badge + uses: tj-actions/coverage-badge-py@v1.8 + + - name: Verify Changed files + uses: tj-actions/verify-changed-files@v12 + id: changed_files + with: + files: coverage.svg + + - name: Commit files + if: steps.changed_files.outputs.files_changed == 'true' + run: | + git config --local user.email "github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git add coverage.svg + git commit -m "Updated coverage.svg" + + - name: Push changes + if: steps.changed_files.outputs.files_changed == 'true' + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.CI_PUSH_TOKEN }} + branch: ${{ github.ref }} diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 732b54abd..f22e8f224 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -23,13 +23,16 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest - pip install pytest-mock + pip install pytest==6.2.4 + pip install pytest-mock==3.6.1 + pip install coverage pip install black==22.3.0 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Check formatting with black run: | black --check . - - name: Test with pytest + - name: Test with pytest and check coverage run: | - pytest -v tests/unit_test.py + coverage run -m --source=src pytest -v tests/unit_test.py + coverage=$(coverage report -m | tail -1 | tail -c 4 | head -c 2) + if (( $coverage < 90 )); then exit 1; else echo "Coverage passed, ${coverage}%"; fi diff --git a/.gitignore b/.gitignore index 3a5ed1679..846fc76e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ dist/ .python-version __pycache__/ +.coverage diff --git a/README.md b/README.md index 362a691b9..c02423a93 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,11 @@ For testing, make sure to have installed: NOTE: Functional tests coming soon, will live in `tests/func_test.py` +For checking code coverage while testing: + - Start by installing `coverage` (can be done via `pip`) + - Now instead when testing run `coverage run -m --source=src pytest tests/unit_test.py` + - To then view a code coverage report w/ missing lines, run `coverage report -m` + For formatting: - Currently using black v22.3.0 for format checking - To install, run `pip install black==22.3.0` diff --git a/requirements.txt b/requirements.txt index 83fc1c519..d0bbd2c90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ openshift-client==1.0.18 rich==12.5.1 -ray==2.1.0 \ No newline at end of file +ray[default]==2.1.0 diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index aac52da40..d279b874d 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -110,7 +110,7 @@ def down(self): oc.invoke("delete", ["AppWrapper", self.app_wrapper_name]) self.config.auth.logout() - def status(self, print_to_console: bool = True): + def status(self, print_to_console: bool = True): # pragma: no cover """ TO BE UPDATED: Will soon return (and print by default) the cluster's status, from AppWrapper submission to setup completion. All resource @@ -151,7 +151,7 @@ def cluster_dashboard_uri(self, namespace: str = "default") -> str: return "Dashboard route not available yet. Did you run cluster.up()?" # checks whether the ray cluster is ready - def is_ready(self, print_to_console: bool = True): + def is_ready(self, print_to_console: bool = True): # pragma: no cover """ TO BE DEPRECATED: functionality will be added into cluster.status(). """ @@ -228,7 +228,7 @@ def job_logs(self, job_id: str) -> str: return client.get_job_logs(job_id) -def get_current_namespace() -> str: +def get_current_namespace() -> str: # pragma: no cover """ Returns the user's current working namespace. """ @@ -236,7 +236,9 @@ def get_current_namespace() -> str: return namespace -def list_all_clusters(namespace: str, print_to_console: bool = True): +def list_all_clusters( + namespace: str, print_to_console: bool = True +): # pragma: no cover """ Returns (and prints by default) a list of all clusters in a given namespace. """ @@ -246,7 +248,7 @@ def list_all_clusters(namespace: str, print_to_console: bool = True): return clusters -def list_all_queued(namespace: str, print_to_console: bool = True): +def list_all_queued(namespace: str, print_to_console: bool = True): # pragma: no cover """ Returns (and prints by default) a list of all currently queued-up AppWrappers in a given namespace. @@ -262,14 +264,18 @@ def list_all_queued(namespace: str, print_to_console: bool = True): # private methods -def _app_wrapper_status(name, namespace="default") -> Optional[AppWrapper]: +def _app_wrapper_status( + name, namespace="default" +) -> Optional[AppWrapper]: # pragma: no cover with oc.project(namespace), oc.timeout(10 * 60): cluster = oc.selector(f"appwrapper/{name}").object() if cluster: return _map_to_app_wrapper(cluster) -def _ray_cluster_status(name, namespace="default") -> Optional[RayCluster]: +def _ray_cluster_status( + name, namespace="default" +) -> Optional[RayCluster]: # pragma: no cover # FIXME should we check the appwrapper first cluster = None try: @@ -283,7 +289,7 @@ def _ray_cluster_status(name, namespace="default") -> Optional[RayCluster]: return cluster -def _get_ray_clusters(namespace="default") -> List[RayCluster]: +def _get_ray_clusters(namespace="default") -> List[RayCluster]: # pragma: no cover list_of_clusters = [] with oc.project(namespace), oc.timeout(10 * 60): @@ -296,7 +302,7 @@ def _get_ray_clusters(namespace="default") -> List[RayCluster]: def _get_app_wrappers( namespace="default", filter=List[AppWrapperStatus] -) -> List[AppWrapper]: +) -> List[AppWrapper]: # pragma: no cover list_of_app_wrappers = [] with oc.project(namespace), oc.timeout(10 * 60): @@ -311,7 +317,7 @@ def _get_app_wrappers( return list_of_app_wrappers -def _map_to_ray_cluster(cluster) -> RayCluster: +def _map_to_ray_cluster(cluster) -> RayCluster: # pragma: no cover cluster_model = cluster.model with oc.project(cluster.namespace()), oc.timeout(10 * 60): @@ -342,7 +348,7 @@ def _map_to_ray_cluster(cluster) -> RayCluster: ) -def _map_to_app_wrapper(cluster) -> AppWrapper: +def _map_to_app_wrapper(cluster) -> AppWrapper: # pragma: no cover cluster_model = cluster.model return AppWrapper( name=cluster.name(), diff --git a/src/codeflare_sdk/utils/generate_yaml.py b/src/codeflare_sdk/utils/generate_yaml.py index cd6f0a435..b80e83e51 100755 --- a/src/codeflare_sdk/utils/generate_yaml.py +++ b/src/codeflare_sdk/utils/generate_yaml.py @@ -240,7 +240,7 @@ def generate_appwrapper( return outfile -def main(): +def main(): # pragma: no cover parser = argparse.ArgumentParser(description="Generate user AppWrapper") parser.add_argument( "--name", @@ -348,5 +348,5 @@ def main(): return outfile -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover main() diff --git a/src/codeflare_sdk/utils/pretty_print.py b/src/codeflare_sdk/utils/pretty_print.py index cf1d41833..6083a9fbd 100644 --- a/src/codeflare_sdk/utils/pretty_print.py +++ b/src/codeflare_sdk/utils/pretty_print.py @@ -27,12 +27,12 @@ from ..cluster.model import RayCluster, AppWrapper, RayClusterStatus -def print_no_resources_found(): +def print_no_resources_found(): # pragma: no cover console = Console() console.print(Panel("[red]No resources found")) -def print_app_wrappers_status(app_wrappers: List[AppWrapper]): +def print_app_wrappers_status(app_wrappers: List[AppWrapper]): # pragma: no cover if not app_wrappers: print_no_resources_found() return # shortcircuit @@ -53,7 +53,7 @@ def print_app_wrappers_status(app_wrappers: List[AppWrapper]): console.print(Panel.fit(table)) -def print_clusters(clusters: List[RayCluster], verbose=True): +def print_clusters(clusters: List[RayCluster], verbose=True): # pragma: no cover if not clusters: print_no_resources_found() return # shortcircuit diff --git a/tests/test-case-cmd.yaml b/tests/test-case-cmd.yaml new file mode 100644 index 000000000..6782c50f5 --- /dev/null +++ b/tests/test-case-cmd.yaml @@ -0,0 +1,150 @@ +apiVersion: mcad.ibm.com/v1beta1 +kind: AppWrapper +metadata: + name: unit-cmd-cluster + namespace: default +spec: + priority: 9 + resources: + GenericItems: + - custompodresources: + - limits: + cpu: 2 + memory: 8G + nvidia.com/gpu: 0 + replicas: 1 + requests: + cpu: 2 + memory: 8G + nvidia.com/gpu: 0 + - limits: + cpu: 1 + memory: 2G + nvidia.com/gpu: 1 + replicas: 2 + requests: + cpu: 1 + memory: 2G + nvidia.com/gpu: 1 + generictemplate: + apiVersion: ray.io/v1alpha1 + kind: RayCluster + metadata: + labels: + appwrapper.mcad.ibm.com: unit-cmd-cluster + controller-tools.k8s.io: '1.0' + name: unit-cmd-cluster + namespace: default + spec: + autoscalerOptions: + idleTimeoutSeconds: 60 + imagePullPolicy: Always + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 500m + memory: 512Mi + upscalingMode: Default + enableInTreeAutoscaling: false + headGroupSpec: + rayStartParams: + block: 'true' + dashboard-host: 0.0.0.0 + num-gpus: '0' + serviceType: ClusterIP + template: + spec: + containers: + - image: rayproject/ray:latest + imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - ray stop + name: ray-head + ports: + - containerPort: 6379 + name: gcs + - containerPort: 8265 + name: dashboard + - containerPort: 10001 + name: client + resources: + limits: + cpu: 2 + memory: 8G + nvidia.com/gpu: 0 + requests: + cpu: 2 + memory: 8G + nvidia.com/gpu: 0 + rayVersion: 1.12.0 + workerGroupSpecs: + - groupName: small-group-unit-cmd-cluster + maxReplicas: 2 + minReplicas: 2 + rayStartParams: + block: 'true' + num-gpus: '1' + replicas: 2 + template: + metadata: + annotations: + key: value + labels: + key: value + spec: + containers: + - env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: rayproject/ray:latest + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - ray stop + name: machine-learning + resources: + limits: + cpu: 1 + memory: 2G + nvidia.com/gpu: 1 + requests: + cpu: 1 + memory: 2G + nvidia.com/gpu: 1 + initContainers: + - command: + - sh + - -c + - until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; + do echo waiting for myservice; sleep 2; done + image: busybox:1.28 + name: init-myservice + replicas: 1 + - generictemplate: + apiVersion: route.openshift.io/v1 + kind: Route + metadata: + labels: + odh-ray-cluster-service: unit-cmd-cluster-head-svc + name: ray-dashboard-unit-cmd-cluster + namespace: default + spec: + port: + targetPort: dashboard + to: + kind: Service + name: unit-cmd-cluster-head-svc + replica: 1 + Items: [] diff --git a/tests/test-case.yaml b/tests/test-case.yaml new file mode 100644 index 000000000..bdaf1b722 --- /dev/null +++ b/tests/test-case.yaml @@ -0,0 +1,170 @@ +apiVersion: mcad.ibm.com/v1beta1 +kind: AppWrapper +metadata: + labels: + orderedinstance: cpu.small_gpu.large + name: unit-test-cluster + namespace: ns +spec: + priority: 9 + resources: + GenericItems: + - custompodresources: + - limits: + cpu: 2 + memory: 8G + nvidia.com/gpu: 0 + replicas: 1 + requests: + cpu: 2 + memory: 8G + nvidia.com/gpu: 0 + - limits: + cpu: 4 + memory: 6G + nvidia.com/gpu: 7 + replicas: 2 + requests: + cpu: 3 + memory: 5G + nvidia.com/gpu: 7 + generictemplate: + apiVersion: ray.io/v1alpha1 + kind: RayCluster + metadata: + labels: + appwrapper.mcad.ibm.com: unit-test-cluster + controller-tools.k8s.io: '1.0' + name: unit-test-cluster + namespace: ns + spec: + autoscalerOptions: + idleTimeoutSeconds: 60 + imagePullPolicy: Always + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 500m + memory: 512Mi + upscalingMode: Default + enableInTreeAutoscaling: false + headGroupSpec: + rayStartParams: + block: 'true' + dashboard-host: 0.0.0.0 + num-gpus: '0' + serviceType: ClusterIP + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: unit-test-cluster + operator: In + values: + - unit-test-cluster + containers: + - image: ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103 + imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - ray stop + name: ray-head + ports: + - containerPort: 6379 + name: gcs + - containerPort: 8265 + name: dashboard + - containerPort: 10001 + name: client + resources: + limits: + cpu: 2 + memory: 8G + nvidia.com/gpu: 0 + requests: + cpu: 2 + memory: 8G + nvidia.com/gpu: 0 + rayVersion: 1.12.0 + workerGroupSpecs: + - groupName: small-group-unit-test-cluster + maxReplicas: 2 + minReplicas: 2 + rayStartParams: + block: 'true' + num-gpus: '7' + replicas: 2 + template: + metadata: + annotations: + key: value + labels: + key: value + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: unit-test-cluster + operator: In + values: + - unit-test-cluster + containers: + - env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103 + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - ray stop + name: machine-learning + resources: + limits: + cpu: 4 + memory: 6G + nvidia.com/gpu: 7 + requests: + cpu: 3 + memory: 5G + nvidia.com/gpu: 7 + initContainers: + - command: + - sh + - -c + - until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; + do echo waiting for myservice; sleep 2; done + image: busybox:1.28 + name: init-myservice + replicas: 1 + - generictemplate: + apiVersion: route.openshift.io/v1 + kind: Route + metadata: + labels: + odh-ray-cluster-service: unit-test-cluster-head-svc + name: ray-dashboard-unit-test-cluster + namespace: ns + spec: + port: + targetPort: dashboard + to: + kind: Service + name: unit-test-cluster-head-svc + replica: 1 + Items: [] diff --git a/tests/unit_test.py b/tests/unit_test.py index c4a2b2a48..a7ee0514e 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -14,13 +14,28 @@ from pathlib import Path import sys +import filecmp +import os parent = Path(__file__).resolve().parents[1] sys.path.append(str(parent) + "/src") -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration -from codeflare_sdk.cluster.auth import TokenAuthentication, PasswordUserAuthentication +from codeflare_sdk.cluster.cluster import ( + Cluster, + ClusterConfiguration, + get_current_namespace, + list_all_clusters, + list_all_queued, +) +from codeflare_sdk.cluster.auth import ( + TokenAuthentication, + PasswordUserAuthentication, + Authentication, +) +from codeflare_sdk.utils.generate_yaml import main import openshift +from openshift import OpenShiftPythonException +import ray import pytest @@ -37,27 +52,46 @@ def att_side_effect(self): return self.high_level_operation +def att_side_effect_tls(self): + if "--insecure-skip-tls-verify" in self.high_level_operation[1]: + return self.high_level_operation + else: + raise OpenShiftPythonException( + "The server uses a certificate signed by unknown authority" + ) + + def test_token_auth_creation(): try: token_auth = TokenAuthentication() assert token_auth.token == None assert token_auth.server == None + assert token_auth.skip_tls == False token_auth = TokenAuthentication("token") assert token_auth.token == "token" assert token_auth.server == None + assert token_auth.skip_tls == False token_auth = TokenAuthentication("token", "server") assert token_auth.token == "token" assert token_auth.server == "server" + assert token_auth.skip_tls == False token_auth = TokenAuthentication("token", server="server") assert token_auth.token == "token" assert token_auth.server == "server" + assert token_auth.skip_tls == False token_auth = TokenAuthentication(token="token", server="server") assert token_auth.token == "token" assert token_auth.server == "server" + assert token_auth.skip_tls == False + + token_auth = TokenAuthentication(token="token", server="server", skip_tls=True) + assert token_auth.token == "token" + assert token_auth.server == "server" + assert token_auth.skip_tls == True except Exception: assert 0 == 1 @@ -76,6 +110,24 @@ def test_token_auth_login_logout(mocker): assert token_auth.logout() == ("logout",) +def test_token_auth_login_tls(mocker): + mocker.patch("openshift.invoke", side_effect=arg_side_effect) + mock_res = mocker.patch.object(openshift.Result, "out") + mock_res.side_effect = lambda: att_side_effect_tls(fake_res) + + # FIXME - Pytest mocker not allowing caught exception + # token_auth = TokenAuthentication(token="testtoken", server="testserver") + # assert token_auth.login() == "Error: certificate auth failure, please set `skip_tls=True` in TokenAuthentication" + + token_auth = TokenAuthentication( + token="testtoken", server="testserver", skip_tls=True + ) + assert token_auth.login() == ( + "login", + ["--token=testtoken", "--server=testserver:6443", "--insecure-skip-tls-verify"], + ) + + def test_passwd_auth_creation(): try: passwd_auth = PasswordUserAuthentication() @@ -111,3 +163,144 @@ def test_passwd_auth_login_logout(mocker): token_auth = PasswordUserAuthentication(username="user", password="passwd") assert token_auth.login() == ("user", "passwd") assert token_auth.logout() == ("logout",) + + +def test_auth_coverage(): + abstract = Authentication() + abstract.login() + abstract.logout() + + +def test_config_creation(): + config = ClusterConfiguration( + name="unit-test-cluster", + namespace="ns", + min_worker=1, + max_worker=2, + min_cpus=3, + max_cpus=4, + min_memory=5, + max_memory=6, + gpu=7, + instascale=True, + machine_types=["cpu.small", "gpu.large"], + auth=TokenAuthentication(token="testtoken", server="testserver"), + ) + + assert config.name == "unit-test-cluster" and config.namespace == "ns" + assert config.min_worker == 1 and config.max_worker == 2 + assert config.min_cpus == 3 and config.max_cpus == 4 + assert config.min_memory == 5 and config.max_memory == 6 + assert config.gpu == 7 + assert ( + config.image + == "ghcr.io/foundation-model-stack/base:ray2.1.0-py38-gpu-pytorch1.12.0cu116-20221213-193103" + ) + assert config.template == f"{parent}/src/codeflare_sdk/templates/new-template.yaml" + assert config.instascale + assert config.machine_types == ["cpu.small", "gpu.large"] + assert config.auth.__class__ == TokenAuthentication + return config + + +def test_cluster_creation(): + cluster = Cluster(test_config_creation()) + assert cluster.app_wrapper_yaml == "unit-test-cluster.yaml" + assert cluster.app_wrapper_name == "unit-test-cluster" + assert filecmp.cmp( + "unit-test-cluster.yaml", f"{parent}/tests/test-case.yaml", shallow=True + ) + return cluster + + +def arg_check_apply_effect(*args): + assert args[0] == "apply" + assert args[1] == ["-f", "unit-test-cluster.yaml"] + + +def arg_check_del_effect(*args): + assert args[0] == "delete" + assert args[1] == ["AppWrapper", "unit-test-cluster"] + + +def test_cluster_up_down(mocker): + mocker.patch( + "codeflare_sdk.cluster.auth.TokenAuthentication.login", return_value="ignore" + ) + mocker.patch( + "codeflare_sdk.cluster.auth.TokenAuthentication.logout", return_value="ignore" + ) + mocker.patch("openshift.invoke", side_effect=arg_check_apply_effect) + cluster = test_cluster_creation() + cluster.up() + mocker.patch("openshift.invoke", side_effect=arg_check_del_effect) + cluster.down() + + +def out_route(self): + return "ray-dashboard-raycluster-autoscaler-ns.apps.cluster.awsroute.org ray-dashboard-unit-test-cluster-ns.apps.cluster.awsroute.org" + + +def test_cluster_uris(mocker): + mocker.patch("openshift.invoke", return_value=fake_res) + mock_res = mocker.patch.object(openshift.Result, "out") + mock_res.side_effect = lambda: out_route(fake_res) + + cluster = test_cluster_creation() + assert cluster.cluster_uri() == "ray://unit-test-cluster-head-svc.ns.svc:10001" + assert ( + cluster.cluster_dashboard_uri() + == "http://ray-dashboard-unit-test-cluster-ns.apps.cluster.awsroute.org" + ) + cluster.config.name = "fake" + assert ( + cluster.cluster_dashboard_uri() + == "Dashboard route not available yet. Did you run cluster.up()?" + ) + + +def ray_addr(self): + return self._address + + +def test_ray_job_wrapping(mocker): + mocker.patch("openshift.invoke", return_value=fake_res) + mock_res = mocker.patch.object(openshift.Result, "out") + mock_res.side_effect = lambda: out_route(fake_res) + cluster = test_cluster_creation() + + mocker.patch( + "ray.job_submission.JobSubmissionClient._check_connection_and_version_with_url", + return_value="None", + ) + # mocker.patch("ray.job_submission.JobSubmissionClient.list_jobs", side_effect=ray_addr) + mock_res = mocker.patch.object( + ray.job_submission.JobSubmissionClient, "list_jobs", autospec=True + ) + mock_res.side_effect = ray_addr + assert cluster.list_jobs() == cluster.cluster_dashboard_uri() + # cluster.job_status() + # cluster.job_logs() + + +def test_get_namespace(mocker): + pass + + +def test_list_clusters(mocker): + pass + + +def test_list_queue(mocker): + pass + + +def test_cmd_line_generation(): + os.system( + f"python3 {parent}/src/codeflare_sdk/utils/generate_yaml.py --name=unit-cmd-cluster --min-cpu=1 --max-cpu=1 --min-memory=2 --max-memory=2 --gpu=1 --workers=2 --template=src/codeflare_sdk/templates/new-template.yaml" + ) + assert filecmp.cmp( + "unit-cmd-cluster.yaml", f"{parent}/tests/test-case-cmd.yaml", shallow=True + ) + os.remove("unit-test-cluster.yaml") + os.remove("unit-cmd-cluster.yaml")