diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 9b22e0582..59dbec862 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -112,10 +112,6 @@ jobs: - name: Run e2e tests run: | - export CODEFLARE_TEST_TIMEOUT_SHORT=1m - export CODEFLARE_TEST_TIMEOUT_MEDIUM=5m - export CODEFLARE_TEST_TIMEOUT_LONG=15m - export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV @@ -123,7 +119,7 @@ jobs: pip install poetry poetry install --with test,docs echo "Running e2e tests..." - poetry run pytest -v -s ./tests/e2e/mnist_raycluster_sdk_test.py > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1 + poetry run pytest -v -s ./tests/e2e -m kind > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1 - name: Print CodeFlare operator logs if: always() && steps.deploy.outcome == 'success' diff --git a/docs/e2e.md b/docs/e2e.md index ce04c7691..469647ca8 100644 --- a/docs/e2e.md +++ b/docs/e2e.md @@ -10,7 +10,6 @@ Pre-requisite for KinD clusters: please add in your local `/etc/hosts` file `127 ``` make kind-e2e export CLUSTER_HOSTNAME=kind - export CODEFLARE_TEST_TIMEOUT_LONG=20m make deploy -e IMG=quay.io/project-codeflare/codeflare-operator:v1.1.0 make setup-e2e ``` @@ -77,3 +76,11 @@ Pre-requisite for KinD clusters: please add in your local `/etc/hosts` file `127 poetry install --with test,docs poetry run pytest -v -s ./tests/e2e/mnist_raycluster_sdk_test.py ``` + - To run the multiple tests based on the cluster environment, we can run the e2e tests by marking -m with cluster environment (kind or openshift) + ``` + poetry run pytest -v -s ./tests/e2e -m openshift + ``` + - By default tests configured with timeout of `15 minutes`. If necessary, we can override the timeout using `--timeout` option + ``` + poetry run pytest -v -s ./tests/e2e -m openshift --timeout=1200 + ``` diff --git a/poetry.lock b/poetry.lock index 5496961b6..a9db5ff67 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "aiohttp" version = "3.9.1" description = "Async http client/server framework (asyncio)" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -100,6 +101,7 @@ speedups = ["Brotli", "aiodns", "brotlicffi"] name = "aiohttp-cors" version = "0.7.0" description = "CORS support for aiohttp" +category = "main" optional = false python-versions = "*" files = [ @@ -114,6 +116,7 @@ aiohttp = ">=1.1" name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -128,6 +131,7 @@ frozenlist = ">=1.1.0" name = "ansicon" version = "1.89.0" description = "Python wrapper for loading Jason Hood's ANSICON" +category = "main" optional = false python-versions = "*" files = [ @@ -139,6 +143,7 @@ files = [ name = "async-timeout" version = "4.0.3" description = "Timeout context manager for asyncio programs" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -150,6 +155,7 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -168,6 +174,7 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "bcrypt" version = "4.0.1" description = "Modern password hashing for your software and your servers" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -202,6 +209,7 @@ typecheck = ["mypy"] name = "blessed" version = "1.20.0" description = "Easy, practical library for making terminal apps, by providing an elegant, well-documented interface to Colors, Keyboard input, and screen Positioning capabilities." +category = "main" optional = false python-versions = ">=2.7" files = [ @@ -218,6 +226,7 @@ wcwidth = ">=0.1.4" name = "cachetools" version = "5.3.1" description = "Extensible memoizing collections and decorators" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -229,6 +238,7 @@ files = [ name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -240,6 +250,7 @@ files = [ name = "cffi" version = "1.16.0" description = "Foreign Function Interface for Python calling C code." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -304,6 +315,7 @@ pycparser = "*" name = "charset-normalizer" version = "3.3.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -403,6 +415,7 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -417,6 +430,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "codeflare-torchx" version = "0.6.0.dev1" description = "TorchX SDK and Components" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -445,6 +459,7 @@ ray = ["ray (>=1.12.1)"] name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -456,6 +471,7 @@ files = [ name = "colorful" version = "0.5.5" description = "Terminal string styling done right, in Python." +category = "main" optional = false python-versions = "*" files = [ @@ -470,6 +486,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "commonmark" version = "0.9.1" description = "Python parser for the CommonMark Markdown spec" +category = "main" optional = false python-versions = "*" files = [ @@ -484,6 +501,7 @@ test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] name = "coverage" version = "7.2.7" description = "Code coverage measurement for Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -556,6 +574,7 @@ toml = ["tomli"] name = "cryptography" version = "40.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -597,6 +616,7 @@ tox = ["tox"] name = "distlib" version = "0.3.7" description = "Distribution utilities" +category = "main" optional = false python-versions = "*" files = [ @@ -608,6 +628,7 @@ files = [ name = "docker" version = "6.1.3" description = "A Python library for the Docker Engine API." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -629,6 +650,7 @@ ssh = ["paramiko (>=2.4.3)"] name = "docstring-parser" version = "0.8.1" description = "\"Parse Python docstrings in reST, Google and Numpydoc format\"" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -639,6 +661,7 @@ files = [ name = "exceptiongroup" version = "1.1.3" description = "Backport of PEP 654 (exception groups)" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -653,6 +676,7 @@ test = ["pytest (>=6)"] name = "executing" version = "1.2.0" description = "Get the currently executing AST node of a frame, and other information" +category = "main" optional = false python-versions = "*" files = [ @@ -667,6 +691,7 @@ tests = ["asttokens", "littleutils", "pytest", "rich"] name = "filelock" version = "3.12.4" description = "A platform independent file lock." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -683,6 +708,7 @@ typing = ["typing-extensions (>=4.7.1)"] name = "frozenlist" version = "1.4.0" description = "A list-like structure which implements collections.abc.MutableSequence" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -753,6 +779,7 @@ files = [ name = "fsspec" version = "2023.9.2" description = "File-system specification" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -788,6 +815,7 @@ tqdm = ["tqdm"] name = "google-api-core" version = "2.15.0" description = "Google API client core library" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -810,6 +838,7 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] name = "google-auth" version = "2.23.3" description = "Google Authentication Library" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -833,6 +862,7 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] name = "googleapis-common-protos" version = "1.62.0" description = "Common protobufs used in Google APIs" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -850,6 +880,7 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] name = "gpustat" version = "1.1.1" description = "An utility to monitor NVIDIA GPU status and usage" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -869,6 +900,7 @@ test = ["mockito (>=1.2.1)", "pytest (>=5.4.1)", "pytest-runner"] name = "grpcio" version = "1.60.0" description = "HTTP/2-based RPC framework" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -935,6 +967,7 @@ protobuf = ["grpcio-tools (>=1.60.0)"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -946,6 +979,7 @@ files = [ name = "importlib-metadata" version = "6.8.0" description = "Read metadata from Python packages" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -965,6 +999,7 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs name = "importlib-resources" version = "6.1.0" description = "Read resources from Python packages" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -983,6 +1018,7 @@ testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -994,6 +1030,7 @@ files = [ name = "jinxed" version = "1.2.1" description = "Jinxed Terminal Library" +category = "main" optional = false python-versions = "*" files = [ @@ -1008,6 +1045,7 @@ ansicon = {version = "*", markers = "platform_system == \"Windows\""} name = "jsonschema" version = "4.19.1" description = "An implementation of JSON Schema validation for Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1031,6 +1069,7 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "jsonschema-specifications" version = "2023.7.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1046,6 +1085,7 @@ referencing = ">=0.28.0" name = "kubernetes" version = "26.1.0" description = "Kubernetes python client" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1063,7 +1103,7 @@ requests-oauthlib = "*" setuptools = ">=21.0.0" six = ">=1.9.0" urllib3 = ">=1.24.2" -websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.dev0 || >=0.43.dev0" +websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.0 || >=0.43.0" [package.extras] adal = ["adal (>=1.0.2)"] @@ -1072,6 +1112,7 @@ adal = ["adal (>=1.0.2)"] name = "mako" version = "1.2.4" description = "A super-fast templating language that borrows the best ideas from the existing templating languages." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1091,6 +1132,7 @@ testing = ["pytest"] name = "markdown" version = "3.5" description = "Python implementation of John Gruber's Markdown." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1109,6 +1151,7 @@ testing = ["coverage", "pyyaml"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1178,6 +1221,7 @@ files = [ name = "msgpack" version = "1.0.7" description = "MessagePack serializer" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1243,6 +1287,7 @@ files = [ name = "multidict" version = "6.0.4" description = "multidict implementation" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1326,6 +1371,7 @@ files = [ name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1337,6 +1383,7 @@ files = [ name = "numpy" version = "1.24.4" description = "Fundamental package for array computing in Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1374,6 +1421,7 @@ files = [ name = "nvidia-ml-py" version = "12.535.133" description = "Python Bindings for the NVIDIA Management Library" +category = "main" optional = false python-versions = "*" files = [ @@ -1385,6 +1433,7 @@ files = [ name = "oauthlib" version = "3.2.2" description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1401,6 +1450,7 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] name = "opencensus" version = "0.11.3" description = "A stats collection and distributed tracing framework" +category = "main" optional = false python-versions = "*" files = [ @@ -1416,6 +1466,7 @@ opencensus-context = ">=0.1.3" name = "opencensus-context" version = "0.1.3" description = "OpenCensus Runtime Context" +category = "main" optional = false python-versions = "*" files = [ @@ -1427,6 +1478,7 @@ files = [ name = "openshift-client" version = "1.0.18" description = "OpenShift python client" +category = "main" optional = false python-versions = "*" files = [ @@ -1443,6 +1495,7 @@ six = "*" name = "packaging" version = "23.2" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1454,6 +1507,7 @@ files = [ name = "pandas" version = "2.0.3" description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1487,8 +1541,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -1521,6 +1575,7 @@ xml = ["lxml (>=4.6.3)"] name = "paramiko" version = "3.3.1" description = "SSH2 protocol library" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1542,6 +1597,7 @@ invoke = ["invoke (>=2.0)"] name = "pdoc3" version = "0.10.0" description = "Auto-generate API documentation for Python projects." +category = "dev" optional = false python-versions = ">= 3.6" files = [ @@ -1557,6 +1613,7 @@ markdown = ">=3.0" name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1568,6 +1625,7 @@ files = [ name = "platformdirs" version = "3.11.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1583,6 +1641,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pluggy" version = "1.3.0" description = "plugin and hook calling mechanisms for python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1598,6 +1657,7 @@ testing = ["pytest", "pytest-benchmark"] name = "prometheus-client" version = "0.19.0" description = "Python client for the Prometheus monitoring system." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1612,6 +1672,7 @@ twisted = ["twisted"] name = "protobuf" version = "4.24.4" description = "" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1634,6 +1695,7 @@ files = [ name = "psutil" version = "5.9.6" description = "Cross-platform lib for process and system monitoring in Python." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -1662,6 +1724,7 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] name = "py-spy" version = "0.3.14" description = "Sampling profiler for Python programs" +category = "main" optional = false python-versions = "*" files = [ @@ -1678,6 +1741,7 @@ files = [ name = "pyarrow" version = "14.0.1" description = "Python library for Apache Arrow" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1726,6 +1790,7 @@ numpy = ">=1.16.6" name = "pyasn1" version = "0.5.0" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -1737,6 +1802,7 @@ files = [ name = "pyasn1-modules" version = "0.3.0" description = "A collection of ASN.1-based protocols modules" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -1751,6 +1817,7 @@ pyasn1 = ">=0.4.6,<0.6.0" name = "pycparser" version = "2.21" description = "C parser in Python" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1762,6 +1829,7 @@ files = [ name = "pydantic" version = "1.10.13" description = "Data validation and settings management using python type hints" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1814,6 +1882,7 @@ email = ["email-validator (>=1.0.3)"] name = "pygments" version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1828,6 +1897,7 @@ plugins = ["importlib-metadata"] name = "pynacl" version = "1.5.0" description = "Python binding to the Networking and Cryptography (NaCl) library" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1854,6 +1924,7 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] name = "pyre-extensions" version = "0.0.30" description = "Type system extensions for use with the pyre type checker" +category = "main" optional = false python-versions = "*" files = [ @@ -1869,6 +1940,7 @@ typing-inspect = "*" name = "pytest" version = "7.4.0" description = "pytest: simple powerful testing with Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1891,6 +1963,7 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-mock" version = "3.11.1" description = "Thin-wrapper around the mock package for easier use with pytest" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1904,10 +1977,26 @@ pytest = ">=5.0" [package.extras] dev = ["pre-commit", "pytest-asyncio", "tox"] +[[package]] +name = "pytest-timeout" +version = "2.2.0" +description = "pytest plugin to abort hanging tests" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-timeout-2.2.0.tar.gz", hash = "sha256:3b0b95dabf3cb50bac9ef5ca912fa0cfc286526af17afc806824df20c2f72c90"}, + {file = "pytest_timeout-2.2.0-py3-none-any.whl", hash = "sha256:bde531e096466f49398a59f2dde76fa78429a09a12411466f88a07213e220de2"}, +] + +[package.dependencies] +pytest = ">=5.0.0" + [[package]] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1922,6 +2011,7 @@ six = ">=1.5" name = "pytz" version = "2023.3.post1" description = "World timezone definitions, modern and historical" +category = "main" optional = false python-versions = "*" files = [ @@ -1933,6 +2023,7 @@ files = [ name = "pywin32" version = "306" description = "Python for Window Extensions" +category = "main" optional = false python-versions = "*" files = [ @@ -1956,6 +2047,7 @@ files = [ name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2016,6 +2108,7 @@ files = [ name = "ray" version = "2.7.0" description = "Ray provides a simple, universal API for building distributed applications." +category = "main" optional = false python-versions = "*" files = [ @@ -2063,8 +2156,8 @@ jsonschema = "*" msgpack = ">=1.0.0,<2.0.0" numpy = [ {version = ">=1.16", markers = "python_version < \"3.9\""}, - {version = ">=1.20", optional = true, markers = "extra == \"data\""}, {version = ">=1.19.3", markers = "python_version >= \"3.9\""}, + {version = ">=1.20", optional = true, markers = "extra == \"data\""}, ] opencensus = {version = "*", optional = true, markers = "extra == \"default\""} packaging = "*" @@ -2097,6 +2190,7 @@ tune = ["fsspec", "pandas", "pyarrow (>=6.0.1)", "requests", "tensorboardX (>=1. name = "referencing" version = "0.30.2" description = "JSON Referencing + Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2112,6 +2206,7 @@ rpds-py = ">=0.7.0" name = "requests" version = "2.31.0" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2133,6 +2228,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "requests-oauthlib" version = "1.3.1" description = "OAuthlib authentication support for Requests." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2151,6 +2247,7 @@ rsa = ["oauthlib[signedtoken] (>=3.0.0)"] name = "rich" version = "12.6.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" optional = false python-versions = ">=3.6.3,<4.0.0" files = [ @@ -2170,6 +2267,7 @@ jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] name = "rpds-py" version = "0.10.4" description = "Python bindings to Rust's persistent data structures (rpds)" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2278,6 +2376,7 @@ files = [ name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" +category = "main" optional = false python-versions = ">=3.6,<4" files = [ @@ -2292,6 +2391,7 @@ pyasn1 = ">=0.1.3" name = "setuptools" version = "68.2.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2308,6 +2408,7 @@ testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jar name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2319,6 +2420,7 @@ files = [ name = "smart-open" version = "6.4.0" description = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)" +category = "main" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -2340,6 +2442,7 @@ webhdfs = ["requests"] name = "tabulate" version = "0.9.0" description = "Pretty-print tabular data" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2354,6 +2457,7 @@ widechars = ["wcwidth"] name = "tomli" version = "2.0.1" description = "A lil' TOML parser" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2365,6 +2469,7 @@ files = [ name = "typing-extensions" version = "4.8.0" description = "Backported and Experimental Type Hints for Python 3.8+" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2376,6 +2481,7 @@ files = [ name = "typing-inspect" version = "0.9.0" description = "Runtime inspection utilities for typing module." +category = "main" optional = false python-versions = "*" files = [ @@ -2391,6 +2497,7 @@ typing-extensions = ">=3.7.4" name = "tzdata" version = "2023.3" description = "Provider of IANA time zone data" +category = "main" optional = false python-versions = ">=2" files = [ @@ -2402,6 +2509,7 @@ files = [ name = "urllib3" version = "1.26.17" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -2418,6 +2526,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "virtualenv" version = "20.21.0" description = "Virtual Python Environment builder" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2438,6 +2547,7 @@ test = ["covdefaults (>=2.2.2)", "coverage (>=7.1)", "coverage-enable-subprocess name = "wcwidth" version = "0.2.12" description = "Measures the displayed width of unicode strings in a terminal" +category = "main" optional = false python-versions = "*" files = [ @@ -2449,6 +2559,7 @@ files = [ name = "websocket-client" version = "1.6.4" description = "WebSocket client for Python with low level API options" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2465,6 +2576,7 @@ test = ["websockets"] name = "yarl" version = "1.9.4" description = "Yet another URL library" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2568,6 +2680,7 @@ multidict = ">=4.0" name = "zipp" version = "3.17.0" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2582,4 +2695,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "75531b507aa2ee5b0514864aa92fbb127ef52a4faf119b5affdfbc3c694c5b03" +content-hash = "e7fa79bd035b5bffcb1668c0a8cf8fa4e869a614b934a09100ab1d53338fe11b" diff --git a/pyproject.toml b/pyproject.toml index 66dd6cd02..e262b5270 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,9 +43,15 @@ optional = true pytest = "7.4.0" coverage = "7.2.7" pytest-mock = "3.11.1" +pytest-timeout = "2.2.0" [tool.pytest.ini_options] filterwarnings = [ "ignore::DeprecationWarning:pkg_resources", "ignore:pkg_resources is deprecated as an API:DeprecationWarning", ] +markers = [ + "kind", + "openshift" +] +timeout = 900 diff --git a/tests/e2e/mnist_raycluster_sdk_oauth_test.py b/tests/e2e/mnist_raycluster_sdk_oauth_test.py new file mode 100644 index 000000000..0cee617de --- /dev/null +++ b/tests/e2e/mnist_raycluster_sdk_oauth_test.py @@ -0,0 +1,144 @@ +import requests + +from time import sleep + +from torchx.specs.api import AppState, is_terminal + +from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration +from codeflare_sdk.job.jobs import DDPJobDefinition + +import pytest + +from support import * + +# This test Creates a Ray cluster with openshift_oauth enable and covers the Ray Job submission with authentication and without authentication functionality + + +@pytest.mark.openshift +class TestRayClusterSDKOauth: + def setup_method(self): + initialize_kubernetes_client(self) + + def teardown_method(self): + delete_namespace(self) + + def test_mnist_ray_cluster_sdk_auth(self): + self.setup_method() + create_namespace(self) + self.run_mnist_raycluster_sdk_oauth() + + def run_mnist_raycluster_sdk_oauth(self): + ray_image = get_ray_image() + + cluster = Cluster( + ClusterConfiguration( + name="mnist", + namespace=self.namespace, + num_workers=1, + head_cpus="500m", + head_memory=2, + min_cpus="500m", + max_cpus=1, + min_memory=1, + max_memory=2, + num_gpus=0, + instascale=False, + image=ray_image, + openshift_oauth=True, + ) + ) + + cluster.up() + self.assert_appwrapper_exists() + + cluster.status() + + cluster.wait_ready() + + cluster.status() + + cluster.details() + + self.assert_jobsubmit_withoutLogin(cluster) + + self.assert_jobsubmit_withlogin(cluster) + + # Assertions + + def assert_jobsubmit_withoutLogin(self, cluster): + dashboard_url = cluster.cluster_dashboard_uri() + jobdata = { + "entrypoint": "python mnist.py", + "runtime_env": { + "working_dir": "./tests/e2e/", + "pip": "mnist_pip_requirements.txt", + }, + } + try: + response = requests.post( + dashboard_url + "/api/jobs/", verify=False, json=jobdata + ) + if response.status_code == 403: + assert True + else: + response.raise_for_status() + assert False + + except Exception as e: + print(f"An unexpected error occurred. Error: {e}") + assert False + + def assert_jobsubmit_withlogin(self, cluster): + self.assert_appwrapper_exists() + jobdef = DDPJobDefinition( + name="mnist", + script="./tests/e2e/mnist.py", + scheduler_args={"requirements": "./tests/e2e/mnist_pip_requirements.txt"}, + ) + job = jobdef.submit(cluster) + + done = False + time = 0 + timeout = 900 + while not done: + status = job.status() + if is_terminal(status.state): + break + if not done: + print(status) + if timeout and time >= timeout: + raise TimeoutError(f"job has timed out after waiting {timeout}s") + sleep(5) + time += 5 + + print(job.status()) + self.assert_job_completion(status) + + print(job.logs()) + + cluster.down() + + def assert_appwrapper_exists(self): + try: + self.custom_api.get_namespaced_custom_object( + "workload.codeflare.dev", + "v1beta1", + self.namespace, + "appwrappers", + "mnist", + ) + print( + f"AppWrapper 'mnist' has been created in the namespace: '{self.namespace}'" + ) + assert True + except Exception as e: + print(f"AppWrapper 'mnist' has not been created. Error: {e}") + assert False + + def assert_job_completion(self, status): + if status.state == AppState.SUCCEEDED: + print(f"Job has completed: '{status.state}'") + assert True + else: + print(f"Job has completed: '{status.state}'") + assert False diff --git a/tests/e2e/mnist_raycluster_sdk_test.py b/tests/e2e/mnist_raycluster_sdk_test.py index 26f76b602..76a5e260f 100644 --- a/tests/e2e/mnist_raycluster_sdk_test.py +++ b/tests/e2e/mnist_raycluster_sdk_test.py @@ -14,40 +14,26 @@ import pytest -from support import random_choice, get_ray_image +from support import * # Creates a Ray cluster, and trains the MNIST dataset using the CodeFlare SDK. # Asserts creation of AppWrapper, RayCluster, and successful completion of the training job. # Covers successfull installation of CodeFlare-SDK +@pytest.mark.kind +@pytest.mark.openshift class TestMNISTRayClusterSDK: def setup_method(self): - # Load the kube config from the environment or Kube config file. - config.load_kube_config() - - # Initialize Kubernetes client - self.api_instance = client.CoreV1Api() - self.custom_api = kubernetes.client.CustomObjectsApi( - self.api_instance.api_client - ) + initialize_kubernetes_client(self) def teardown_method(self): - if hasattr(self, "namespace"): - self.api_instance.delete_namespace(self.namespace) + delete_namespace(self) def test_mnist_ray_cluster_sdk(self): - self.create_test_namespace() + create_namespace(self) self.run_mnist_raycluster_sdk() - def create_test_namespace(self): - self.namespace = f"test-ns-{random_choice()}" - namespace_body = client.V1Namespace( - metadata=client.V1ObjectMeta(name=self.namespace) - ) - self.api_instance.create_namespace(namespace_body) - return self.namespace - def run_mnist_raycluster_sdk(self): ray_image = get_ray_image() host = os.getenv("CLUSTER_HOSTNAME") diff --git a/tests/e2e/support.py b/tests/e2e/support.py index 303b03c8a..1ab45c192 100644 --- a/tests/e2e/support.py +++ b/tests/e2e/support.py @@ -1,6 +1,8 @@ import os import random import string +from kubernetes import client, config +import kubernetes.client def get_ray_image(): @@ -11,3 +13,23 @@ def get_ray_image(): def random_choice(): alphabet = string.ascii_lowercase + string.digits return "".join(random.choices(alphabet, k=5)) + + +def create_namespace(self): + self.namespace = f"test-ns-{random_choice()}" + namespace_body = client.V1Namespace( + metadata=client.V1ObjectMeta(name=self.namespace) + ) + self.api_instance.create_namespace(namespace_body) + + +def delete_namespace(self): + if hasattr(self, "namespace"): + self.api_instance.delete_namespace(self.namespace) + + +def initialize_kubernetes_client(self): + config.load_kube_config() + # Initialize Kubernetes client + self.api_instance = client.CoreV1Api() + self.custom_api = kubernetes.client.CustomObjectsApi(self.api_instance.api_client)