diff --git a/.github/workflows/integration-gateway.yml b/.github/workflows/integration-gateway.yml new file mode 100644 index 0000000..a531308 --- /dev/null +++ b/.github/workflows/integration-gateway.yml @@ -0,0 +1,93 @@ +name: API Gateway integration tests with Pytest + +on: + push: + branches: [main] + pull_request: + branches: [main] + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +env: + GATEWAY_CHECKOUT_DIR: "gateway" + S3_ENDPOINT: "https://s3.fr-par.scw.cloud" + S3_REGION: "fr-par" + +jobs: + test-deployed-gateway: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: ./.github/actions/setup-poetry + + - uses: actions/checkout@v3 + with: + repository: scaleway/serverless-gateway + path: ${{ env.GATEWAY_CHECKOUT_DIR }} + + - name: Install Scaleway CLI + # Should point to GATEWAY_CHECKOUT_DIR but env is unusable here. + # See: https://docs.github.com/en/actions/learn-github-actions/contexts#env-context + uses: ./gateway/.github/actions/setup-scaleway-cli + with: + scw-version: "2.13.0" + scw-access-key: ${{ secrets.SCW_ACCESS_KEY }} + scw-secret-key: ${{ secrets.SCW_SECRET_KEY }} + scw-default-project-id: ${{ secrets.SCW_DEFAULT_PROJECT_ID }} + scw-default-organization-id: ${{ secrets.SCW_DEFAULT_ORGANIZATION_ID }} + + - name: Create Gateway namespace + working-directory: ${{ env.GATEWAY_CHECKOUT_DIR }} + run: | + make create-namespace + until [ $(make check-namespace -s) == ready ]; do sleep 10; done + + - name: Create Gateway container + working-directory: ${{ env.GATEWAY_CHECKOUT_DIR }} + # We need to truncate gateway.env as it will override our env vars + run: | + truncate -s 0 gateway.env + make create-container + make deploy-container + until [ $(make check-container -s) == ready ]; do sleep 10; done + env: + SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }} + SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }} + S3_BUCKET_NAME: ${{ secrets.GATEWAY_S3_BUCKET_NAME }} + + - name: Install s3cmd + run: pip install s3cmd + + - name: Create S3 bucket + working-directory: ${{ env.GATEWAY_CHECKOUT_DIR }} + run: | + make set-up-s3-cli + make create-s3-bucket + env: + S3_BUCKET_NAME: ${{ secrets.GATEWAY_S3_BUCKET_NAME }} + + - name: Run integration tests + run: | + pushd $GATEWAY_CHECKOUT_DIR + export GATEWAY_HOST=$(make get-gateway-endpoint -s) + popd + poetry run pytest tests/integrations/gateway -n $(nproc --all) + env: + SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }} + SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }} + GATEWAY_S3_BUCKET_NAME: ${{ secrets.GATEWAY_S3_BUCKET_NAME }} + + - name: Delete S3 bucket + working-directory: ${{ env.GATEWAY_CHECKOUT_DIR }} + run: make delete-bucket + env: + S3_BUCKET_NAME: ${{ secrets.GATEWAY_S3_BUCKET_NAME }} + if: always() + + - name: Delete Gateway namespace and container + working-directory: ${{ env.GATEWAY_CHECKOUT_DIR }} + run: make delete-namespace + if: always() diff --git a/.github/workflows/pytest-integration.yml b/.github/workflows/pytest-integration.yml index 41d788b..fd15a3e 100644 --- a/.github/workflows/pytest-integration.yml +++ b/.github/workflows/pytest-integration.yml @@ -5,7 +5,7 @@ on: push: branches: [main] pull_request: - branches: [main] + types: [opened, synchronize, reopened, ready_for_review] permissions: contents: read @@ -28,7 +28,7 @@ jobs: - name: Test with pytest working-directory: tests - run: poetry run pytest integrations -n $(nproc --all) + run: poetry run pytest integrations -n $(nproc --all) --ignore=integrations/gateway env: SCW_DEFAULT_ORGANIZATION_ID: ${{ secrets.SCW_DEFAULT_ORGANIZATION_ID }} SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 12a707b..41efff1 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -6,6 +6,7 @@ on: branches: [main] pull_request: branches: [main] + types: [opened, synchronize, reopened, ready_for_review] permissions: contents: read diff --git a/poetry.lock b/poetry.lock index fb6aba3..40e530c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -14,14 +14,14 @@ files = [ [[package]] name = "astroid" -version = "2.15.0" +version = "2.15.2" description = "An abstract syntax tree for Python with inference support." category = "dev" optional = false python-versions = ">=3.7.2" files = [ - {file = "astroid-2.15.0-py3-none-any.whl", hash = "sha256:e3e4d0ffc2d15d954065579689c36aac57a339a4679a679579af6401db4d3fdb"}, - {file = "astroid-2.15.0.tar.gz", hash = "sha256:525f126d5dc1b8b0b6ee398b33159105615d92dc4a17f2cd064125d57f6186fa"}, + {file = "astroid-2.15.2-py3-none-any.whl", hash = "sha256:dea89d9f99f491c66ac9c04ebddf91e4acf8bd711722175fe6245c0725cc19bb"}, + {file = "astroid-2.15.2.tar.gz", hash = "sha256:6e61b85c891ec53b07471aec5878f4ac6446a41e590ede0f2ce095f39f7d49dd"}, ] [package.dependencies] @@ -63,6 +63,46 @@ files = [ {file = "Babel-2.12.1.tar.gz", hash = "sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455"}, ] +[[package]] +name = "boto3" +version = "1.26.105" +description = "The AWS SDK for Python" +category = "dev" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "boto3-1.26.105-py3-none-any.whl", hash = "sha256:f4951f8162905b96fd045e32853ba8cf707042faac846a23910817c508ef27d7"}, + {file = "boto3-1.26.105.tar.gz", hash = "sha256:2914776e0138530ec6464d0e2f05b4aa18e9212ac920c48472f8a93650feaed2"}, +] + +[package.dependencies] +botocore = ">=1.29.105,<1.30.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.6.0,<0.7.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.29.105" +description = "Low-level, data-driven core of boto 3." +category = "dev" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "botocore-1.29.105-py3-none-any.whl", hash = "sha256:06a2838daad3f346cba5460d0d3deb198225b556ff9ca729798d787fadbdebde"}, + {file = "botocore-1.29.105.tar.gz", hash = "sha256:17c82391dfd6aaa8f96fbbb08cad2c2431ef3cda0ece89e6e6ba444c5eed45c2"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<1.27" + +[package.extras] +crt = ["awscrt (==0.16.9)"] + [[package]] name = "certifi" version = "2022.12.7" @@ -240,14 +280,14 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.1.0" +version = "1.1.1" description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, - {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, ] [package.extras] @@ -270,30 +310,30 @@ testing = ["pre-commit"] [[package]] name = "filelock" -version = "3.9.0" +version = "3.10.7" description = "A platform independent file lock." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "filelock-3.9.0-py3-none-any.whl", hash = "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d"}, - {file = "filelock-3.9.0.tar.gz", hash = "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de"}, + {file = "filelock-3.10.7-py3-none-any.whl", hash = "sha256:bde48477b15fde2c7e5a0713cbe72721cb5a5ad32ee0b8f419907960b9d75536"}, + {file = "filelock-3.10.7.tar.gz", hash = "sha256:892be14aa8efc01673b5ed6589dbccb95f9a8596f0507e232626155495c18105"}, ] [package.extras] -docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2022.12.7)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] [[package]] name = "identify" -version = "2.5.19" +version = "2.5.22" description = "File identification library for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "identify-2.5.19-py2.py3-none-any.whl", hash = "sha256:3ee3533e7f6f5023157fbebbd5687bb4b698ce6f305259e0d24b2d7d9efb72bc"}, - {file = "identify-2.5.19.tar.gz", hash = "sha256:4102ecd051f6884449e7359e55b38ba6cd7aafb6ef27b8e2b38495a5723ea106"}, + {file = "identify-2.5.22-py2.py3-none-any.whl", hash = "sha256:f0faad595a4687053669c112004178149f6c326db71ee999ae4636685753ad2f"}, + {file = "identify-2.5.22.tar.gz", hash = "sha256:f7a93d6cf98e29bd07663c60728e7a4057615068d7a639d132dc883b2d54d31e"}, ] [package.extras] @@ -325,14 +365,14 @@ files = [ [[package]] name = "importlib-metadata" -version = "6.0.0" +version = "6.1.0" description = "Read metadata from Python packages" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "importlib_metadata-6.0.0-py3-none-any.whl", hash = "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad"}, - {file = "importlib_metadata-6.0.0.tar.gz", hash = "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"}, + {file = "importlib_metadata-6.1.0-py3-none-any.whl", hash = "sha256:ff80f3b5394912eb1b108fcfd444dc78b7f1f3e16b16188054bd01cb9cb86f09"}, + {file = "importlib_metadata-6.1.0.tar.gz", hash = "sha256:43ce9281e097583d758c2c708c4376371261a02c34682491a8e98352365aad20"}, ] [package.dependencies] @@ -391,6 +431,18 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "lazy-object-proxy" version = "1.9.0" @@ -622,19 +674,19 @@ files = [ [[package]] name = "platformdirs" -version = "3.1.0" +version = "3.2.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.1.0-py3-none-any.whl", hash = "sha256:13b08a53ed71021350c9e300d4ea8668438fb0046ab3937ac9a29913a1a1350a"}, - {file = "platformdirs-3.1.0.tar.gz", hash = "sha256:accc3665857288317f32c7bebb5a8e482ba717b474f3fc1d18ca7f9214be0cef"}, + {file = "platformdirs-3.2.0-py3-none-any.whl", hash = "sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e"}, + {file = "platformdirs-3.2.0.tar.gz", hash = "sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08"}, ] [package.extras] docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pluggy" @@ -654,14 +706,14 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pre-commit" -version = "3.2.1" +version = "3.2.2" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "pre_commit-3.2.1-py2.py3-none-any.whl", hash = "sha256:a06a7fcce7f420047a71213c175714216498b49ebc81fe106f7716ca265f5bb6"}, - {file = "pre_commit-3.2.1.tar.gz", hash = "sha256:b5aee7d75dbba21ee161ba641b01e7ae10c5b91967ebf7b2ab0dfae12d07e1f1"}, + {file = "pre_commit-3.2.2-py2.py3-none-any.whl", hash = "sha256:0b4210aea813fe81144e87c5a291f09ea66f199f367fa1df41b55e1d26e1e2b4"}, + {file = "pre_commit-3.2.2.tar.gz", hash = "sha256:5b808fcbda4afbccf6d6633a56663fed35b6c2bc08096fd3d47ce197ac351d9d"}, ] [package.dependencies] @@ -688,18 +740,18 @@ plugins = ["importlib-metadata"] [[package]] name = "pylint" -version = "2.17.1" +version = "2.17.2" description = "python code static checker" category = "dev" optional = false python-versions = ">=3.7.2" files = [ - {file = "pylint-2.17.1-py3-none-any.whl", hash = "sha256:8660a54e3f696243d644fca98f79013a959c03f979992c1ab59c24d3f4ec2700"}, - {file = "pylint-2.17.1.tar.gz", hash = "sha256:d4d009b0116e16845533bc2163493d6681846ac725eab8ca8014afb520178ddd"}, + {file = "pylint-2.17.2-py3-none-any.whl", hash = "sha256:001cc91366a7df2970941d7e6bbefcbf98694e00102c1f121c531a814ddc2ea8"}, + {file = "pylint-2.17.2.tar.gz", hash = "sha256:1b647da5249e7c279118f657ca28b6aaebb299f86bf92affc632acf199f7adbb"}, ] [package.dependencies] -astroid = ">=2.15.0,<=2.17.0-dev0" +astroid = ">=2.15.2,<=2.17.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.2", markers = "python_version < \"3.11\""}, @@ -884,6 +936,24 @@ urllib3 = ">=1.25.10" [package.extras] tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] +[[package]] +name = "s3transfer" +version = "0.6.0" +description = "An Amazon S3 Transfer Manager" +category = "dev" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, +] + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + [[package]] name = "scaleway" version = "0.10.0" @@ -901,14 +971,14 @@ scaleway-core = ">=0,<1" [[package]] name = "scaleway-core" -version = "0.9.0" +version = "0.11.0" description = "Scaleway SDK for Python" category = "main" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "scaleway_core-0.9.0-py3-none-any.whl", hash = "sha256:77754e1172ca0c5b3143ab4a104188c4341acb44d231da00bcc75c2654759957"}, - {file = "scaleway_core-0.9.0.tar.gz", hash = "sha256:2046aef7dbccc379bcbac5e074f841e6a3531b93260f0fdf54840571c6cd73ae"}, + {file = "scaleway_core-0.11.0-py3-none-any.whl", hash = "sha256:1864b20fd73ff32dc75921854969c125570e08a9d556db8a9f5782659500c7b7"}, + {file = "scaleway_core-0.11.0.tar.gz", hash = "sha256:2ded3121c7df4ec0dfd044bc4e59ff078d1fb43a4f1ec3534af74686005d51ce"}, ] [package.dependencies] @@ -918,14 +988,14 @@ requests = ">=2.28.1,<3.0.0" [[package]] name = "setuptools" -version = "67.6.0" +version = "67.6.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-67.6.0-py3-none-any.whl", hash = "sha256:b78aaa36f6b90a074c1fa651168723acbf45d14cb1196b6f02c0fd07f17623b2"}, - {file = "setuptools-67.6.0.tar.gz", hash = "sha256:2ee892cd5f29f3373097f5a814697e397cf3ce313616df0af11231e2ad118077"}, + {file = "setuptools-67.6.1-py3-none-any.whl", hash = "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"}, + {file = "setuptools-67.6.1.tar.gz", hash = "sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a"}, ] [package.extras] @@ -1063,18 +1133,18 @@ test = ["html5lib", "pytest"] [[package]] name = "sphinxcontrib-jquery" -version = "2.0.0" +version = "4.1" description = "Extension to include jQuery on newer Sphinx releases" category = "dev" optional = false python-versions = ">=2.7" files = [ - {file = "sphinxcontrib-jquery-2.0.0.tar.gz", hash = "sha256:8fb65f6dba84bf7bcd1aea1f02ab3955ac34611d838bcc95d4983b805b234daa"}, - {file = "sphinxcontrib_jquery-2.0.0-py3-none-any.whl", hash = "sha256:ed47fa425c338ffebe3c37e1cdb56e30eb806116b85f01055b158c7057fdb995"}, + {file = "sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a"}, + {file = "sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae"}, ] [package.dependencies] -setuptools = "*" +Sphinx = ">=1.8" [[package]] name = "sphinxcontrib-jsmath" @@ -1137,26 +1207,26 @@ files = [ [[package]] name = "tomlkit" -version = "0.11.6" +version = "0.11.7" description = "Style preserving TOML library" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "tomlkit-0.11.6-py3-none-any.whl", hash = "sha256:07de26b0d8cfc18f871aec595fda24d95b08fef89d147caa861939f37230bf4b"}, - {file = "tomlkit-0.11.6.tar.gz", hash = "sha256:71b952e5721688937fb02cf9d354dbcf0785066149d2855e44531ebdd2b65d73"}, + {file = "tomlkit-0.11.7-py3-none-any.whl", hash = "sha256:5325463a7da2ef0c6bbfefb62a3dc883aebe679984709aee32a317907d0a8d3c"}, + {file = "tomlkit-0.11.7.tar.gz", hash = "sha256:f392ef70ad87a672f02519f99967d28a4d3047133e2d1df936511465fbb3791d"}, ] [[package]] name = "types-pyyaml" -version = "6.0.12.8" +version = "6.0.12.9" description = "Typing stubs for PyYAML" category = "dev" optional = false python-versions = "*" files = [ - {file = "types-PyYAML-6.0.12.8.tar.gz", hash = "sha256:19304869a89d49af00be681e7b267414df213f4eb89634c4495fa62e8f942b9f"}, - {file = "types_PyYAML-6.0.12.8-py3-none-any.whl", hash = "sha256:5314a4b2580999b2ea06b2e5f9a7763d860d6e09cdf21c0e9561daa9cbd60178"}, + {file = "types-PyYAML-6.0.12.9.tar.gz", hash = "sha256:c51b1bd6d99ddf0aa2884a7a328810ebf70a4262c292195d3f4f9a0005f9eeb6"}, + {file = "types_PyYAML-6.0.12.9-py3-none-any.whl", hash = "sha256:5aed5aa66bd2d2e158f75dda22b059570ede988559f030cf294871d3b647e3e8"}, ] [[package]] @@ -1173,14 +1243,14 @@ files = [ [[package]] name = "urllib3" -version = "1.26.14" +version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, - {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, + {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, + {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, ] [package.extras] @@ -1190,14 +1260,14 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.20.0" +version = "20.21.0" description = "Virtual Python Environment builder" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.20.0-py3-none-any.whl", hash = "sha256:3c22fa5a7c7aa106ced59934d2c20a2ecb7f49b4130b8bf444178a16b880fa45"}, - {file = "virtualenv-20.20.0.tar.gz", hash = "sha256:a8a4b8ca1e28f864b7514a253f98c1d62b64e31e77325ba279248c65fb4fcef4"}, + {file = "virtualenv-20.21.0-py3-none-any.whl", hash = "sha256:31712f8f2a17bd06234fa97fdf19609e789dd4e3e4bf108c3da71d710651adbc"}, + {file = "virtualenv-20.21.0.tar.gz", hash = "sha256:f50e3e60f990a0757c9b68333c9fdaa72d7188caa417f96af9e52407831a3b68"}, ] [package.dependencies] @@ -1313,4 +1383,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "fb1a54ffeadf7d598aaf1a0b240dc43d8b2e453dc751dbea7822135e4f40431a" +content-hash = "1c00ad55d86008145a2eed716575af4f85571f5b2ec1d9427617b9a724094655" diff --git a/pyproject.toml b/pyproject.toml index 0d7e6b6..9d19d41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ pytest-xdist = "^3.1.0" pylint = "^2.15.10" pylint-per-file-ignores = "^1.1.0" responses = ">=0.22,<0.24" +boto3 = "^1.26.97" [tool.poetry.group.doc] optional = true @@ -60,13 +61,19 @@ myst_parser = ">=0.18.1,<1.1.0" sphinx = "^5.3.0" sphinx_rtd_theme = "^1.1.1" +[tool.pytest.ini_options] +filterwarnings = [ + "ignore:.*pkg_resources\\.declare_namespace.*:DeprecationWarning", + "ignore:::pkg_resources", +] + [tool.pylint] load-plugins = ["pylint_per_file_ignores"] disable = "missing-module-docstring" # Commented Black formatted code. max-line-length = 89 # Short and common names. e is commonly used for exceptions. -good-names = "i,fp,e" +good-names = "e,fp,i,s,s3" # Classes with a single responsibility are fine. min-public-methods = 1 diff --git a/scw_serverless/app.py b/scw_serverless/app.py index 24f919e..4a6e9de 100644 --- a/scw_serverless/app.py +++ b/scw_serverless/app.py @@ -19,7 +19,6 @@ class Serverless: :param service_name: name of the namespace :param env: namespace level environment variables :param secret: namespace level secrets - :param gateway_domains: domains to be supported by the gateway """ def __init__( @@ -27,11 +26,9 @@ def __init__( service_name: str, env: Optional[dict[str, Any]] = None, secret: Optional[dict[str, Any]] = None, - gateway_domains: Optional[list[str]] = None, ): self.functions: list[Function] = [] self.service_name: str = service_name - self.gateway_domains: list[str] = gateway_domains if gateway_domains else [] self.env = env self.secret = secret @@ -108,7 +105,7 @@ def get(self, url: str, **kwargs: Unpack[FunctionKwargs]) -> Callable: Requires an API gateway """ - kwargs |= {"url": url, "methods": [HTTPMethod.GET]} + kwargs |= {"relative_url": url, "http_methods": [HTTPMethod.GET]} return self.func(**kwargs) def post(self, url: str, **kwargs: Unpack[FunctionKwargs]) -> Callable: @@ -120,7 +117,7 @@ def post(self, url: str, **kwargs: Unpack[FunctionKwargs]) -> Callable: Requires an API gateway """ - kwargs |= {"url": url, "methods": [HTTPMethod.POST]} + kwargs |= {"relative_url": url, "http_methods": [HTTPMethod.POST]} return self.func(**kwargs) def put(self, url: str, **kwargs: Unpack[FunctionKwargs]) -> Callable: @@ -132,7 +129,7 @@ def put(self, url: str, **kwargs: Unpack[FunctionKwargs]) -> Callable: Requires an API gateway """ - kwargs |= {"url": url, "methods": [HTTPMethod.PUT]} + kwargs |= {"relative_url": url, "http_methods": [HTTPMethod.PUT]} return self.func(**kwargs) def delete(self, url: str, **kwargs: Unpack[FunctionKwargs]) -> Callable: @@ -144,7 +141,7 @@ def delete(self, url: str, **kwargs: Unpack[FunctionKwargs]) -> Callable: Requires an API gateway """ - kwargs |= {"url": url, "methods": [HTTPMethod.DELETE]} + kwargs |= {"relative_url": url, "http_methods": [HTTPMethod.DELETE]} return self.func(**kwargs) def patch(self, url: str, **kwargs: Unpack[FunctionKwargs]) -> Callable: @@ -156,5 +153,5 @@ def patch(self, url: str, **kwargs: Unpack[FunctionKwargs]) -> Callable: Requires an API gateway """ - kwargs |= {"url": url, "methods": [HTTPMethod.PATCH]} + kwargs |= {"relative_url": url, "http_methods": [HTTPMethod.PATCH]} return self.func(**kwargs) diff --git a/scw_serverless/cli.py b/scw_serverless/cli.py index 9a01f9d..8b86483 100644 --- a/scw_serverless/cli.py +++ b/scw_serverless/cli.py @@ -4,12 +4,10 @@ import click -from scw_serverless.config.generators.serverless_framework import ( - ServerlessFrameworkGenerator, -) -from scw_serverless.config.generators.terraform import TerraformGenerator +from scw_serverless.config import generators from scw_serverless.dependencies_manager import DependenciesManager from scw_serverless.deploy import backends +from scw_serverless.gateway.gateway_manager import GatewayManager from scw_serverless.logger import DEFAULT, get_logger from scw_serverless.utils.credentials import DEFAULT_REGION, get_scw_client from scw_serverless.utils.loader import get_app_instance @@ -40,6 +38,34 @@ def cli() -> None: # pylint: disable=too-many-arguments,too-many-locals @cli.command() @CLICK_ARG_FILE +@click.option( + "--backend", + "-b", + "backend", + default="api", + type=click.Choice(["api", "serverless"], case_sensitive=False), + show_default=True, + help="Select the backend used to deploy", +) +@click.option( + "--no-single-source", + "single_source", + is_flag=True, + default=True, + help="Do not remove functions not present in the code being deployed", +) +@click.option( + "--gateway-url", + "gateway_url", + default=None, + help="URL of a deployed API Gateway.", +) +@click.option( + "--gateway-api-key", + "gateway_api_key", + default=None, + help="API key used to manage the routes of the API Gateway.", +) @click.option( "--profile", "-p", @@ -67,26 +93,12 @@ def cli() -> None: default=None, help=f"Region to deploy to. Default: {DEFAULT_REGION}", ) -@click.option( - "--no-single-source", - "single_source", - is_flag=True, - default=True, - help="Do not remove functions not present in the code being deployed", -) -@click.option( - "--backend", - "-b", - "backend", - default="api", - type=click.Choice(["api", "serverless"], case_sensitive=False), - show_default=True, - help="Select the backend used to deploy", -) def deploy( file: Path, backend: Literal["api", "serverless"], single_source: bool, + gateway_url: Optional[str] = None, + gateway_api_key: Optional[str] = None, profile_name: Optional[str] = None, secret_key: Optional[str] = None, project_id: Optional[str] = None, @@ -123,6 +135,28 @@ def deploy( deploy_backend.deploy() + needs_gateway = any(function.gateway_route for function in app_instance.functions) + if not needs_gateway: + return + + if not gateway_url: + raise RuntimeError( + "Your application requires an API Gateway but no gateway URL was provided" + ) + if not gateway_api_key: + raise RuntimeError( + "Your application requires an API Gateway but " + + "no gateway API key was provided to manage routes" + ) + + manager = GatewayManager( + app_instance=app_instance, + gateway_url=gateway_url, + gateway_api_key=gateway_api_key, + sdk_client=client, + ) + manager.update_routes() + @cli.command() @CLICK_ARG_FILE @@ -157,17 +191,18 @@ def generate(file: Path, target: str, save: str) -> None: if not os.path.exists(save): os.mkdir(save) + generator = None if target == "serverless": - serverless_framework_generator = ServerlessFrameworkGenerator(app_instance) - serverless_framework_generator.write(save) + generator = generators.ServerlessFrameworkGenerator(app_instance) elif target == "terraform": - terraform_generator = TerraformGenerator( + generator = generators.TerraformGenerator( app_instance, deps_manager=DependenciesManager( file.parent.resolve(), file.parent.resolve() ), ) - terraform_generator.write(save) + if generator: + generator.write(save) get_logger().success(f"Done! Generated configuration file saved in {save}") diff --git a/scw_serverless/config/function.py b/scw_serverless/config/function.py index f25035c..7618817 100644 --- a/scw_serverless/config/function.py +++ b/scw_serverless/config/function.py @@ -61,8 +61,8 @@ class FunctionKwargs(TypedDict, total=False): description: str http_option: HTTPOption # Parameters for the Gateway - url: str - methods: list[HTTPMethod] + relative_url: str + http_methods: list[HTTPMethod] # Triggers triggers: list[Trigger] @@ -107,8 +107,8 @@ def from_handler( if args_http_option := args.get("http_option"): http_option = sdk.FunctionHttpOption(args_http_option) gateway_route = None - if url := args.get("url"): - gateway_route = GatewayRoute(url, methods=args.get("methods")) + if url := args.get("relative_url"): + gateway_route = GatewayRoute(url, http_methods=args.get("http_methods")) return Function( name=to_valid_fn_name(handler.__name__), diff --git a/scw_serverless/config/route.py b/scw_serverless/config/route.py index fc557ee..e77bdb4 100644 --- a/scw_serverless/config/route.py +++ b/scw_serverless/config/route.py @@ -1,6 +1,8 @@ from dataclasses import dataclass from enum import Enum -from typing import Optional +from typing import Any, Optional + +from scw_serverless.config.utils import _SerializableDataClass class HTTPMethod(Enum): @@ -17,8 +19,25 @@ class HTTPMethod(Enum): @dataclass -class GatewayRoute: +class GatewayRoute(_SerializableDataClass): """Route to a function.""" - path: str - methods: Optional[list[HTTPMethod]] = None + relative_url: str + http_methods: Optional[list[HTTPMethod]] = None + target: Optional[str] = None + + def validate(self) -> None: + """Validate a route.""" + if not self.relative_url: + raise RuntimeError("Route relative_url must be defined") + if not self.target: + raise RuntimeError("Route target must be defined") + for method in self.http_methods or []: + if method not in HTTPMethod: + raise RuntimeError(f"Route contains invalid method {method.value}") + + def asdict(self) -> dict[str, Any]: + serialized = super().asdict() + if self.http_methods: + serialized["http_methods"] = [method.value for method in self.http_methods] + return serialized diff --git a/scw_serverless/dependencies_manager.py b/scw_serverless/dependencies_manager.py index b07a2ca..4f7a130 100644 --- a/scw_serverless/dependencies_manager.py +++ b/scw_serverless/dependencies_manager.py @@ -2,6 +2,7 @@ import pathlib import subprocess import sys +from importlib.metadata import version from typing import Optional from scw_serverless.logger import get_logger @@ -71,7 +72,8 @@ def _check_for_scw_serverless(self): not self.pkg_path.exists() or not self.pkg_path.joinpath(__package__).exists() ): - self._run_pip_install("scw_serverless") + # Installs the current version with pip + self._run_pip_install(f"{__package__}=={version(__package__)}") def _run_pip_install(self, *args: str): python_path = sys.executable @@ -84,6 +86,7 @@ def _run_pip_install(self, *args: str): "--target", str(self.pkg_path.resolve()), ] + try: subprocess.run( command, diff --git a/scw_serverless/gateway/__init__.py b/scw_serverless/gateway/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scw_serverless/gateway/gateway_api_client.py b/scw_serverless/gateway/gateway_api_client.py new file mode 100644 index 0000000..6f8dc5c --- /dev/null +++ b/scw_serverless/gateway/gateway_api_client.py @@ -0,0 +1,46 @@ +import requests + +from scw_serverless.config.route import GatewayRoute + + +class GatewayAPIClient: + """A client for the API to manage routes on the Gateway.""" + + def __init__(self, gateway_url: str, gateway_api_key: str): + self.url = gateway_url + "/scw" + + self.session = requests.Session() + self.session.headers["X-Auth-Token"] = gateway_api_key + + def get_all(self) -> list[GatewayRoute]: + """Get all previously defined routes.""" + + resp = self.session.get(self.url) + resp.raise_for_status() + + endpoints = resp.json()["endpoints"] + routes = [] + for endpoint in endpoints: + routes.append( + GatewayRoute( + relative_url=endpoint["relative_url"], + http_methods=endpoint.get("http_methods"), + target=endpoint["target"], + ) + ) + + return routes + + def create_route(self, route: GatewayRoute) -> None: + """Create a route on the API Gateway.""" + + route.validate() + + resp = self.session.post(self.url, json=route.asdict()) + resp.raise_for_status() + + def delete_route(self, route: GatewayRoute) -> None: + """Delete a route on the API Gateway.""" + + resp = self.session.delete(self.url, json=route.asdict()) + resp.raise_for_status() diff --git a/scw_serverless/gateway/gateway_manager.py b/scw_serverless/gateway/gateway_manager.py new file mode 100644 index 0000000..ced4d1f --- /dev/null +++ b/scw_serverless/gateway/gateway_manager.py @@ -0,0 +1,73 @@ +import scaleway.function.v1beta1 as sdk +from scaleway import Client + +from scw_serverless.app import Serverless +from scw_serverless.gateway.gateway_api_client import GatewayAPIClient + + +class GatewayManager: + """Apply the configured routes to an existing API Gateway.""" + + def __init__( + self, + app_instance: Serverless, + gateway_url: str, + gateway_api_key: str, + sdk_client: Client, + ): + self.app_instance = app_instance + self.api = sdk.FunctionV1Beta1API(sdk_client) + self.gateway_client = GatewayAPIClient( + gateway_url=gateway_url, gateway_api_key=gateway_api_key + ) + + def _list_created_functions(self) -> dict[str, sdk.Function]: + """Get the list of created functions.""" + namespace_name = self.app_instance.service_name + namespaces = self.api.list_namespaces_all(name=namespace_name) + if not namespaces: + raise RuntimeError( + f"Could not find a namespace with name: {namespace_name}" + ) + if len(namespaces) > 1: + namespaces_ids = ", ".join([ns.id for ns in namespaces]) + raise RuntimeWarning( + f"Foud multiple namespaces with name {namespace_name}: {namespaces_ids}" + ) + + namespace_id = namespaces[0].id + return { + function.name: function + for function in self.api.list_functions_all(namespace_id=namespace_id) + } + + def update_routes(self) -> None: + """Update the Gateway routes configured by the functions.""" + created_functions = self._list_created_functions() + routed_functions = [ + function + for function in self.app_instance.functions + if function.gateway_route + ] + + # The Gateway deletes routes based on the relative_url, + # so we need to cleanup all routes at the start, + # otherwise we might accidentally delete a route we previously created. + # If it has the same relative_url but different http methods. + for function in routed_functions: + self.gateway_client.delete_route(function.gateway_route) # type: ignore + + for function in routed_functions: + if function.name not in created_functions: + raise RuntimeError( + f"Could not update route to function {function.name} " + + "because it was not deployed" + ) + + target = "https://" + created_functions[function.name].domain_name + function.gateway_route.target = target # type: ignore + + for function in routed_functions: + if not function.gateway_route: + continue + self.gateway_client.create_route(function.gateway_route) diff --git a/scw_serverless/version.py b/scw_serverless/version.py deleted file mode 100644 index 901e511..0000000 --- a/scw_serverless/version.py +++ /dev/null @@ -1 +0,0 @@ -VERSION = "0.0.1" diff --git a/tests/app_fixtures/routed_functions.py b/tests/app_fixtures/routed_functions.py new file mode 100644 index 0000000..eb59bd1 --- /dev/null +++ b/tests/app_fixtures/routed_functions.py @@ -0,0 +1,33 @@ +from typing import Any + +from scw_serverless.app import Serverless + +NAMESPACE_NAME = "integration-tests-gateway" +MESSAGES = { + "/health": "I'm fine!", + "/messages": "Could not find any message", +} + +app = Serverless(NAMESPACE_NAME) + + +@app.get(url="/health") +def health(_event: dict[str, Any], _context: dict[str, Any]): + return MESSAGES["/health"] + + +@app.get(url="/messages") +def get_messages(_event: dict[str, Any], _context: dict[str, Any]): + return MESSAGES["/messages"] + + +@app.post(url="/messages/new") +def post_message(event: dict[str, Any], _context: dict[str, Any]): + return {"statusCode": 200, "body": f'Message {event["body"]} successfully created!'} + + +@app.put(url="/messages/") +def put_message(event: dict[str, Any], _context: dict[str, Any]): + path: str = event["path"] + message = path.removeprefix("/messages/") + return {"statusCode": 200, "body": f"Message {message} successfully created!"} diff --git a/tests/constants.py b/tests/constants.py index 2f6caf5..ccb104b 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,11 +1,21 @@ import os from pathlib import Path -DEFAULT_REGION = "fr-par" +from scaleway_core.bridge.region import REGION_FR_PAR + +DEFAULT_REGION = REGION_FR_PAR SCALEWAY_API_URL = "https://api.scaleway.com/" +SCALEWAY_FNC_API_URL = SCALEWAY_API_URL + f"functions/v1beta1/regions/{DEFAULT_REGION}" COLD_START_TIMEOUT = 20 +GATEWAY_HOST = os.getenv("GATEWAY_HOST") +GATEWAY_S3_REGION = os.getenv("S3_REGION", str(DEFAULT_REGION)) +GATEWAY_S3_BUCKET_ENDPOINT = os.getenv( + "S3_ENDPOINT", f"https://s3.{DEFAULT_REGION}.scw.cloud" +) +GATEWAY_S3_BUCKET_NAME = os.getenv("GATEWAY_S3_BUCKET_NAME") + TESTS_DIR = os.path.realpath(os.path.dirname(__file__)) APP_FIXTURES_PATH = Path(TESTS_DIR, "app_fixtures") diff --git a/tests/integrations/deploy/test_api_backend.py b/tests/integrations/deploy/test_api_backend.py index da675ff..3b722d7 100644 --- a/tests/integrations/deploy/test_api_backend.py +++ b/tests/integrations/deploy/test_api_backend.py @@ -1,12 +1,11 @@ # pylint: disable=unused-import,redefined-outer-name # fixture - import scaleway.function.v1beta1 as sdk from tests import constants from tests.app_fixtures import app, app_updated, multiple_functions -from tests.integrations.deploy.deploy_wrapper import run_deploy_command +from tests.integrations.deploy_wrapper import run_deploy_command from tests.integrations.project_fixture import scaleway_project # noqa -from tests.integrations.utils import create_client, trigger_function +from tests.integrations.utils import create_client, trigger_function, wait_for_body_text def test_integration_deploy(scaleway_project: str): # noqa @@ -50,13 +49,8 @@ def test_integration_deploy_existing_function(scaleway_project: str): # noqa app_path=constants.APP_FIXTURES_PATH / "app_updated.py", ) - import time - - time.sleep(60) - # Check updated message content - resp = trigger_function(url) - assert resp.text == app_updated.MESSAGE + wait_for_body_text(url, app_updated.MESSAGE) # Check updated description function = api.get_function(function_id=function.id) diff --git a/tests/integrations/deploy/test_sf_backend.py b/tests/integrations/deploy/test_sf_backend.py index 3be8ebd..34da199 100644 --- a/tests/integrations/deploy/test_sf_backend.py +++ b/tests/integrations/deploy/test_sf_backend.py @@ -1,12 +1,11 @@ # pylint: disable=unused-import,redefined-outer-name # fixture - import scaleway.function.v1beta1 as sdk from tests import constants from tests.app_fixtures import app, app_updated -from tests.integrations.deploy.deploy_wrapper import run_deploy_command +from tests.integrations.deploy_wrapper import run_deploy_command from tests.integrations.project_fixture import scaleway_project # noqa -from tests.integrations.utils import create_client, trigger_function +from tests.integrations.utils import create_client, trigger_function, wait_for_body_text def test_integration_deploy_serverless_backend(scaleway_project: str): # noqa @@ -22,6 +21,8 @@ def test_integration_deploy_serverless_backend(scaleway_project: str): # noqa assert resp.text == app.MESSAGE +# I think duplication for assertions is fine and more flexible +# pylint: disable=duplicate-code def test_integration_deploy_existing_function_serverless_backend( scaleway_project: str, # noqa ): @@ -47,16 +48,11 @@ def test_integration_deploy_existing_function_serverless_backend( # Deploy twice in a row url, *_ = run_deploy_command( client, - app_path=constants.APP_FIXTURES_PATH.joinpath("app_updated.py"), + app_path=constants.APP_FIXTURES_PATH / "app_updated.py", ) - import time - - time.sleep(30) - # Check updated message content - resp = trigger_function(url) - assert resp.text == app_updated.MESSAGE + wait_for_body_text(url, app_updated.MESSAGE) # Check updated description function = api.get_function(function_id=function.id) diff --git a/tests/integrations/deploy/deploy_wrapper.py b/tests/integrations/deploy_wrapper.py similarity index 78% rename from tests/integrations/deploy/deploy_wrapper.py rename to tests/integrations/deploy_wrapper.py index 6840231..1fa392f 100644 --- a/tests/integrations/deploy/deploy_wrapper.py +++ b/tests/integrations/deploy_wrapper.py @@ -6,13 +6,16 @@ from scaleway import Client -from ..utils import run_cli, trigger_function +from tests.integrations.utils import run_cli, trigger_function FunctionUrl = str def run_deploy_command( - client: Client, app_path: Path, backend: Literal["serverless", "api"] = "api" + client: Client, + app_path: Path, + *args, + backend: Literal["serverless", "api"] = "api", ) -> list[FunctionUrl]: """Run deploy command with a specific backend.""" @@ -22,7 +25,9 @@ def run_deploy_command( with tempfile.TemporaryDirectory() as directory: shutil.copytree(src=app_dir, dst=directory, dirs_exist_ok=True) - ret = run_cli(client, directory, ["deploy", app_path.name, "-b", backend]) + cmd = ["deploy", app_path.name, "-b", backend] + cmd.extend(args) + ret = run_cli(client, directory, cmd) assert ret.returncode == 0, f"Non-null return code: {ret}" diff --git a/tests/integrations/gateway/test_gateway.py b/tests/integrations/gateway/test_gateway.py new file mode 100644 index 0000000..a5d2695 --- /dev/null +++ b/tests/integrations/gateway/test_gateway.py @@ -0,0 +1,60 @@ +# pylint: disable=unused-import,redefined-outer-name # fixture +import requests + +from tests import constants +from tests.app_fixtures.routed_functions import MESSAGES +from tests.integrations.deploy_wrapper import run_deploy_command +from tests.integrations.gateway_fixtures import auth_key # noqa +from tests.integrations.project_fixture import scaleway_project # noqa +from tests.integrations.utils import create_client + + +def test_integration_gateway(scaleway_project: str, auth_key: str): # noqa + client = create_client() + client.default_project_id = scaleway_project + + gateway_url = f"https://{constants.GATEWAY_HOST}" + + run_deploy_command( + client, + constants.APP_FIXTURES_PATH / "routed_functions.py", + "--gateway-url", + gateway_url, + "--gateway-api-key", + auth_key, + ) + + # Check general routing configuration + resp = requests.get( + url=gateway_url + "/health", timeout=constants.COLD_START_TIMEOUT + ) + assert resp.status_code == 200 + assert resp.text == MESSAGES["/health"] + + # Test with common prefix with configured routes + resp = requests.get( + url=gateway_url + "/messages", timeout=constants.COLD_START_TIMEOUT + ) + assert resp.status_code == 200 + assert resp.text == MESSAGES["/messages"] + + # Check a route with a method that is not configured + resp = requests.post( + url=gateway_url + "/messages", timeout=constants.COLD_START_TIMEOUT + ) + assert resp.status_code == 404 + + resp = requests.post( + url=gateway_url + "/messages/new", + timeout=constants.COLD_START_TIMEOUT, + data="welcome", + ) + assert resp.status_code == 200 + assert "welcome" in resp.text + + resp = requests.put( + url=gateway_url + "/messages/welcome", + timeout=constants.COLD_START_TIMEOUT, + ) + assert resp.status_code == 200 + assert "welcome" in resp.text diff --git a/tests/integrations/gateway_fixtures.py b/tests/integrations/gateway_fixtures.py new file mode 100644 index 0000000..b8adc55 --- /dev/null +++ b/tests/integrations/gateway_fixtures.py @@ -0,0 +1,36 @@ +import boto3 +import pytest +import requests + +from tests import constants + +from .utils import create_client + + +@pytest.fixture() +def auth_key() -> str: + assert constants.GATEWAY_HOST, "Gateway needs to be configured." + + client = create_client() + + response = requests.post( + "https://" + constants.GATEWAY_HOST + "/token", + timeout=constants.COLD_START_TIMEOUT, + ) + response.raise_for_status() + + s3 = boto3.resource( + "s3", + region_name=constants.GATEWAY_S3_BUCKET_NAME, + endpoint_url=constants.GATEWAY_S3_BUCKET_ENDPOINT, + aws_access_key_id=client.access_key, + aws_secret_access_key=client.secret_key, + ) + + objects = sorted( + s3.Bucket(constants.GATEWAY_S3_BUCKET_NAME).objects.all(), # type: ignore + key=lambda obj: obj.last_modified, + reverse=True, + ) + key = objects[0].key + return key diff --git a/tests/integrations/utils.py b/tests/integrations/utils.py index c4f442d..fc2c54b 100644 --- a/tests/integrations/utils.py +++ b/tests/integrations/utils.py @@ -1,6 +1,7 @@ import os import shutil import subprocess +import time import requests from requests.adapters import HTTPAdapter, Retry @@ -9,6 +10,7 @@ from tests import constants CLI_COMMAND = "scw-serverless" +RETRY_INTERVAL = 10 def create_client() -> Client: @@ -44,3 +46,16 @@ def trigger_function(url: str, max_retries: int = 5) -> requests.Response: req = session.get(url, timeout=constants.COLD_START_TIMEOUT) req.raise_for_status() return req + + +def wait_for_body_text(url: str, body: str, max_retries: int = 10) -> requests.Response: + last_body = None + for _ in range(max_retries): + resp = trigger_function(url) + if resp.text == body: + return resp + last_body = resp.text + time.sleep(RETRY_INTERVAL) + raise RuntimeError( + f"Max retries {max_retries} for url {url} to match body {body}, got: {last_body}" + ) diff --git a/tests/test_deploy/test_backends/test_scaleway_api_backend.py b/tests/test_deploy/test_backends/test_scaleway_api_backend.py index 1eddfee..a264d76 100644 --- a/tests/test_deploy/test_backends/test_scaleway_api_backend.py +++ b/tests/test_deploy/test_backends/test_scaleway_api_backend.py @@ -6,7 +6,6 @@ import scaleway.function.v1beta1 as sdk from responses import matchers from scaleway import Client -from scaleway_core.bridge.region import REGION_FR_PAR from scw_serverless.app import Serverless from scw_serverless.config import Function @@ -14,8 +13,6 @@ from scw_serverless.triggers import CronTrigger from tests import constants -FNC_API_URL = constants.SCALEWAY_API_URL + f"functions/v1beta1/regions/{REGION_FR_PAR}" - # pylint: disable=redefined-outer-name # fixture @pytest.fixture @@ -39,7 +36,7 @@ def get_test_backend() -> ScalewayApiBackend: access_key="SCWXXXXXXXXXXXXXXXXX", # The uuid is validated secret_key="498cce73-2a07-4e8c-b8ef-8f988e3c6929", # nosec # fake data - default_region=REGION_FR_PAR, + default_region=constants.DEFAULT_REGION, ) backend = ScalewayApiBackend(app, client, True) @@ -63,7 +60,7 @@ def test_scaleway_api_backend_deploy_function(mocked_responses: responses.Reques # Looking for existing namespace mocked_responses.get( - FNC_API_URL + "/namespaces", + constants.SCALEWAY_FNC_API_URL + "/namespaces", json={"namespaces": []}, ) namespace = { @@ -72,15 +69,17 @@ def test_scaleway_api_backend_deploy_function(mocked_responses: responses.Reques "secret_environment_variables": [], # Otherwise breaks the marshalling } # Creating namespace - mocked_responses.post(FNC_API_URL + "/namespaces", json=namespace) + mocked_responses.post( + constants.SCALEWAY_FNC_API_URL + "/namespaces", json=namespace + ) # Polling its status mocked_responses.get( - f'{FNC_API_URL}/namespaces/{namespace["id"]}', + f'{ constants.SCALEWAY_FNC_API_URL}/namespaces/{namespace["id"]}', json=namespace | {"status": sdk.NamespaceStatus.READY}, ) # Looking for existing function mocked_responses.get( - FNC_API_URL + "/functions", + constants.SCALEWAY_FNC_API_URL + "/functions", match=[ matchers.query_param_matcher({"namespace_id": namespace["id"], "page": 1}) ], @@ -93,7 +92,7 @@ def test_scaleway_api_backend_deploy_function(mocked_responses: responses.Reques "secret_environment_variables": [], } mocked_responses.post( - FNC_API_URL + "/functions", + constants.SCALEWAY_FNC_API_URL + "/functions", match=[ matchers.json_params_matcher( { @@ -109,7 +108,7 @@ def test_scaleway_api_backend_deploy_function(mocked_responses: responses.Reques ], json=mocked_fn, ) - test_fn_api_url = f'{FNC_API_URL}/functions/{mocked_fn["id"]}' + test_fn_api_url = f'{constants.SCALEWAY_FNC_API_URL}/functions/{mocked_fn["id"]}' mocked_responses.get( test_fn_api_url + "/upload-url", json={"url": "https://url"}, @@ -146,7 +145,7 @@ def test_scaleway_api_backend_deploy_function_with_trigger( # Looking for existing namespace mocked_responses.get( - FNC_API_URL + "/namespaces", + constants.SCALEWAY_FNC_API_URL + "/namespaces", json={"namespaces": []}, ) namespace = { @@ -155,15 +154,17 @@ def test_scaleway_api_backend_deploy_function_with_trigger( "secret_environment_variables": [], # Otherwise breaks the marshalling } # Creating namespace - mocked_responses.post(FNC_API_URL + "/namespaces", json=namespace) + mocked_responses.post( + constants.SCALEWAY_FNC_API_URL + "/namespaces", json=namespace + ) # Polling its status mocked_responses.get( - f'{FNC_API_URL}/namespaces/{namespace["id"]}', + f'{ constants.SCALEWAY_FNC_API_URL}/namespaces/{namespace["id"]}', json=namespace | {"status": sdk.NamespaceStatus.READY}, ) # Looking for existing function mocked_responses.get( - FNC_API_URL + "/functions", + constants.SCALEWAY_FNC_API_URL + "/functions", match=[ matchers.query_param_matcher({"namespace_id": namespace["id"], "page": 1}) ], @@ -176,7 +177,7 @@ def test_scaleway_api_backend_deploy_function_with_trigger( "secret_environment_variables": [], } mocked_responses.post( - FNC_API_URL + "/functions", + constants.SCALEWAY_FNC_API_URL + "/functions", match=[ matchers.json_params_matcher( { @@ -192,7 +193,7 @@ def test_scaleway_api_backend_deploy_function_with_trigger( ], json=mocked_fn, ) - test_fn_api_url = f'{FNC_API_URL}/functions/{mocked_fn["id"]}' + test_fn_api_url = f'{ constants.SCALEWAY_FNC_API_URL}/functions/{mocked_fn["id"]}' mocked_responses.get( test_fn_api_url + "/upload-url", json={"url": "https://url"}, @@ -208,7 +209,7 @@ def test_scaleway_api_backend_deploy_function_with_trigger( ) # Looking for existing cron mocked_responses.get( - FNC_API_URL + "/crons", + constants.SCALEWAY_FNC_API_URL + "/crons", match=[ matchers.query_param_matcher({"function_id": mocked_fn["id"], "page": 1}) ], @@ -216,7 +217,7 @@ def test_scaleway_api_backend_deploy_function_with_trigger( ) cron = {"id": "cron-id"} mocked_responses.post( - FNC_API_URL + "/crons", + constants.SCALEWAY_FNC_API_URL + "/crons", match=[ matchers.json_params_matcher( { @@ -231,7 +232,7 @@ def test_scaleway_api_backend_deploy_function_with_trigger( ) # Poll the status mocked_responses.get( - f'{FNC_API_URL}/crons/{cron["id"]}', + f'{ constants.SCALEWAY_FNC_API_URL}/crons/{cron["id"]}', json=mocked_fn | {"status": sdk.CronStatus.READY}, ) backend.deploy() diff --git a/tests/test_gateway/test_gateway_manager.py b/tests/test_gateway/test_gateway_manager.py new file mode 100644 index 0000000..1334372 --- /dev/null +++ b/tests/test_gateway/test_gateway_manager.py @@ -0,0 +1,110 @@ +import pytest +import responses +import scaleway.function.v1beta1 as sdk +from responses.matchers import header_matcher, json_params_matcher, query_param_matcher +from scaleway import Client + +from scw_serverless.app import Serverless +from scw_serverless.config import Function +from scw_serverless.config.route import GatewayRoute, HTTPMethod +from scw_serverless.gateway.gateway_manager import GatewayManager +from tests import constants + +HELLO_WORLD_MOCK_DOMAIN = ( + "helloworldfunctionnawns8i8vo-hello-world.functions.fnc.fr-par.scw.cloud" +) +MOCK_GATEWAY_URL = "https://my-gateway-domain.com" +MOCK_GATEWAY_API_KEY = "7tfxBRB^vJbBcR5s#*RE" +MOCK_UUID = "xxxxxxxx-xxxx-Mxxx-Nxxx-xxxxxxxxxxxx" +PROJECT_ID = "projecti-xxxx-Mxxx-Nxxx-xxxxxxxxxxxx" + + +# pylint: disable=redefined-outer-name # fixture +@pytest.fixture +def mocked_responses(): + with responses.RequestsMock() as rsps: + yield rsps + + +@pytest.fixture() +def app_gateway_manager() -> GatewayManager: + app = Serverless("test-namespace") + client = Client( + access_key="SCWXXXXXXXXXXXXXXXXX", + # The uuid is validated + secret_key="498cce73-2a07-4e8c-b8ef-8f988e3c6929", # nosec # false positive + default_region=constants.DEFAULT_REGION, + ) + return GatewayManager(app, MOCK_GATEWAY_URL, MOCK_GATEWAY_API_KEY, client) + + +def test_gateway_manager_update_routes( + app_gateway_manager: GatewayManager, mocked_responses: responses.RequestsMock +): + function = Function( + name="test-function", + handler="handler", + runtime=sdk.FunctionRuntime.PYTHON311, + gateway_route=GatewayRoute( + relative_url="/hello", http_methods=[HTTPMethod.GET] + ), + ) + app_gateway_manager.app_instance.functions = [function] + + namespace = { + "id": "namespace-id", + "name": app_gateway_manager.app_instance.service_name, + "secret_environment_variables": [], # Otherwise breaks the marshalling + } + # Looking for existing namespace + mocked_responses.get( + constants.SCALEWAY_FNC_API_URL + "/namespaces", + json={"namespaces": [namespace]}, + ) + # We have to provide a stop gap otherwise list_namepaces_all() will keep + # making API calls. + mocked_responses.get( + constants.SCALEWAY_FNC_API_URL + "/namespaces", + json={"namespaces": []}, + ) + + mocked_responses.get( + constants.SCALEWAY_FNC_API_URL + "/functions", + match=[query_param_matcher({"namespace_id": namespace["id"], "page": 1})], + json={ + "functions": [ + { + "name": function.name, + "domain_name": HELLO_WORLD_MOCK_DOMAIN, + "secret_environment_variables": [], + } + ] + }, + ) + mocked_responses.get( + constants.SCALEWAY_FNC_API_URL + "/functions", + match=[query_param_matcher({"namespace_id": namespace["id"], "page": 2})], + json={"functions": []}, + ) + + # We should attempt to delete the route + mocked_responses.delete( + MOCK_GATEWAY_URL + "/scw", # type: ignore + match=[ + header_matcher({"X-Auth-Token": MOCK_GATEWAY_API_KEY}), + json_params_matcher(params=function.gateway_route.asdict()), # type: ignore + ], + ) + # We should attempt to create the route + mocked_responses.post( + MOCK_GATEWAY_URL + "/scw", # type: ignore + match=[ + header_matcher({"X-Auth-Token": MOCK_GATEWAY_API_KEY}), + json_params_matcher( + params=function.gateway_route.asdict() # type: ignore + | {"target": "https://" + HELLO_WORLD_MOCK_DOMAIN} + ), + ], + ) + + app_gateway_manager.update_routes()