chore: use different workflow for gateway ci

Author: cyclimse

.github/workflows/integration-gateway.yml

@@ -1,29 +1,89 @@
----
-name: API Gateway integration tests with Pytest
+name: test with deployed gateway
 
 on:
   push:
     branches: [main]
   pull_request:
     branches: [main]
+    types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:
   contents: read
 
+env:
+  GATEWAY_CHECKOUT_DIR: "gateway"
+  S3_ENDPOINT: "https://s3.fr-par.scw.cloud"
+  S3_REGION: "fr-par"
+
 jobs:
-  integration-gateway:
-    runs-on: self-hosted
-    container: nikolaik/python-nodejs:python3.10-nodejs18
+  test-deployed-gateway:
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
       - uses: ./.github/actions/setup-poetry
 
-      - name: Test with pytest
-        working-directory: tests
-        run: poetry run pytest integrations/gateway -n $(nproc --all)
+      - uses: actions/checkout@v3
+        with:
+          repository: scaleway/serverless-gateway
+          path: $GATEWAY_CHECKOUT_DIR
+
+      - name: Install Scaleway CLI
+        uses: ./$GATEWAY_CHECKOUT_DIR/.github/actions/setup-scaleway-cli
+        with:
+          scw-version: "2.13.0"
+          scw-access-key: ${{ secrets.SCW_ACCESS_KEY }}
+          scw-secret-key: ${{ secrets.SCW_SECRET_KEY }}
+          scw-default-project-id: ${{ secrets.SCW_DEFAULT_PROJECT_ID }}
+          scw-default-organization-id: ${{ secrets.SCW_DEFAULT_ORGANIZATION_ID }}
+
+      - name: Create Gateway namespace
+        working-directory: $GATEWAY_CHECKOUT_DIR
+        run: |
+          make create-namespace
+          until [ $(make check-namespace -s) == ready ]; do sleep 10; done
+
+      - name: Create Gateway container
+        working-directory: $GATEWAY_CHECKOUT_DIR
+        # We need to truncate gateway.env as it will override our env vars
+        run: |
+          truncate -s 0 gateway.env
+          make create-container
+          make deploy-container
+          until [ $(make check-container -s) == ready ]; do sleep 10; done
         env:
-          SCW_DEFAULT_ORGANIZATION_ID: ${{ secrets.SCW_DEFAULT_ORGANIZATION_ID }}
+          SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
           SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
+          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
+
+      - name: Install s3cmd
+        run: pip install s3cmd
+
+      - name: Create S3 bucket
+        working-directory: $GATEWAY_CHECKOUT_DIR
+        run: |
+          make set-up-s3-cli
+          make create-s3-bucket
+        env:
+          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
+
+      - name: Run integration tests
+        run: |
+          pushd $GATEWAY_CHECKOUT_DIR
+          export GATEWAY_HOST=$(make get-gateway-host -s)
+          popd
+          poetry run pytest integrations/gateway -n $(nproc --all)
+        env:
           SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
-          API_GATEWAY_S3_BUCKET: ${{ secrets.API_GATEWAY_S3_BUCKET }}
+          SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
+          GATEWAY_S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
+
+      - name: Delete S3 bucket
+        run: make delete-bucket
+        env:
+          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
+        if: always()
+
+      - name: Delete Gateway namespace and container
+        run: make delete-namespace
+        if: always()

.github/workflows/pytest-integration.yml

@@ -5,7 +5,7 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
+    types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:
   contents: read

.github/workflows/pytest.yml

@@ -6,6 +6,7 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+    types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:
   contents: read

tests/constants.py

@@ -9,14 +9,10 @@
 
 COLD_START_TIMEOUT = 20
 
-API_GATEWAY_IMAGE_TAG = "0.0.5"
-API_GATEWAY_IMAGE = (
-    f"registry.hub.docker.com/shillakerscw/scw-sls-gw:{API_GATEWAY_IMAGE_TAG}"
-)
-API_GATEWAY_MEMORY_LIMIT = 2048
-API_GATEWAY_S3_REGION = REGION_FR_PAR
-API_GATEWAY_S3_BUCKET_ENDPOINT = f"https://s3.{REGION_FR_PAR}.scw.cloud"
-API_GATEWAY_S3_BUCKET = os.getenv("API_GATEWAY_S3_BUCKET")
+GATEWAY_HOST = os.getenv("GATEWAY_HOST")
+GATEWAY_S3_REGION = REGION_FR_PAR
+GATEWAY_S3_BUCKET_ENDPOINT = f"https://s3.{REGION_FR_PAR}.scw.cloud"
+GATEWAY_S3_BUCKET_NAME = os.getenv("GATEWAY_S3_BUCKET")
 
 TESTS_DIR = os.path.realpath(os.path.dirname(__file__))
 

tests/integrations/gateway/test_gateway.py

@@ -1,22 +1,19 @@
 # pylint: disable=unused-import,redefined-outer-name # fixture
 import requests
-from scaleway.container.v1beta1 import Container
 
 from tests import constants
 from tests.app_fixtures.routed_functions import MESSAGES
 from tests.integrations.deploy_wrapper import run_deploy_command
-from tests.integrations.gateway_fixtures import api_gateway, auth_key  # noqa
+from tests.integrations.gateway_fixtures import auth_key  # noqa
 from tests.integrations.project_fixture import scaleway_project  # noqa
 from tests.integrations.utils import create_client
 
 
-def test_integration_gateway(
-    scaleway_project: str, api_gateway: Container, auth_key: str  # noqa
-):
+def test_integration_gateway(scaleway_project: str, auth_key: str):  # noqa
     client = create_client()
     client.default_project_id = scaleway_project
 
-    gateway_url = f"https://{api_gateway.domain_name}"
+    gateway_url = f"https://{constants.GATEWAY_HOST}"
 
     run_deploy_command(
         client,

tests/integrations/gateway_fixtures.py

@@ -1,93 +1,36 @@
-import typing as t
-
 import boto3
 import pytest
 import requests
-import scaleway.container.v1beta1 as sdk
-from scaleway import Client
 
 from tests import constants
 
 from .utils import create_client
 
 
 @pytest.fixture()
-def api_gateway(scaleway_project: str) -> t.Iterator[sdk.Container]:
-    client = create_client()
-    container = None
-    try:
-        container = _deploy_gateway(scaleway_project, client)
-        yield container
-    finally:
-        if container:
-            _cleanup_gateway(client, container)
-
+def auth_key() -> str:
+    assert constants.GATEWAY_HOST, "Gateway needs to be configured."
 
-@pytest.fixture()
-def auth_key(
-    api_gateway: sdk.Container,  # pylint: disable=redefined-outer-name
-) -> str:
     client = create_client()
 
     response = requests.post(
-        "https://" + api_gateway.domain_name + "/token",
+        "https://" + constants.GATEWAY_HOST + "/token",
         timeout=constants.COLD_START_TIMEOUT,
     )
     response.raise_for_status()
 
     s3 = boto3.resource(
         "s3",
-        region_name=constants.API_GATEWAY_S3_BUCKET,
-        endpoint_url=constants.API_GATEWAY_S3_BUCKET_ENDPOINT,
+        region_name=constants.GATEWAY_S3_BUCKET_NAME,
+        endpoint_url=constants.GATEWAY_S3_BUCKET_ENDPOINT,
         aws_access_key_id=client.access_key,
         aws_secret_access_key=client.secret_key,
     )
 
     objects = sorted(
-        s3.Bucket(constants.API_GATEWAY_S3_BUCKET).objects.all(),  # type: ignore
+        s3.Bucket(constants.GATEWAY_S3_BUCKET_NAME).objects.all(),  # type: ignore
         key=lambda obj: obj.last_modified,
         reverse=True,
     )
     key = objects[0].key
     return key
-
-
-def _deploy_gateway(project_id: str, client: Client) -> sdk.Container:
-    assert (
-        constants.API_GATEWAY_S3_BUCKET
-    ), "S3 bucket needs to be configured to deploy Gateway"
-
-    api = sdk.ContainerV1Beta1API(client)
-    namespace = api.create_namespace(name="gateway", project_id=project_id)
-    api.wait_for_namespace(namespace_id=namespace.id)
-
-    container = api.create_container(
-        namespace_id=namespace.id,
-        min_scale=1,
-        max_scale=1,
-        protocol=sdk.ContainerProtocol.HTTP1,
-        registry_image=constants.API_GATEWAY_IMAGE,
-        privacy=sdk.ContainerPrivacy.PUBLIC,
-        http_option=sdk.ContainerHttpOption.REDIRECTED,
-        memory_limit=constants.API_GATEWAY_MEMORY_LIMIT,
-        secret_environment_variables=[
-            sdk.Secret("SCW_ACCESS_KEY", client.access_key),
-            sdk.Secret("SCW_SECRET_KEY", client.secret_key),
-        ],
-        environment_variables={
-            "S3_REGION": constants.API_GATEWAY_S3_REGION,
-            "S3_ENDPOINT": constants.API_GATEWAY_S3_BUCKET_ENDPOINT,
-            "S3_BUCKET_NAME": constants.API_GATEWAY_S3_BUCKET,
-        },
-    )
-    api.deploy_container(container_id=container.id)
-    container = api.wait_for_container(container_id=container.id)
-
-    return container
-
-
-def _cleanup_gateway(client: Client, container: sdk.Container):
-    """Delete all Scaleway resources created in the temporary project."""
-    api = sdk.ContainerV1Beta1API(client)
-    api.delete_container(container_id=container.id)
-    api.delete_namespace(namespace_id=container.namespace_id)