fix: can import multiple modules (#43)

* fix: can import multiple modules

* feat(examples): multiple modules example

* chore: forgot debug stacktrace

Author: cyclimse

docs/source/examples.rst

@@ -9,6 +9,9 @@ To showcase the framework and how it can used to deploy serverless applications,
 :doc:`../examples/cron`
     Function that gets triggered periodically.
 
+:doc:`../examples/multiple_modules`
+    An application with multiple handlers organized in different modules.
+
 :doc:`../examples/github_actions`
     A GitHub action workflow to deploy your application with `scw_serverless`.
 
@@ -23,5 +26,6 @@ To showcase the framework and how it can used to deploy serverless applications,
 
     ../examples/hello_world
     ../examples/cron
+    ../examples/multiple_modules
     ../examples/github_actions
     ../examples/pr_notifier

docs/source/examples/multiple_modules.md

@@ -0,0 +1,13 @@
+```{include} ../../../examples/multiple_modules/README.md
+```
+
+## [Sources](https://github.com/scaleway/serverless-api-project/blob/main/examples/multiple_modules/app.py)
+
+```{literalinclude} ../../../examples/multiple_modules/app.py
+```
+
+```{literalinclude} ../../../examples/multiple_modules/upload.py
+```
+
+```{literalinclude} ../../../examples/multiple_modules/query.py
+```

docs/source/examples/pr_notifier.md

@@ -1,7 +1,7 @@
 ```{include} ../../../examples/pr_notifier/README.md
 ```
 
-## [Source](https://github.com/scaleway/serverless-api-project/blob/main/examples/github_actions/deploy.yml)
+## [Source](https://github.com/scaleway/serverless-api-project/blob/main/examples/pr_notifier/notifier.py)
 
 ```{literalinclude} ../../../examples/pr_notifier/notifier.py
 ```

examples/multiple_functions/handler.py

@@ -0,0 +1,31 @@
+# [Multiple Modules](https://github.com/scaleway/serverless-api-project/tree/main/examples/multiple_modules)
+
+An app to upload and query files to S3 Glacier on Scaleway split in multiple modules.
+
+The upload endpoint allows you to upload files to Glacier via the `file` form-data key:
+
+```console
+echo -e "Hello world!\n My contents will be stored in a bunker!" > myfile.dat
+curl -F file=@myfile.dat <upload-function-url>
+```
+
+This example is there to showcase how to split handlers into different Python modules.
+
+## Deploying
+
+Deployment can be done with `scw_serverless`:
+
+```console
+pip install scw_serverless
+scw-serverless deploy app.py
+```
+
+## Configuration
+
+Here's all the environments variables that needs to be passed when deploying:
+
+|     Variable     |               Description               |      Required      |
+|:----------------:|:---------------------------------------:|:------------------:|
+| `SCW_SECRET_KEY` |   Secret key to use for S3 operations   | :heavy_check_mark: |
+| `SCW_ACCESS_KEY` |   Access key to use for S3 operations   | :heavy_check_mark: |
+|   `S3_BUCKET`    | Name of the bucket to store files into. | :heavy_check_mark: |

examples/multiple_modules/README.md

@@ -0,0 +1,18 @@
+import logging
+import os
+
+from scw_serverless.app import Serverless
+
+logging.basicConfig(level=logging.INFO)
+
+app = Serverless(
+    "multiple-modules",
+    secret={
+        "SCW_ACCESS_KEY": os.environ["SCW_ACCESS_KEY"],
+        "SCW_SECRET_KEY": os.environ["SCW_SECRET_KEY"],
+    },
+    env={"S3_BUCKET": os.environ["S3_BUCKET"]},
+)
+
+import query  # noqa
+import upload  # pylint: disable=all # noqa

examples/multiple_modules/app.py

@@ -0,0 +1,36 @@
+import json
+import os
+from typing import Any
+
+from app import app
+from s3 import bucket
+
+
+@app.func(
+    description="List objects in S3 uploads.",
+    privacy="public",
+    env={"LIMIT": "100"},
+    min_scale=0,
+    max_scale=2,
+    memory_limit=128,
+    timeout="300s",
+)
+def query(_event: dict[str, Any], _context: dict[str, Any]) -> dict[str, Any]:
+    """A handler to list objects in a S3 bucket."""
+
+    response = []
+
+    for obj in bucket.objects.limit(count=int(os.environ["LIMIT"])):
+        response.append(
+            {
+                "name": obj.key,
+                "last_modified": obj.last_modified.strftime("%m/%d/%Y, %H:%M:%S"),
+                "storage_class": obj.storage_class,
+            }
+        )
+
+    return {
+        "statusCode": 200,
+        "headers": {"Content-Type": "application/json"},
+        "body": json.dumps(response),
+    }

examples/multiple_modules/query.py

@@ -0,0 +1,3 @@
+boto3~=1.26
+scw-serverless~=0.0.3
+streaming_form_data~=1.11.0

examples/multiple_modules/requirements.txt

@@ -0,0 +1,14 @@
+import os
+
+import boto3
+
+s3 = boto3.resource(
+    "s3",
+    region_name="fr-par",
+    use_ssl=True,
+    endpoint_url="https://s3.fr-par.scw.cloud",
+    aws_access_key_id=os.environ["SCW_ACCESS_KEY"],
+    aws_secret_access_key=os.environ["SCW_SECRET_KEY"],
+)
+
+bucket = s3.Bucket(os.environ["S3_BUCKET"])

examples/multiple_modules/s3.py

@@ -0,0 +1,31 @@
+import logging
+from typing import Any
+
+from app import app
+from s3 import bucket
+from streaming_form_data import StreamingFormDataParser
+from streaming_form_data.targets import ValueTarget
+
+
+@app.func()
+def upload(event: dict[str, Any], _context: dict[str, Any]) -> dict[str, Any]:
+    """Upload form data to S3 Glacier."""
+
+    headers = event["headers"]
+    parser = StreamingFormDataParser(headers=headers)
+
+    target = ValueTarget()
+    parser.register("file", target)
+
+    body: str = event["body"]
+    parser.data_received(body.encode("utf-8"))
+
+    if not (len(target.value) > 0 and target.multipart_filename):
+        return {"statusCode": 400}
+
+    name = target.multipart_filename
+
+    logging.info("Uploading file %s to Glacier on %s", name, bucket.name)
+    bucket.put_object(Key=name, Body=target.value, StorageClass="GLACIER")
+
+    return {"statusCode": 200}

examples/multiple_modules/upload.py

@@ -40,16 +40,15 @@ Once you have deployed the functions, they can be setup as webhooks on your repo
 
 Here's all the environments variables that needs to be passed when deploying:
 
-| Variable | Description  | Required |
-| :---:   | :---: | :---: |
-| `SCW_SECRET_KEY` | Secret key to use for S3 operations |  :heavy_check_mark: |
-| `SCW_ACCESS_KEY` | Access key to use for S3 operations |  :heavy_check_mark: |
-| `SCW_SECRET_KEY` | Secret key to use for S3 operations |  :heavy_check_mark: |
-| `S3_BUCKET` | Name of the bucket to store opened PRs into. |  :heavy_check_mark: |
-| `SLACK_TOKEN` | Slack token. See below for details on scope. |  :heavy_check_mark: |
-| `SLACK_CHANNEL` | Channel ID of the Slack channel to send messages to |  :heavy_check_mark: |
-| `GITLAB_EMAIL_DOMAIN` | Will be appended to GitLab usernames to create a valid email. Emails are converted to Slack IDs to ping developers in the reminder |   |
-| `REMINDER_SCHEDULE` | CRON schedule to trigger the reminder |   |
+|       Variable        |                                                            Description                                                             |      Required      |
+|:---------------------:|:----------------------------------------------------------------------------------------------------------------------------------:|:------------------:|
+|   `SCW_SECRET_KEY`    |                                                Secret key to use for S3 operations                                                 | :heavy_check_mark: |
+|   `SCW_ACCESS_KEY`    |                                                Access key to use for S3 operations                                                 | :heavy_check_mark: |
+|      `S3_BUCKET`      |                                            Name of the bucket to store opened PRs into.                                            | :heavy_check_mark: |
+|     `SLACK_TOKEN`     |                                            Slack token. See below for details on scope.                                            | :heavy_check_mark: |
+|    `SLACK_CHANNEL`    |                                        Channel ID of the Slack channel to send messages to                                         | :heavy_check_mark: |
+| `GITLAB_EMAIL_DOMAIN` | Will be appended to GitLab usernames to create a valid email. Emails are converted to Slack IDs to ping developers in the reminder |                    |
+|  `REMINDER_SCHEDULE`  |                                               CRON schedule to trigger the reminder                                                |                    |
 
 ### Creating the Slack application
 

examples/pr_notifier/README.md

@@ -1,12 +1,9 @@
-import importlib.util
-import inspect
 import os
 from pathlib import Path
 from typing import Literal, Optional
 
 import click
 
-from scw_serverless.app import Serverless
 from scw_serverless.config.generators.serverless_framework import (
     ServerlessFrameworkGenerator,
 )
@@ -16,6 +13,7 @@
 from scw_serverless.logger import DEFAULT, get_logger
 from scw_serverless.utils.config import Config
 from scw_serverless.utils.credentials import DEFAULT_REGION, get_scw_client
+from scw_serverless.utils.loader import get_app_instance
 
 CLICK_ARG_FILE = click.argument(
     "file",
@@ -211,35 +209,6 @@ def generate(file: Path, target: str, save: str) -> None:
     get_logger().success(f"Done! Generated configuration file saved in {save}")
 
 
-def get_app_instance(file: Path) -> Serverless:
-    """Load the app instance from the client module."""
-
-    file = file.resolve()
-    module_name = (
-        str(file.relative_to(Path(".").resolve())).removesuffix(".py").replace("/", ".")
-    )
-
-    # Importing user's app
-    spec = importlib.util.spec_from_file_location(module_name, file)
-    user_app = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(user_app)
-
-    app_instance = None
-
-    for member in inspect.getmembers(user_app):
-        if isinstance(member[1], Serverless):
-            # Ok. Found the variable now, need to use it
-            app_instance = member[1]
-
-    if not app_instance:  # No variable with type "Serverless" found
-        raise RuntimeError(
-            f"""Unable to locate an instance of serverless App
-            in the provided file: {file}."""
-        )
-
-    return app_instance
-
-
 def main() -> int:
     """Entrypoint for click"""
     # Set logging level to DEFAULT. (ignore debug)

scw_serverless/cli.py

@@ -0,0 +1,52 @@
+import importlib.util
+import inspect
+import sys
+from pathlib import Path
+
+from scw_serverless.app import Serverless
+
+
+def get_module_name(file: Path) -> str:
+    """Extract the module name from the path."""
+
+    file = file.resolve()
+    return (
+        str(file.relative_to(Path(".").resolve())).removesuffix(".py").replace("/", ".")
+    )
+
+
+def get_app_instance(file: Path) -> Serverless:
+    """Load the app instance from the client module."""
+
+    module_name = get_module_name(file)
+    parent_directory = str(file.parent.resolve())
+
+    spec = importlib.util.spec_from_file_location(
+        module_name,
+        str(file.resolve()),
+        submodule_search_locations=[parent_directory],
+    )
+
+    if not spec or not spec.loader:
+        raise ImportError(
+            f"Can't find module {module_name} at location {file.resolve()}"
+        )
+
+    user_app = importlib.util.module_from_spec(spec)
+    sys.path.append(parent_directory)
+    sys.modules[module_name] = user_app
+    spec.loader.exec_module(user_app)
+
+    app_instance = None
+    for member in inspect.getmembers(user_app):
+        if isinstance(member[1], Serverless):
+            # Ok. Found the variable now, need to use it
+            app_instance = member[1]
+
+    if not app_instance:  # No variable with type "Serverless" found
+        raise RuntimeError(
+            f"""Unable to locate an instance of serverless App
+            in the provided file: {file}."""
+        )
+
+    return app_instance