From ee24a47df65faf0993673917a9d4d6cf66d7779a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andy=20M=C3=A9ry?= Date: Thu, 23 Feb 2023 15:38:19 +0100 Subject: [PATCH 1/3] fix: can import multiple modules --- examples/multiple_functions/app.py | 6 +++ examples/multiple_functions/first.py | 12 ++++++ examples/multiple_functions/handler.py | 44 ---------------------- examples/multiple_functions/second.py | 22 +++++++++++ scw_serverless/cli.py | 45 ++++------------------ scw_serverless/utils/loader.py | 52 ++++++++++++++++++++++++++ 6 files changed, 99 insertions(+), 82 deletions(-) create mode 100644 examples/multiple_functions/app.py create mode 100644 examples/multiple_functions/first.py delete mode 100644 examples/multiple_functions/handler.py create mode 100644 examples/multiple_functions/second.py create mode 100644 scw_serverless/utils/loader.py diff --git a/examples/multiple_functions/app.py b/examples/multiple_functions/app.py new file mode 100644 index 0000000..d0021d5 --- /dev/null +++ b/examples/multiple_functions/app.py @@ -0,0 +1,6 @@ +from scw_serverless.app import Serverless + +app = Serverless("multiple-functions") + +import first # pylint: disable=all # noqa +import second # noqa diff --git a/examples/multiple_functions/first.py b/examples/multiple_functions/first.py new file mode 100644 index 0000000..1ac4ebd --- /dev/null +++ b/examples/multiple_functions/first.py @@ -0,0 +1,12 @@ +from typing import Any + +from app import app + + +@app.func() +def handle(_event: dict[str, Any], _context: dict[str, Any]) -> dict[str, Any]: + """A simple function that greets people.""" + return { + "message": "This function is handled by Scaleway" + + "functions using Serverless API Framework" + } diff --git a/examples/multiple_functions/handler.py b/examples/multiple_functions/handler.py deleted file mode 100644 index 039e882..0000000 --- a/examples/multiple_functions/handler.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -from typing import Any - -from scw_serverless.app import Serverless - -app = Serverless("multiple-functions") - - -@app.func() -def handle(_event: dict[str, Any], _context: dict[str, Any]) -> dict[str, Any]: - """handle a request to the function - Args: - event (dict): request params - context (dict): function call metadata - """ - - return { - "message": "This function is handled by Scaleway" - + "functions using Serverless API Framework" - } - - -@app.func( - description="Say hi", - privacy="public", - env={"CUSTOM_NAME": "everyone"}, - secret={"SECRET_VALUE": "***"}, - min_scale=0, - max_scale=2, - memory_limit=128, - timeout="300s", - custom_domains=["hello.functions.scaleway"], -) -def hello(_event: dict[str, Any], _context: dict[str, Any]) -> dict[str, Any]: - """handle a request to the function - Args: - event (dict): request params - context (dict): function call metadata - """ - - return { - "message": f"Hello {os.getenv('CUSTOM_NAME')}" - + "from Scaleway functions using Serverless API Framework" - } diff --git a/examples/multiple_functions/second.py b/examples/multiple_functions/second.py new file mode 100644 index 0000000..2644436 --- /dev/null +++ b/examples/multiple_functions/second.py @@ -0,0 +1,22 @@ +import os +from typing import Any + +from app import app + + +@app.func( + description="Happy Coding!", + privacy="public", + env={"CUSTOM_NAME": "everyone"}, + secret={"SECRET_VALUE": "***"}, + min_scale=0, + max_scale=2, + memory_limit=128, + timeout="300s", +) +def hello(_event: dict[str, Any], _context: dict[str, Any]) -> dict[str, Any]: + """A simple function that greets people.""" + return { + "message": f"Hello {os.getenv('CUSTOM_NAME')}" + + "from Scaleway functions using Serverless API Framework" + } diff --git a/scw_serverless/cli.py b/scw_serverless/cli.py index 2600649..88d043e 100644 --- a/scw_serverless/cli.py +++ b/scw_serverless/cli.py @@ -1,12 +1,9 @@ -import importlib.util -import inspect import os from pathlib import Path from typing import Literal, Optional import click -from scw_serverless.app import Serverless from scw_serverless.config.generators.serverless_framework import ( ServerlessFrameworkGenerator, ) @@ -16,6 +13,7 @@ from scw_serverless.logger import DEFAULT, get_logger from scw_serverless.utils.config import Config from scw_serverless.utils.credentials import DEFAULT_REGION, get_scw_client +from scw_serverless.utils.loader import get_app_instance CLICK_ARG_FILE = click.argument( "file", @@ -211,42 +209,13 @@ def generate(file: Path, target: str, save: str) -> None: get_logger().success(f"Done! Generated configuration file saved in {save}") -def get_app_instance(file: Path) -> Serverless: - """Load the app instance from the client module.""" - - file = file.resolve() - module_name = ( - str(file.relative_to(Path(".").resolve())).removesuffix(".py").replace("/", ".") - ) - - # Importing user's app - spec = importlib.util.spec_from_file_location(module_name, file) - user_app = importlib.util.module_from_spec(spec) - spec.loader.exec_module(user_app) - - app_instance = None - - for member in inspect.getmembers(user_app): - if isinstance(member[1], Serverless): - # Ok. Found the variable now, need to use it - app_instance = member[1] - - if not app_instance: # No variable with type "Serverless" found - raise RuntimeError( - f"""Unable to locate an instance of serverless App - in the provided file: {file}.""" - ) - - return app_instance - - def main() -> int: """Entrypoint for click""" # Set logging level to DEFAULT. (ignore debug) get_logger().set_level(DEFAULT) - try: - cli() - return 0 - except Exception as exception: # pylint: disable=broad-except - get_logger().critical(str(exception)) - return 2 + + cli() + return 0 + # except Exception as exception: # pylint: disable=broad-except + # get_logger().critical(str(exception)) + # return 2 diff --git a/scw_serverless/utils/loader.py b/scw_serverless/utils/loader.py new file mode 100644 index 0000000..7e12c82 --- /dev/null +++ b/scw_serverless/utils/loader.py @@ -0,0 +1,52 @@ +import importlib.util +import inspect +import sys +from pathlib import Path + +from scw_serverless.app import Serverless + + +def get_module_name(file: Path) -> str: + """Extract the module name from the path.""" + + file = file.resolve() + return ( + str(file.relative_to(Path(".").resolve())).removesuffix(".py").replace("/", ".") + ) + + +def get_app_instance(file: Path) -> Serverless: + """Load the app instance from the client module.""" + + module_name = get_module_name(file) + parent_directory = str(file.parent.resolve()) + + spec = importlib.util.spec_from_file_location( + module_name, + str(file.resolve()), + submodule_search_locations=[parent_directory], + ) + + if not spec or not spec.loader: + raise ImportError( + f"Can't find module {module_name} at location {file.resolve()}" + ) + + user_app = importlib.util.module_from_spec(spec) + sys.path.append(parent_directory) + sys.modules[module_name] = user_app + spec.loader.exec_module(user_app) + + app_instance = None + for member in inspect.getmembers(user_app): + if isinstance(member[1], Serverless): + # Ok. Found the variable now, need to use it + app_instance = member[1] + + if not app_instance: # No variable with type "Serverless" found + raise RuntimeError( + f"""Unable to locate an instance of serverless App + in the provided file: {file}.""" + ) + + return app_instance From 109eb5faa0eccf48ea90ce498fdd1fec910d3ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andy=20M=C3=A9ry?= Date: Fri, 24 Feb 2023 14:16:43 +0100 Subject: [PATCH 2/3] feat(examples): multiple modules example --- docs/source/examples.rst | 4 +++ docs/source/examples/multiple_modules.md | 13 ++++++++ docs/source/examples/pr_notifier.md | 2 +- examples/multiple_functions/app.py | 6 ---- examples/multiple_functions/first.py | 12 -------- examples/multiple_functions/second.py | 22 ------------- examples/multiple_modules/README.md | 31 +++++++++++++++++++ examples/multiple_modules/app.py | 18 +++++++++++ examples/multiple_modules/query.py | 36 ++++++++++++++++++++++ examples/multiple_modules/requirements.txt | 3 ++ examples/multiple_modules/s3.py | 14 +++++++++ examples/multiple_modules/upload.py | 31 +++++++++++++++++++ examples/pr_notifier/README.md | 19 ++++++------ 13 files changed, 160 insertions(+), 51 deletions(-) create mode 100644 docs/source/examples/multiple_modules.md delete mode 100644 examples/multiple_functions/app.py delete mode 100644 examples/multiple_functions/first.py delete mode 100644 examples/multiple_functions/second.py create mode 100644 examples/multiple_modules/README.md create mode 100644 examples/multiple_modules/app.py create mode 100644 examples/multiple_modules/query.py create mode 100644 examples/multiple_modules/requirements.txt create mode 100644 examples/multiple_modules/s3.py create mode 100644 examples/multiple_modules/upload.py diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 4d330f9..247a476 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -9,6 +9,9 @@ To showcase the framework and how it can used to deploy serverless applications, :doc:`../examples/cron` Function that gets triggered periodically. +:doc:`../examples/multiple_modules` + An application with multiple handlers organized in different modules. + :doc:`../examples/github_actions` A GitHub action workflow to deploy your application with `scw_serverless`. @@ -23,5 +26,6 @@ To showcase the framework and how it can used to deploy serverless applications, ../examples/hello_world ../examples/cron + ../examples/multiple_modules ../examples/github_actions ../examples/pr_notifier diff --git a/docs/source/examples/multiple_modules.md b/docs/source/examples/multiple_modules.md new file mode 100644 index 0000000..75aaa1b --- /dev/null +++ b/docs/source/examples/multiple_modules.md @@ -0,0 +1,13 @@ +```{include} ../../../examples/multiple_modules/README.md +``` + +## [Sources](https://github.com/scaleway/serverless-api-project/blob/main/examples/multiple_modules/app.py) + +```{literalinclude} ../../../examples/multiple_modules/app.py +``` + +```{literalinclude} ../../../examples/multiple_modules/upload.py +``` + +```{literalinclude} ../../../examples/multiple_modules/query.py +``` diff --git a/docs/source/examples/pr_notifier.md b/docs/source/examples/pr_notifier.md index a49e05b..af60b62 100644 --- a/docs/source/examples/pr_notifier.md +++ b/docs/source/examples/pr_notifier.md @@ -1,7 +1,7 @@ ```{include} ../../../examples/pr_notifier/README.md ``` -## [Source](https://github.com/scaleway/serverless-api-project/blob/main/examples/github_actions/deploy.yml) +## [Source](https://github.com/scaleway/serverless-api-project/blob/main/examples/pr_notifier/notifier.py) ```{literalinclude} ../../../examples/pr_notifier/notifier.py ``` diff --git a/examples/multiple_functions/app.py b/examples/multiple_functions/app.py deleted file mode 100644 index d0021d5..0000000 --- a/examples/multiple_functions/app.py +++ /dev/null @@ -1,6 +0,0 @@ -from scw_serverless.app import Serverless - -app = Serverless("multiple-functions") - -import first # pylint: disable=all # noqa -import second # noqa diff --git a/examples/multiple_functions/first.py b/examples/multiple_functions/first.py deleted file mode 100644 index 1ac4ebd..0000000 --- a/examples/multiple_functions/first.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Any - -from app import app - - -@app.func() -def handle(_event: dict[str, Any], _context: dict[str, Any]) -> dict[str, Any]: - """A simple function that greets people.""" - return { - "message": "This function is handled by Scaleway" - + "functions using Serverless API Framework" - } diff --git a/examples/multiple_functions/second.py b/examples/multiple_functions/second.py deleted file mode 100644 index 2644436..0000000 --- a/examples/multiple_functions/second.py +++ /dev/null @@ -1,22 +0,0 @@ -import os -from typing import Any - -from app import app - - -@app.func( - description="Happy Coding!", - privacy="public", - env={"CUSTOM_NAME": "everyone"}, - secret={"SECRET_VALUE": "***"}, - min_scale=0, - max_scale=2, - memory_limit=128, - timeout="300s", -) -def hello(_event: dict[str, Any], _context: dict[str, Any]) -> dict[str, Any]: - """A simple function that greets people.""" - return { - "message": f"Hello {os.getenv('CUSTOM_NAME')}" - + "from Scaleway functions using Serverless API Framework" - } diff --git a/examples/multiple_modules/README.md b/examples/multiple_modules/README.md new file mode 100644 index 0000000..72c7e9c --- /dev/null +++ b/examples/multiple_modules/README.md @@ -0,0 +1,31 @@ +# [Multiple Modules](https://github.com/scaleway/serverless-api-project/tree/main/examples/multiple_modules) + +An app to upload and query files to S3 Glacier on Scaleway split in multiple modules. + +The upload endpoint allows you to upload files to Glacier via the `file` form-data key: + +```console +echo -e "Hello world!\n My contents will be stored in a bunker!" > myfile.dat +curl -F file=@myfile.dat +``` + +This example is there to showcase how to split handlers into different Python modules. + +## Deploying + +Deployment can be done with `scw_serverless`: + +```console +pip install scw_serverless +scw-serverless deploy app.py +``` + +## Configuration + +Here's all the environments variables that needs to be passed when deploying: + +| Variable | Description | Required | +|:----------------:|:---------------------------------------:|:------------------:| +| `SCW_SECRET_KEY` | Secret key to use for S3 operations | :heavy_check_mark: | +| `SCW_ACCESS_KEY` | Access key to use for S3 operations | :heavy_check_mark: | +| `S3_BUCKET` | Name of the bucket to store files into. | :heavy_check_mark: | diff --git a/examples/multiple_modules/app.py b/examples/multiple_modules/app.py new file mode 100644 index 0000000..36af6d6 --- /dev/null +++ b/examples/multiple_modules/app.py @@ -0,0 +1,18 @@ +import logging +import os + +from scw_serverless.app import Serverless + +logging.basicConfig(level=logging.INFO) + +app = Serverless( + "multiple-modules", + secret={ + "SCW_ACCESS_KEY": os.environ["SCW_ACCESS_KEY"], + "SCW_SECRET_KEY": os.environ["SCW_SECRET_KEY"], + }, + env={"S3_BUCKET": os.environ["S3_BUCKET"]}, +) + +import query # noqa +import upload # pylint: disable=all # noqa diff --git a/examples/multiple_modules/query.py b/examples/multiple_modules/query.py new file mode 100644 index 0000000..75c7d1f --- /dev/null +++ b/examples/multiple_modules/query.py @@ -0,0 +1,36 @@ +import json +import os +from typing import Any + +from app import app +from s3 import bucket + + +@app.func( + description="List objects in S3 uploads.", + privacy="public", + env={"LIMIT": "100"}, + min_scale=0, + max_scale=2, + memory_limit=128, + timeout="300s", +) +def query(_event: dict[str, Any], _context: dict[str, Any]) -> dict[str, Any]: + """A handler to list objects in a S3 bucket.""" + + response = [] + + for obj in bucket.objects.limit(count=int(os.environ["LIMIT"])): + response.append( + { + "name": obj.key, + "last_modified": obj.last_modified.strftime("%m/%d/%Y, %H:%M:%S"), + "storage_class": obj.storage_class, + } + ) + + return { + "statusCode": 200, + "headers": {"Content-Type": "application/json"}, + "body": json.dumps(response), + } diff --git a/examples/multiple_modules/requirements.txt b/examples/multiple_modules/requirements.txt new file mode 100644 index 0000000..f54a4c1 --- /dev/null +++ b/examples/multiple_modules/requirements.txt @@ -0,0 +1,3 @@ +boto3~=1.26 +scw-serverless~=0.0.3 +streaming_form_data~=1.11.0 diff --git a/examples/multiple_modules/s3.py b/examples/multiple_modules/s3.py new file mode 100644 index 0000000..c6d926a --- /dev/null +++ b/examples/multiple_modules/s3.py @@ -0,0 +1,14 @@ +import os + +import boto3 + +s3 = boto3.resource( + "s3", + region_name="fr-par", + use_ssl=True, + endpoint_url="https://s3.fr-par.scw.cloud", + aws_access_key_id=os.environ["SCW_ACCESS_KEY"], + aws_secret_access_key=os.environ["SCW_SECRET_KEY"], +) + +bucket = s3.Bucket(os.environ["S3_BUCKET"]) diff --git a/examples/multiple_modules/upload.py b/examples/multiple_modules/upload.py new file mode 100644 index 0000000..e0b7df9 --- /dev/null +++ b/examples/multiple_modules/upload.py @@ -0,0 +1,31 @@ +import logging +from typing import Any + +from app import app +from s3 import bucket +from streaming_form_data import StreamingFormDataParser +from streaming_form_data.targets import ValueTarget + + +@app.func() +def upload(event: dict[str, Any], _context: dict[str, Any]) -> dict[str, Any]: + """Upload form data to S3 Glacier.""" + + headers = event["headers"] + parser = StreamingFormDataParser(headers=headers) + + target = ValueTarget() + parser.register("file", target) + + body: str = event["body"] + parser.data_received(body.encode("utf-8")) + + if not (len(target.value) > 0 and target.multipart_filename): + return {"statusCode": 400} + + name = target.multipart_filename + + logging.info("Uploading file %s to Glacier on %s", name, bucket.name) + bucket.put_object(Key=name, Body=target.value, StorageClass="GLACIER") + + return {"statusCode": 200} diff --git a/examples/pr_notifier/README.md b/examples/pr_notifier/README.md index 11269c0..fe703ad 100644 --- a/examples/pr_notifier/README.md +++ b/examples/pr_notifier/README.md @@ -40,16 +40,15 @@ Once you have deployed the functions, they can be setup as webhooks on your repo Here's all the environments variables that needs to be passed when deploying: -| Variable | Description | Required | -| :---: | :---: | :---: | -| `SCW_SECRET_KEY` | Secret key to use for S3 operations | :heavy_check_mark: | -| `SCW_ACCESS_KEY` | Access key to use for S3 operations | :heavy_check_mark: | -| `SCW_SECRET_KEY` | Secret key to use for S3 operations | :heavy_check_mark: | -| `S3_BUCKET` | Name of the bucket to store opened PRs into. | :heavy_check_mark: | -| `SLACK_TOKEN` | Slack token. See below for details on scope. | :heavy_check_mark: | -| `SLACK_CHANNEL` | Channel ID of the Slack channel to send messages to | :heavy_check_mark: | -| `GITLAB_EMAIL_DOMAIN` | Will be appended to GitLab usernames to create a valid email. Emails are converted to Slack IDs to ping developers in the reminder | | -| `REMINDER_SCHEDULE` | CRON schedule to trigger the reminder | | +| Variable | Description | Required | +|:---------------------:|:----------------------------------------------------------------------------------------------------------------------------------:|:------------------:| +| `SCW_SECRET_KEY` | Secret key to use for S3 operations | :heavy_check_mark: | +| `SCW_ACCESS_KEY` | Access key to use for S3 operations | :heavy_check_mark: | +| `S3_BUCKET` | Name of the bucket to store opened PRs into. | :heavy_check_mark: | +| `SLACK_TOKEN` | Slack token. See below for details on scope. | :heavy_check_mark: | +| `SLACK_CHANNEL` | Channel ID of the Slack channel to send messages to | :heavy_check_mark: | +| `GITLAB_EMAIL_DOMAIN` | Will be appended to GitLab usernames to create a valid email. Emails are converted to Slack IDs to ping developers in the reminder | | +| `REMINDER_SCHEDULE` | CRON schedule to trigger the reminder | | ### Creating the Slack application From 2295ac41d667871991e0f8ee42fecf45d7d42cd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andy=20M=C3=A9ry?= Date: Fri, 24 Feb 2023 14:24:13 +0100 Subject: [PATCH 3/3] chore: forgot debug stacktrace --- scw_serverless/cli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scw_serverless/cli.py b/scw_serverless/cli.py index 88d043e..531209b 100644 --- a/scw_serverless/cli.py +++ b/scw_serverless/cli.py @@ -213,9 +213,9 @@ def main() -> int: """Entrypoint for click""" # Set logging level to DEFAULT. (ignore debug) get_logger().set_level(DEFAULT) - - cli() - return 0 - # except Exception as exception: # pylint: disable=broad-except - # get_logger().critical(str(exception)) - # return 2 + try: + cli() + return 0 + except Exception as exception: # pylint: disable=broad-except + get_logger().critical(str(exception)) + return 2