From 71fefeeaedabc4937aef3e674eea9c836d662db8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andy=20M=C3=A9ry?= Date: Fri, 24 Mar 2023 16:59:21 +0100 Subject: [PATCH 1/3] feat(functions): add python s3 form data upload example --- README.md | 4 +- .../node-upload-file-s3-multipart/README.md | 12 ++-- .../python-upload-file-s3-multipart/README.md | 45 ++++++++++++ .../python-upload-file-s3-multipart/app.py | 71 +++++++++++++++++++ .../requirements-dev.txt | 3 + .../requirements.txt | 3 + 6 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 functions/python-upload-file-s3-multipart/README.md create mode 100644 functions/python-upload-file-s3-multipart/app.py create mode 100644 functions/python-upload-file-s3-multipart/requirements-dev.txt create mode 100644 functions/python-upload-file-s3-multipart/requirements.txt diff --git a/README.md b/README.md index e14d64e..0b9c8fb 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,11 @@ Table of Contents: | **[Node MultiPart Upload to S3](functions/node-upload-file-s3-multipart/README.md)**
A function to upload file from form-data to S3. | node19 | [Serverless Framework] | | **[Python ChatBot](functions/python-dependencies/README.md)**
A chatbot example with ChatterBot. | python310 | [Serverless Framework] | | **[Python Dependencies](functions/python-dependencies/README.md)**
Example showing how to use Python requirements with Serverless Framework. | python310 | [Serverless Framework] | +| **[Python MultiPart Upload to S3](functions/python-upload-file-s3-multipart/README.md)**
A function to upload file from form-data to S3. | python311 | [Python API Framework] | | **[Redis TLS](functions/redis-tls/README.md)**
How to connect a function to a Scaleway Redis cluster with TLS enabled. | python310 | [Terraform] | | **[Rust MNIST](functions/rust-mnist/README.md)**
A Rust function to recognize hand-written digits with a simple neural network. | rust165 | [Serverless Framework] | | **[Terraform Python](functions/terraform-python-example/README.md)**
A Python function deployed with Terraform. | python310 | [Terraform] | -| **[PostgeSQL Node](functions/postgre-sql-node/README.md)**
A Node function to connect and interact with PostgreSQL database. | node18 | [Serverless Framework] | +| **[PostgeSQL Node](functions/postgre-sql-node/README.md)**
A Node function to connect and interact with PostgreSQL database. | node18 | [Serverless Framework] | ### 📦 Containers @@ -61,6 +62,7 @@ Table of Contents: [Serverless Framework]: https://github.com/scaleway/serverless-scaleway-functions [Terraform]: https://registry.terraform.io/providers/scaleway/scaleway/latest/docs +[Python API Framework]: https://github.com/scaleway/serverless-api-project ## Contributing diff --git a/functions/node-upload-file-s3-multipart/README.md b/functions/node-upload-file-s3-multipart/README.md index 6983781..c2e6ab9 100644 --- a/functions/node-upload-file-s3-multipart/README.md +++ b/functions/node-upload-file-s3-multipart/README.md @@ -1,13 +1,13 @@ # Node function to read files and upload to S3 -This function does the following steps : +This function does the following steps: * Read a file from an HTTP request * Send the file to an S3 bucket ## Requirements -If you want to enable S3 upload, ensure to create a bucket and have the following secrets variables available in your environment: +Ensure to create a bucket and have the following secrets variables available in your environment: ```env S3_REGION = # Default: fr-par @@ -16,8 +16,12 @@ SCW_SECRET_KEY = BUCKET_NAME = ``` +## Running + To call the function (replace `README.md` with the file you want to upload): -```sh -curl -X POST -F "data=@README.md" -F "another=@package.json" http://localhost:8080 +```console +serverless deploy + +curl -X POST -F "data=@README.md" -F "another=@package.json" ``` diff --git a/functions/python-upload-file-s3-multipart/README.md b/functions/python-upload-file-s3-multipart/README.md new file mode 100644 index 0000000..dcb836d --- /dev/null +++ b/functions/python-upload-file-s3-multipart/README.md @@ -0,0 +1,45 @@ +# Python function to upload files to S3 + +This function does the following steps: + +* Read a file from an HTTP request form +* Send the file to long-term storage with Glacier for S3 + +## Requirements + +This example uses the [Python API Framework](https://github.com/scaleway/serverless-api-project) to deploy the function. + +If needed, create a bucket and provide the following variables in your environment: + +```env +export SCW_ACCESS_KEY = +export SCW_SECRET_KEY = +export BUCKET_NAME = +``` + +## Running + +### Running locally + +This examples uses [Serverless Offline Python](https://github.com/scaleway/serverless-functions-python) and can be executed locally: + +```console +pip install -r requirements-dev.txt + +python app.py +``` + +The upload endpoint allows you to upload files to Glacier via the `file` form-data key: + +```console +echo -e "Hello world!\n My contents will be stored in a bunker!" > myfile.dat +curl -F file=@myfile.dat localhost:8080 +``` + +### Deploying with the API Framework + +Deployment can be done with `scw_serverless`: + +```console +scw_serverless deploy app.py +``` diff --git a/functions/python-upload-file-s3-multipart/app.py b/functions/python-upload-file-s3-multipart/app.py new file mode 100644 index 0000000..d707a74 --- /dev/null +++ b/functions/python-upload-file-s3-multipart/app.py @@ -0,0 +1,71 @@ +from typing import TYPE_CHECKING +import logging +import os + +from scw_serverless import Serverless +if TYPE_CHECKING: + from scaleway_functions_python.framework.v1.hints import Context, Event, Response + +import boto3 +from streaming_form_data import StreamingFormDataParser +from streaming_form_data.targets import ValueTarget + +SCW_ACCESS_KEY = os.environ["SCW_ACCESS_KEY"] +SCW_SECRET_KEY = os.environ["SCW_SECRET_KEY"] +BUCKET_NAME = os.environ["BUCKET_NAME"] + +app = Serverless( + "s3-utilities", + secret={ + "SCW_ACCESS_KEY": SCW_ACCESS_KEY, + "SCW_SECRET_KEY": SCW_SECRET_KEY, + }, + env={ + "BUCKET_NAME": BUCKET_NAME, + "PYTHONUNBUFFERED": "1", + }, +) + +s3 = boto3.resource( + "s3", + region_name="fr-par", + use_ssl=True, + endpoint_url="https://s3.fr-par.scw.cloud", + aws_access_key_id=SCW_ACCESS_KEY, + aws_secret_access_key=SCW_SECRET_KEY, +) + +bucket = s3.Bucket(BUCKET_NAME) + +logging.basicConfig(level=logging.INFO) + + +@app.func() +def upload(event: "Event", _context: "Context") -> "Response": + """Upload form data to S3 Glacier.""" + + headers = event["headers"] + parser = StreamingFormDataParser(headers=headers) + + # Defined a target to handle files uploaded with the "file" key + target = ValueTarget() + parser.register("file", target) + + body: str = event["body"] + parser.data_received(body.encode("utf-8")) + + if not (len(target.value) > 0 and target.multipart_filename): + return {"statusCode": 400} + + name = target.multipart_filename + + logging.info("Uploading file %s to Glacier on %s", name, bucket.name) + bucket.put_object(Key=name, Body=target.value, StorageClass="GLACIER") + + return {"statusCode": 200, "body": f"Successfully uploaded {name} to bucket!"} + + +if __name__ == "__main__": + from scaleway_functions_python import local + + local.serve_handler(upload) diff --git a/functions/python-upload-file-s3-multipart/requirements-dev.txt b/functions/python-upload-file-s3-multipart/requirements-dev.txt new file mode 100644 index 0000000..65a1f9b --- /dev/null +++ b/functions/python-upload-file-s3-multipart/requirements-dev.txt @@ -0,0 +1,3 @@ +-r requirements.txt + +scaleway-functions-python~=0.0.1 diff --git a/functions/python-upload-file-s3-multipart/requirements.txt b/functions/python-upload-file-s3-multipart/requirements.txt new file mode 100644 index 0000000..4f1bbfa --- /dev/null +++ b/functions/python-upload-file-s3-multipart/requirements.txt @@ -0,0 +1,3 @@ +scw-serverless~=0.0.4 +boto3~=1.26 +streaming_form_data~=1.11.0 From dbab09ffb7836b8b7fa79fbb2499d5aab3a03a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andy=20M=C3=A9ry?= Date: Fri, 24 Mar 2023 17:01:31 +0100 Subject: [PATCH 2/3] refactor: add variable for storage class --- functions/python-upload-file-s3-multipart/app.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/functions/python-upload-file-s3-multipart/app.py b/functions/python-upload-file-s3-multipart/app.py index d707a74..41d7728 100644 --- a/functions/python-upload-file-s3-multipart/app.py +++ b/functions/python-upload-file-s3-multipart/app.py @@ -14,6 +14,10 @@ SCW_SECRET_KEY = os.environ["SCW_SECRET_KEY"] BUCKET_NAME = os.environ["BUCKET_NAME"] +# Files will be uploaded to cold storage +# See: https://www.scaleway.com/en/glacier-cold-storage/ +STORAGE_CLASS = "GLACIER" + app = Serverless( "s3-utilities", secret={ @@ -60,7 +64,7 @@ def upload(event: "Event", _context: "Context") -> "Response": name = target.multipart_filename logging.info("Uploading file %s to Glacier on %s", name, bucket.name) - bucket.put_object(Key=name, Body=target.value, StorageClass="GLACIER") + bucket.put_object(Key=name, Body=target.value, StorageClass=STORAGE_CLASS) return {"statusCode": 200, "body": f"Successfully uploaded {name} to bucket!"} From d37234f96863851583b1a820dbd1218d07bb5d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andy=20M=C3=A9ry?= Date: Mon, 27 Mar 2023 13:27:28 +0200 Subject: [PATCH 3/3] refactor: apply suggestions from code review Co-authored-by: Reda Noureddine <58738453+redanrd@users.noreply.github.com> --- functions/python-upload-file-s3-multipart/README.md | 4 ++-- .../python-upload-file-s3-multipart/requirements-dev.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/python-upload-file-s3-multipart/README.md b/functions/python-upload-file-s3-multipart/README.md index dcb836d..f6c117a 100644 --- a/functions/python-upload-file-s3-multipart/README.md +++ b/functions/python-upload-file-s3-multipart/README.md @@ -21,7 +21,7 @@ export BUCKET_NAME = ### Running locally -This examples uses [Serverless Offline Python](https://github.com/scaleway/serverless-functions-python) and can be executed locally: +This examples uses [Serverless Functions Python Framework](https://github.com/scaleway/serverless-functions-python) and can be executed locally: ```console pip install -r requirements-dev.txt @@ -32,7 +32,7 @@ python app.py The upload endpoint allows you to upload files to Glacier via the `file` form-data key: ```console -echo -e "Hello world!\n My contents will be stored in a bunker!" > myfile.dat +echo -e 'Hello world!\n My contents will be stored in a bunker!' > myfile.dat curl -F file=@myfile.dat localhost:8080 ``` diff --git a/functions/python-upload-file-s3-multipart/requirements-dev.txt b/functions/python-upload-file-s3-multipart/requirements-dev.txt index 65a1f9b..f4fc021 100644 --- a/functions/python-upload-file-s3-multipart/requirements-dev.txt +++ b/functions/python-upload-file-s3-multipart/requirements-dev.txt @@ -1,3 +1,3 @@ -r requirements.txt -scaleway-functions-python~=0.0.1 +scaleway-functions-python~=0.1.0