From 4c962158ec2799d37507ff875558b9b736e5058d Mon Sep 17 00:00:00 2001 From: Salakar Date: Wed, 12 Apr 2023 11:07:37 +0100 Subject: [PATCH] feat: scheduler fn support --- samples/basic_scheduler/.firebaserc | 5 + samples/basic_scheduler/.gitignore | 66 +++++++++ samples/basic_scheduler/__init__.py | 3 + samples/basic_scheduler/firebase.json | 11 ++ samples/basic_scheduler/functions/.gitignore | 13 ++ samples/basic_scheduler/functions/main.py | 12 ++ .../functions/requirements.txt | 8 ++ src/firebase_functions/options.py | 86 +++++++++++- src/firebase_functions/private/manifest.py | 28 +++- src/firebase_functions/scheduler_fn.py | 131 ++++++++++++++++++ tests/test_scheduler_fn.py | 120 ++++++++++++++++ 11 files changed, 475 insertions(+), 8 deletions(-) create mode 100644 samples/basic_scheduler/.firebaserc create mode 100644 samples/basic_scheduler/.gitignore create mode 100644 samples/basic_scheduler/__init__.py create mode 100644 samples/basic_scheduler/firebase.json create mode 100644 samples/basic_scheduler/functions/.gitignore create mode 100644 samples/basic_scheduler/functions/main.py create mode 100644 samples/basic_scheduler/functions/requirements.txt create mode 100644 src/firebase_functions/scheduler_fn.py create mode 100644 tests/test_scheduler_fn.py diff --git a/samples/basic_scheduler/.firebaserc b/samples/basic_scheduler/.firebaserc new file mode 100644 index 0000000..ad27d4b --- /dev/null +++ b/samples/basic_scheduler/.firebaserc @@ -0,0 +1,5 @@ +{ + "projects": { + "default": "python-functions-testing" + } +} diff --git a/samples/basic_scheduler/.gitignore b/samples/basic_scheduler/.gitignore new file mode 100644 index 0000000..dbb58ff --- /dev/null +++ b/samples/basic_scheduler/.gitignore @@ -0,0 +1,66 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +firebase-debug.log* +firebase-debug.*.log* + +# Firebase cache +.firebase/ + +# Firebase config + +# Uncomment this if you'd like others to create their own Firebase project. +# For a team working on the same Firebase project(s), it is recommended to leave +# it commented so all members can deploy to the same project(s) in .firebaserc. +# .firebaserc + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env diff --git a/samples/basic_scheduler/__init__.py b/samples/basic_scheduler/__init__.py new file mode 100644 index 0000000..2340b04 --- /dev/null +++ b/samples/basic_scheduler/__init__.py @@ -0,0 +1,3 @@ +# Required to avoid a 'duplicate modules' mypy error +# in monorepos that have multiple main.py files. +# https://github.com/python/mypy/issues/4008 diff --git a/samples/basic_scheduler/firebase.json b/samples/basic_scheduler/firebase.json new file mode 100644 index 0000000..7bbd899 --- /dev/null +++ b/samples/basic_scheduler/firebase.json @@ -0,0 +1,11 @@ +{ + "functions": [ + { + "source": "functions", + "codebase": "default", + "ignore": [ + "venv" + ] + } + ] +} diff --git a/samples/basic_scheduler/functions/.gitignore b/samples/basic_scheduler/functions/.gitignore new file mode 100644 index 0000000..34cef6b --- /dev/null +++ b/samples/basic_scheduler/functions/.gitignore @@ -0,0 +1,13 @@ +# pyenv +.python-version + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Environments +.env +.venv +venv/ +venv.bak/ +__pycache__ diff --git a/samples/basic_scheduler/functions/main.py b/samples/basic_scheduler/functions/main.py new file mode 100644 index 0000000..a20ea84 --- /dev/null +++ b/samples/basic_scheduler/functions/main.py @@ -0,0 +1,12 @@ +"""Firebase Scheduled Cloud Functions example.""" + +from firebase_functions import scheduler_fn + + +@scheduler_fn.on_schedule( + schedule="* * * * *", + timezone=scheduler_fn.Timezone("America/Los_Angeles"), +) +def example(event: scheduler_fn.ScheduledEvent) -> None: + print(event.job_name) + print(event.schedule_time) diff --git a/samples/basic_scheduler/functions/requirements.txt b/samples/basic_scheduler/functions/requirements.txt new file mode 100644 index 0000000..8977a41 --- /dev/null +++ b/samples/basic_scheduler/functions/requirements.txt @@ -0,0 +1,8 @@ +# Not published yet, +# firebase-functions-python >= 0.0.1 +# so we use a relative path during development: +./../../../ +# Or switch to git ref for deployment testing: +# git+https://github.com/firebase/firebase-functions-python.git@main#egg=firebase-functions + +firebase-admin >= 6.0.1 diff --git a/src/firebase_functions/options.py b/src/firebase_functions/options.py index 967c7cc..d8ba3ad 100644 --- a/src/firebase_functions/options.py +++ b/src/firebase_functions/options.py @@ -20,12 +20,16 @@ import dataclasses as _dataclasses import re as _re import typing as _typing +from zoneinfo import ZoneInfo as _ZoneInfo import firebase_functions.private.manifest as _manifest import firebase_functions.private.util as _util import firebase_functions.private.path_pattern as _path_pattern from firebase_functions.params import SecretParam, Expression +Timezone = _ZoneInfo +"""An alias of the zoneinfo.ZoneInfo for convenience.""" + RESET_VALUE = _util.Sentinel( "Special configuration value to reset configuration to platform default.") """Special configuration value to reset configuration to platform default.""" @@ -409,7 +413,7 @@ def _endpoint( maxDispatchesPerSecond=self.rate_limits.max_dispatches_per_second, ) if self.rate_limits is not None else None - retry_config: _manifest.RetryConfig | None = _manifest.RetryConfig( + retry_config: _manifest.RetryConfigTasks | None = _manifest.RetryConfigTasks( maxAttempts=self.retry_config.max_attempts, maxRetrySeconds=self.retry_config.max_retry_seconds, maxBackoffSeconds=self.retry_config.max_backoff_seconds, @@ -498,6 +502,86 @@ def _endpoint( **kwargs, event_filters=event_filters, event_type=event_type)))) +@_dataclasses.dataclass(frozen=True, kw_only=True) +class ScheduleOptions(RuntimeOptions): + """ + Options that can be set on a Schedule trigger. + """ + + schedule: str + """ + The schedule, in Unix Crontab or AppEngine syntax. + """ + + timezone: Timezone | Expression[str] | _util.Sentinel | None = None + """ + The timezone that the schedule executes in. + """ + + retry_count: int | Expression[int] | _util.Sentinel | None = None + """ + The number of retry attempts for a failed run. + """ + + max_retry_seconds: int | Expression[int] | _util.Sentinel | None = None + """ + The time limit for retrying. + """ + + max_backoff_seconds: int | Expression[int] | _util.Sentinel | None = None + """ + The maximum amount of time to wait between attempts. + """ + + max_doublings: int | Expression[int] | _util.Sentinel | None = None + """ + The maximum number of times to double the backoff between + retries. + """ + + min_backoff_seconds: int | Expression[int] | _util.Sentinel | None = None + """ + The minimum time to wait between attempts. + """ + + def _endpoint( + self, + **kwargs, + ) -> _manifest.ManifestEndpoint: + retry_config: _manifest.RetryConfigScheduler = _manifest.RetryConfigScheduler( + retryCount=self.retry_count, + maxRetrySeconds=self.max_retry_seconds, + maxBackoffSeconds=self.max_backoff_seconds, + maxDoublings=self.max_doublings, + minBackoffSeconds=self.min_backoff_seconds, + ) + time_zone: str | Expression[str] | _util.Sentinel | None = None + if isinstance(self.timezone, Timezone): + time_zone = self.timezone.key + else: + time_zone = self.timezone + + kwargs_merged = { + **_dataclasses.asdict(super()._endpoint(**kwargs)), + "scheduleTrigger": + _manifest.ScheduleTrigger( + schedule=self.schedule, + timeZone=time_zone, + retryConfig=retry_config, + ), + } + return _manifest.ManifestEndpoint( + **_typing.cast(_typing.Dict, kwargs_merged)) + + def _required_apis(self) -> list[_manifest.ManifestRequiredApi]: + return [ + _manifest.ManifestRequiredApi( + api="cloudscheduler.googleapis.com", + reason="Needed for scheduled functions.", + ) + ] + + @_dataclasses.dataclass(frozen=True, kw_only=True) class StorageOptions(RuntimeOptions): """ diff --git a/src/firebase_functions/private/manifest.py b/src/firebase_functions/private/manifest.py index 6617911..b923302 100644 --- a/src/firebase_functions/private/manifest.py +++ b/src/firebase_functions/private/manifest.py @@ -65,12 +65,10 @@ class EventTrigger(_typing.TypedDict): _util.Sentinel] -class RetryConfig(_typing.TypedDict): +class RetryConfigBase(_typing.TypedDict): """ Retry configuration for a endpoint. """ - maxAttempts: _typing_extensions.NotRequired[int | _params.Expression[int] | - _util.Sentinel | None] maxRetrySeconds: _typing_extensions.NotRequired[int | _params.Expression[int] | _util.Sentinel | None] @@ -84,6 +82,22 @@ class RetryConfig(_typing.TypedDict): _util.Sentinel | None] +class RetryConfigTasks(RetryConfigBase): + """ + Retry configuration for a task. + """ + maxAttempts: _typing_extensions.NotRequired[int | _params.Expression[int] | + _util.Sentinel | None] + + +class RetryConfigScheduler(RetryConfigBase): + """ + Retry configuration for a schedule. + """ + retryCount: _typing_extensions.NotRequired[int | _params.Expression[int] | + _util.Sentinel | None] + + class RateLimits(_typing.TypedDict): maxConcurrentDispatches: int | _params.Expression[ int] | _util.Sentinel | None @@ -96,14 +110,14 @@ class TaskQueueTrigger(_typing.TypedDict): Trigger definitions for RPCs servers using the HTTP protocol defined at https://firebase.google.com/docs/functions/callable-reference """ - retryConfig: RetryConfig | None + retryConfig: RetryConfigTasks | None rateLimits: RateLimits | None class ScheduleTrigger(_typing.TypedDict): - schedule: _typing_extensions.NotRequired[str | _params.Expression[str]] - timeZone: _typing_extensions.NotRequired[str | _params.Expression[str]] - retryConfig: _typing_extensions.NotRequired[RetryConfig] + schedule: str | _params.Expression[str] + timeZone: str | _params.Expression[str] | _util.Sentinel | None + retryConfig: RetryConfigScheduler | None class BlockingTrigger(_typing.TypedDict): diff --git a/src/firebase_functions/scheduler_fn.py b/src/firebase_functions/scheduler_fn.py new file mode 100644 index 0000000..7912b2a --- /dev/null +++ b/src/firebase_functions/scheduler_fn.py @@ -0,0 +1,131 @@ +# Copyright 2022 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Cloud functions to handle Schedule triggers.""" + +import typing as _typing +import dataclasses as _dataclasses +import datetime as _dt +import functools as _functools + +import firebase_functions.options as _options +import firebase_functions.private.util as _util +from functions_framework import logging as _logging +from flask import ( + Request as _Request, + Response as _Response, + make_response as _make_response, +) + +# Export for user convenience. +# pylint: disable=unused-import +from firebase_functions.options import Timezone + + +@_dataclasses.dataclass(frozen=True) +class ScheduledEvent: + """ + A ScheduleEvent that is passed to the function handler + """ + + job_name: str | None + """ + The Cloud Scheduler job name. + Populated via the X-CloudScheduler-JobName header. + If invoked manually, this field is None. + """ + + schedule_time: _dt.datetime + """ + For Cloud Scheduler jobs specified in the unix-cron format, + this is the job schedule time in RFC3339 UTC "Zulu" format. + Populated via the X-CloudScheduler-ScheduleTime header. + + If the schedule is manually triggered, this field will be + the function execution time. + """ + + +_C = _typing.Callable[[ScheduledEvent], None] + + +@_util.copy_func_kwargs(_options.ScheduleOptions) +def on_schedule(**kwargs) -> _typing.Callable[[_C], _Response]: + """ + Creates a handler for tasks sent to a Google Cloud Tasks queue. + Requires a function that takes a CallableRequest. + + Example: + + .. code-block:: python + + from firebase_functions import scheduler_fn + + + @scheduler_fn.on_schedule( + schedule="* * * * *", + timezone=scheduler_fn.Timezone("America/Los_Angeles"), + ) + def example(event: scheduler_fn.ScheduledEvent) -> None: + print(event.job_name) + print(event.schedule_time) + + + :param \\*\\*kwargs: ScheduleOptions options. + :type \\*\\*kwargs: as :exc:`firebase_functions.options.ScheduleOptions` + :rtype: :exc:`typing.Callable` + \\[ \\[ :exc:`firebase_functions.schedule_fn.ScheduledEvent` \\], :exc:`None` \\] + A function that takes a ScheduledEvent and returns nothing. + """ + options = _options.ScheduleOptions(**kwargs) + + def on_schedule_decorator(func: _C): + + @_functools.wraps(func) + def on_schedule_wrapped(request: _Request) -> _Response: + schedule_time: _dt.datetime + schedule_time_str = request.headers.get( + "X-CloudScheduler-ScheduleTime") + if schedule_time_str is None: + schedule_time = _dt.datetime.utcnow() + else: + schedule_time = _dt.datetime.strptime( + schedule_time_str, + "%Y-%m-%dT%H:%M:%S%z", + ) + event = ScheduledEvent( + job_name=request.headers.get("X-CloudScheduler-JobName"), + schedule_time=schedule_time, + ) + try: + func(event) + return _make_response() + # Disable broad exceptions lint since we want to handle all exceptions. + # pylint: disable=broad-except + except Exception as exception: + _logging.exception(exception) + return _make_response(str(exception), 500) + + _util.set_func_endpoint_attr( + on_schedule_wrapped, + # pylint: disable=protected-access + options._endpoint(func_name=func.__name__), + ) + _util.set_required_apis_attr( + on_schedule_wrapped, + # pylint: disable=protected-access + options._required_apis(), + ) + return on_schedule_wrapped + + return on_schedule_decorator diff --git a/tests/test_scheduler_fn.py b/tests/test_scheduler_fn.py new file mode 100644 index 0000000..eb68219 --- /dev/null +++ b/tests/test_scheduler_fn.py @@ -0,0 +1,120 @@ +# Copyright 2022 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Scheduler function tests.""" +import unittest +from unittest.mock import Mock +from datetime import datetime +from flask import Request, Flask +from werkzeug.test import EnvironBuilder +from firebase_functions import scheduler_fn + + +class TestScheduler(unittest.TestCase): + """ + Scheduler function tests. + """ + + def test_on_schedule_decorator(self): + """ + Tests the on_schedule decorator functionality by checking + that the __firebase_endpoint__ attribute is set properly. + """ + + schedule = "* * * * *" + tz = "America/Los_Angeles" + example_func = Mock(__name__="example_func") + decorated_func = scheduler_fn.on_schedule( + schedule="* * * * *", + timezone=scheduler_fn.Timezone(tz))(example_func) + endpoint = getattr(decorated_func, "__firebase_endpoint__") + + self.assertIsNotNone(endpoint) + self.assertIsNotNone(endpoint.scheduleTrigger) + self.assertEqual(endpoint.scheduleTrigger.get("schedule"), schedule) + self.assertEqual(endpoint.scheduleTrigger.get("timeZone"), tz) + + def test_on_schedule_call(self): + """ + Tests to ensure the decorated function is called correctly + with appropriate ScheduledEvent object and returns a 200 + status code if successful. + """ + + with Flask(__name__).test_request_context("/"): + environ = EnvironBuilder( + headers={ + "X-CloudScheduler-JobName": "example-job", + "X-CloudScheduler-ScheduleTime": "2023-04-13T12:00:00-07:00" + }).get_environ() + mock_request = Request(environ) + example_func = Mock(__name__="example_func") + decorated_func = scheduler_fn.on_schedule( + schedule="* * * * *")(example_func) + response = decorated_func(mock_request) + + self.assertEqual(response.status_code, 200) + example_func.assert_called_once_with( + scheduler_fn.ScheduledEvent( + job_name="example-job", + schedule_time=datetime( + 2023, + 4, + 13, + 12, + 0, + tzinfo=scheduler_fn.Timezone("America/Los_Angeles"), + ), + )) + + def test_on_schedule_call_with_no_headers(self): + """ + Tests to ensure that if the function is called manually + then the ScheduledEvent object is populated with the + current time and the job_name is None. + """ + + with Flask(__name__).test_request_context("/"): + environ = EnvironBuilder().get_environ() + mock_request = Request(environ) + example_func = Mock(__name__="example_func") + decorated_func = scheduler_fn.on_schedule( + schedule="* * * * *")(example_func) + response = decorated_func(mock_request) + + self.assertEqual(response.status_code, 200) + self.assertEqual(example_func.call_count, 1) + self.assertIsNone(example_func.call_args[0][0].job_name) + self.assertIsNotNone(example_func.call_args[0][0].schedule_time) + + def test_on_schedule_call_with_exception(self): + """ + Tests to ensure exceptions in the users handler are handled + caught and returns a 500 status code. + """ + + with Flask(__name__).test_request_context("/"): + environ = EnvironBuilder( + headers={ + "X-CloudScheduler-JobName": "example-job", + "X-CloudScheduler-ScheduleTime": "2023-04-13T12:00:00-07:00" + }).get_environ() + mock_request = Request(environ) + example_func = Mock(__name__="example_func", + side_effect=Exception("Test exception")) + decorated_func = scheduler_fn.on_schedule( + schedule="* * * * *")(example_func) + response = decorated_func(mock_request) + + self.assertEqual(response.status_code, 500) + self.assertEqual(response.data, b"Test exception")