From 900a5dda89a4396489d36445653f03854c632d24 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 19 Apr 2025 11:47:57 +0800 Subject: [PATCH 01/19] enable direct response, 5.2.0, deprecation warnings --- CHANGELOG.md | 8 +++++++ stac_fastapi/core/setup.py | 6 +++--- .../stac_fastapi/core/extensions/query.py | 4 ++-- .../stac_fastapi/elasticsearch/app.py | 4 +++- .../elasticsearch/database_logic.py | 9 ++++---- .../opensearch/stac_fastapi/opensearch/app.py | 5 ++++- stac_fastapi/tests/api/test_api.py | 1 + stac_fastapi/tests/conftest.py | 21 ++++++++++++------- 8 files changed, 39 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b727634..97b47f2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Changed +- Migrated Elasticsearch index template creation from legacy `put_template` to composable `put_index_template` API in `database_logic.py`. This resolves deprecation warnings and ensures compatibility with Elasticsearch 7.x and 8.x. +- Updated all Pydantic models to use `ConfigDict` instead of class-based `Config` for Pydantic v2 compatibility. This resolves deprecation warnings and prepares for Pydantic v3. +- Migrated all Pydantic `@root_validator` validators to `@model_validator` for Pydantic v2 compatibility. + ### Added +- Added `enable_direct_response` option to API settings for more flexible response handling. ### Changed +- Updated test suite to use `httpx.ASGITransport(app=...)` for FastAPI app testing (removes deprecation warning). +- Updated stac-fastapi parent libraries to 5.2.0. ### Fixed diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index adde5c82..ddf786b6 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -10,9 +10,9 @@ "attrs>=23.2.0", "pydantic>=2.4.1,<3.0.0", "stac_pydantic~=3.1.0", - "stac-fastapi.api==5.1.1", - "stac-fastapi.extensions==5.1.1", - "stac-fastapi.types==5.1.1", + "stac-fastapi.api==5.2.0", + "stac-fastapi.extensions==5.2.0", + "stac-fastapi.types==5.2.0", "orjson~=3.9.0", "overrides~=7.4.0", "geojson-pydantic~=1.0.0", diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/query.py b/stac_fastapi/core/stac_fastapi/core/extensions/query.py index 3084cbf8..f6e0868d 100644 --- a/stac_fastapi/core/stac_fastapi/core/extensions/query.py +++ b/stac_fastapi/core/stac_fastapi/core/extensions/query.py @@ -10,7 +10,7 @@ from types import DynamicClassAttribute from typing import Any, Callable, Dict, Optional -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, model_validator from stac_pydantic.utils import AutoValueEnum from stac_fastapi.extensions.core.query import QueryExtension as QueryExtensionBase @@ -63,7 +63,7 @@ class QueryExtensionPostRequest(BaseModel): query: Optional[Dict[str, Dict[Operator, Any]]] = None - @root_validator(pre=True) + @model_validator(mode="before") def validate_query_fields(cls, values: Dict) -> Dict: """Validate query fields.""" ... diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 9510eaa6..30fecb50 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -87,7 +87,7 @@ api = StacApi( title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-elasticsearch"), description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-elasticsearch"), - api_version=os.getenv("STAC_FASTAPI_VERSION", "2.1"), + api_version=os.getenv("STAC_FASTAPI_VERSION", "5.2.0"), settings=settings, extensions=extensions, client=CoreClient( @@ -96,6 +96,8 @@ search_get_request_model=create_get_request_model(search_extensions), search_post_request_model=post_request_model, route_dependencies=get_route_dependencies(), + enable_response_models=False, + enable_direct_response=True, ) app = api.app app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index ec84de57..7ab044c2 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -50,19 +50,18 @@ async def create_index_templates() -> None: """ client = AsyncElasticsearchSettings().create_client - await client.indices.put_template( + await client.indices.put_index_template( name=f"template_{COLLECTIONS_INDEX}", body={ "index_patterns": [f"{COLLECTIONS_INDEX}*"], - "mappings": ES_COLLECTIONS_MAPPINGS, + "template": {"mappings": ES_COLLECTIONS_MAPPINGS}, }, ) - await client.indices.put_template( + await client.indices.put_index_template( name=f"template_{ITEMS_INDEX_PREFIX}", body={ "index_patterns": [f"{ITEMS_INDEX_PREFIX}*"], - "settings": ES_ITEMS_SETTINGS, - "mappings": ES_ITEMS_MAPPINGS, + "template": {"settings": ES_ITEMS_SETTINGS, "mappings": ES_ITEMS_MAPPINGS}, }, ) await client.close() diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index 90038302..f157a0dc 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -87,7 +87,7 @@ api = StacApi( title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-opensearch"), description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-opensearch"), - api_version=os.getenv("STAC_FASTAPI_VERSION", "2.1"), + api_version=os.getenv("STAC_FASTAPI_VERSION", "5.2.0"), settings=settings, extensions=extensions, client=CoreClient( @@ -96,10 +96,13 @@ search_get_request_model=create_get_request_model(search_extensions), search_post_request_model=post_request_model, route_dependencies=get_route_dependencies(), + enable_response_models=False, + enable_direct_response=True, ) app = api.app app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") + # Add rate limit setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT")) diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 64545807..fb128f74 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -7,6 +7,7 @@ ROUTES = { "GET /_mgmt/ping", + "GET /_mgmt/health", "GET /docs/oauth2-redirect", "HEAD /docs/oauth2-redirect", "GET /", diff --git a/stac_fastapi/tests/conftest.py b/stac_fastapi/tests/conftest.py index 651cdadb..a82f1485 100644 --- a/stac_fastapi/tests/conftest.py +++ b/stac_fastapi/tests/conftest.py @@ -8,7 +8,8 @@ import pytest import pytest_asyncio from fastapi import Depends, HTTPException, security, status -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient +from pydantic import ConfigDict from stac_pydantic import api from stac_fastapi.api.app import StacApi @@ -85,8 +86,7 @@ def __init__( class TestSettings(AsyncSettings): - class Config: - env_file = ".env.test" + model_config = ConfigDict(env_file=".env.test") settings = TestSettings() @@ -243,7 +243,9 @@ async def app_client(app): await create_index_templates() await create_collection_index() - async with AsyncClient(app=app, base_url="http://test-server") as c: + async with AsyncClient( + transport=ASGITransport(app=app), base_url="http://test-server" + ) as c: yield c @@ -302,7 +304,9 @@ async def app_client_rate_limit(app_rate_limit): await create_index_templates() await create_collection_index() - async with AsyncClient(app=app_rate_limit, base_url="http://test-server") as c: + async with AsyncClient( + transport=ASGITransport(app=app_rate_limit), base_url="http://test-server" + ) as c: yield c @@ -392,7 +396,9 @@ async def app_client_basic_auth(app_basic_auth): await create_index_templates() await create_collection_index() - async with AsyncClient(app=app_basic_auth, base_url="http://test-server") as c: + async with AsyncClient( + transport=ASGITransport(app=app_basic_auth), base_url="http://test-server" + ) as c: yield c @@ -469,6 +475,7 @@ async def route_dependencies_client(route_dependencies_app): await create_collection_index() async with AsyncClient( - app=route_dependencies_app, base_url="http://test-server" + transport=ASGITransport(app=route_dependencies_app), + base_url="http://test-server", ) as c: yield c From 612c9790984300e565798f76ab2113a495fcd62e Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 19 Apr 2025 19:19:11 +0800 Subject: [PATCH 02/19] enable direct response to readme --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d6e648f3..e681d381 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,12 @@ - There is [Postman](https://documenter.getpostman.com/view/12888943/2s8ZDSdRHA) documentation here for examples on how to run some of the API routes locally - after starting the elasticsearch backend via the compose.yml file. - The `/examples` folder shows an example of running stac-fastapi-elasticsearch from PyPI in docker without needing any code from the repository. There is also a Postman collection here that you can load into Postman for testing the API routes. -- For changes, see the [Changelog](CHANGELOG.md) -- We are always welcoming contributions. For the development notes: [Contributing](CONTRIBUTING.md) + +### Performance Note + +The `enable_direct_response` option is provided by the stac-fastapi core library (introduced in stac-fastapi 5.2.0) and is available in this project starting from v4.0.0. When enabled, it allows endpoints to return Starlette Response objects directly, bypassing FastAPI's default serialization for improved performance. + +See: [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) ### To install from PyPI: From bc4ff4d0d4c1cacbf28c1e24740ca49b5e1e50af Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 19 Apr 2025 19:19:28 +0800 Subject: [PATCH 03/19] changelog reorg --- CHANGELOG.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b47f2d..66b873c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,17 +7,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] -### Changed -- Migrated Elasticsearch index template creation from legacy `put_template` to composable `put_index_template` API in `database_logic.py`. This resolves deprecation warnings and ensures compatibility with Elasticsearch 7.x and 8.x. -- Updated all Pydantic models to use `ConfigDict` instead of class-based `Config` for Pydantic v2 compatibility. This resolves deprecation warnings and prepares for Pydantic v3. -- Migrated all Pydantic `@root_validator` validators to `@model_validator` for Pydantic v2 compatibility. - ### Added -- Added `enable_direct_response` option to API settings for more flexible response handling. +- Added `enable_direct_response` option to API settings for more flexible response handling. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) ### Changed -- Updated test suite to use `httpx.ASGITransport(app=...)` for FastAPI app testing (removes deprecation warning). -- Updated stac-fastapi parent libraries to 5.2.0. +- Enabled `enable_direct_response` to improve response performance by bypassing FastAPI's jsonable_encoder and Pydantic serialization when returning Response objects. This change significantly improves performance for large search responses, as profiled in [this issue](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347). Note: This disables automatic Pydantic response validation for affected endpoints, but this is not used in our current implementation. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Updated test suite to use `httpx.ASGITransport(app=...)` for FastAPI app testing (removes deprecation warning). [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Updated stac-fastapi parent libraries to 5.2.0. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Migrated Elasticsearch index template creation from legacy `put_template` to composable `put_index_template` API in `database_logic.py`. This resolves deprecation warnings and ensures compatibility with Elasticsearch 7.x and 8.x. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Updated all Pydantic models to use `ConfigDict` instead of class-based `Config` for Pydantic v2 compatibility. This resolves deprecation warnings and prepares for Pydantic v3. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Migrated all Pydantic `@root_validator` validators to `@model_validator` for Pydantic v2 compatibility. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) ### Fixed From 013820f1ae6c710645f0b5695ca94038a685c44e Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 20 Apr 2025 00:29:50 +0800 Subject: [PATCH 04/19] changelog fix --- CHANGELOG.md | 53 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66b873c4..861f0d62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added + +### Changed + +### Fixed + +## [v4.0.0a2] - 2025-04-20 + ### Added - Added `enable_direct_response` option to API settings for more flexible response handling. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) @@ -350,25 +358,26 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v4.0.0a1...main -[v4.0.0a1]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v4.0.0a0...v4.0.0a1 -[v4.0.0a0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.5...v4.0.0a0 -[v3.2.5]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.4...v3.2.5 -[v3.2.4]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.3...v3.2.4 -[v3.2.3]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.2...v3.2.3 -[v3.2.2]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.1...v3.2.2 -[v3.2.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.0...v3.2.1 -[v3.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.1.0...v3.2.0 -[v3.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.0.0...v3.1.0 -[v3.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.4.1...v3.0.0 -[v2.4.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.4.0...v2.4.1 -[v2.4.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.3.0...v2.4.0 -[v2.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.2.0...v2.3.0 -[v2.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.1.0...v2.2.0 -[v2.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.0.0...v2.1.0 -[v2.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v1.1.0...v2.0.0 -[v1.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v1.0.0...v1.1.0 -[v1.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.3.0...v1.0.0 -[v0.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.2.0...v0.3.0 -[v0.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0...v0.2.0 -[v0.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0 +[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0a2...main +[v4.0.0a2]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0a1...v4.0.0a2 +[v4.0.0a1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0a0...v4.0.0a1 +[v4.0.0a0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.5...v4.0.0a0 +[v3.2.5]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.4...v3.2.5 +[v3.2.4]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.3...v3.2.4 +[v3.2.3]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.2...v3.2.3 +[v3.2.2]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.1...v3.2.2 +[v3.2.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.0...v3.2.1 +[v3.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.1.0...v3.2.0 +[v3.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.0.0...v3.1.0 +[v3.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.4.1...v3.0.0 +[v2.4.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.4.0...v2.4.1 +[v2.4.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.3.0...v2.4.0 +[v2.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.2.0...v2.3.0 +[v2.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.1.0...v2.2.0 +[v2.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v2.0.0...v2.1.0 +[v2.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v1.1.0...v2.0.0 +[v1.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v1.0.0...v1.1.0 +[v1.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v0.3.0...v1.0.0 +[v0.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v0.2.0...v0.3.0 +[v0.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v0.1.0...v0.2.0 +[v0.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v0.1.0 From b9f17ca0f67e6d4cb20af3fdba2130b45a139db5 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 20 Apr 2025 00:30:37 +0800 Subject: [PATCH 05/19] update to v4.0.0a2 --- compose.yml | 4 ++-- examples/auth/compose.basic_auth.yml | 4 ++-- examples/auth/compose.oauth2.yml | 4 ++-- examples/auth/compose.route_dependencies.yml | 4 ++-- examples/rate_limit/compose.rate_limit.yml | 4 ++-- stac_fastapi/core/stac_fastapi/core/version.py | 2 +- stac_fastapi/elasticsearch/setup.py | 2 +- .../elasticsearch/stac_fastapi/elasticsearch/version.py | 2 +- stac_fastapi/opensearch/setup.py | 2 +- stac_fastapi/opensearch/stac_fastapi/opensearch/version.py | 2 +- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/compose.yml b/compose.yml index a66e584f..8f982ccb 100644 --- a/compose.yml +++ b/compose.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -41,7 +41,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/compose.basic_auth.yml b/examples/auth/compose.basic_auth.yml index c3e069ec..88e95fa0 100644 --- a/examples/auth/compose.basic_auth.yml +++ b/examples/auth/compose.basic_auth.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/compose.oauth2.yml b/examples/auth/compose.oauth2.yml index ccd3bb1f..3a295862 100644 --- a/examples/auth/compose.oauth2.yml +++ b/examples/auth/compose.oauth2.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -43,7 +43,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/auth/compose.route_dependencies.yml b/examples/auth/compose.route_dependencies.yml index 0516fccd..08576691 100644 --- a/examples/auth/compose.route_dependencies.yml +++ b/examples/auth/compose.route_dependencies.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/examples/rate_limit/compose.rate_limit.yml b/examples/rate_limit/compose.rate_limit.yml index 3fa902ab..7d4340fb 100644 --- a/examples/rate_limit/compose.rate_limit.yml +++ b/examples/rate_limit/compose.rate_limit.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8080 - RELOAD=true @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=4.0.0a1 + - STAC_FASTAPI_VERSION=4.0.0a2 - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index af49b95b..2c71d558 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a1" +__version__ = "4.0.0a2" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 1377211b..77158e44 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==4.0.0a1", + "stac-fastapi-core==4.0.0a2", "elasticsearch[async]~=8.18.0", "uvicorn~=0.23.0", "starlette>=0.35.0,<0.36.0", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index af49b95b..2c71d558 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a1" +__version__ = "4.0.0a2" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index ece68679..4d718733 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==4.0.0a1", + "stac-fastapi-core==4.0.0a2", "opensearch-py~=2.8.0", "opensearch-py[async]~=2.8.0", "uvicorn~=0.23.0", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index af49b95b..2c71d558 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "4.0.0a1" +__version__ = "4.0.0a2" From afd528ba104302a6f05ca41b539ccb76d486384b Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 20 Apr 2025 00:39:14 +0800 Subject: [PATCH 06/19] changelog fix --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 861f0d62..9a74072c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,10 +16,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [v4.0.0a2] - 2025-04-20 ### Added -- Added `enable_direct_response` option to API settings for more flexible response handling. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Enabled `enable_direct_response` to improve response performance by bypassing FastAPI's jsonable_encoder and Pydantic serialization when returning Response objects. This change significantly improves performance for large search responses, as profiled in [this issue](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) ### Changed -- Enabled `enable_direct_response` to improve response performance by bypassing FastAPI's jsonable_encoder and Pydantic serialization when returning Response objects. This change significantly improves performance for large search responses, as profiled in [this issue](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347). Note: This disables automatic Pydantic response validation for affected endpoints, but this is not used in our current implementation. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - Updated test suite to use `httpx.ASGITransport(app=...)` for FastAPI app testing (removes deprecation warning). [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - Updated stac-fastapi parent libraries to 5.2.0. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - Migrated Elasticsearch index template creation from legacy `put_template` to composable `put_index_template` API in `database_logic.py`. This resolves deprecation warnings and ensures compatibility with Elasticsearch 7.x and 8.x. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) From d641f128a57ce71ef9aac21cbd7077fad8db4a0b Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 20 Apr 2025 18:48:31 +0800 Subject: [PATCH 07/19] move to settings --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py | 2 -- .../elasticsearch/stac_fastapi/elasticsearch/config.py | 4 ++++ stac_fastapi/opensearch/stac_fastapi/opensearch/app.py | 2 -- stac_fastapi/opensearch/stac_fastapi/opensearch/config.py | 4 ++++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 30fecb50..2e26fd83 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -96,8 +96,6 @@ search_get_request_model=create_get_request_model(search_extensions), search_post_request_model=post_request_model, route_dependencies=get_route_dependencies(), - enable_response_models=False, - enable_direct_response=True, ) app = api.app app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index fb9e2e0f..c08ab6e4 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -76,6 +76,8 @@ class ElasticsearchSettings(ApiSettings, ApiBaseSettings): # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} + enable_response_models: bool = False + enable_direct_response: bool = True @property def create_client(self): @@ -89,6 +91,8 @@ class AsyncElasticsearchSettings(ApiSettings, ApiBaseSettings): # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} + enable_response_models: bool = False + enable_direct_response: bool = True @property def create_client(self): diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index f157a0dc..a9cfd200 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -96,8 +96,6 @@ search_get_request_model=create_get_request_model(search_extensions), search_post_request_model=post_request_model, route_dependencies=get_route_dependencies(), - enable_response_models=False, - enable_direct_response=True, ) app = api.app app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py index 6de2ab91..889fb47a 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py @@ -74,6 +74,8 @@ class OpensearchSettings(ApiSettings, ApiBaseSettings): # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} + enable_response_models: bool = False + enable_direct_response: bool = True @property def create_client(self): @@ -87,6 +89,8 @@ class AsyncOpensearchSettings(ApiSettings, ApiBaseSettings): # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} + enable_response_models: bool = False + enable_direct_response: bool = True @property def create_client(self): From 8467be3755b6be9187351790a37c924fd408ac72 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 20 Apr 2025 23:28:54 +0800 Subject: [PATCH 08/19] enable direct response env variable --- CHANGELOG.md | 3 +- README.md | 12 +++++-- .../stac_fastapi/elasticsearch/config.py | 35 +++++++++++++++---- .../stac_fastapi/opensearch/config.py | 35 +++++++++++++++---- 4 files changed, 70 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a74072c..dcc6aaf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [v4.0.0a2] - 2025-04-20 ### Added -- Enabled `enable_direct_response` to improve response performance by bypassing FastAPI's jsonable_encoder and Pydantic serialization when returning Response objects. This change significantly improves performance for large search responses, as profiled in [this issue](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Added support for high-performance direct response mode for both Elasticsearch and Opensearch backends, controlled by the `ENABLE_DIRECT_RESPONSE` environment variable. When enabled (`ENABLE_DIRECT_RESPONSE=true`), endpoints return Starlette Response objects directly, bypassing FastAPI's jsonable_encoder and Pydantic serialization for significantly improved performance on large search responses. **Note:** In this mode, all FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes. Default is `false` for safety. A warning is logged at startup if enabled. See [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) and [PR #359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359). ### Changed - Updated test suite to use `httpx.ASGITransport(app=...)` for FastAPI app testing (removes deprecation warning). [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Migrated Elasticsearch index template creation from legacy `put_template` to composable `put_index_template` API in `database_logic.py`. This resolves deprecation warnings and ensures compatibility with Elasticsearch 7.x and 8.x. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - Updated all Pydantic models to use `ConfigDict` instead of class-based `Config` for Pydantic v2 compatibility. This resolves deprecation warnings and prepares for Pydantic v3. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - Migrated all Pydantic `@root_validator` validators to `@model_validator` for Pydantic v2 compatibility. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Migrated startup event handling from deprecated `@app.on_event("startup")` to FastAPI's recommended lifespan context manager. This removes deprecation warnings and ensures compatibility with future FastAPI versions. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) ### Fixed diff --git a/README.md b/README.md index e681d381..33de03f2 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,13 @@ ### Performance Note -The `enable_direct_response` option is provided by the stac-fastapi core library (introduced in stac-fastapi 5.2.0) and is available in this project starting from v4.0.0. When enabled, it allows endpoints to return Starlette Response objects directly, bypassing FastAPI's default serialization for improved performance. +The `enable_direct_response` option is provided by the stac-fastapi core library (introduced in stac-fastapi 5.2.0) and is available in this project starting from v4.0.0. + +**You can now control this setting via the `ENABLE_DIRECT_RESPONSE` environment variable.** + +When enabled (`ENABLE_DIRECT_RESPONSE=true`), endpoints return Starlette Response objects directly, bypassing FastAPI's default serialization for improved performance. **However, all FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes.** + +This mode is best suited for public or read-only APIs where authentication and custom logic are not required. Default is `false` for safety. See: [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) @@ -80,6 +86,7 @@ file named `.env` in the same directory you run Docker Compose from: ```shell ELASTICSEARCH_VERSION=7.17.1 OPENSEARCH_VERSION=2.11.0 +ENABLE_DIRECT_RESPONSE=false ``` The most recent Elasticsearch 7.x versions should also work. See the [opensearch-py docs](https://github.com/opensearch-project/opensearch-py/blob/main/COMPATIBILITY.md) for compatibility information. @@ -104,7 +111,8 @@ You can customize additional settings in your `.env` file: | `RELOAD` | Enable auto-reload for development. | `true` | Optional | | `STAC_FASTAPI_RATE_LIMIT` | API rate limit per client. | `200/minute` | Optional | | `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional | -| `ELASTICSEARCH_VERSION` | ElasticSearch version | `7.17.1` | Optional | +| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | N/A | Optional | +| `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional | | `OPENSEARCH_VERSION` | OpenSearch version | `2.11.0` | Optional | > [!NOTE] diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index c08ab6e4..08ea2483 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -1,5 +1,6 @@ """API configuration.""" +import logging import os import ssl from typing import Any, Dict, Set @@ -71,13 +72,20 @@ def _es_config() -> Dict[str, Any]: class ElasticsearchSettings(ApiSettings, ApiBaseSettings): - """API settings.""" + """ + API settings. + + Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable. + If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled. + Default is False for safety. + """ - # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} enable_response_models: bool = False - enable_direct_response: bool = True + enable_direct_response: bool = ( + os.getenv("ENABLE_DIRECT_RESPONSE", "false").lower() == "true" + ) @property def create_client(self): @@ -85,14 +93,29 @@ def create_client(self): return Elasticsearch(**_es_config()) +# Warn at import if direct response is enabled +if ElasticsearchSettings.enable_direct_response: + logging.basicConfig(level=logging.WARNING) + logging.warning( + "ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!" + ) + + class AsyncElasticsearchSettings(ApiSettings, ApiBaseSettings): - """API settings.""" + """ + API settings. + + Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable. + If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled. + Default is False for safety. + """ - # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} enable_response_models: bool = False - enable_direct_response: bool = True + enable_direct_response: bool = ( + os.getenv("ENABLE_DIRECT_RESPONSE", "false").lower() == "true" + ) @property def create_client(self): diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py index 889fb47a..00205fa7 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py @@ -1,4 +1,5 @@ """API configuration.""" +import logging import os import ssl from typing import Any, Dict, Set @@ -69,13 +70,20 @@ def _es_config() -> Dict[str, Any]: class OpensearchSettings(ApiSettings, ApiBaseSettings): - """API settings.""" + """ + API settings. + + Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable. + If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled. + Default is False for safety. + """ - # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} enable_response_models: bool = False - enable_direct_response: bool = True + enable_direct_response: bool = ( + os.getenv("ENABLE_DIRECT_RESPONSE", "false").lower() == "true" + ) @property def create_client(self): @@ -83,14 +91,29 @@ def create_client(self): return OpenSearch(**_es_config()) +# Warn at import if direct response is enabled +if OpensearchSettings.enable_direct_response: + logging.basicConfig(level=logging.WARNING) + logging.warning( + "ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!" + ) + + class AsyncOpensearchSettings(ApiSettings, ApiBaseSettings): - """API settings.""" + """ + API settings. + + Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable. + If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled. + Default is False for safety. + """ - # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} enable_response_models: bool = False - enable_direct_response: bool = True + enable_direct_response: bool = ( + os.getenv("ENABLE_DIRECT_RESPONSE", "false").lower() == "true" + ) @property def create_client(self): From 1aede0413d618b6efd6914017d72057088e8d3f3 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 20 Apr 2025 23:29:30 +0800 Subject: [PATCH 09/19] update methods --- .../elasticsearch/database_logic.py | 4 +- .../stac_fastapi/opensearch/database_logic.py | 46 ++++++++----------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 7ab044c2..175dc0e4 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -79,7 +79,7 @@ async def create_collection_index() -> None: await client.options(ignore_status=400).indices.create( index=f"{COLLECTIONS_INDEX}-000001", - aliases={COLLECTIONS_INDEX: {}}, + body={"aliases": {COLLECTIONS_INDEX: {}}}, ) await client.close() @@ -99,7 +99,7 @@ async def create_item_index(collection_id: str): await client.options(ignore_status=400).indices.create( index=f"{index_by_collection_id(collection_id)}-000001", - aliases={index_alias_by_collection_id(collection_id): {}}, + body={"aliases": {index_alias_by_collection_id(collection_id): {}}}, ) await client.close() diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 22e6ffe0..3184fa06 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -9,7 +9,6 @@ import attr from opensearchpy import exceptions, helpers -from opensearchpy.exceptions import TransportError from opensearchpy.helpers.query import Q from opensearchpy.helpers.search import Search from starlette.requests import Request @@ -80,24 +79,21 @@ async def create_collection_index() -> None: """ client = AsyncSearchSettings().create_client - search_body: Dict[str, Any] = { - "aliases": {COLLECTIONS_INDEX: {}}, - } - index = f"{COLLECTIONS_INDEX}-000001" - try: - await client.indices.create(index=index, body=search_body) - except TransportError as e: - if e.status_code == 400: - pass # Ignore 400 status codes - else: - raise e - + exists = await client.indices.exists(index=index) + if not exists: + await client.indices.create( + index=index, + body={ + "aliases": {COLLECTIONS_INDEX: {}}, + "mappings": ES_COLLECTIONS_MAPPINGS, + }, + ) await client.close() -async def create_item_index(collection_id: str): +async def create_item_index(collection_id: str) -> None: """ Create the index for Items. The settings of the index template will be used implicitly. @@ -109,24 +105,22 @@ async def create_item_index(collection_id: str): """ client = AsyncSearchSettings().create_client - search_body: Dict[str, Any] = { - "aliases": {index_alias_by_collection_id(collection_id): {}}, - } - try: + index_name = f"{index_by_collection_id(collection_id)}-000001" + exists = await client.indices.exists(index=index_name) + if not exists: await client.indices.create( - index=f"{index_by_collection_id(collection_id)}-000001", body=search_body + index=index_name, + body={ + "aliases": {index_alias_by_collection_id(collection_id): {}}, + "mappings": ES_ITEMS_MAPPINGS, + "settings": ES_ITEMS_SETTINGS, + }, ) - except TransportError as e: - if e.status_code == 400: - pass # Ignore 400 status codes - else: - raise e - await client.close() -async def delete_item_index(collection_id: str): +async def delete_item_index(collection_id: str) -> None: """Delete the index for items in a collection. Args: From 15bc499672933d591065efa161f8ee33721f26f1 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 20 Apr 2025 23:36:10 +0800 Subject: [PATCH 10/19] fix --- .../elasticsearch/stac_fastapi/elasticsearch/config.py | 2 +- stac_fastapi/opensearch/stac_fastapi/opensearch/config.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index 08ea2483..2e81fa42 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -94,7 +94,7 @@ def create_client(self): # Warn at import if direct response is enabled -if ElasticsearchSettings.enable_direct_response: +if ElasticsearchSettings().enable_direct_response: logging.basicConfig(level=logging.WARNING) logging.warning( "ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!" diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py index 00205fa7..068c3be8 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py @@ -91,8 +91,8 @@ def create_client(self): return OpenSearch(**_es_config()) -# Warn at import if direct response is enabled -if OpensearchSettings.enable_direct_response: +# Warn at import if direct response is enabled (applies to both settings) +if OpensearchSettings().enable_direct_response: logging.basicConfig(level=logging.WARNING) logging.warning( "ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!" From d88cdf483c017fd21106bdfa83e9d0d3e8f12969 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 21 Apr 2025 10:53:54 +0800 Subject: [PATCH 11/19] update es version makefile --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 33de03f2..896db23f 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,8 @@ If you wish to use a different version, put the following in a file named `.env` in the same directory you run Docker Compose from: ```shell -ELASTICSEARCH_VERSION=7.17.1 -OPENSEARCH_VERSION=2.11.0 +ELASTICSEARCH_VERSION=8.11.0 +OPENSEARCH_VERSION=2.11.1 ENABLE_DIRECT_RESPONSE=false ``` The most recent Elasticsearch 7.x versions should also work. See the [opensearch-py docs](https://github.com/opensearch-project/opensearch-py/blob/main/COMPATIBILITY.md) for compatibility information. @@ -111,9 +111,9 @@ You can customize additional settings in your `.env` file: | `RELOAD` | Enable auto-reload for development. | `true` | Optional | | `STAC_FASTAPI_RATE_LIMIT` | API rate limit per client. | `200/minute` | Optional | | `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional | -| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | N/A | Optional | +| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional | | `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional | -| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.0` | Optional | +| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional | > [!NOTE] > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, and `ES_VERIFY_CERTS` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch. From 2ef6813ac87af84c0c422e8ef10420fcc88ad2a4 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 21 Apr 2025 11:03:59 +0800 Subject: [PATCH 12/19] update versions in app --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py | 2 +- stac_fastapi/opensearch/stac_fastapi/opensearch/app.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 2e26fd83..91e239a4 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -87,7 +87,7 @@ api = StacApi( title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-elasticsearch"), description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-elasticsearch"), - api_version=os.getenv("STAC_FASTAPI_VERSION", "5.2.0"), + api_version=os.getenv("STAC_FASTAPI_VERSION", "4.0.0a2"), settings=settings, extensions=extensions, client=CoreClient( diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index a9cfd200..504d5eab 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -87,7 +87,7 @@ api = StacApi( title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-opensearch"), description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-opensearch"), - api_version=os.getenv("STAC_FASTAPI_VERSION", "5.2.0"), + api_version=os.getenv("STAC_FASTAPI_VERSION", "4.0.0a2"), settings=settings, extensions=extensions, client=CoreClient( From 222b5b3413a793aaf927fd61b39caadc6c4e7f7b Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 22 Apr 2025 11:25:16 +0800 Subject: [PATCH 13/19] use get bool env method --- CHANGELOG.md | 3 ++ .../core/stac_fastapi/core/utilities.py | 29 ++++++++++++++ .../stac_fastapi/elasticsearch/config.py | 15 +++---- .../stac_fastapi/opensearch/config.py | 11 +++--- .../elasticsearch/test_direct_response.py | 39 +++++++++++++++++++ 5 files changed, 82 insertions(+), 15 deletions(-) create mode 100644 stac_fastapi/tests/elasticsearch/test_direct_response.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dcc6aaf2..06dd7791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Added support for high-performance direct response mode for both Elasticsearch and Opensearch backends, controlled by the `ENABLE_DIRECT_RESPONSE` environment variable. When enabled (`ENABLE_DIRECT_RESPONSE=true`), endpoints return Starlette Response objects directly, bypassing FastAPI's jsonable_encoder and Pydantic serialization for significantly improved performance on large search responses. **Note:** In this mode, all FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes. Default is `false` for safety. A warning is logged at startup if enabled. See [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) and [PR #359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359). +- Added robust tests for the `ENABLE_DIRECT_RESPONSE` environment variable, covering both Elasticsearch and OpenSearch backends. Tests gracefully handle missing backends by attempting to import both configs and skipping if neither is available. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) ### Changed - Updated test suite to use `httpx.ASGITransport(app=...)` for FastAPI app testing (removes deprecation warning). [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) @@ -25,6 +26,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Updated all Pydantic models to use `ConfigDict` instead of class-based `Config` for Pydantic v2 compatibility. This resolves deprecation warnings and prepares for Pydantic v3. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - Migrated all Pydantic `@root_validator` validators to `@model_validator` for Pydantic v2 compatibility. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) - Migrated startup event handling from deprecated `@app.on_event("startup")` to FastAPI's recommended lifespan context manager. This removes deprecation warnings and ensures compatibility with future FastAPI versions. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) +- Refactored all boolean environment variable parsing in both Elasticsearch and OpenSearch backends to use the shared `get_bool_env` utility. This ensures robust and consistent handling of environment variables such as `ES_USE_SSL`, `ES_HTTP_COMPRESS`, and `ES_VERIFY_CERTS` across both backends. [#359](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/359) + ### Fixed diff --git a/stac_fastapi/core/stac_fastapi/core/utilities.py b/stac_fastapi/core/stac_fastapi/core/utilities.py index d8c69529..e7aafe67 100644 --- a/stac_fastapi/core/stac_fastapi/core/utilities.py +++ b/stac_fastapi/core/stac_fastapi/core/utilities.py @@ -3,6 +3,8 @@ This module contains functions for transforming geospatial coordinates, such as converting bounding boxes to polygon representations. """ +import logging +import os from typing import Any, Dict, List, Optional, Set, Union from stac_fastapi.types.stac import Item @@ -10,6 +12,33 @@ MAX_LIMIT = 10000 +def get_bool_env(name: str, default: bool = False) -> bool: + """ + Retrieve a boolean value from an environment variable. + + Args: + name (str): The name of the environment variable. + default (bool, optional): The default value to use if the variable is not set or unrecognized. Defaults to False. + + Returns: + bool: The boolean value parsed from the environment variable. + """ + value = os.getenv(name, str(default).lower()) + true_values = ("true", "1", "yes", "y") + false_values = ("false", "0", "no", "n") + if value.lower() in true_values: + return True + elif value.lower() in false_values: + return False + else: + logger = logging.getLogger(__name__) + logger.warning( + f"Environment variable '{name}' has unrecognized value '{value}'. " + f"Expected one of {true_values + false_values}. Using default: {default}" + ) + return default + + def bbox2polygon(b0: float, b1: float, b2: float, b3: float) -> List[List[List[float]]]: """Transform a bounding box represented by its four coordinates `b0`, `b1`, `b2`, and `b3` into a polygon. diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index 2e81fa42..8043e04a 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -9,12 +9,13 @@ from elasticsearch import AsyncElasticsearch, Elasticsearch # type: ignore from stac_fastapi.core.base_settings import ApiBaseSettings +from stac_fastapi.core.utilities import get_bool_env from stac_fastapi.types.config import ApiSettings def _es_config() -> Dict[str, Any]: # Determine the scheme (http or https) - use_ssl = os.getenv("ES_USE_SSL", "true").lower() == "true" + use_ssl = get_bool_env("ES_USE_SSL", default=True) scheme = "https" if use_ssl else "http" # Configure the hosts parameter with the correct scheme @@ -45,7 +46,7 @@ def _es_config() -> Dict[str, Any]: config["headers"] = headers - http_compress = os.getenv("ES_HTTP_COMPRESS", "true").lower() == "true" + http_compress = get_bool_env("ES_HTTP_COMPRESS", default=True) if http_compress: config["http_compress"] = True @@ -55,7 +56,7 @@ def _es_config() -> Dict[str, Any]: # Include SSL settings if using https config["ssl_version"] = ssl.TLSVersion.TLSv1_3 # type: ignore - config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" # type: ignore + config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) # type: ignore # Include CA Certificates if verifying certs if config["verify_certs"]: @@ -83,9 +84,7 @@ class ElasticsearchSettings(ApiSettings, ApiBaseSettings): forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} enable_response_models: bool = False - enable_direct_response: bool = ( - os.getenv("ENABLE_DIRECT_RESPONSE", "false").lower() == "true" - ) + enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False) @property def create_client(self): @@ -113,9 +112,7 @@ class AsyncElasticsearchSettings(ApiSettings, ApiBaseSettings): forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} enable_response_models: bool = False - enable_direct_response: bool = ( - os.getenv("ENABLE_DIRECT_RESPONSE", "false").lower() == "true" - ) + enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False) @property def create_client(self): diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py index 068c3be8..e6832f10 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py @@ -8,12 +8,13 @@ from opensearchpy import AsyncOpenSearch, OpenSearch from stac_fastapi.core.base_settings import ApiBaseSettings +from stac_fastapi.core.utilities import get_bool_env from stac_fastapi.types.config import ApiSettings def _es_config() -> Dict[str, Any]: # Determine the scheme (http or https) - use_ssl = os.getenv("ES_USE_SSL", "true").lower() == "true" + use_ssl = get_bool_env("ES_USE_SSL", default=True) scheme = "https" if use_ssl else "http" # Configure the hosts parameter with the correct scheme @@ -34,7 +35,7 @@ def _es_config() -> Dict[str, Any]: "headers": {"accept": "application/json", "Content-Type": "application/json"}, } - http_compress = os.getenv("ES_HTTP_COMPRESS", "true").lower() == "true" + http_compress = get_bool_env("ES_HTTP_COMPRESS", default=True) if http_compress: config["http_compress"] = True @@ -44,7 +45,7 @@ def _es_config() -> Dict[str, Any]: # Include SSL settings if using https config["ssl_version"] = ssl.PROTOCOL_SSLv23 # type: ignore - config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" # type: ignore + config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) # type: ignore # Include CA Certificates if verifying certs if config["verify_certs"]: @@ -81,9 +82,7 @@ class OpensearchSettings(ApiSettings, ApiBaseSettings): forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} enable_response_models: bool = False - enable_direct_response: bool = ( - os.getenv("ENABLE_DIRECT_RESPONSE", "false").lower() == "true" - ) + enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False) @property def create_client(self): diff --git a/stac_fastapi/tests/elasticsearch/test_direct_response.py b/stac_fastapi/tests/elasticsearch/test_direct_response.py new file mode 100644 index 00000000..bbbceb56 --- /dev/null +++ b/stac_fastapi/tests/elasticsearch/test_direct_response.py @@ -0,0 +1,39 @@ +import importlib + +import pytest + + +def get_settings_class(): + """ + Try to import ElasticsearchSettings or OpenSearchSettings, whichever is available. + Returns a tuple: (settings_class, config_module) + """ + try: + config = importlib.import_module("stac_fastapi.elasticsearch.config") + importlib.reload(config) + return config.ElasticsearchSettings, config + except ModuleNotFoundError: + try: + config = importlib.import_module("stac_fastapi.opensearch.config") + importlib.reload(config) + return config.OpensearchSettings, config + except ModuleNotFoundError: + pytest.skip( + "Neither Elasticsearch nor OpenSearch config module is available." + ) + + +def test_enable_direct_response_true(monkeypatch): + """Test that ENABLE_DIRECT_RESPONSE env var enables direct response config.""" + monkeypatch.setenv("ENABLE_DIRECT_RESPONSE", "true") + settings_class, _ = get_settings_class() + settings = settings_class() + assert settings.enable_direct_response is True + + +def test_enable_direct_response_false(monkeypatch): + """Test that ENABLE_DIRECT_RESPONSE env var disables direct response config.""" + monkeypatch.setenv("ENABLE_DIRECT_RESPONSE", "false") + settings_class, _ = get_settings_class() + settings = settings_class() + assert settings.enable_direct_response is False From aa3291816a21fb3c008d66b94eacd45d425bb4ba Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 22 Apr 2025 11:52:17 +0800 Subject: [PATCH 14/19] remove ignores --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py | 2 +- stac_fastapi/opensearch/stac_fastapi/opensearch/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index 8043e04a..ff660921 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -56,7 +56,7 @@ def _es_config() -> Dict[str, Any]: # Include SSL settings if using https config["ssl_version"] = ssl.TLSVersion.TLSv1_3 # type: ignore - config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) # type: ignore + config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) # Include CA Certificates if verifying certs if config["verify_certs"]: diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py index e6832f10..37211827 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py @@ -45,7 +45,7 @@ def _es_config() -> Dict[str, Any]: # Include SSL settings if using https config["ssl_version"] = ssl.PROTOCOL_SSLv23 # type: ignore - config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) # type: ignore + config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) # Include CA Certificates if verifying certs if config["verify_certs"]: From f1c38d811cf03b59ec988075177424f0dc2dc9a6 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 22 Apr 2025 12:15:34 +0800 Subject: [PATCH 15/19] log async --- .../stac_fastapi/elasticsearch/config.py | 19 ++++++++------- .../stac_fastapi/opensearch/config.py | 23 ++++++++++--------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index ff660921..74ed831a 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -92,14 +92,6 @@ def create_client(self): return Elasticsearch(**_es_config()) -# Warn at import if direct response is enabled -if ElasticsearchSettings().enable_direct_response: - logging.basicConfig(level=logging.WARNING) - logging.warning( - "ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!" - ) - - class AsyncElasticsearchSettings(ApiSettings, ApiBaseSettings): """ API settings. @@ -118,3 +110,14 @@ class AsyncElasticsearchSettings(ApiSettings, ApiBaseSettings): def create_client(self): """Create async elasticsearch client.""" return AsyncElasticsearch(**_es_config()) + + +# Warn at import if direct response is enabled (applies to either settings class) +if ( + ElasticsearchSettings().enable_direct_response + or AsyncElasticsearchSettings().enable_direct_response +): + logging.basicConfig(level=logging.WARNING) + logging.warning( + "ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!" + ) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py index 37211827..25700b12 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py @@ -90,14 +90,6 @@ def create_client(self): return OpenSearch(**_es_config()) -# Warn at import if direct response is enabled (applies to both settings) -if OpensearchSettings().enable_direct_response: - logging.basicConfig(level=logging.WARNING) - logging.warning( - "ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!" - ) - - class AsyncOpensearchSettings(ApiSettings, ApiBaseSettings): """ API settings. @@ -110,11 +102,20 @@ class AsyncOpensearchSettings(ApiSettings, ApiBaseSettings): forbidden_fields: Set[str] = _forbidden_fields indexed_fields: Set[str] = {"datetime"} enable_response_models: bool = False - enable_direct_response: bool = ( - os.getenv("ENABLE_DIRECT_RESPONSE", "false").lower() == "true" - ) + enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False) @property def create_client(self): """Create async elasticsearch client.""" return AsyncOpenSearch(**_es_config()) + + +# Warn at import if direct response is enabled (applies to either settings class) +if ( + OpensearchSettings().enable_direct_response + or AsyncOpensearchSettings().enable_direct_response +): + logging.basicConfig(level=logging.WARNING) + logging.warning( + "ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!" + ) From b89621c48ea3faa83419d1c1db4bcbe93f6bedb8 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 22 Apr 2025 12:18:38 +0800 Subject: [PATCH 16/19] remove ignore test --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index 74ed831a..3652b06b 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -55,7 +55,7 @@ def _es_config() -> Dict[str, Any]: return config # Include SSL settings if using https - config["ssl_version"] = ssl.TLSVersion.TLSv1_3 # type: ignore + config["ssl_version"] = ssl.TLSVersion.TLSv1_3 config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) # Include CA Certificates if verifying certs From f4e9c3f82368ba4a9dd15c737ade86c6f55f9b87 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 22 Apr 2025 12:22:26 +0800 Subject: [PATCH 17/19] ignore --- stac_fastapi/opensearch/stac_fastapi/opensearch/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py index 25700b12..00498468 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py @@ -44,7 +44,7 @@ def _es_config() -> Dict[str, Any]: return config # Include SSL settings if using https - config["ssl_version"] = ssl.PROTOCOL_SSLv23 # type: ignore + config["ssl_version"] = ssl.PROTOCOL_SSLv23 config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) # Include CA Certificates if verifying certs From 1e7346fd81cfc93293993034de3abc718f1f0c5d Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 22 Apr 2025 12:48:31 +0800 Subject: [PATCH 18/19] more type fixes --- stac_fastapi/core/stac_fastapi/core/core.py | 9 ++++++--- .../stac_fastapi/elasticsearch/config.py | 3 ++- .../stac_fastapi/elasticsearch/database_logic.py | 15 ++++++++------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 16197da3..3ac14efc 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -334,7 +334,7 @@ async def item_collection( search=search, limit=limit, sort=None, - token=token, # type: ignore + token=token, collection_ids=[collection_id], ) @@ -633,7 +633,7 @@ async def post_search( items, maybe_count, next_token = await self.database.execute_search( search=search, limit=limit, - token=search_request.token, # type: ignore + token=search_request.token, sort=sort, collection_ids=search_request.collections, ) @@ -701,7 +701,10 @@ async def create_item( database=self.database, settings=self.settings ) processed_items = [ - bulk_client.preprocess_item(item, base_url, BulkTransactionMethod.INSERT) for item in item["features"] # type: ignore + bulk_client.preprocess_item( + item, base_url, BulkTransactionMethod.INSERT + ) + for item in item["features"] ] await self.database.bulk_async( diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index 3652b06b..2044a4b2 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -6,8 +6,9 @@ from typing import Any, Dict, Set import certifi +from elasticsearch._async.client import AsyncElasticsearch -from elasticsearch import AsyncElasticsearch, Elasticsearch # type: ignore +from elasticsearch import Elasticsearch # type: ignore[attr-defined] from stac_fastapi.core.base_settings import ApiBaseSettings from stac_fastapi.core.utilities import get_bool_env from stac_fastapi.types.config import ApiSettings diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 175dc0e4..f57ef9bb 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -8,10 +8,11 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Type import attr +import elasticsearch.helpers as helpers from elasticsearch.dsl import Q, Search +from elasticsearch.exceptions import NotFoundError as ESNotFoundError from starlette.requests import Request -from elasticsearch import exceptions, helpers # type: ignore from stac_fastapi.core.base_database_logic import BaseDatabaseLogic from stac_fastapi.core.database_logic import ( COLLECTIONS_INDEX, @@ -271,7 +272,7 @@ async def get_one_item(self, collection_id: str, item_id: str) -> Dict: index=index_alias_by_collection_id(collection_id), id=mk_item_id(item_id, collection_id), ) - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError( f"Item {item_id} does not exist inside Collection {collection_id}" ) @@ -511,7 +512,7 @@ async def execute_search( try: es_response = await search_task - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError(f"Collections '{collection_ids}' do not exist") hits = es_response["hits"]["hits"] @@ -594,7 +595,7 @@ def _fill_aggregation_parameters(name: str, agg: dict) -> dict: try: db_response = await search_task - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError(f"Collections '{collection_ids}' do not exist") return db_response @@ -720,7 +721,7 @@ async def delete_item( id=mk_item_id(item_id, collection_id), refresh=refresh, ) - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError( f"Item {item_id} in collection {collection_id} not found" ) @@ -740,7 +741,7 @@ async def get_items_mapping(self, collection_id: str) -> Dict[str, Any]: index=index_name, allow_no_indices=False ) return mapping.body - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError(f"Mapping for index {index_name} not found") async def create_collection(self, collection: Collection, refresh: bool = False): @@ -791,7 +792,7 @@ async def find_collection(self, collection_id: str) -> Collection: collection = await self.client.get( index=COLLECTIONS_INDEX, id=collection_id ) - except exceptions.NotFoundError: + except ESNotFoundError: raise NotFoundError(f"Collection {collection_id} not found") return collection["_source"] From 2fd11fe742a9cfa25694dd45d432151854428846 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 23 Apr 2025 11:44:59 +0800 Subject: [PATCH 19/19] use lifespan context manager --- .../stac_fastapi/elasticsearch/app.py | 21 ++++++++++++------ .../opensearch/stac_fastapi/opensearch/app.py | 22 ++++++++++++------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 91e239a4..9aefca43 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -1,6 +1,9 @@ """FastAPI application.""" import os +from contextlib import asynccontextmanager + +from fastapi import FastAPI from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model @@ -97,17 +100,21 @@ search_post_request_model=post_request_model, route_dependencies=get_route_dependencies(), ) -app = api.app -app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") - -# Add rate limit -setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT")) -@app.on_event("startup") -async def _startup_event() -> None: +@asynccontextmanager +async def lifespan(app: FastAPI): + """Lifespan handler for FastAPI app. Initializes index templates and collections at startup.""" await create_index_templates() await create_collection_index() + yield + + +app = api.app +app.router.lifespan_context = lifespan +app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") +# Add rate limit +setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT")) def run() -> None: diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index 504d5eab..07c48e67 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -1,6 +1,9 @@ """FastAPI application.""" import os +from contextlib import asynccontextmanager + +from fastapi import FastAPI from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model @@ -97,18 +100,21 @@ search_post_request_model=post_request_model, route_dependencies=get_route_dependencies(), ) -app = api.app -app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") - - -# Add rate limit -setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT")) -@app.on_event("startup") -async def _startup_event() -> None: +@asynccontextmanager +async def lifespan(app: FastAPI): + """Lifespan handler for FastAPI app. Initializes index templates and collections at startup.""" await create_index_templates() await create_collection_index() + yield + + +app = api.app +app.router.lifespan_context = lifespan +app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "") +# Add rate limit +setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT")) def run() -> None: