From 5e99c8ab3394199adf185d489034b898dee79ed8 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Wed, 14 May 2025 12:05:18 +0800
Subject: [PATCH 01/29] sfeos helpers module
---
.github/workflows/cicd.yml | 4 +
.github/workflows/publish.yml | 12 ++
Makefile | 3 +-
dockerfiles/Dockerfile.ci.es | 1 +
dockerfiles/Dockerfile.ci.os | 1 +
dockerfiles/Dockerfile.deploy.es | 1 +
dockerfiles/Dockerfile.deploy.os | 1 +
dockerfiles/Dockerfile.dev.es | 1 +
dockerfiles/Dockerfile.dev.os | 1 +
dockerfiles/Dockerfile.docs | 1 +
stac_fastapi/core/stac_fastapi/core/core.py | 160 +-----------------
.../core/stac_fastapi/core/utilities.py | 40 -----
stac_fastapi/elasticsearch/setup.py | 1 +
.../stac_fastapi/elasticsearch/app.py | 2 +-
.../stac_fastapi/elasticsearch/config.py | 2 +-
.../elasticsearch/database_logic.py | 17 +-
stac_fastapi/opensearch/setup.py | 1 +
.../opensearch/stac_fastapi/opensearch/app.py | 2 +-
.../stac_fastapi/opensearch/config.py | 2 +-
.../stac_fastapi/opensearch/database_logic.py | 17 +-
stac_fastapi/sfeos_helpers/README.md | 1 +
stac_fastapi/sfeos_helpers/setup.cfg | 2 +
stac_fastapi/sfeos_helpers/setup.py | 34 ++++
.../stac_fastapi/sfeos_helpers/filter.py | 143 ++++++++++++++++
.../stac_fastapi/sfeos_helpers/mappings.py} | 28 ++-
.../stac_fastapi/sfeos_helpers/utilities.py | 45 +++++
.../stac_fastapi/sfeos_helpers/version.py | 2 +
27 files changed, 303 insertions(+), 222 deletions(-)
create mode 120000 stac_fastapi/sfeos_helpers/README.md
create mode 100644 stac_fastapi/sfeos_helpers/setup.cfg
create mode 100644 stac_fastapi/sfeos_helpers/setup.py
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
rename stac_fastapi/{core/stac_fastapi/core/database_logic.py => sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py} (90%)
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py
diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
index 864b52e3..abf6ebfa 100644
--- a/.github/workflows/cicd.yml
+++ b/.github/workflows/cicd.yml
@@ -96,6 +96,10 @@ jobs:
run: |
pip install ./stac_fastapi/core
+ - name: Install helpers library stac-fastapi
+ run: |
+ pip install ./stac_fastapi/sfeos_helpers
+
- name: Install elasticsearch stac-fastapi
run: |
pip install ./stac_fastapi/elasticsearch[dev,server]
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index eb84e7fc..8ed81ce6 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -35,6 +35,18 @@ jobs:
# Publish to PyPI
twine upload dist/*
+ - name: Build and publish sfeos_helpers
+ working-directory: stac_fastapi/sfeos_helpers
+ env:
+ TWINE_USERNAME: "__token__"
+ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+ run: |
+ # Build package
+ python setup.py sdist bdist_wheel
+
+ # Publish to PyPI
+ twine upload dist/*
+
- name: Build and publish stac-fastapi-elasticsearch
working-directory: stac_fastapi/elasticsearch
env:
diff --git a/Makefile b/Makefile
index 3440b7a2..5896e734 100644
--- a/Makefile
+++ b/Makefile
@@ -95,7 +95,8 @@ pybase-install:
pip install -e ./stac_fastapi/api[dev] && \
pip install -e ./stac_fastapi/types[dev] && \
pip install -e ./stac_fastapi/extensions[dev] && \
- pip install -e ./stac_fastapi/core
+ pip install -e ./stac_fastapi/core && \
+ pip install -e ./stac_fastapi/sfeos_helpers
.PHONY: install-es
install-es: pybase-install
diff --git a/dockerfiles/Dockerfile.ci.es b/dockerfiles/Dockerfile.ci.es
index a6fb6a53..5bd3853b 100644
--- a/dockerfiles/Dockerfile.ci.es
+++ b/dockerfiles/Dockerfile.ci.es
@@ -12,6 +12,7 @@ RUN apt-get update && \
COPY . /app/
RUN pip3 install --no-cache-dir -e ./stac_fastapi/core && \
+ pip3 install --no-cache-dir -e ./stac_fastapi/sfeos_helpers && \
pip3 install --no-cache-dir ./stac_fastapi/elasticsearch[server]
USER root
diff --git a/dockerfiles/Dockerfile.ci.os b/dockerfiles/Dockerfile.ci.os
index a046a3b6..e359f1a8 100644
--- a/dockerfiles/Dockerfile.ci.os
+++ b/dockerfiles/Dockerfile.ci.os
@@ -12,6 +12,7 @@ RUN apt-get update && \
COPY . /app/
RUN pip3 install --no-cache-dir -e ./stac_fastapi/core && \
+ pip3 install --no-cache-dir -e ./stac_fastapi/sfeos_helpers && \
pip3 install --no-cache-dir ./stac_fastapi/opensearch[server]
USER root
diff --git a/dockerfiles/Dockerfile.deploy.es b/dockerfiles/Dockerfile.deploy.es
index 2eab7b9d..2a6fc4fc 100644
--- a/dockerfiles/Dockerfile.deploy.es
+++ b/dockerfiles/Dockerfile.deploy.es
@@ -13,6 +13,7 @@ WORKDIR /app
COPY . /app
RUN pip install --no-cache-dir -e ./stac_fastapi/core
+RUN pip install --no-cache-dir -e ./stac_fastapi/sfeos_helpers
RUN pip install --no-cache-dir ./stac_fastapi/elasticsearch[server]
EXPOSE 8080
diff --git a/dockerfiles/Dockerfile.deploy.os b/dockerfiles/Dockerfile.deploy.os
index 035b181e..8a532f0c 100644
--- a/dockerfiles/Dockerfile.deploy.os
+++ b/dockerfiles/Dockerfile.deploy.os
@@ -13,6 +13,7 @@ WORKDIR /app
COPY . /app
RUN pip install --no-cache-dir -e ./stac_fastapi/core
+RUN pip install --no-cache-dir -e ./stac_fastapi/sfeos_helpers
RUN pip install --no-cache-dir ./stac_fastapi/opensearch[server]
EXPOSE 8080
diff --git a/dockerfiles/Dockerfile.dev.es b/dockerfiles/Dockerfile.dev.es
index 009f9681..7a01aca8 100644
--- a/dockerfiles/Dockerfile.dev.es
+++ b/dockerfiles/Dockerfile.dev.es
@@ -16,4 +16,5 @@ WORKDIR /app
COPY . /app
RUN pip install --no-cache-dir -e ./stac_fastapi/core
+RUN pip install --no-cache-dir -e ./stac_fastapi/sfeos_helpers
RUN pip install --no-cache-dir -e ./stac_fastapi/elasticsearch[dev,server]
diff --git a/dockerfiles/Dockerfile.dev.os b/dockerfiles/Dockerfile.dev.os
index d9dc8b0a..28012dfb 100644
--- a/dockerfiles/Dockerfile.dev.os
+++ b/dockerfiles/Dockerfile.dev.os
@@ -16,4 +16,5 @@ WORKDIR /app
COPY . /app
RUN pip install --no-cache-dir -e ./stac_fastapi/core
+RUN pip install --no-cache-dir -e ./stac_fastapi/sfeos_helpers
RUN pip install --no-cache-dir -e ./stac_fastapi/opensearch[dev,server]
diff --git a/dockerfiles/Dockerfile.docs b/dockerfiles/Dockerfile.docs
index f1fe63b8..aa080c7c 100644
--- a/dockerfiles/Dockerfile.docs
+++ b/dockerfiles/Dockerfile.docs
@@ -12,6 +12,7 @@ WORKDIR /opt/src
RUN python -m pip install \
stac_fastapi/core \
+ stac_fastapi/sfeos_helpers \
stac_fastapi/elasticsearch \
stac_fastapi/opensearch
diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py
index 987acdf6..415d202b 100644
--- a/stac_fastapi/core/stac_fastapi/core/core.py
+++ b/stac_fastapi/core/stac_fastapi/core/core.py
@@ -1,11 +1,10 @@
"""Core client."""
import logging
-from collections import deque
from datetime import datetime as datetime_type
from datetime import timezone
from enum import Enum
-from typing import Any, Dict, List, Literal, Optional, Set, Type, Union
+from typing import Dict, List, Optional, Set, Type, Union
from urllib.parse import unquote_plus, urljoin
import attr
@@ -26,7 +25,6 @@
from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
from stac_fastapi.core.session import Session
from stac_fastapi.core.utilities import filter_fields
-from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient
from stac_fastapi.extensions.third_party.bulk_transactions import (
BaseBulkTransactionsClient,
BulkTransactionMethod,
@@ -947,159 +945,3 @@ def bulk_item_insert(
logger.info(f"Bulk sync operation succeeded with {success} actions.")
return f"Successfully added/updated {success} Items. {attempted - success} errors occurred."
-
-
-_DEFAULT_QUERYABLES: Dict[str, Dict[str, Any]] = {
- "id": {
- "description": "ID",
- "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id",
- },
- "collection": {
- "description": "Collection",
- "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/then/properties/collection",
- },
- "geometry": {
- "description": "Geometry",
- "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/1/oneOf/0/properties/geometry",
- },
- "datetime": {
- "description": "Acquisition Timestamp",
- "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/datetime",
- },
- "created": {
- "description": "Creation Timestamp",
- "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/created",
- },
- "updated": {
- "description": "Creation Timestamp",
- "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/updated",
- },
- "cloud_cover": {
- "description": "Cloud Cover",
- "$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fields/properties/eo:cloud_cover",
- },
- "cloud_shadow_percentage": {
- "title": "Cloud Shadow Percentage",
- "description": "Cloud Shadow Percentage",
- "type": "number",
- "minimum": 0,
- "maximum": 100,
- },
- "nodata_pixel_percentage": {
- "title": "No Data Pixel Percentage",
- "description": "No Data Pixel Percentage",
- "type": "number",
- "minimum": 0,
- "maximum": 100,
- },
-}
-
-_ES_MAPPING_TYPE_TO_JSON: Dict[
- str, Literal["string", "number", "boolean", "object", "array", "null"]
-] = {
- "date": "string",
- "date_nanos": "string",
- "keyword": "string",
- "match_only_text": "string",
- "text": "string",
- "wildcard": "string",
- "byte": "number",
- "double": "number",
- "float": "number",
- "half_float": "number",
- "long": "number",
- "scaled_float": "number",
- "short": "number",
- "token_count": "number",
- "unsigned_long": "number",
- "geo_point": "object",
- "geo_shape": "object",
- "nested": "array",
-}
-
-
-@attr.s
-class EsAsyncBaseFiltersClient(AsyncBaseFiltersClient):
- """Defines a pattern for implementing the STAC filter extension."""
-
- database: BaseDatabaseLogic = attr.ib()
-
- async def get_queryables(
- self, collection_id: Optional[str] = None, **kwargs
- ) -> Dict[str, Any]:
- """Get the queryables available for the given collection_id.
-
- If collection_id is None, returns the intersection of all
- queryables over all collections.
-
- This base implementation returns a blank queryable schema. This is not allowed
- under OGC CQL but it is allowed by the STAC API Filter Extension
-
- https://github.com/radiantearth/stac-api-spec/tree/master/fragments/filter#queryables
-
- Args:
- collection_id (str, optional): The id of the collection to get queryables for.
- **kwargs: additional keyword arguments
-
- Returns:
- Dict[str, Any]: A dictionary containing the queryables for the given collection.
- """
- queryables: Dict[str, Any] = {
- "$schema": "https://json-schema.org/draft/2019-09/schema",
- "$id": "https://stac-api.example.com/queryables",
- "type": "object",
- "title": "Queryables for STAC API",
- "description": "Queryable names for the STAC API Item Search filter.",
- "properties": _DEFAULT_QUERYABLES,
- "additionalProperties": True,
- }
- if not collection_id:
- return queryables
-
- properties: Dict[str, Any] = queryables["properties"]
- queryables.update(
- {
- "properties": properties,
- "additionalProperties": False,
- }
- )
-
- mapping_data = await self.database.get_items_mapping(collection_id)
- mapping_properties = next(iter(mapping_data.values()))["mappings"]["properties"]
- stack = deque(mapping_properties.items())
-
- while stack:
- field_name, field_def = stack.popleft()
-
- # Iterate over nested fields
- field_properties = field_def.get("properties")
- if field_properties:
- # Fields in Item Properties should be exposed with their un-prefixed names,
- # and not require expressions to prefix them with properties,
- # e.g., eo:cloud_cover instead of properties.eo:cloud_cover.
- if field_name == "properties":
- stack.extend(field_properties.items())
- else:
- stack.extend(
- (f"{field_name}.{k}", v) for k, v in field_properties.items()
- )
-
- # Skip non-indexed or disabled fields
- field_type = field_def.get("type")
- if not field_type or not field_def.get("enabled", True):
- continue
-
- # Generate field properties
- field_result = _DEFAULT_QUERYABLES.get(field_name, {})
- properties[field_name] = field_result
-
- field_name_human = field_name.replace("_", " ").title()
- field_result.setdefault("title", field_name_human)
-
- field_type_json = _ES_MAPPING_TYPE_TO_JSON.get(field_type, field_type)
- field_result.setdefault("type", field_type_json)
-
- if field_type in {"date", "date_nanos"}:
- field_result.setdefault("format", "date-time")
-
- return queryables
diff --git a/stac_fastapi/core/stac_fastapi/core/utilities.py b/stac_fastapi/core/stac_fastapi/core/utilities.py
index d4a35109..be197f71 100644
--- a/stac_fastapi/core/stac_fastapi/core/utilities.py
+++ b/stac_fastapi/core/stac_fastapi/core/utilities.py
@@ -12,46 +12,6 @@
MAX_LIMIT = 10000
-def validate_refresh(value: Union[str, bool]) -> str:
- """
- Validate the `refresh` parameter value.
-
- Args:
- value (Union[str, bool]): The `refresh` parameter value, which can be a string or a boolean.
-
- Returns:
- str: The validated value of the `refresh` parameter, which can be "true", "false", or "wait_for".
- """
- logger = logging.getLogger(__name__)
-
- # Handle boolean-like values using get_bool_env
- if isinstance(value, bool) or value in {
- "true",
- "false",
- "1",
- "0",
- "yes",
- "no",
- "y",
- "n",
- }:
- is_true = get_bool_env("DATABASE_REFRESH", default=value)
- return "true" if is_true else "false"
-
- # Normalize to lowercase for case-insensitivity
- value = value.lower()
-
- # Handle "wait_for" explicitly
- if value == "wait_for":
- return "wait_for"
-
- # Log a warning for invalid values and default to "false"
- logger.warning(
- f"Invalid value for `refresh`: '{value}'. Expected 'true', 'false', or 'wait_for'. Defaulting to 'false'."
- )
- return "false"
-
-
def get_bool_env(name: str, default: Union[bool, str] = False) -> bool:
"""
Retrieve a boolean value from an environment variable.
diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py
index fe12fb07..e24aa5d6 100644
--- a/stac_fastapi/elasticsearch/setup.py
+++ b/stac_fastapi/elasticsearch/setup.py
@@ -7,6 +7,7 @@
install_requires = [
"stac-fastapi-core==4.1.0",
+ "sfeos-helpers==4.1.0",
"elasticsearch[async]~=8.18.0",
"uvicorn~=0.23.0",
"starlette>=0.35.0,<0.36.0",
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py
index 6747af39..96b0cb6f 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py
@@ -11,7 +11,6 @@
from stac_fastapi.core.core import (
BulkTransactionsClient,
CoreClient,
- EsAsyncBaseFiltersClient,
TransactionsClient,
)
from stac_fastapi.core.extensions import QueryExtension
@@ -40,6 +39,7 @@
TransactionExtension,
)
from stac_fastapi.extensions.third_party import BulkTransactionExtension
+from stac_fastapi.sfeos_helpers.filter import EsAsyncBaseFiltersClient
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py
index accbe8cc..1321a0f7 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py
@@ -10,7 +10,7 @@
from elasticsearch import Elasticsearch # type: ignore[attr-defined]
from stac_fastapi.core.base_settings import ApiBaseSettings
-from stac_fastapi.core.utilities import get_bool_env, validate_refresh
+from stac_fastapi.sfeos_helpers.utilities import get_bool_env, validate_refresh
from stac_fastapi.types.config import ApiSettings
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
index 7afbb58d..9385fe86 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
@@ -14,7 +14,14 @@
from starlette.requests import Request
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
-from stac_fastapi.core.database_logic import (
+from stac_fastapi.core.extensions import filter
+from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
+from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon
+from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
+from stac_fastapi.elasticsearch.config import (
+ ElasticsearchSettings as SyncElasticsearchSettings,
+)
+from stac_fastapi.sfeos_helpers.mappings import (
COLLECTIONS_INDEX,
DEFAULT_SORT,
ES_COLLECTIONS_MAPPINGS,
@@ -29,13 +36,7 @@
mk_actions,
mk_item_id,
)
-from stac_fastapi.core.extensions import filter
-from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
-from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon, validate_refresh
-from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
-from stac_fastapi.elasticsearch.config import (
- ElasticsearchSettings as SyncElasticsearchSettings,
-)
+from stac_fastapi.sfeos_helpers.utilities import validate_refresh
from stac_fastapi.types.errors import ConflictError, NotFoundError
from stac_fastapi.types.stac import Collection, Item
diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py
index ab9e4018..c5aefaff 100644
--- a/stac_fastapi/opensearch/setup.py
+++ b/stac_fastapi/opensearch/setup.py
@@ -7,6 +7,7 @@
install_requires = [
"stac-fastapi-core==4.1.0",
+ "sfeos-helpers==4.1.0",
"opensearch-py~=2.8.0",
"opensearch-py[async]~=2.8.0",
"uvicorn~=0.23.0",
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py
index 99e56ff9..6f4488a8 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py
@@ -11,7 +11,6 @@
from stac_fastapi.core.core import (
BulkTransactionsClient,
CoreClient,
- EsAsyncBaseFiltersClient,
TransactionsClient,
)
from stac_fastapi.core.extensions import QueryExtension
@@ -40,6 +39,7 @@
create_collection_index,
create_index_templates,
)
+from stac_fastapi.sfeos_helpers.filter import EsAsyncBaseFiltersClient
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py
index 3a53ffdf..f707cf45 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py
@@ -8,7 +8,7 @@
from opensearchpy import AsyncOpenSearch, OpenSearch
from stac_fastapi.core.base_settings import ApiBaseSettings
-from stac_fastapi.core.utilities import get_bool_env, validate_refresh
+from stac_fastapi.sfeos_helpers.utilities import get_bool_env, validate_refresh
from stac_fastapi.types.config import ApiSettings
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
index 5b9510f3..2f4c1806 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
@@ -14,7 +14,14 @@
from starlette.requests import Request
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
-from stac_fastapi.core.database_logic import (
+from stac_fastapi.core.extensions import filter
+from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
+from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon
+from stac_fastapi.opensearch.config import (
+ AsyncOpensearchSettings as AsyncSearchSettings,
+)
+from stac_fastapi.opensearch.config import OpensearchSettings as SyncSearchSettings
+from stac_fastapi.sfeos_helpers.mappings import (
COLLECTIONS_INDEX,
DEFAULT_SORT,
ES_COLLECTIONS_MAPPINGS,
@@ -29,13 +36,7 @@
mk_actions,
mk_item_id,
)
-from stac_fastapi.core.extensions import filter
-from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
-from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon, validate_refresh
-from stac_fastapi.opensearch.config import (
- AsyncOpensearchSettings as AsyncSearchSettings,
-)
-from stac_fastapi.opensearch.config import OpensearchSettings as SyncSearchSettings
+from stac_fastapi.sfeos_helpers.utilities import validate_refresh
from stac_fastapi.types.errors import ConflictError, NotFoundError
from stac_fastapi.types.stac import Collection, Item
diff --git a/stac_fastapi/sfeos_helpers/README.md b/stac_fastapi/sfeos_helpers/README.md
new file mode 120000
index 00000000..fe840054
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/README.md
@@ -0,0 +1 @@
+../../README.md
\ No newline at end of file
diff --git a/stac_fastapi/sfeos_helpers/setup.cfg b/stac_fastapi/sfeos_helpers/setup.cfg
new file mode 100644
index 00000000..a3210acb
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/setup.cfg
@@ -0,0 +1,2 @@
+[metadata]
+version = attr: stac_fastapi.sfeos_helpers.version.__version__
diff --git a/stac_fastapi/sfeos_helpers/setup.py b/stac_fastapi/sfeos_helpers/setup.py
new file mode 100644
index 00000000..8b26c9de
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/setup.py
@@ -0,0 +1,34 @@
+"""stac_fastapi: helpers elasticsearch/ opensearch module."""
+
+from setuptools import find_namespace_packages, setup
+
+with open("README.md") as f:
+ desc = f.read()
+
+install_requires = [
+ "stac-fastapi.core==4.1.0",
+]
+
+setup(
+ name="sfeos_helpers",
+ description="Helper library for the Elasticsearch and Opensearch stac-fastapi backends.",
+ long_description=desc,
+ long_description_content_type="text/markdown",
+ python_requires=">=3.9",
+ classifiers=[
+ "Intended Audience :: Developers",
+ "Intended Audience :: Information Technology",
+ "Intended Audience :: Science/Research",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
+ "License :: OSI Approved :: MIT License",
+ ],
+ url="https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch",
+ license="MIT",
+ packages=find_namespace_packages(),
+ zip_safe=False,
+ install_requires=install_requires,
+)
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
new file mode 100644
index 00000000..a2c39845
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
@@ -0,0 +1,143 @@
+"""Shared filter extension methods for stac-fastapi elasticsearch and opensearch backends."""
+
+from collections import deque
+from typing import Any, Dict, Optional
+
+import attr
+
+from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
+from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient
+
+from .mappings import ES_MAPPING_TYPE_TO_JSON
+
+_DEFAULT_QUERYABLES: Dict[str, Dict[str, Any]] = {
+ "id": {
+ "description": "ID",
+ "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id",
+ },
+ "collection": {
+ "description": "Collection",
+ "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/then/properties/collection",
+ },
+ "geometry": {
+ "description": "Geometry",
+ "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/1/oneOf/0/properties/geometry",
+ },
+ "datetime": {
+ "description": "Acquisition Timestamp",
+ "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/datetime",
+ },
+ "created": {
+ "description": "Creation Timestamp",
+ "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/created",
+ },
+ "updated": {
+ "description": "Creation Timestamp",
+ "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/updated",
+ },
+ "cloud_cover": {
+ "description": "Cloud Cover",
+ "$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fields/properties/eo:cloud_cover",
+ },
+ "cloud_shadow_percentage": {
+ "title": "Cloud Shadow Percentage",
+ "description": "Cloud Shadow Percentage",
+ "type": "number",
+ "minimum": 0,
+ "maximum": 100,
+ },
+ "nodata_pixel_percentage": {
+ "title": "No Data Pixel Percentage",
+ "description": "No Data Pixel Percentage",
+ "type": "number",
+ "minimum": 0,
+ "maximum": 100,
+ },
+}
+
+
+@attr.s
+class EsAsyncBaseFiltersClient(AsyncBaseFiltersClient):
+ """Defines a pattern for implementing the STAC filter extension."""
+
+ database: BaseDatabaseLogic = attr.ib()
+
+ async def get_queryables(
+ self, collection_id: Optional[str] = None, **kwargs
+ ) -> Dict[str, Any]:
+ """Get the queryables available for the given collection_id.
+
+ If collection_id is None, returns the intersection of all
+ queryables over all collections.
+
+ This base implementation returns a blank queryable schema. This is not allowed
+ under OGC CQL but it is allowed by the STAC API Filter Extension
+
+ https://github.com/radiantearth/stac-api-spec/tree/master/fragments/filter#queryables
+
+ Args:
+ collection_id (str, optional): The id of the collection to get queryables for.
+ **kwargs: additional keyword arguments
+
+ Returns:
+ Dict[str, Any]: A dictionary containing the queryables for the given collection.
+ """
+ queryables: Dict[str, Any] = {
+ "$schema": "https://json-schema.org/draft/2019-09/schema",
+ "$id": "https://stac-api.example.com/queryables",
+ "type": "object",
+ "title": "Queryables for STAC API",
+ "description": "Queryable names for the STAC API Item Search filter.",
+ "properties": _DEFAULT_QUERYABLES,
+ "additionalProperties": True,
+ }
+ if not collection_id:
+ return queryables
+
+ properties: Dict[str, Any] = queryables["properties"]
+ queryables.update(
+ {
+ "properties": properties,
+ "additionalProperties": False,
+ }
+ )
+
+ mapping_data = await self.database.get_items_mapping(collection_id)
+ mapping_properties = next(iter(mapping_data.values()))["mappings"]["properties"]
+ stack = deque(mapping_properties.items())
+
+ while stack:
+ field_name, field_def = stack.popleft()
+
+ # Iterate over nested fields
+ field_properties = field_def.get("properties")
+ if field_properties:
+ # Fields in Item Properties should be exposed with their un-prefixed names,
+ # and not require expressions to prefix them with properties,
+ # e.g., eo:cloud_cover instead of properties.eo:cloud_cover.
+ if field_name == "properties":
+ stack.extend(field_properties.items())
+ else:
+ stack.extend(
+ (f"{field_name}.{k}", v) for k, v in field_properties.items()
+ )
+
+ # Skip non-indexed or disabled fields
+ field_type = field_def.get("type")
+ if not field_type or not field_def.get("enabled", True):
+ continue
+
+ # Generate field properties
+ field_result = _DEFAULT_QUERYABLES.get(field_name, {})
+ properties[field_name] = field_result
+
+ field_name_human = field_name.replace("_", " ").title()
+ field_result.setdefault("title", field_name_human)
+
+ field_type_json = ES_MAPPING_TYPE_TO_JSON.get(field_type, field_type)
+ field_result.setdefault("type", field_type_json)
+
+ if field_type in {"date", "date_nanos"}:
+ field_result.setdefault("format", "date-time")
+
+ return queryables
diff --git a/stac_fastapi/core/stac_fastapi/core/database_logic.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py
similarity index 90%
rename from stac_fastapi/core/stac_fastapi/core/database_logic.py
rename to stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py
index 85ebcf21..246c27d6 100644
--- a/stac_fastapi/core/stac_fastapi/core/database_logic.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py
@@ -1,8 +1,8 @@
-"""Database logic core."""
+"""Shared mappings for stac-fastapi elasticsearch and opensearch backends."""
import os
from functools import lru_cache
-from typing import Any, Dict, List, Optional, Protocol
+from typing import Any, Dict, List, Literal, Optional, Protocol
from stac_fastapi.types.stac import Item
@@ -145,6 +145,30 @@ class Geometry(Protocol): # noqa
}
+ES_MAPPING_TYPE_TO_JSON: Dict[
+ str, Literal["string", "number", "boolean", "object", "array", "null"]
+] = {
+ "date": "string",
+ "date_nanos": "string",
+ "keyword": "string",
+ "match_only_text": "string",
+ "text": "string",
+ "wildcard": "string",
+ "byte": "number",
+ "double": "number",
+ "float": "number",
+ "half_float": "number",
+ "long": "number",
+ "scaled_float": "number",
+ "short": "number",
+ "token_count": "number",
+ "unsigned_long": "number",
+ "geo_point": "object",
+ "geo_shape": "object",
+ "nested": "array",
+}
+
+
@lru_cache(256)
def index_by_collection_id(collection_id: str) -> str:
"""
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py
new file mode 100644
index 00000000..fa0bd194
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py
@@ -0,0 +1,45 @@
+"""Shared utilities functions for stac-fastapi elasticsearch and opensearch backends."""
+import logging
+from typing import Union
+
+from stac_fastapi.core.utilities import get_bool_env
+
+
+def validate_refresh(value: Union[str, bool]) -> str:
+ """
+ Validate the `refresh` parameter value.
+
+ Args:
+ value (Union[str, bool]): The `refresh` parameter value, which can be a string or a boolean.
+
+ Returns:
+ str: The validated value of the `refresh` parameter, which can be "true", "false", or "wait_for".
+ """
+ logger = logging.getLogger(__name__)
+
+ # Handle boolean-like values using get_bool_env
+ if isinstance(value, bool) or value in {
+ "true",
+ "false",
+ "1",
+ "0",
+ "yes",
+ "no",
+ "y",
+ "n",
+ }:
+ is_true = get_bool_env("DATABASE_REFRESH", default=value)
+ return "true" if is_true else "false"
+
+ # Normalize to lowercase for case-insensitivity
+ value = value.lower()
+
+ # Handle "wait_for" explicitly
+ if value == "wait_for":
+ return "wait_for"
+
+ # Log a warning for invalid values and default to "false"
+ logger.warning(
+ f"Invalid value for `refresh`: '{value}'. Expected 'true', 'false', or 'wait_for'. Defaulting to 'false'."
+ )
+ return "false"
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py
new file mode 100644
index 00000000..e42ce685
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py
@@ -0,0 +1,2 @@
+"""library version."""
+__version__ = "4.1.0"
From d3d4b16d77a681a5d87b58ba7be79c9a01d81971 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Wed, 14 May 2025 14:31:28 +0800
Subject: [PATCH 02/29] move date fns, fliter fns
---
stac_fastapi/core/stac_fastapi/core/core.py | 91 +---------
.../core/stac_fastapi/core/datetime_utils.py | 89 +++++++++-
.../stac_fastapi/core/extensions/filter.py | 147 +++++-----------
.../elasticsearch/database_logic.py | 2 +-
.../stac_fastapi/opensearch/database_logic.py | 2 +-
.../stac_fastapi/sfeos_helpers/filter.py | 158 +++++++++++++-----
6 files changed, 251 insertions(+), 238 deletions(-)
diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py
index 415d202b..3bfe32f8 100644
--- a/stac_fastapi/core/stac_fastapi/core/core.py
+++ b/stac_fastapi/core/stac_fastapi/core/core.py
@@ -4,7 +4,7 @@
from datetime import datetime as datetime_type
from datetime import timezone
from enum import Enum
-from typing import Dict, List, Optional, Set, Type, Union
+from typing import List, Optional, Set, Type, Union
from urllib.parse import unquote_plus, urljoin
import attr
@@ -21,6 +21,7 @@
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
from stac_fastapi.core.base_settings import ApiBaseSettings
+from stac_fastapi.core.datetime_utils import format_datetime_range, return_date
from stac_fastapi.core.models.links import PagingLinks
from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
from stac_fastapi.core.session import Session
@@ -35,7 +36,6 @@
from stac_fastapi.types.core import AsyncBaseCoreClient, AsyncBaseTransactionsClient
from stac_fastapi.types.extension import ApiExtension
from stac_fastapi.types.requests import get_base_url
-from stac_fastapi.types.rfc3339 import DateTimeType, rfc3339_str_to_datetime
from stac_fastapi.types.search import BaseSearchPostRequest
logger = logging.getLogger(__name__)
@@ -316,7 +316,7 @@ async def item_collection(
)
if datetime:
- datetime_search = self._return_date(datetime)
+ datetime_search = return_date(datetime)
search = self.database.apply_datetime_filter(
search=search, datetime_search=datetime_search
)
@@ -372,87 +372,6 @@ async def get_item(
)
return self.item_serializer.db_to_stac(item, base_url)
- @staticmethod
- def _return_date(
- interval: Optional[Union[DateTimeType, str]]
- ) -> Dict[str, Optional[str]]:
- """
- Convert a date interval.
-
- (which may be a datetime, a tuple of one or two datetimes a string
- representing a datetime or range, or None) into a dictionary for filtering
- search results with Elasticsearch.
-
- This function ensures the output dictionary contains 'gte' and 'lte' keys,
- even if they are set to None, to prevent KeyError in the consuming logic.
-
- Args:
- interval (Optional[Union[DateTimeType, str]]): The date interval, which might be a single datetime,
- a tuple with one or two datetimes, a string, or None.
-
- Returns:
- dict: A dictionary representing the date interval for use in filtering search results,
- always containing 'gte' and 'lte' keys.
- """
- result: Dict[str, Optional[str]] = {"gte": None, "lte": None}
-
- if interval is None:
- return result
-
- if isinstance(interval, str):
- if "/" in interval:
- parts = interval.split("/")
- result["gte"] = parts[0] if parts[0] != ".." else None
- result["lte"] = (
- parts[1] if len(parts) > 1 and parts[1] != ".." else None
- )
- else:
- converted_time = interval if interval != ".." else None
- result["gte"] = result["lte"] = converted_time
- return result
-
- if isinstance(interval, datetime_type):
- datetime_iso = interval.isoformat()
- result["gte"] = result["lte"] = datetime_iso
- elif isinstance(interval, tuple):
- start, end = interval
- # Ensure datetimes are converted to UTC and formatted with 'Z'
- if start:
- result["gte"] = start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
- if end:
- result["lte"] = end.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
-
- return result
-
- def _format_datetime_range(self, date_str: str) -> str:
- """
- Convert a datetime range string into a normalized UTC string for API requests using rfc3339_str_to_datetime.
-
- Args:
- date_str (str): A string containing two datetime values separated by a '/'.
-
- Returns:
- str: A string formatted as 'YYYY-MM-DDTHH:MM:SSZ/YYYY-MM-DDTHH:MM:SSZ', with '..' used if any element is None.
- """
-
- def normalize(dt):
- dt = dt.strip()
- if not dt or dt == "..":
- return ".."
- dt_obj = rfc3339_str_to_datetime(dt)
- dt_utc = dt_obj.astimezone(timezone.utc)
- return dt_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
-
- if not isinstance(date_str, str):
- return "../.."
- if "/" not in date_str:
- return f"{normalize(date_str)}/{normalize(date_str)}"
- try:
- start, end = date_str.split("/", 1)
- except Exception:
- return "../.."
- return f"{normalize(start)}/{normalize(end)}"
-
async def get_search(
self,
request: Request,
@@ -504,7 +423,7 @@ async def get_search(
}
if datetime:
- base_args["datetime"] = self._format_datetime_range(date_str=datetime)
+ base_args["datetime"] = format_datetime_range(date_str=datetime)
if intersects:
base_args["intersects"] = orjson.loads(unquote_plus(intersects))
@@ -574,7 +493,7 @@ async def post_search(
)
if search_request.datetime:
- datetime_search = self._return_date(search_request.datetime)
+ datetime_search = return_date(search_request.datetime)
search = self.database.apply_datetime_filter(
search=search, datetime_search=datetime_search
)
diff --git a/stac_fastapi/core/stac_fastapi/core/datetime_utils.py b/stac_fastapi/core/stac_fastapi/core/datetime_utils.py
index 3d6dd663..8e965614 100644
--- a/stac_fastapi/core/stac_fastapi/core/datetime_utils.py
+++ b/stac_fastapi/core/stac_fastapi/core/datetime_utils.py
@@ -1,5 +1,90 @@
-"""A few datetime methods."""
-from datetime import datetime, timezone
+"""Utility functions to handle datetime parsing."""
+from datetime import datetime
+from datetime import datetime as datetime_type
+from datetime import timezone
+from typing import Dict, Optional, Union
+
+from stac_fastapi.types.rfc3339 import DateTimeType, rfc3339_str_to_datetime
+
+
+def return_date(
+ interval: Optional[Union[DateTimeType, str]]
+) -> Dict[str, Optional[str]]:
+ """
+ Convert a date interval.
+
+ (which may be a datetime, a tuple of one or two datetimes a string
+ representing a datetime or range, or None) into a dictionary for filtering
+ search results with Elasticsearch.
+
+ This function ensures the output dictionary contains 'gte' and 'lte' keys,
+ even if they are set to None, to prevent KeyError in the consuming logic.
+
+ Args:
+ interval (Optional[Union[DateTimeType, str]]): The date interval, which might be a single datetime,
+ a tuple with one or two datetimes, a string, or None.
+
+ Returns:
+ dict: A dictionary representing the date interval for use in filtering search results,
+ always containing 'gte' and 'lte' keys.
+ """
+ result: Dict[str, Optional[str]] = {"gte": None, "lte": None}
+
+ if interval is None:
+ return result
+
+ if isinstance(interval, str):
+ if "/" in interval:
+ parts = interval.split("/")
+ result["gte"] = parts[0] if parts[0] != ".." else None
+ result["lte"] = parts[1] if len(parts) > 1 and parts[1] != ".." else None
+ else:
+ converted_time = interval if interval != ".." else None
+ result["gte"] = result["lte"] = converted_time
+ return result
+
+ if isinstance(interval, datetime_type):
+ datetime_iso = interval.isoformat()
+ result["gte"] = result["lte"] = datetime_iso
+ elif isinstance(interval, tuple):
+ start, end = interval
+ # Ensure datetimes are converted to UTC and formatted with 'Z'
+ if start:
+ result["gte"] = start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+ if end:
+ result["lte"] = end.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+
+ return result
+
+
+def format_datetime_range(date_str: str) -> str:
+ """
+ Convert a datetime range string into a normalized UTC string for API requests using rfc3339_str_to_datetime.
+
+ Args:
+ date_str (str): A string containing two datetime values separated by a '/'.
+
+ Returns:
+ str: A string formatted as 'YYYY-MM-DDTHH:MM:SSZ/YYYY-MM-DDTHH:MM:SSZ', with '..' used if any element is None.
+ """
+
+ def normalize(dt):
+ dt = dt.strip()
+ if not dt or dt == "..":
+ return ".."
+ dt_obj = rfc3339_str_to_datetime(dt)
+ dt_utc = dt_obj.astimezone(timezone.utc)
+ return dt_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+ if not isinstance(date_str, str):
+ return "../.."
+ if "/" not in date_str:
+ return f"{normalize(date_str)}/{normalize(date_str)}"
+ try:
+ start, end = date_str.split("/", 1)
+ except Exception:
+ return "../.."
+ return f"{normalize(start)}/{normalize(end)}"
# Borrowed from pystac - https://github.com/stac-utils/pystac/blob/f5e4cf4a29b62e9ef675d4a4dac7977b09f53c8f/pystac/utils.py#L370-L394
diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/filter.py b/stac_fastapi/core/stac_fastapi/core/extensions/filter.py
index a74eff99..34adc3e6 100644
--- a/stac_fastapi/core/stac_fastapi/core/extensions/filter.py
+++ b/stac_fastapi/core/stac_fastapi/core/extensions/filter.py
@@ -17,6 +17,51 @@
from enum import Enum
from typing import Any, Dict
+DEFAULT_QUERYABLES: Dict[str, Dict[str, Any]] = {
+ "id": {
+ "description": "ID",
+ "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id",
+ },
+ "collection": {
+ "description": "Collection",
+ "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/then/properties/collection",
+ },
+ "geometry": {
+ "description": "Geometry",
+ "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/1/oneOf/0/properties/geometry",
+ },
+ "datetime": {
+ "description": "Acquisition Timestamp",
+ "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/datetime",
+ },
+ "created": {
+ "description": "Creation Timestamp",
+ "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/created",
+ },
+ "updated": {
+ "description": "Creation Timestamp",
+ "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/updated",
+ },
+ "cloud_cover": {
+ "description": "Cloud Cover",
+ "$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fields/properties/eo:cloud_cover",
+ },
+ "cloud_shadow_percentage": {
+ "title": "Cloud Shadow Percentage",
+ "description": "Cloud Shadow Percentage",
+ "type": "number",
+ "minimum": 0,
+ "maximum": 100,
+ },
+ "nodata_pixel_percentage": {
+ "title": "No Data Pixel Percentage",
+ "description": "No Data Pixel Percentage",
+ "type": "number",
+ "minimum": 0,
+ "maximum": 100,
+ },
+}
+
_cql2_like_patterns = re.compile(r"\\.|[%_]|\\$")
_valid_like_substitutions = {
"\\\\": "\\",
@@ -115,105 +160,3 @@ def to_es_field(field: str) -> str:
str: The mapped field name suitable for Elasticsearch queries.
"""
return queryables_mapping.get(field, field)
-
-
-def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
- """
- Transform a simplified CQL2 query structure to an Elasticsearch compatible query DSL.
-
- Args:
- query (Dict[str, Any]): The query dictionary containing 'op' and 'args'.
-
- Returns:
- Dict[str, Any]: The corresponding Elasticsearch query in the form of a dictionary.
- """
- if query["op"] in [LogicalOp.AND, LogicalOp.OR, LogicalOp.NOT]:
- bool_type = {
- LogicalOp.AND: "must",
- LogicalOp.OR: "should",
- LogicalOp.NOT: "must_not",
- }[query["op"]]
- return {"bool": {bool_type: [to_es(sub_query) for sub_query in query["args"]]}}
-
- elif query["op"] in [
- ComparisonOp.EQ,
- ComparisonOp.NEQ,
- ComparisonOp.LT,
- ComparisonOp.LTE,
- ComparisonOp.GT,
- ComparisonOp.GTE,
- ]:
- range_op = {
- ComparisonOp.LT: "lt",
- ComparisonOp.LTE: "lte",
- ComparisonOp.GT: "gt",
- ComparisonOp.GTE: "gte",
- }
-
- field = to_es_field(query["args"][0]["property"])
- value = query["args"][1]
- if isinstance(value, dict) and "timestamp" in value:
- value = value["timestamp"]
- if query["op"] == ComparisonOp.EQ:
- return {"range": {field: {"gte": value, "lte": value}}}
- elif query["op"] == ComparisonOp.NEQ:
- return {
- "bool": {
- "must_not": [{"range": {field: {"gte": value, "lte": value}}}]
- }
- }
- else:
- return {"range": {field: {range_op[query["op"]]: value}}}
- else:
- if query["op"] == ComparisonOp.EQ:
- return {"term": {field: value}}
- elif query["op"] == ComparisonOp.NEQ:
- return {"bool": {"must_not": [{"term": {field: value}}]}}
- else:
- return {"range": {field: {range_op[query["op"]]: value}}}
-
- elif query["op"] == ComparisonOp.IS_NULL:
- field = to_es_field(query["args"][0]["property"])
- return {"bool": {"must_not": {"exists": {"field": field}}}}
-
- elif query["op"] == AdvancedComparisonOp.BETWEEN:
- field = to_es_field(query["args"][0]["property"])
- gte, lte = query["args"][1], query["args"][2]
- if isinstance(gte, dict) and "timestamp" in gte:
- gte = gte["timestamp"]
- if isinstance(lte, dict) and "timestamp" in lte:
- lte = lte["timestamp"]
- return {"range": {field: {"gte": gte, "lte": lte}}}
-
- elif query["op"] == AdvancedComparisonOp.IN:
- field = to_es_field(query["args"][0]["property"])
- values = query["args"][1]
- if not isinstance(values, list):
- raise ValueError(f"Arg {values} is not a list")
- return {"terms": {field: values}}
-
- elif query["op"] == AdvancedComparisonOp.LIKE:
- field = to_es_field(query["args"][0]["property"])
- pattern = cql2_like_to_es(query["args"][1])
- return {"wildcard": {field: {"value": pattern, "case_insensitive": True}}}
-
- elif query["op"] in [
- SpatialOp.S_INTERSECTS,
- SpatialOp.S_CONTAINS,
- SpatialOp.S_WITHIN,
- SpatialOp.S_DISJOINT,
- ]:
- field = to_es_field(query["args"][0]["property"])
- geometry = query["args"][1]
-
- relation_mapping = {
- SpatialOp.S_INTERSECTS: "intersects",
- SpatialOp.S_CONTAINS: "contains",
- SpatialOp.S_WITHIN: "within",
- SpatialOp.S_DISJOINT: "disjoint",
- }
-
- relation = relation_mapping[query["op"]]
- return {"geo_shape": {field: {"shape": geometry, "relation": relation}}}
-
- return {}
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
index 9385fe86..855399ef 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
@@ -14,13 +14,13 @@
from starlette.requests import Request
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
-from stac_fastapi.core.extensions import filter
from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon
from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
from stac_fastapi.elasticsearch.config import (
ElasticsearchSettings as SyncElasticsearchSettings,
)
+from stac_fastapi.sfeos_helpers import filter
from stac_fastapi.sfeos_helpers.mappings import (
COLLECTIONS_INDEX,
DEFAULT_SORT,
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
index 2f4c1806..b483ff2a 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
@@ -14,13 +14,13 @@
from starlette.requests import Request
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
-from stac_fastapi.core.extensions import filter
from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon
from stac_fastapi.opensearch.config import (
AsyncOpensearchSettings as AsyncSearchSettings,
)
from stac_fastapi.opensearch.config import OpensearchSettings as SyncSearchSettings
+from stac_fastapi.sfeos_helpers import filter
from stac_fastapi.sfeos_helpers.mappings import (
COLLECTIONS_INDEX,
DEFAULT_SORT,
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
index a2c39845..aba463a9 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
@@ -6,54 +6,120 @@
import attr
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
+from stac_fastapi.core.extensions.filter import (
+ DEFAULT_QUERYABLES,
+ AdvancedComparisonOp,
+ ComparisonOp,
+ LogicalOp,
+ SpatialOp,
+ cql2_like_to_es,
+ to_es_field,
+)
from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient
from .mappings import ES_MAPPING_TYPE_TO_JSON
-_DEFAULT_QUERYABLES: Dict[str, Dict[str, Any]] = {
- "id": {
- "description": "ID",
- "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id",
- },
- "collection": {
- "description": "Collection",
- "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/then/properties/collection",
- },
- "geometry": {
- "description": "Geometry",
- "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/1/oneOf/0/properties/geometry",
- },
- "datetime": {
- "description": "Acquisition Timestamp",
- "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/datetime",
- },
- "created": {
- "description": "Creation Timestamp",
- "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/created",
- },
- "updated": {
- "description": "Creation Timestamp",
- "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/updated",
- },
- "cloud_cover": {
- "description": "Cloud Cover",
- "$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fields/properties/eo:cloud_cover",
- },
- "cloud_shadow_percentage": {
- "title": "Cloud Shadow Percentage",
- "description": "Cloud Shadow Percentage",
- "type": "number",
- "minimum": 0,
- "maximum": 100,
- },
- "nodata_pixel_percentage": {
- "title": "No Data Pixel Percentage",
- "description": "No Data Pixel Percentage",
- "type": "number",
- "minimum": 0,
- "maximum": 100,
- },
-}
+
+def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
+ """
+ Transform a simplified CQL2 query structure to an Elasticsearch compatible query DSL.
+
+ Args:
+ query (Dict[str, Any]): The query dictionary containing 'op' and 'args'.
+
+ Returns:
+ Dict[str, Any]: The corresponding Elasticsearch query in the form of a dictionary.
+ """
+ if query["op"] in [LogicalOp.AND, LogicalOp.OR, LogicalOp.NOT]:
+ bool_type = {
+ LogicalOp.AND: "must",
+ LogicalOp.OR: "should",
+ LogicalOp.NOT: "must_not",
+ }[query["op"]]
+ return {"bool": {bool_type: [to_es(sub_query) for sub_query in query["args"]]}}
+
+ elif query["op"] in [
+ ComparisonOp.EQ,
+ ComparisonOp.NEQ,
+ ComparisonOp.LT,
+ ComparisonOp.LTE,
+ ComparisonOp.GT,
+ ComparisonOp.GTE,
+ ]:
+ range_op = {
+ ComparisonOp.LT: "lt",
+ ComparisonOp.LTE: "lte",
+ ComparisonOp.GT: "gt",
+ ComparisonOp.GTE: "gte",
+ }
+
+ field = to_es_field(query["args"][0]["property"])
+ value = query["args"][1]
+ if isinstance(value, dict) and "timestamp" in value:
+ value = value["timestamp"]
+ if query["op"] == ComparisonOp.EQ:
+ return {"range": {field: {"gte": value, "lte": value}}}
+ elif query["op"] == ComparisonOp.NEQ:
+ return {
+ "bool": {
+ "must_not": [{"range": {field: {"gte": value, "lte": value}}}]
+ }
+ }
+ else:
+ return {"range": {field: {range_op[query["op"]]: value}}}
+ else:
+ if query["op"] == ComparisonOp.EQ:
+ return {"term": {field: value}}
+ elif query["op"] == ComparisonOp.NEQ:
+ return {"bool": {"must_not": [{"term": {field: value}}]}}
+ else:
+ return {"range": {field: {range_op[query["op"]]: value}}}
+
+ elif query["op"] == ComparisonOp.IS_NULL:
+ field = to_es_field(query["args"][0]["property"])
+ return {"bool": {"must_not": {"exists": {"field": field}}}}
+
+ elif query["op"] == AdvancedComparisonOp.BETWEEN:
+ field = to_es_field(query["args"][0]["property"])
+ gte, lte = query["args"][1], query["args"][2]
+ if isinstance(gte, dict) and "timestamp" in gte:
+ gte = gte["timestamp"]
+ if isinstance(lte, dict) and "timestamp" in lte:
+ lte = lte["timestamp"]
+ return {"range": {field: {"gte": gte, "lte": lte}}}
+
+ elif query["op"] == AdvancedComparisonOp.IN:
+ field = to_es_field(query["args"][0]["property"])
+ values = query["args"][1]
+ if not isinstance(values, list):
+ raise ValueError(f"Arg {values} is not a list")
+ return {"terms": {field: values}}
+
+ elif query["op"] == AdvancedComparisonOp.LIKE:
+ field = to_es_field(query["args"][0]["property"])
+ pattern = cql2_like_to_es(query["args"][1])
+ return {"wildcard": {field: {"value": pattern, "case_insensitive": True}}}
+
+ elif query["op"] in [
+ SpatialOp.S_INTERSECTS,
+ SpatialOp.S_CONTAINS,
+ SpatialOp.S_WITHIN,
+ SpatialOp.S_DISJOINT,
+ ]:
+ field = to_es_field(query["args"][0]["property"])
+ geometry = query["args"][1]
+
+ relation_mapping = {
+ SpatialOp.S_INTERSECTS: "intersects",
+ SpatialOp.S_CONTAINS: "contains",
+ SpatialOp.S_WITHIN: "within",
+ SpatialOp.S_DISJOINT: "disjoint",
+ }
+
+ relation = relation_mapping[query["op"]]
+ return {"geo_shape": {field: {"shape": geometry, "relation": relation}}}
+
+ return {}
@attr.s
@@ -88,7 +154,7 @@ async def get_queryables(
"type": "object",
"title": "Queryables for STAC API",
"description": "Queryable names for the STAC API Item Search filter.",
- "properties": _DEFAULT_QUERYABLES,
+ "properties": DEFAULT_QUERYABLES,
"additionalProperties": True,
}
if not collection_id:
@@ -128,7 +194,7 @@ async def get_queryables(
continue
# Generate field properties
- field_result = _DEFAULT_QUERYABLES.get(field_name, {})
+ field_result = DEFAULT_QUERYABLES.get(field_name, {})
properties[field_name] = field_result
field_name_human = field_name.replace("_", " ").title()
From d5b50091b71cd69430438374c6e8aa298210a2fc Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Thu, 15 May 2025 13:35:03 +0800
Subject: [PATCH 03/29] db logic helpers
---
.../elasticsearch/database_logic.py | 23 ++----------
.../stac_fastapi/opensearch/database_logic.py | 21 ++---------
.../sfeos_helpers/database_logic_helpers.py | 37 +++++++++++++++++++
stac_fastapi/tests/database/test_database.py | 23 ++++--------
4 files changed, 52 insertions(+), 52 deletions(-)
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
index 855399ef..7b0fed6f 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
@@ -21,12 +21,12 @@
ElasticsearchSettings as SyncElasticsearchSettings,
)
from stac_fastapi.sfeos_helpers import filter
+from stac_fastapi.sfeos_helpers.database_logic_helpers import (
+ create_index_templates_shared,
+)
from stac_fastapi.sfeos_helpers.mappings import (
COLLECTIONS_INDEX,
DEFAULT_SORT,
- ES_COLLECTIONS_MAPPINGS,
- ES_ITEMS_MAPPINGS,
- ES_ITEMS_SETTINGS,
ITEM_INDICES,
ITEMS_INDEX_PREFIX,
Geometry,
@@ -51,22 +51,7 @@ async def create_index_templates() -> None:
None
"""
- client = AsyncElasticsearchSettings().create_client
- await client.indices.put_index_template(
- name=f"template_{COLLECTIONS_INDEX}",
- body={
- "index_patterns": [f"{COLLECTIONS_INDEX}*"],
- "template": {"mappings": ES_COLLECTIONS_MAPPINGS},
- },
- )
- await client.indices.put_index_template(
- name=f"template_{ITEMS_INDEX_PREFIX}",
- body={
- "index_patterns": [f"{ITEMS_INDEX_PREFIX}*"],
- "template": {"settings": ES_ITEMS_SETTINGS, "mappings": ES_ITEMS_MAPPINGS},
- },
- )
- await client.close()
+ await create_index_templates_shared(settings=AsyncElasticsearchSettings())
async def create_collection_index() -> None:
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
index b483ff2a..8e360e72 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
@@ -21,6 +21,9 @@
)
from stac_fastapi.opensearch.config import OpensearchSettings as SyncSearchSettings
from stac_fastapi.sfeos_helpers import filter
+from stac_fastapi.sfeos_helpers.database_logic_helpers import (
+ create_index_templates_shared,
+)
from stac_fastapi.sfeos_helpers.mappings import (
COLLECTIONS_INDEX,
DEFAULT_SORT,
@@ -51,23 +54,7 @@ async def create_index_templates() -> None:
None
"""
- client = AsyncSearchSettings().create_client
- await client.indices.put_template(
- name=f"template_{COLLECTIONS_INDEX}",
- body={
- "index_patterns": [f"{COLLECTIONS_INDEX}*"],
- "mappings": ES_COLLECTIONS_MAPPINGS,
- },
- )
- await client.indices.put_template(
- name=f"template_{ITEMS_INDEX_PREFIX}",
- body={
- "index_patterns": [f"{ITEMS_INDEX_PREFIX}*"],
- "settings": ES_ITEMS_SETTINGS,
- "mappings": ES_ITEMS_MAPPINGS,
- },
- )
- await client.close()
+ await create_index_templates_shared(settings=AsyncSearchSettings())
async def create_collection_index() -> None:
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
new file mode 100644
index 00000000..06bb785a
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
@@ -0,0 +1,37 @@
+"""Shared code for elasticsearch/ opensearch database logic."""
+
+from typing import Any
+
+from stac_fastapi.sfeos_helpers.mappings import (
+ COLLECTIONS_INDEX,
+ ES_COLLECTIONS_MAPPINGS,
+ ES_ITEMS_MAPPINGS,
+ ES_ITEMS_SETTINGS,
+ ITEMS_INDEX_PREFIX,
+)
+
+
+async def create_index_templates_shared(settings: Any) -> None:
+ """
+ Create index templates for the Collection and Item indices.
+
+ Returns:
+ None
+
+ """
+ client = settings.create_client
+ await client.indices.put_index_template(
+ name=f"template_{COLLECTIONS_INDEX}",
+ body={
+ "index_patterns": [f"{COLLECTIONS_INDEX}*"],
+ "template": {"mappings": ES_COLLECTIONS_MAPPINGS},
+ },
+ )
+ await client.indices.put_index_template(
+ name=f"template_{ITEMS_INDEX_PREFIX}",
+ body={
+ "index_patterns": [f"{ITEMS_INDEX_PREFIX}*"],
+ "template": {"settings": ES_ITEMS_SETTINGS, "mappings": ES_ITEMS_MAPPINGS},
+ },
+ )
+ await client.close()
diff --git a/stac_fastapi/tests/database/test_database.py b/stac_fastapi/tests/database/test_database.py
index a5a01e60..b70bde2f 100644
--- a/stac_fastapi/tests/database/test_database.py
+++ b/stac_fastapi/tests/database/test_database.py
@@ -1,25 +1,16 @@
-import os
import uuid
import pytest
from stac_pydantic import api
-from ..conftest import MockRequest, database
+from stac_fastapi.sfeos_helpers.mappings import (
+ COLLECTIONS_INDEX,
+ ES_COLLECTIONS_MAPPINGS,
+ ES_ITEMS_MAPPINGS,
+ index_alias_by_collection_id,
+)
-if os.getenv("BACKEND", "elasticsearch").lower() == "opensearch":
- from stac_fastapi.opensearch.database_logic import (
- COLLECTIONS_INDEX,
- ES_COLLECTIONS_MAPPINGS,
- ES_ITEMS_MAPPINGS,
- index_alias_by_collection_id,
- )
-else:
- from stac_fastapi.elasticsearch.database_logic import (
- COLLECTIONS_INDEX,
- ES_COLLECTIONS_MAPPINGS,
- ES_ITEMS_MAPPINGS,
- index_alias_by_collection_id,
- )
+from ..conftest import MockRequest, database
@pytest.mark.asyncio
From 14d88b2a8d9b2bdb3a7483f8d8e4f50c5c44989d Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Fri, 16 May 2025 15:54:50 +0800
Subject: [PATCH 04/29] update version
---
stac_fastapi/sfeos_helpers/setup.py | 2 +-
.../sfeos_helpers/stac_fastapi/sfeos_helpers/version.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/stac_fastapi/sfeos_helpers/setup.py b/stac_fastapi/sfeos_helpers/setup.py
index 8b26c9de..6a1223ea 100644
--- a/stac_fastapi/sfeos_helpers/setup.py
+++ b/stac_fastapi/sfeos_helpers/setup.py
@@ -6,7 +6,7 @@
desc = f.read()
install_requires = [
- "stac-fastapi.core==4.1.0",
+ "stac-fastapi.core==4.2.0",
]
setup(
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py
index e42ce685..1cd0ed04 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py
@@ -1,2 +1,2 @@
"""library version."""
-__version__ = "4.1.0"
+__version__ = "4.2.0"
From 67e1c4c17f1c4f2341e46c9aed117771deb4c55d Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Fri, 16 May 2025 19:16:43 +0800
Subject: [PATCH 05/29] move to_es filter update to sfeos_helpers
---
.../stac_fastapi/core/extensions/filter.py | 108 ------------------
.../stac_fastapi/sfeos_helpers/filter.py | 22 ++--
2 files changed, 14 insertions(+), 116 deletions(-)
diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/filter.py b/stac_fastapi/core/stac_fastapi/core/extensions/filter.py
index ddc6e258..ad07e2b9 100644
--- a/stac_fastapi/core/stac_fastapi/core/extensions/filter.py
+++ b/stac_fastapi/core/stac_fastapi/core/extensions/filter.py
@@ -147,111 +147,3 @@ def to_es_field(queryables_mapping: Dict[str, Any], field: str) -> str:
str: The mapped field name suitable for Elasticsearch queries.
"""
return queryables_mapping.get(field, field)
-
-
-def to_es(queryables_mapping: Dict[str, Any], query: Dict[str, Any]) -> Dict[str, Any]:
- """
- Transform a simplified CQL2 query structure to an Elasticsearch compatible query DSL.
-
- Args:
- query (Dict[str, Any]): The query dictionary containing 'op' and 'args'.
-
- Returns:
- Dict[str, Any]: The corresponding Elasticsearch query in the form of a dictionary.
- """
- if query["op"] in [LogicalOp.AND, LogicalOp.OR, LogicalOp.NOT]:
- bool_type = {
- LogicalOp.AND: "must",
- LogicalOp.OR: "should",
- LogicalOp.NOT: "must_not",
- }[query["op"]]
- return {
- "bool": {
- bool_type: [
- to_es(queryables_mapping, sub_query) for sub_query in query["args"]
- ]
- }
- }
-
- elif query["op"] in [
- ComparisonOp.EQ,
- ComparisonOp.NEQ,
- ComparisonOp.LT,
- ComparisonOp.LTE,
- ComparisonOp.GT,
- ComparisonOp.GTE,
- ]:
- range_op = {
- ComparisonOp.LT: "lt",
- ComparisonOp.LTE: "lte",
- ComparisonOp.GT: "gt",
- ComparisonOp.GTE: "gte",
- }
-
- field = to_es_field(queryables_mapping, query["args"][0]["property"])
- value = query["args"][1]
- if isinstance(value, dict) and "timestamp" in value:
- value = value["timestamp"]
- if query["op"] == ComparisonOp.EQ:
- return {"range": {field: {"gte": value, "lte": value}}}
- elif query["op"] == ComparisonOp.NEQ:
- return {
- "bool": {
- "must_not": [{"range": {field: {"gte": value, "lte": value}}}]
- }
- }
- else:
- return {"range": {field: {range_op[query["op"]]: value}}}
- else:
- if query["op"] == ComparisonOp.EQ:
- return {"term": {field: value}}
- elif query["op"] == ComparisonOp.NEQ:
- return {"bool": {"must_not": [{"term": {field: value}}]}}
- else:
- return {"range": {field: {range_op[query["op"]]: value}}}
-
- elif query["op"] == ComparisonOp.IS_NULL:
- field = to_es_field(queryables_mapping, query["args"][0]["property"])
- return {"bool": {"must_not": {"exists": {"field": field}}}}
-
- elif query["op"] == AdvancedComparisonOp.BETWEEN:
- field = to_es_field(queryables_mapping, query["args"][0]["property"])
- gte, lte = query["args"][1], query["args"][2]
- if isinstance(gte, dict) and "timestamp" in gte:
- gte = gte["timestamp"]
- if isinstance(lte, dict) and "timestamp" in lte:
- lte = lte["timestamp"]
- return {"range": {field: {"gte": gte, "lte": lte}}}
-
- elif query["op"] == AdvancedComparisonOp.IN:
- field = to_es_field(queryables_mapping, query["args"][0]["property"])
- values = query["args"][1]
- if not isinstance(values, list):
- raise ValueError(f"Arg {values} is not a list")
- return {"terms": {field: values}}
-
- elif query["op"] == AdvancedComparisonOp.LIKE:
- field = to_es_field(queryables_mapping, query["args"][0]["property"])
- pattern = cql2_like_to_es(query["args"][1])
- return {"wildcard": {field: {"value": pattern, "case_insensitive": True}}}
-
- elif query["op"] in [
- SpatialOp.S_INTERSECTS,
- SpatialOp.S_CONTAINS,
- SpatialOp.S_WITHIN,
- SpatialOp.S_DISJOINT,
- ]:
- field = to_es_field(queryables_mapping, query["args"][0]["property"])
- geometry = query["args"][1]
-
- relation_mapping = {
- SpatialOp.S_INTERSECTS: "intersects",
- SpatialOp.S_CONTAINS: "contains",
- SpatialOp.S_WITHIN: "within",
- SpatialOp.S_DISJOINT: "disjoint",
- }
-
- relation = relation_mapping[query["op"]]
- return {"geo_shape": {field: {"shape": geometry, "relation": relation}}}
-
- return {}
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
index aba463a9..72b43d65 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
@@ -20,7 +20,7 @@
from .mappings import ES_MAPPING_TYPE_TO_JSON
-def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
+def to_es(queryables_mapping: Dict[str, Any], query: Dict[str, Any]) -> Dict[str, Any]:
"""
Transform a simplified CQL2 query structure to an Elasticsearch compatible query DSL.
@@ -36,7 +36,13 @@ def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
LogicalOp.OR: "should",
LogicalOp.NOT: "must_not",
}[query["op"]]
- return {"bool": {bool_type: [to_es(sub_query) for sub_query in query["args"]]}}
+ return {
+ "bool": {
+ bool_type: [
+ to_es(queryables_mapping, sub_query) for sub_query in query["args"]
+ ]
+ }
+ }
elif query["op"] in [
ComparisonOp.EQ,
@@ -53,7 +59,7 @@ def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
ComparisonOp.GTE: "gte",
}
- field = to_es_field(query["args"][0]["property"])
+ field = to_es_field(queryables_mapping, query["args"][0]["property"])
value = query["args"][1]
if isinstance(value, dict) and "timestamp" in value:
value = value["timestamp"]
@@ -76,11 +82,11 @@ def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
return {"range": {field: {range_op[query["op"]]: value}}}
elif query["op"] == ComparisonOp.IS_NULL:
- field = to_es_field(query["args"][0]["property"])
+ field = to_es_field(queryables_mapping, query["args"][0]["property"])
return {"bool": {"must_not": {"exists": {"field": field}}}}
elif query["op"] == AdvancedComparisonOp.BETWEEN:
- field = to_es_field(query["args"][0]["property"])
+ field = to_es_field(queryables_mapping, query["args"][0]["property"])
gte, lte = query["args"][1], query["args"][2]
if isinstance(gte, dict) and "timestamp" in gte:
gte = gte["timestamp"]
@@ -89,14 +95,14 @@ def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
return {"range": {field: {"gte": gte, "lte": lte}}}
elif query["op"] == AdvancedComparisonOp.IN:
- field = to_es_field(query["args"][0]["property"])
+ field = to_es_field(queryables_mapping, query["args"][0]["property"])
values = query["args"][1]
if not isinstance(values, list):
raise ValueError(f"Arg {values} is not a list")
return {"terms": {field: values}}
elif query["op"] == AdvancedComparisonOp.LIKE:
- field = to_es_field(query["args"][0]["property"])
+ field = to_es_field(queryables_mapping, query["args"][0]["property"])
pattern = cql2_like_to_es(query["args"][1])
return {"wildcard": {field: {"value": pattern, "case_insensitive": True}}}
@@ -106,7 +112,7 @@ def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
SpatialOp.S_WITHIN,
SpatialOp.S_DISJOINT,
]:
- field = to_es_field(query["args"][0]["property"])
+ field = to_es_field(queryables_mapping, query["args"][0]["property"])
geometry = query["args"][1]
relation_mapping = {
From 7ab88d4c3b10cf682f5ff21d729349c27ee6c3c9 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Fri, 16 May 2025 19:35:38 +0800
Subject: [PATCH 06/29] es filter methods to sfeos helpers
---
.../stac_fastapi/core/extensions/filter.py | 46 ++-----------------
.../stac_fastapi/sfeos_helpers/filter.py | 45 +++++++++++++++++-
.../tests/extensions/test_cql2_like_to_es.py | 2 +-
3 files changed, 47 insertions(+), 46 deletions(-)
diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/filter.py b/stac_fastapi/core/stac_fastapi/core/extensions/filter.py
index ad07e2b9..dba80bf4 100644
--- a/stac_fastapi/core/stac_fastapi/core/extensions/filter.py
+++ b/stac_fastapi/core/stac_fastapi/core/extensions/filter.py
@@ -1,4 +1,4 @@
-"""Filter extension logic for es conversion."""
+"""Filter extension logic for conversion."""
# """
# Implements Filter Extension.
@@ -62,8 +62,8 @@
},
}
-_cql2_like_patterns = re.compile(r"\\.|[%_]|\\$")
-_valid_like_substitutions = {
+cql2_like_patterns = re.compile(r"\\.|[%_]|\\$")
+valid_like_substitutions = {
"\\\\": "\\",
"\\%": "%",
"\\_": "_",
@@ -72,33 +72,6 @@
}
-def _replace_like_patterns(match: re.Match) -> str:
- pattern = match.group()
- try:
- return _valid_like_substitutions[pattern]
- except KeyError:
- raise ValueError(f"'{pattern}' is not a valid escape sequence")
-
-
-def cql2_like_to_es(string: str) -> str:
- """
- Convert CQL2 "LIKE" characters to Elasticsearch "wildcard" characters.
-
- Args:
- string (str): The string containing CQL2 wildcard characters.
-
- Returns:
- str: The converted string with Elasticsearch compatible wildcards.
-
- Raises:
- ValueError: If an invalid escape sequence is encountered.
- """
- return _cql2_like_patterns.sub(
- repl=_replace_like_patterns,
- string=string,
- )
-
-
class LogicalOp(str, Enum):
"""Enumeration for logical operators used in constructing Elasticsearch queries."""
@@ -134,16 +107,3 @@ class SpatialOp(str, Enum):
S_CONTAINS = "s_contains"
S_WITHIN = "s_within"
S_DISJOINT = "s_disjoint"
-
-
-def to_es_field(queryables_mapping: Dict[str, Any], field: str) -> str:
- """
- Map a given field to its corresponding Elasticsearch field according to a predefined mapping.
-
- Args:
- field (str): The field name from a user query or filter.
-
- Returns:
- str: The mapped field name suitable for Elasticsearch queries.
- """
- return queryables_mapping.get(field, field)
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
index 72b43d65..d2b890db 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
@@ -1,5 +1,6 @@
"""Shared filter extension methods for stac-fastapi elasticsearch and opensearch backends."""
+import re
from collections import deque
from typing import Any, Dict, Optional
@@ -12,14 +13,54 @@
ComparisonOp,
LogicalOp,
SpatialOp,
- cql2_like_to_es,
- to_es_field,
+ cql2_like_patterns,
+ valid_like_substitutions,
)
from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient
from .mappings import ES_MAPPING_TYPE_TO_JSON
+def _replace_like_patterns(match: re.Match) -> str:
+ pattern = match.group()
+ try:
+ return valid_like_substitutions[pattern]
+ except KeyError:
+ raise ValueError(f"'{pattern}' is not a valid escape sequence")
+
+
+def cql2_like_to_es(string: str) -> str:
+ """
+ Convert CQL2 "LIKE" characters to Elasticsearch "wildcard" characters.
+
+ Args:
+ string (str): The string containing CQL2 wildcard characters.
+
+ Returns:
+ str: The converted string with Elasticsearch compatible wildcards.
+
+ Raises:
+ ValueError: If an invalid escape sequence is encountered.
+ """
+ return cql2_like_patterns.sub(
+ repl=_replace_like_patterns,
+ string=string,
+ )
+
+
+def to_es_field(queryables_mapping: Dict[str, Any], field: str) -> str:
+ """
+ Map a given field to its corresponding Elasticsearch field according to a predefined mapping.
+
+ Args:
+ field (str): The field name from a user query or filter.
+
+ Returns:
+ str: The mapped field name suitable for Elasticsearch queries.
+ """
+ return queryables_mapping.get(field, field)
+
+
def to_es(queryables_mapping: Dict[str, Any], query: Dict[str, Any]) -> Dict[str, Any]:
"""
Transform a simplified CQL2 query structure to an Elasticsearch compatible query DSL.
diff --git a/stac_fastapi/tests/extensions/test_cql2_like_to_es.py b/stac_fastapi/tests/extensions/test_cql2_like_to_es.py
index 96d51272..2125eeed 100644
--- a/stac_fastapi/tests/extensions/test_cql2_like_to_es.py
+++ b/stac_fastapi/tests/extensions/test_cql2_like_to_es.py
@@ -1,6 +1,6 @@
import pytest
-from stac_fastapi.core.extensions.filter import cql2_like_to_es
+from stac_fastapi.sfeos_helpers.filter import cql2_like_to_es
@pytest.mark.parametrize(
From 9ab8e99a0172d46d7c165c842e641c310a7c000e Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Sun, 18 May 2025 19:46:30 +0800
Subject: [PATCH 07/29] create shared filter methods
---
.../elasticsearch/database_logic.py | 58 +++++------
.../stac_fastapi/opensearch/database_logic.py | 58 +++++------
.../sfeos_helpers/database_logic_helpers.py | 96 ++++++++++++++++++-
3 files changed, 154 insertions(+), 58 deletions(-)
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
index 1698f6b5..9c4e8366 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
@@ -22,7 +22,10 @@
)
from stac_fastapi.sfeos_helpers import filter
from stac_fastapi.sfeos_helpers.database_logic_helpers import (
+ apply_free_text_filter_shared,
+ apply_intersects_filter_shared,
create_index_templates_shared,
+ populate_sort_shared,
)
from stac_fastapi.sfeos_helpers.mappings import (
COLLECTIONS_INDEX,
@@ -483,21 +486,8 @@ def apply_intersects_filter(
Notes:
A geo_shape filter is added to the search object, set to intersect with the specified geometry.
"""
- return search.filter(
- Q(
- {
- "geo_shape": {
- "geometry": {
- "shape": {
- "type": intersects.type.lower(),
- "coordinates": intersects.coordinates,
- },
- "relation": "intersects",
- }
- }
- }
- )
- )
+ filter = apply_intersects_filter_shared(intersects=intersects)
+ return search.filter(Q(filter))
@staticmethod
def apply_stacql_filter(search: Search, op: str, field: str, value: float):
@@ -523,14 +513,21 @@ def apply_stacql_filter(search: Search, op: str, field: str, value: float):
@staticmethod
def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]]):
- """Database logic to perform query for search endpoint."""
- if free_text_queries is not None:
- free_text_query_string = '" OR properties.\\*:"'.join(free_text_queries)
- search = search.query(
- "query_string", query=f'properties.\\*:"{free_text_query_string}"'
- )
+ """Create a free text query for Elasticsearch queries.
- return search
+ This method delegates to the shared implementation in apply_free_text_filter_shared.
+
+ Args:
+ search (Search): The search object to apply the query to.
+ free_text_queries (Optional[List[str]]): A list of text strings to search for in the properties.
+
+ Returns:
+ Search: The search object with the free text query applied, or the original search
+ object if no free_text_queries were provided.
+ """
+ return apply_free_text_filter_shared(
+ search=search, free_text_queries=free_text_queries
+ )
async def apply_cql2_filter(
self, search: Search, _filter: Optional[Dict[str, Any]]
@@ -561,11 +558,18 @@ async def apply_cql2_filter(
@staticmethod
def populate_sort(sortby: List) -> Optional[Dict[str, Dict[str, str]]]:
- """Database logic to sort search instance."""
- if sortby:
- return {s.field: {"order": s.direction} for s in sortby}
- else:
- return None
+ """Create a sort configuration for Elasticsearch queries.
+
+ This method delegates to the shared implementation in populate_sort_shared.
+
+ Args:
+ sortby (List): A list of sort specifications, each containing a field and direction.
+
+ Returns:
+ Optional[Dict[str, Dict[str, str]]]: A dictionary mapping field names to sort direction
+ configurations, or None if no sort was specified.
+ """
+ return populate_sort_shared(sortby=sortby)
async def execute_search(
self,
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
index 6f48cbe1..3115a56d 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
@@ -22,7 +22,10 @@
from stac_fastapi.opensearch.config import OpensearchSettings as SyncSearchSettings
from stac_fastapi.sfeos_helpers import filter
from stac_fastapi.sfeos_helpers.database_logic_helpers import (
+ apply_free_text_filter_shared,
+ apply_intersects_filter_shared,
create_index_templates_shared,
+ populate_sort_shared,
)
from stac_fastapi.sfeos_helpers.mappings import (
COLLECTIONS_INDEX,
@@ -340,14 +343,21 @@ def apply_collections_filter(search: Search, collection_ids: List[str]):
@staticmethod
def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]]):
- """Database logic to perform query for search endpoint."""
- if free_text_queries is not None:
- free_text_query_string = '" OR properties.\\*:"'.join(free_text_queries)
- search = search.query(
- "query_string", query=f'properties.\\*:"{free_text_query_string}"'
- )
+ """Create a free text query for OpenSearch queries.
- return search
+ This method delegates to the shared implementation in apply_free_text_filter_shared.
+
+ Args:
+ search (Search): The search object to apply the query to.
+ free_text_queries (Optional[List[str]]): A list of text strings to search for in the properties.
+
+ Returns:
+ Search: The search object with the free text query applied, or the original search
+ object if no free_text_queries were provided.
+ """
+ return apply_free_text_filter_shared(
+ search=search, free_text_queries=free_text_queries
+ )
@staticmethod
def apply_datetime_filter(search: Search, datetime_search):
@@ -513,21 +523,8 @@ def apply_intersects_filter(
Notes:
A geo_shape filter is added to the search object, set to intersect with the specified geometry.
"""
- return search.filter(
- Q(
- {
- "geo_shape": {
- "geometry": {
- "shape": {
- "type": intersects.type.lower(),
- "coordinates": intersects.coordinates,
- },
- "relation": "intersects",
- }
- }
- }
- )
- )
+ filter = apply_intersects_filter_shared(intersects=intersects)
+ return search.filter(Q(filter))
@staticmethod
def apply_stacql_filter(search: Search, op: str, field: str, value: float):
@@ -580,11 +577,18 @@ async def apply_cql2_filter(
@staticmethod
def populate_sort(sortby: List) -> Optional[Dict[str, Dict[str, str]]]:
- """Database logic to sort search instance."""
- if sortby:
- return {s.field: {"order": s.direction} for s in sortby}
- else:
- return None
+ """Create a sort configuration for OpenSearch queries.
+
+ This method delegates to the shared implementation in populate_sort_shared.
+
+ Args:
+ sortby (List): A list of sort specifications, each containing a field and direction.
+
+ Returns:
+ Optional[Dict[str, Dict[str, str]]]: A dictionary mapping field names to sort direction
+ configurations, or None if no sort was specified.
+ """
+ return populate_sort_shared(sortby=sortby)
async def execute_search(
self,
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
index 06bb785a..a8e80658 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
@@ -1,6 +1,6 @@
"""Shared code for elasticsearch/ opensearch database logic."""
-from typing import Any
+from typing import Any, Dict, List, Optional
from stac_fastapi.sfeos_helpers.mappings import (
COLLECTIONS_INDEX,
@@ -8,16 +8,27 @@
ES_ITEMS_MAPPINGS,
ES_ITEMS_SETTINGS,
ITEMS_INDEX_PREFIX,
+ Geometry,
)
async def create_index_templates_shared(settings: Any) -> None:
- """
- Create index templates for the Collection and Item indices.
+ """Create index templates for Elasticsearch/OpenSearch Collection and Item indices.
+
+ Args:
+ settings (Any): The settings object containing the client configuration.
+ Must have a create_client attribute that returns an Elasticsearch/OpenSearch client.
Returns:
- None
+ None: This function doesn't return any value but creates index templates in the database.
+ Notes:
+ This function creates two index templates:
+ 1. A template for the Collections index with the appropriate mappings
+ 2. A template for the Items indices with both settings and mappings
+
+ These templates ensure that any new indices created with matching patterns
+ will automatically have the correct structure.
"""
client = settings.create_client
await client.indices.put_index_template(
@@ -35,3 +46,80 @@ async def create_index_templates_shared(settings: Any) -> None:
},
)
await client.close()
+
+
+def apply_free_text_filter_shared(
+ search: Any, free_text_queries: Optional[List[str]]
+) -> Any:
+ """Create a free text query for Elasticsearch/OpenSearch.
+
+ Args:
+ search (Any): The search object to apply the query to.
+ free_text_queries (Optional[List[str]]): A list of text strings to search for in the properties.
+
+ Returns:
+ Any: The search object with the free text query applied, or the original search
+ object if no free_text_queries were provided.
+
+ Notes:
+ This function creates a query_string query that searches for the specified text strings
+ in all properties of the documents. The query strings are joined with OR operators.
+ """
+ if free_text_queries is not None:
+ free_text_query_string = '" OR properties.\\*:"'.join(free_text_queries)
+ search = search.query(
+ "query_string", query=f'properties.\\*:"{free_text_query_string}"'
+ )
+
+ return search
+
+
+def apply_intersects_filter_shared(
+ intersects: Geometry,
+) -> Dict[str, Dict]:
+ """Create a geo_shape filter for intersecting geometry.
+
+ Args:
+ intersects (Geometry): The intersecting geometry, represented as a GeoJSON-like object.
+
+ Returns:
+ Dict[str, Dict]: A dictionary containing the geo_shape filter configuration
+ that can be used with Elasticsearch/OpenSearch Q objects.
+
+ Notes:
+ This function creates a geo_shape filter configuration to find documents that intersect
+ with the specified geometry. The returned dictionary should be wrapped in a Q object
+ when applied to a search.
+ """
+ return {
+ "geo_shape": {
+ "geometry": {
+ "shape": {
+ "type": intersects.type.lower(),
+ "coordinates": intersects.coordinates,
+ },
+ "relation": "intersects",
+ }
+ }
+ }
+
+
+def populate_sort_shared(sortby: List) -> Optional[Dict[str, Dict[str, str]]]:
+ """Create a sort configuration for Elasticsearch/OpenSearch queries.
+
+ Args:
+ sortby (List): A list of sort specifications, each containing a field and direction.
+
+ Returns:
+ Optional[Dict[str, Dict[str, str]]]: A dictionary mapping field names to sort direction
+ configurations, or None if no sort was specified.
+
+ Notes:
+ This function transforms a list of sort specifications into the format required by
+ Elasticsearch/OpenSearch for sorting query results. The returned dictionary can be
+ directly used in search requests.
+ """
+ if sortby:
+ return {s.field: {"order": s.direction} for s in sortby}
+ else:
+ return None
From 30ea1c7625c63192fdf11f4024109429a41dd165 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Sun, 18 May 2025 20:22:44 +0800
Subject: [PATCH 08/29] shared delete items index
---
.../elasticsearch/database_logic.py | 18 +++++-------
.../stac_fastapi/opensearch/database_logic.py | 18 +++++-------
.../sfeos_helpers/database_logic_helpers.py | 29 +++++++++++++++++++
3 files changed, 43 insertions(+), 22 deletions(-)
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
index 9c4e8366..1d9e8a7d 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
@@ -25,6 +25,7 @@
apply_free_text_filter_shared,
apply_intersects_filter_shared,
create_index_templates_shared,
+ delete_item_index_shared,
populate_sort_shared,
)
from stac_fastapi.sfeos_helpers.mappings import (
@@ -99,18 +100,13 @@ async def delete_item_index(collection_id: str):
Args:
collection_id (str): The ID of the collection whose items index will be deleted.
- """
- client = AsyncElasticsearchSettings().create_client
- name = index_alias_by_collection_id(collection_id)
- resolved = await client.indices.resolve_index(name=name)
- if "aliases" in resolved and resolved["aliases"]:
- [alias] = resolved["aliases"]
- await client.indices.delete_alias(index=alias["indices"], name=alias["name"])
- await client.indices.delete(index=alias["indices"])
- else:
- await client.indices.delete(index=name)
- await client.close()
+ Notes:
+ This function delegates to the shared implementation in delete_item_index_shared.
+ """
+ await delete_item_index_shared(
+ settings=AsyncElasticsearchSettings(), collection_id=collection_id
+ )
@attr.s
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
index 3115a56d..34308acb 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
@@ -25,6 +25,7 @@
apply_free_text_filter_shared,
apply_intersects_filter_shared,
create_index_templates_shared,
+ delete_item_index_shared,
populate_sort_shared,
)
from stac_fastapi.sfeos_helpers.mappings import (
@@ -116,18 +117,13 @@ async def delete_item_index(collection_id: str) -> None:
Args:
collection_id (str): The ID of the collection whose items index will be deleted.
- """
- client = AsyncSearchSettings().create_client
- name = index_alias_by_collection_id(collection_id)
- resolved = await client.indices.resolve_index(name=name)
- if "aliases" in resolved and resolved["aliases"]:
- [alias] = resolved["aliases"]
- await client.indices.delete_alias(index=alias["indices"], name=alias["name"])
- await client.indices.delete(index=alias["indices"])
- else:
- await client.indices.delete(index=name)
- await client.close()
+ Notes:
+ This function delegates to the shared implementation in delete_item_index_shared.
+ """
+ await delete_item_index_shared(
+ settings=AsyncSearchSettings(), collection_id=collection_id
+ )
@attr.s
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
index a8e80658..690c4195 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
@@ -9,6 +9,7 @@
ES_ITEMS_SETTINGS,
ITEMS_INDEX_PREFIX,
Geometry,
+ index_alias_by_collection_id,
)
@@ -123,3 +124,31 @@ def populate_sort_shared(sortby: List) -> Optional[Dict[str, Dict[str, str]]]:
return {s.field: {"order": s.direction} for s in sortby}
else:
return None
+
+
+async def delete_item_index_shared(settings: Any, collection_id: str) -> None:
+ """Delete the index for items in a collection.
+
+ Args:
+ settings (Any): The settings object containing the client configuration.
+ Must have a create_client attribute that returns an Elasticsearch/OpenSearch client.
+ collection_id (str): The ID of the collection whose items index will be deleted.
+
+ Returns:
+ None: This function doesn't return any value but deletes an item index in the database.
+
+ Notes:
+ This function deletes an item index and its alias. It first resolves the alias to find
+ the actual index name, then deletes both the alias and the index.
+ """
+ client = settings.create_client
+
+ name = index_alias_by_collection_id(collection_id)
+ resolved = await client.indices.resolve_index(name=name)
+ if "aliases" in resolved and resolved["aliases"]:
+ [alias] = resolved["aliases"]
+ await client.indices.delete_alias(index=alias["indices"], name=alias["name"])
+ await client.indices.delete(index=alias["indices"])
+ else:
+ await client.indices.delete(index=name)
+ await client.close()
From e6743815322006401174729efe5c5b47c56db6e4 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Sun, 18 May 2025 23:03:33 +0800
Subject: [PATCH 09/29] share aggregations mapping
---
.../elasticsearch/database_logic.py | 72 +------------------
.../stac_fastapi/opensearch/database_logic.py | 72 +------------------
.../stac_fastapi/sfeos_helpers/mappings.py | 71 ++++++++++++++++++
3 files changed, 75 insertions(+), 140 deletions(-)
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
index 1d9e8a7d..e2cbffe6 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
@@ -29,6 +29,7 @@
populate_sort_shared,
)
from stac_fastapi.sfeos_helpers.mappings import (
+ AGGREGATION_MAPPING,
COLLECTIONS_INDEX,
DEFAULT_SORT,
ITEM_INDICES,
@@ -135,76 +136,7 @@ def __attrs_post_init__(self):
extensions: List[str] = attr.ib(default=attr.Factory(list))
- aggregation_mapping: Dict[str, Dict[str, Any]] = {
- "total_count": {"value_count": {"field": "id"}},
- "collection_frequency": {"terms": {"field": "collection", "size": 100}},
- "platform_frequency": {"terms": {"field": "properties.platform", "size": 100}},
- "cloud_cover_frequency": {
- "range": {
- "field": "properties.eo:cloud_cover",
- "ranges": [
- {"to": 5},
- {"from": 5, "to": 15},
- {"from": 15, "to": 40},
- {"from": 40},
- ],
- }
- },
- "datetime_frequency": {
- "date_histogram": {
- "field": "properties.datetime",
- "calendar_interval": "month",
- }
- },
- "datetime_min": {"min": {"field": "properties.datetime"}},
- "datetime_max": {"max": {"field": "properties.datetime"}},
- "grid_code_frequency": {
- "terms": {
- "field": "properties.grid:code",
- "missing": "none",
- "size": 10000,
- }
- },
- "sun_elevation_frequency": {
- "histogram": {"field": "properties.view:sun_elevation", "interval": 5}
- },
- "sun_azimuth_frequency": {
- "histogram": {"field": "properties.view:sun_azimuth", "interval": 5}
- },
- "off_nadir_frequency": {
- "histogram": {"field": "properties.view:off_nadir", "interval": 5}
- },
- "centroid_geohash_grid_frequency": {
- "geohash_grid": {
- "field": "properties.proj:centroid",
- "precision": 1,
- }
- },
- "centroid_geohex_grid_frequency": {
- "geohex_grid": {
- "field": "properties.proj:centroid",
- "precision": 0,
- }
- },
- "centroid_geotile_grid_frequency": {
- "geotile_grid": {
- "field": "properties.proj:centroid",
- "precision": 0,
- }
- },
- "geometry_geohash_grid_frequency": {
- "geohash_grid": {
- "field": "geometry",
- "precision": 1,
- }
- },
- "geometry_geotile_grid_frequency": {
- "geotile_grid": {
- "field": "geometry",
- "precision": 0,
- }
- },
- }
+ aggregation_mapping: Dict[str, Dict[str, Any]] = AGGREGATION_MAPPING
"""CORE LOGIC"""
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
index 34308acb..d4cc8532 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
@@ -29,6 +29,7 @@
populate_sort_shared,
)
from stac_fastapi.sfeos_helpers.mappings import (
+ AGGREGATION_MAPPING,
COLLECTIONS_INDEX,
DEFAULT_SORT,
ES_COLLECTIONS_MAPPINGS,
@@ -148,76 +149,7 @@ def __attrs_post_init__(self):
extensions: List[str] = attr.ib(default=attr.Factory(list))
- aggregation_mapping: Dict[str, Dict[str, Any]] = {
- "total_count": {"value_count": {"field": "id"}},
- "collection_frequency": {"terms": {"field": "collection", "size": 100}},
- "platform_frequency": {"terms": {"field": "properties.platform", "size": 100}},
- "cloud_cover_frequency": {
- "range": {
- "field": "properties.eo:cloud_cover",
- "ranges": [
- {"to": 5},
- {"from": 5, "to": 15},
- {"from": 15, "to": 40},
- {"from": 40},
- ],
- }
- },
- "datetime_frequency": {
- "date_histogram": {
- "field": "properties.datetime",
- "calendar_interval": "month",
- }
- },
- "datetime_min": {"min": {"field": "properties.datetime"}},
- "datetime_max": {"max": {"field": "properties.datetime"}},
- "grid_code_frequency": {
- "terms": {
- "field": "properties.grid:code",
- "missing": "none",
- "size": 10000,
- }
- },
- "sun_elevation_frequency": {
- "histogram": {"field": "properties.view:sun_elevation", "interval": 5}
- },
- "sun_azimuth_frequency": {
- "histogram": {"field": "properties.view:sun_azimuth", "interval": 5}
- },
- "off_nadir_frequency": {
- "histogram": {"field": "properties.view:off_nadir", "interval": 5}
- },
- "centroid_geohash_grid_frequency": {
- "geohash_grid": {
- "field": "properties.proj:centroid",
- "precision": 1,
- }
- },
- "centroid_geohex_grid_frequency": {
- "geohex_grid": {
- "field": "properties.proj:centroid",
- "precision": 0,
- }
- },
- "centroid_geotile_grid_frequency": {
- "geotile_grid": {
- "field": "properties.proj:centroid",
- "precision": 0,
- }
- },
- "geometry_geohash_grid_frequency": {
- "geohash_grid": {
- "field": "geometry",
- "precision": 1,
- }
- },
- "geometry_geotile_grid_frequency": {
- "geotile_grid": {
- "field": "geometry",
- "precision": 0,
- }
- },
- }
+ aggregation_mapping: Dict[str, Dict[str, Any]] = AGGREGATION_MAPPING
"""CORE LOGIC"""
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py
index 246c27d6..afd2f740 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py
@@ -144,6 +144,77 @@ class Geometry(Protocol): # noqa
},
}
+# Shared aggregation mapping for both Elasticsearch and OpenSearch
+AGGREGATION_MAPPING: Dict[str, Dict[str, Any]] = {
+ "total_count": {"value_count": {"field": "id"}},
+ "collection_frequency": {"terms": {"field": "collection", "size": 100}},
+ "platform_frequency": {"terms": {"field": "properties.platform", "size": 100}},
+ "cloud_cover_frequency": {
+ "range": {
+ "field": "properties.eo:cloud_cover",
+ "ranges": [
+ {"to": 5},
+ {"from": 5, "to": 15},
+ {"from": 15, "to": 40},
+ {"from": 40},
+ ],
+ }
+ },
+ "datetime_frequency": {
+ "date_histogram": {
+ "field": "properties.datetime",
+ "calendar_interval": "month",
+ }
+ },
+ "datetime_min": {"min": {"field": "properties.datetime"}},
+ "datetime_max": {"max": {"field": "properties.datetime"}},
+ "grid_code_frequency": {
+ "terms": {
+ "field": "properties.grid:code",
+ "missing": "none",
+ "size": 10000,
+ }
+ },
+ "sun_elevation_frequency": {
+ "histogram": {"field": "properties.view:sun_elevation", "interval": 5}
+ },
+ "sun_azimuth_frequency": {
+ "histogram": {"field": "properties.view:sun_azimuth", "interval": 5}
+ },
+ "off_nadir_frequency": {
+ "histogram": {"field": "properties.view:off_nadir", "interval": 5}
+ },
+ "centroid_geohash_grid_frequency": {
+ "geohash_grid": {
+ "field": "properties.proj:centroid",
+ "precision": 1,
+ }
+ },
+ "centroid_geohex_grid_frequency": {
+ "geohex_grid": {
+ "field": "properties.proj:centroid",
+ "precision": 0,
+ }
+ },
+ "centroid_geotile_grid_frequency": {
+ "geotile_grid": {
+ "field": "properties.proj:centroid",
+ "precision": 0,
+ }
+ },
+ "geometry_geohash_grid_frequency": {
+ "geohash_grid": {
+ "field": "geometry",
+ "precision": 1,
+ }
+ },
+ "geometry_geotile_grid_frequency": {
+ "geotile_grid": {
+ "field": "geometry",
+ "precision": 0,
+ }
+ },
+}
ES_MAPPING_TYPE_TO_JSON: Dict[
str, Literal["string", "number", "boolean", "object", "array", "null"]
From b956bae83a5f059e8a49fc64b13a55f4bbfa7ddb Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Sun, 18 May 2025 23:21:25 +0800
Subject: [PATCH 10/29] move logic to utilities
---
.../elasticsearch/database_logic.py | 4 +-
.../stac_fastapi/opensearch/database_logic.py | 4 +-
.../sfeos_helpers/database_logic_helpers.py | 2 +-
.../stac_fastapi/sfeos_helpers/mappings.py | 92 +----------------
.../stac_fastapi/sfeos_helpers/utilities.py | 98 ++++++++++++++++++-
stac_fastapi/tests/database/test_database.py | 2 +-
6 files changed, 106 insertions(+), 96 deletions(-)
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
index e2cbffe6..8b4d7e81 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
@@ -35,13 +35,15 @@
ITEM_INDICES,
ITEMS_INDEX_PREFIX,
Geometry,
+)
+from stac_fastapi.sfeos_helpers.utilities import (
index_alias_by_collection_id,
index_by_collection_id,
indices,
mk_actions,
mk_item_id,
+ validate_refresh,
)
-from stac_fastapi.sfeos_helpers.utilities import validate_refresh
from stac_fastapi.types.errors import ConflictError, NotFoundError
from stac_fastapi.types.stac import Collection, Item
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
index d4cc8532..26e7cd0f 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
@@ -38,13 +38,15 @@
ITEM_INDICES,
ITEMS_INDEX_PREFIX,
Geometry,
+)
+from stac_fastapi.sfeos_helpers.utilities import (
index_alias_by_collection_id,
index_by_collection_id,
indices,
mk_actions,
mk_item_id,
+ validate_refresh,
)
-from stac_fastapi.sfeos_helpers.utilities import validate_refresh
from stac_fastapi.types.errors import ConflictError, NotFoundError
from stac_fastapi.types.stac import Collection, Item
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
index 690c4195..f91f3ddb 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
@@ -9,8 +9,8 @@
ES_ITEMS_SETTINGS,
ITEMS_INDEX_PREFIX,
Geometry,
- index_alias_by_collection_id,
)
+from stac_fastapi.sfeos_helpers.utilities import index_alias_by_collection_id
async def create_index_templates_shared(settings: Any) -> None:
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py
index afd2f740..9aa2844e 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py
@@ -1,10 +1,7 @@
"""Shared mappings for stac-fastapi elasticsearch and opensearch backends."""
import os
-from functools import lru_cache
-from typing import Any, Dict, List, Literal, Optional, Protocol
-
-from stac_fastapi.types.stac import Item
+from typing import Any, Dict, Literal, Protocol
# stac_pydantic classes extend _GeometryBase, which doesn't have a type field,
@@ -238,90 +235,3 @@ class Geometry(Protocol): # noqa
"geo_shape": "object",
"nested": "array",
}
-
-
-@lru_cache(256)
-def index_by_collection_id(collection_id: str) -> str:
- """
- Translate a collection id into an Elasticsearch index name.
-
- Args:
- collection_id (str): The collection id to translate into an index name.
-
- Returns:
- str: The index name derived from the collection id.
- """
- cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
- return (
- f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{collection_id.encode('utf-8').hex()}"
- )
-
-
-@lru_cache(256)
-def index_alias_by_collection_id(collection_id: str) -> str:
- """
- Translate a collection id into an Elasticsearch index alias.
-
- Args:
- collection_id (str): The collection id to translate into an index alias.
-
- Returns:
- str: The index alias derived from the collection id.
- """
- cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
- return f"{ITEMS_INDEX_PREFIX}{cleaned}"
-
-
-def indices(collection_ids: Optional[List[str]]) -> str:
- """
- Get a comma-separated string of index names for a given list of collection ids.
-
- Args:
- collection_ids: A list of collection ids.
-
- Returns:
- A string of comma-separated index names. If `collection_ids` is empty, returns the default indices.
- """
- return (
- ",".join(map(index_alias_by_collection_id, collection_ids))
- if collection_ids
- else ITEM_INDICES
- )
-
-
-def mk_item_id(item_id: str, collection_id: str) -> str:
- """Create the document id for an Item in Elasticsearch.
-
- Args:
- item_id (str): The id of the Item.
- collection_id (str): The id of the Collection that the Item belongs to.
-
- Returns:
- str: The document id for the Item, combining the Item id and the Collection id, separated by a `|` character.
- """
- return f"{item_id}|{collection_id}"
-
-
-def mk_actions(collection_id: str, processed_items: List[Item]) -> List[Dict[str, Any]]:
- """Create Elasticsearch bulk actions for a list of processed items.
-
- Args:
- collection_id (str): The identifier for the collection the items belong to.
- processed_items (List[Item]): The list of processed items to be bulk indexed.
-
- Returns:
- List[Dict[str, Union[str, Dict]]]: The list of bulk actions to be executed,
- each action being a dictionary with the following keys:
- - `_index`: the index to store the document in.
- - `_id`: the document's identifier.
- - `_source`: the source of the document.
- """
- index_alias = index_alias_by_collection_id(collection_id)
- return [
- {
- "_index": index_alias,
- "_id": mk_item_id(item["id"], item["collection"]),
- "_source": item,
- }
- for item in processed_items
- ]
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py
index fa0bd194..910732ac 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py
@@ -1,9 +1,18 @@
"""Shared utilities functions for stac-fastapi elasticsearch and opensearch backends."""
import logging
-from typing import Union
+from functools import lru_cache
+from typing import Any, Dict, List, Optional, Union
from stac_fastapi.core.utilities import get_bool_env
+# Import constants from mappings
+from stac_fastapi.sfeos_helpers.mappings import (
+ _ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE,
+ ITEM_INDICES,
+ ITEMS_INDEX_PREFIX,
+)
+from stac_fastapi.types.stac import Item
+
def validate_refresh(value: Union[str, bool]) -> str:
"""
@@ -43,3 +52,90 @@ def validate_refresh(value: Union[str, bool]) -> str:
f"Invalid value for `refresh`: '{value}'. Expected 'true', 'false', or 'wait_for'. Defaulting to 'false'."
)
return "false"
+
+
+@lru_cache(256)
+def index_by_collection_id(collection_id: str) -> str:
+ """
+ Translate a collection id into an Elasticsearch index name.
+
+ Args:
+ collection_id (str): The collection id to translate into an index name.
+
+ Returns:
+ str: The index name derived from the collection id.
+ """
+ cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
+ return (
+ f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{collection_id.encode('utf-8').hex()}"
+ )
+
+
+@lru_cache(256)
+def index_alias_by_collection_id(collection_id: str) -> str:
+ """
+ Translate a collection id into an Elasticsearch index alias.
+
+ Args:
+ collection_id (str): The collection id to translate into an index alias.
+
+ Returns:
+ str: The index alias derived from the collection id.
+ """
+ cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
+ return f"{ITEMS_INDEX_PREFIX}{cleaned}"
+
+
+def indices(collection_ids: Optional[List[str]]) -> str:
+ """
+ Get a comma-separated string of index names for a given list of collection ids.
+
+ Args:
+ collection_ids: A list of collection ids.
+
+ Returns:
+ A string of comma-separated index names. If `collection_ids` is empty, returns the default indices.
+ """
+ return (
+ ",".join(map(index_alias_by_collection_id, collection_ids))
+ if collection_ids
+ else ITEM_INDICES
+ )
+
+
+def mk_item_id(item_id: str, collection_id: str) -> str:
+ """Create the document id for an Item in Elasticsearch.
+
+ Args:
+ item_id (str): The id of the Item.
+ collection_id (str): The id of the Collection that the Item belongs to.
+
+ Returns:
+ str: The document id for the Item, combining the Item id and the Collection id, separated by a `|` character.
+ """
+ return f"{item_id}|{collection_id}"
+
+
+def mk_actions(collection_id: str, processed_items: List[Item]) -> List[Dict[str, Any]]:
+ """Create Elasticsearch bulk actions for a list of processed items.
+
+ Args:
+ collection_id (str): The identifier for the collection the items belong to.
+ processed_items (List[Item]): The list of processed items to be bulk indexed.
+
+ Returns:
+ List[Dict[str, Union[str, Dict]]]: The list of bulk actions to be executed,
+ each action being a dictionary with the following keys:
+ - `_index`: the index to store the document in.
+ - `_id`: the document's identifier.
+ - `_source`: the source of the document.
+ """
+ index_alias = index_alias_by_collection_id(collection_id)
+ return [
+ {
+ "_index": index_alias,
+ "_id": mk_item_id(item["id"], item["collection"]),
+ "_source": item,
+ }
+ for item in processed_items
+ ]
diff --git a/stac_fastapi/tests/database/test_database.py b/stac_fastapi/tests/database/test_database.py
index b70bde2f..12196186 100644
--- a/stac_fastapi/tests/database/test_database.py
+++ b/stac_fastapi/tests/database/test_database.py
@@ -7,8 +7,8 @@
COLLECTIONS_INDEX,
ES_COLLECTIONS_MAPPINGS,
ES_ITEMS_MAPPINGS,
- index_alias_by_collection_id,
)
+from stac_fastapi.sfeos_helpers.utilities import index_alias_by_collection_id
from ..conftest import MockRequest, database
From 149be4ebc72f3eb483fb8de16d618c25858c0c28 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Sun, 18 May 2025 23:34:02 +0800
Subject: [PATCH 11/29] share queryables mapping
---
.../elasticsearch/database_logic.py | 18 +++--------
.../stac_fastapi/opensearch/database_logic.py | 18 +++--------
.../sfeos_helpers/database_logic_helpers.py | 32 +++++++++++++++++++
3 files changed, 40 insertions(+), 28 deletions(-)
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
index 8b4d7e81..9781c677 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
@@ -26,6 +26,7 @@
apply_intersects_filter_shared,
create_index_templates_shared,
delete_item_index_shared,
+ get_queryables_mapping_shared,
populate_sort_shared,
)
from stac_fastapi.sfeos_helpers.mappings import (
@@ -219,23 +220,12 @@ async def get_queryables_mapping(self, collection_id: str = "*") -> dict:
Returns:
dict: A dictionary containing the Queryables mappings.
"""
- queryables_mapping = {}
-
mappings = await self.client.indices.get_mapping(
index=f"{ITEMS_INDEX_PREFIX}{collection_id}",
)
-
- for mapping in mappings.values():
- fields = mapping["mappings"].get("properties", {})
- properties = fields.pop("properties", {}).get("properties", {}).keys()
-
- for field_key in fields:
- queryables_mapping[field_key] = field_key
-
- for property_key in properties:
- queryables_mapping[property_key] = f"properties.{property_key}"
-
- return queryables_mapping
+ return await get_queryables_mapping_shared(
+ collection_id=collection_id, mappings=mappings
+ )
@staticmethod
def make_search():
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
index 26e7cd0f..b1703bf1 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
@@ -26,6 +26,7 @@
apply_intersects_filter_shared,
create_index_templates_shared,
delete_item_index_shared,
+ get_queryables_mapping_shared,
populate_sort_shared,
)
from stac_fastapi.sfeos_helpers.mappings import (
@@ -238,23 +239,12 @@ async def get_queryables_mapping(self, collection_id: str = "*") -> dict:
Returns:
dict: A dictionary containing the Queryables mappings.
"""
- queryables_mapping = {}
-
mappings = await self.client.indices.get_mapping(
index=f"{ITEMS_INDEX_PREFIX}{collection_id}",
)
-
- for mapping in mappings.values():
- fields = mapping["mappings"].get("properties", {})
- properties = fields.pop("properties", {}).get("properties", {}).keys()
-
- for field_key in fields:
- queryables_mapping[field_key] = field_key
-
- for property_key in properties:
- queryables_mapping[property_key] = f"properties.{property_key}"
-
- return queryables_mapping
+ return await get_queryables_mapping_shared(
+ collection_id=collection_id, mappings=mappings
+ )
@staticmethod
def make_search():
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
index f91f3ddb..42dd068f 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
@@ -152,3 +152,35 @@ async def delete_item_index_shared(settings: Any, collection_id: str) -> None:
else:
await client.indices.delete(index=name)
await client.close()
+
+
+async def get_queryables_mapping_shared(
+ mappings: Dict[str, Dict[str, Any]], collection_id: str = "*"
+) -> Dict[str, str]:
+ """Retrieve mapping of Queryables for search.
+
+ Args:
+ mappings (Dict[str, Dict[str, Any]]): The mapping information returned from
+ Elasticsearch/OpenSearch client's indices.get_mapping() method.
+ Expected structure is {index_name: {"mappings": {...}}}.
+ collection_id (str, optional): The id of the Collection the Queryables
+ belongs to. Defaults to "*".
+
+ Returns:
+ Dict[str, str]: A dictionary containing the Queryables mappings, where keys are
+ field names and values are the corresponding paths in the Elasticsearch/OpenSearch
+ document structure.
+ """
+ queryables_mapping = {}
+
+ for mapping in mappings.values():
+ fields = mapping["mappings"].get("properties", {})
+ properties = fields.pop("properties", {}).get("properties", {}).keys()
+
+ for field_key in fields:
+ queryables_mapping[field_key] = field_key
+
+ for property_key in properties:
+ queryables_mapping[property_key] = f"properties.{property_key}"
+
+ return queryables_mapping
From 9a681680c01bffa80945ea5c9c863f7a910853e8 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Mon, 19 May 2025 13:08:11 +0800
Subject: [PATCH 12/29] organization
---
.../sfeos_helpers/database_logic_helpers.py | 89 +++++++++++++------
.../stac_fastapi/sfeos_helpers/filter.py | 40 ++++++++-
.../stac_fastapi/sfeos_helpers/mappings.py | 27 +++++-
.../stac_fastapi/sfeos_helpers/utilities.py | 37 +++++++-
4 files changed, 163 insertions(+), 30 deletions(-)
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
index 42dd068f..941b8fcf 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
@@ -1,4 +1,25 @@
-"""Shared code for elasticsearch/ opensearch database logic."""
+"""Shared code for elasticsearch/ opensearch database logic.
+
+This module contains shared functions used by both the Elasticsearch and OpenSearch
+implementations of STAC FastAPI for database operations. It helps reduce code duplication
+and ensures consistent behavior between the two implementations.
+
+The sfeos_helpers package is organized as follows:
+- database_logic_helpers.py: Shared database operations (this file)
+- filter.py: Shared filter extension implementation
+- mappings.py: Shared constants and mapping definitions
+- utilities.py: Shared utility functions
+
+When adding new functionality to this package, consider:
+1. Will this code be used by both Elasticsearch and OpenSearch implementations?
+2. Is the functionality stable and unlikely to diverge between implementations?
+3. Is the function well-documented with clear input/output contracts?
+
+Function Naming Conventions:
+- All shared functions should end with `_shared` to clearly indicate they're meant to be used by both implementations
+- Function names should be descriptive and indicate their purpose
+- Parameter names should be consistent across similar functions
+"""
from typing import Any, Dict, List, Optional
@@ -12,6 +33,10 @@
)
from stac_fastapi.sfeos_helpers.utilities import index_alias_by_collection_id
+# ============================================================================
+# Index Management Functions
+# ============================================================================
+
async def create_index_templates_shared(settings: Any) -> None:
"""Create index templates for Elasticsearch/OpenSearch Collection and Item indices.
@@ -49,6 +74,39 @@ async def create_index_templates_shared(settings: Any) -> None:
await client.close()
+async def delete_item_index_shared(settings: Any, collection_id: str) -> None:
+ """Delete the index for items in a collection.
+
+ Args:
+ settings (Any): The settings object containing the client configuration.
+ Must have a create_client attribute that returns an Elasticsearch/OpenSearch client.
+ collection_id (str): The ID of the collection whose items index will be deleted.
+
+ Returns:
+ None: This function doesn't return any value but deletes an item index in the database.
+
+ Notes:
+ This function deletes an item index and its alias. It first resolves the alias to find
+ the actual index name, then deletes both the alias and the index.
+ """
+ client = settings.create_client
+
+ name = index_alias_by_collection_id(collection_id)
+ resolved = await client.indices.resolve_index(name=name)
+ if "aliases" in resolved and resolved["aliases"]:
+ [alias] = resolved["aliases"]
+ await client.indices.delete_alias(index=alias["indices"], name=alias["name"])
+ await client.indices.delete(index=alias["indices"])
+ else:
+ await client.indices.delete(index=name)
+ await client.close()
+
+
+# ============================================================================
+# Query Building Functions
+# ============================================================================
+
+
def apply_free_text_filter_shared(
search: Any, free_text_queries: Optional[List[str]]
) -> Any:
@@ -126,32 +184,9 @@ def populate_sort_shared(sortby: List) -> Optional[Dict[str, Dict[str, str]]]:
return None
-async def delete_item_index_shared(settings: Any, collection_id: str) -> None:
- """Delete the index for items in a collection.
-
- Args:
- settings (Any): The settings object containing the client configuration.
- Must have a create_client attribute that returns an Elasticsearch/OpenSearch client.
- collection_id (str): The ID of the collection whose items index will be deleted.
-
- Returns:
- None: This function doesn't return any value but deletes an item index in the database.
-
- Notes:
- This function deletes an item index and its alias. It first resolves the alias to find
- the actual index name, then deletes both the alias and the index.
- """
- client = settings.create_client
-
- name = index_alias_by_collection_id(collection_id)
- resolved = await client.indices.resolve_index(name=name)
- if "aliases" in resolved and resolved["aliases"]:
- [alias] = resolved["aliases"]
- await client.indices.delete_alias(index=alias["indices"], name=alias["name"])
- await client.indices.delete(index=alias["indices"])
- else:
- await client.indices.delete(index=name)
- await client.close()
+# ============================================================================
+# Mapping Functions
+# ============================================================================
async def get_queryables_mapping_shared(
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
index d2b890db..be6f5dad 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
@@ -1,4 +1,28 @@
-"""Shared filter extension methods for stac-fastapi elasticsearch and opensearch backends."""
+"""Shared filter extension methods for stac-fastapi elasticsearch and opensearch backends.
+
+This module provides shared functionality for implementing the STAC API Filter Extension
+with Elasticsearch and OpenSearch. It includes:
+
+1. Functions for converting CQL2 queries to Elasticsearch/OpenSearch query DSL
+2. Helper functions for field mapping and query transformation
+3. Base implementation of the AsyncBaseFiltersClient for Elasticsearch/OpenSearch
+
+The sfeos_helpers package is organized as follows:
+- database_logic_helpers.py: Shared database operations
+- filter.py: Shared filter extension implementation (this file)
+- mappings.py: Shared constants and mapping definitions
+- utilities.py: Shared utility functions
+
+When adding new functionality to this package, consider:
+1. Will this code be used by both Elasticsearch and OpenSearch implementations?
+2. Is the functionality stable and unlikely to diverge between implementations?
+3. Is the function well-documented with clear input/output contracts?
+
+Function Naming Conventions:
+- All shared functions should end with `_shared` to clearly indicate they're meant to be used by both implementations
+- Function names should be descriptive and indicate their purpose
+- Parameter names should be consistent across similar functions
+"""
import re
from collections import deque
@@ -20,6 +44,10 @@
from .mappings import ES_MAPPING_TYPE_TO_JSON
+# ============================================================================
+# CQL2 Pattern Conversion Helpers
+# ============================================================================
+
def _replace_like_patterns(match: re.Match) -> str:
pattern = match.group()
@@ -48,6 +76,11 @@ def cql2_like_to_es(string: str) -> str:
)
+# ============================================================================
+# Query Transformation Functions
+# ============================================================================
+
+
def to_es_field(queryables_mapping: Dict[str, Any], field: str) -> str:
"""
Map a given field to its corresponding Elasticsearch field according to a predefined mapping.
@@ -169,6 +202,11 @@ def to_es(queryables_mapping: Dict[str, Any], query: Dict[str, Any]) -> Dict[str
return {}
+# ============================================================================
+# Filter Client Implementation
+# ============================================================================
+
+
@attr.s
class EsAsyncBaseFiltersClient(AsyncBaseFiltersClient):
"""Defines a pattern for implementing the STAC filter extension."""
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py
index 9aa2844e..476d656a 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py
@@ -1,4 +1,29 @@
-"""Shared mappings for stac-fastapi elasticsearch and opensearch backends."""
+"""Shared mappings for stac-fastapi elasticsearch and opensearch backends.
+
+This module contains shared constants, mappings, and type definitions used by both
+the Elasticsearch and OpenSearch implementations of STAC FastAPI. It includes:
+
+1. Index name constants and character translation tables
+2. Mapping definitions for Collections and Items
+3. Aggregation mappings for search queries
+4. Type conversion mappings between Elasticsearch/OpenSearch and JSON Schema types
+
+The sfeos_helpers package is organized as follows:
+- database_logic_helpers.py: Shared database operations
+- filter.py: Shared filter extension implementation
+- mappings.py: Shared constants and mapping definitions (this file)
+- utilities.py: Shared utility functions
+
+When adding new functionality to this package, consider:
+1. Will this code be used by both Elasticsearch and OpenSearch implementations?
+2. Is the functionality stable and unlikely to diverge between implementations?
+3. Is the function well-documented with clear input/output contracts?
+
+Function Naming Conventions:
+- All shared functions should end with `_shared` to clearly indicate they're meant to be used by both implementations
+- Function names should be descriptive and indicate their purpose
+- Parameter names should be consistent across similar functions
+"""
import os
from typing import Any, Dict, Literal, Protocol
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py
index 910732ac..7317d9e5 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py
@@ -1,4 +1,25 @@
-"""Shared utilities functions for stac-fastapi elasticsearch and opensearch backends."""
+"""Shared utilities functions for stac-fastapi elasticsearch and opensearch backends.
+
+This module contains general utility functions used by both the Elasticsearch and OpenSearch
+implementations of STAC FastAPI. These functions handle common tasks like parameter validation,
+index naming, and document ID generation.
+
+The sfeos_helpers package is organized as follows:
+- database_logic_helpers.py: Shared database operations
+- filter.py: Shared filter extension implementation
+- mappings.py: Shared constants and mapping definitions
+- utilities.py: Shared utility functions (this file)
+
+When adding new functionality to this package, consider:
+1. Will this code be used by both Elasticsearch and OpenSearch implementations?
+2. Is the functionality stable and unlikely to diverge between implementations?
+3. Is the function well-documented with clear input/output contracts?
+
+Function Naming Conventions:
+- All shared functions should end with `_shared` to clearly indicate they're meant to be used by both implementations
+- Function names should be descriptive and indicate their purpose
+- Parameter names should be consistent across similar functions
+"""
import logging
from functools import lru_cache
from typing import Any, Dict, List, Optional, Union
@@ -13,6 +34,10 @@
)
from stac_fastapi.types.stac import Item
+# ============================================================================
+# Parameter Validation
+# ============================================================================
+
def validate_refresh(value: Union[str, bool]) -> str:
"""
@@ -54,6 +79,11 @@ def validate_refresh(value: Union[str, bool]) -> str:
return "false"
+# ============================================================================
+# Index and Document ID Utilities
+# ============================================================================
+
+
@lru_cache(256)
def index_by_collection_id(collection_id: str) -> str:
"""
@@ -103,6 +133,11 @@ def indices(collection_ids: Optional[List[str]]) -> str:
)
+# ============================================================================
+# Document ID and Action Generation
+# ============================================================================
+
+
def mk_item_id(item_id: str, collection_id: str) -> str:
"""Create the document id for an Item in Elasticsearch.
From c0d5aac621440bf6e904877f86b14938d52334c2 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Mon, 19 May 2025 17:18:02 +0800
Subject: [PATCH 13/29] create filter package
---
.../stac_fastapi/sfeos_helpers/filter.py | 294 ------------------
.../sfeos_helpers/filter/README.md | 27 ++
.../sfeos_helpers/filter/__init__.py | 37 +++
.../sfeos_helpers/filter/client.py | 98 ++++++
.../stac_fastapi/sfeos_helpers/filter/cql2.py | 35 +++
.../sfeos_helpers/filter/transform.py | 133 ++++++++
6 files changed, 330 insertions(+), 294 deletions(-)
delete mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/README.md
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/__init__.py
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/client.py
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/cql2.py
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/transform.py
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
deleted file mode 100644
index be6f5dad..00000000
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py
+++ /dev/null
@@ -1,294 +0,0 @@
-"""Shared filter extension methods for stac-fastapi elasticsearch and opensearch backends.
-
-This module provides shared functionality for implementing the STAC API Filter Extension
-with Elasticsearch and OpenSearch. It includes:
-
-1. Functions for converting CQL2 queries to Elasticsearch/OpenSearch query DSL
-2. Helper functions for field mapping and query transformation
-3. Base implementation of the AsyncBaseFiltersClient for Elasticsearch/OpenSearch
-
-The sfeos_helpers package is organized as follows:
-- database_logic_helpers.py: Shared database operations
-- filter.py: Shared filter extension implementation (this file)
-- mappings.py: Shared constants and mapping definitions
-- utilities.py: Shared utility functions
-
-When adding new functionality to this package, consider:
-1. Will this code be used by both Elasticsearch and OpenSearch implementations?
-2. Is the functionality stable and unlikely to diverge between implementations?
-3. Is the function well-documented with clear input/output contracts?
-
-Function Naming Conventions:
-- All shared functions should end with `_shared` to clearly indicate they're meant to be used by both implementations
-- Function names should be descriptive and indicate their purpose
-- Parameter names should be consistent across similar functions
-"""
-
-import re
-from collections import deque
-from typing import Any, Dict, Optional
-
-import attr
-
-from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
-from stac_fastapi.core.extensions.filter import (
- DEFAULT_QUERYABLES,
- AdvancedComparisonOp,
- ComparisonOp,
- LogicalOp,
- SpatialOp,
- cql2_like_patterns,
- valid_like_substitutions,
-)
-from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient
-
-from .mappings import ES_MAPPING_TYPE_TO_JSON
-
-# ============================================================================
-# CQL2 Pattern Conversion Helpers
-# ============================================================================
-
-
-def _replace_like_patterns(match: re.Match) -> str:
- pattern = match.group()
- try:
- return valid_like_substitutions[pattern]
- except KeyError:
- raise ValueError(f"'{pattern}' is not a valid escape sequence")
-
-
-def cql2_like_to_es(string: str) -> str:
- """
- Convert CQL2 "LIKE" characters to Elasticsearch "wildcard" characters.
-
- Args:
- string (str): The string containing CQL2 wildcard characters.
-
- Returns:
- str: The converted string with Elasticsearch compatible wildcards.
-
- Raises:
- ValueError: If an invalid escape sequence is encountered.
- """
- return cql2_like_patterns.sub(
- repl=_replace_like_patterns,
- string=string,
- )
-
-
-# ============================================================================
-# Query Transformation Functions
-# ============================================================================
-
-
-def to_es_field(queryables_mapping: Dict[str, Any], field: str) -> str:
- """
- Map a given field to its corresponding Elasticsearch field according to a predefined mapping.
-
- Args:
- field (str): The field name from a user query or filter.
-
- Returns:
- str: The mapped field name suitable for Elasticsearch queries.
- """
- return queryables_mapping.get(field, field)
-
-
-def to_es(queryables_mapping: Dict[str, Any], query: Dict[str, Any]) -> Dict[str, Any]:
- """
- Transform a simplified CQL2 query structure to an Elasticsearch compatible query DSL.
-
- Args:
- query (Dict[str, Any]): The query dictionary containing 'op' and 'args'.
-
- Returns:
- Dict[str, Any]: The corresponding Elasticsearch query in the form of a dictionary.
- """
- if query["op"] in [LogicalOp.AND, LogicalOp.OR, LogicalOp.NOT]:
- bool_type = {
- LogicalOp.AND: "must",
- LogicalOp.OR: "should",
- LogicalOp.NOT: "must_not",
- }[query["op"]]
- return {
- "bool": {
- bool_type: [
- to_es(queryables_mapping, sub_query) for sub_query in query["args"]
- ]
- }
- }
-
- elif query["op"] in [
- ComparisonOp.EQ,
- ComparisonOp.NEQ,
- ComparisonOp.LT,
- ComparisonOp.LTE,
- ComparisonOp.GT,
- ComparisonOp.GTE,
- ]:
- range_op = {
- ComparisonOp.LT: "lt",
- ComparisonOp.LTE: "lte",
- ComparisonOp.GT: "gt",
- ComparisonOp.GTE: "gte",
- }
-
- field = to_es_field(queryables_mapping, query["args"][0]["property"])
- value = query["args"][1]
- if isinstance(value, dict) and "timestamp" in value:
- value = value["timestamp"]
- if query["op"] == ComparisonOp.EQ:
- return {"range": {field: {"gte": value, "lte": value}}}
- elif query["op"] == ComparisonOp.NEQ:
- return {
- "bool": {
- "must_not": [{"range": {field: {"gte": value, "lte": value}}}]
- }
- }
- else:
- return {"range": {field: {range_op[query["op"]]: value}}}
- else:
- if query["op"] == ComparisonOp.EQ:
- return {"term": {field: value}}
- elif query["op"] == ComparisonOp.NEQ:
- return {"bool": {"must_not": [{"term": {field: value}}]}}
- else:
- return {"range": {field: {range_op[query["op"]]: value}}}
-
- elif query["op"] == ComparisonOp.IS_NULL:
- field = to_es_field(queryables_mapping, query["args"][0]["property"])
- return {"bool": {"must_not": {"exists": {"field": field}}}}
-
- elif query["op"] == AdvancedComparisonOp.BETWEEN:
- field = to_es_field(queryables_mapping, query["args"][0]["property"])
- gte, lte = query["args"][1], query["args"][2]
- if isinstance(gte, dict) and "timestamp" in gte:
- gte = gte["timestamp"]
- if isinstance(lte, dict) and "timestamp" in lte:
- lte = lte["timestamp"]
- return {"range": {field: {"gte": gte, "lte": lte}}}
-
- elif query["op"] == AdvancedComparisonOp.IN:
- field = to_es_field(queryables_mapping, query["args"][0]["property"])
- values = query["args"][1]
- if not isinstance(values, list):
- raise ValueError(f"Arg {values} is not a list")
- return {"terms": {field: values}}
-
- elif query["op"] == AdvancedComparisonOp.LIKE:
- field = to_es_field(queryables_mapping, query["args"][0]["property"])
- pattern = cql2_like_to_es(query["args"][1])
- return {"wildcard": {field: {"value": pattern, "case_insensitive": True}}}
-
- elif query["op"] in [
- SpatialOp.S_INTERSECTS,
- SpatialOp.S_CONTAINS,
- SpatialOp.S_WITHIN,
- SpatialOp.S_DISJOINT,
- ]:
- field = to_es_field(queryables_mapping, query["args"][0]["property"])
- geometry = query["args"][1]
-
- relation_mapping = {
- SpatialOp.S_INTERSECTS: "intersects",
- SpatialOp.S_CONTAINS: "contains",
- SpatialOp.S_WITHIN: "within",
- SpatialOp.S_DISJOINT: "disjoint",
- }
-
- relation = relation_mapping[query["op"]]
- return {"geo_shape": {field: {"shape": geometry, "relation": relation}}}
-
- return {}
-
-
-# ============================================================================
-# Filter Client Implementation
-# ============================================================================
-
-
-@attr.s
-class EsAsyncBaseFiltersClient(AsyncBaseFiltersClient):
- """Defines a pattern for implementing the STAC filter extension."""
-
- database: BaseDatabaseLogic = attr.ib()
-
- async def get_queryables(
- self, collection_id: Optional[str] = None, **kwargs
- ) -> Dict[str, Any]:
- """Get the queryables available for the given collection_id.
-
- If collection_id is None, returns the intersection of all
- queryables over all collections.
-
- This base implementation returns a blank queryable schema. This is not allowed
- under OGC CQL but it is allowed by the STAC API Filter Extension
-
- https://github.com/radiantearth/stac-api-spec/tree/master/fragments/filter#queryables
-
- Args:
- collection_id (str, optional): The id of the collection to get queryables for.
- **kwargs: additional keyword arguments
-
- Returns:
- Dict[str, Any]: A dictionary containing the queryables for the given collection.
- """
- queryables: Dict[str, Any] = {
- "$schema": "https://json-schema.org/draft/2019-09/schema",
- "$id": "https://stac-api.example.com/queryables",
- "type": "object",
- "title": "Queryables for STAC API",
- "description": "Queryable names for the STAC API Item Search filter.",
- "properties": DEFAULT_QUERYABLES,
- "additionalProperties": True,
- }
- if not collection_id:
- return queryables
-
- properties: Dict[str, Any] = queryables["properties"]
- queryables.update(
- {
- "properties": properties,
- "additionalProperties": False,
- }
- )
-
- mapping_data = await self.database.get_items_mapping(collection_id)
- mapping_properties = next(iter(mapping_data.values()))["mappings"]["properties"]
- stack = deque(mapping_properties.items())
-
- while stack:
- field_name, field_def = stack.popleft()
-
- # Iterate over nested fields
- field_properties = field_def.get("properties")
- if field_properties:
- # Fields in Item Properties should be exposed with their un-prefixed names,
- # and not require expressions to prefix them with properties,
- # e.g., eo:cloud_cover instead of properties.eo:cloud_cover.
- if field_name == "properties":
- stack.extend(field_properties.items())
- else:
- stack.extend(
- (f"{field_name}.{k}", v) for k, v in field_properties.items()
- )
-
- # Skip non-indexed or disabled fields
- field_type = field_def.get("type")
- if not field_type or not field_def.get("enabled", True):
- continue
-
- # Generate field properties
- field_result = DEFAULT_QUERYABLES.get(field_name, {})
- properties[field_name] = field_result
-
- field_name_human = field_name.replace("_", " ").title()
- field_result.setdefault("title", field_name_human)
-
- field_type_json = ES_MAPPING_TYPE_TO_JSON.get(field_type, field_type)
- field_result.setdefault("type", field_type_json)
-
- if field_type in {"date", "date_nanos"}:
- field_result.setdefault("format", "date-time")
-
- return queryables
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/README.md b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/README.md
new file mode 100644
index 00000000..d3b09167
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/README.md
@@ -0,0 +1,27 @@
+# STAC FastAPI Filter Package
+
+This package contains shared filter extension functionality used by both the Elasticsearch and OpenSearch
+implementations of STAC FastAPI. It helps reduce code duplication and ensures consistent behavior
+between the two implementations.
+
+## Package Structure
+
+The filter package is organized into three main modules:
+
+- **cql2.py**: Contains functions for converting CQL2 patterns to Elasticsearch/OpenSearch compatible formats
+ - [cql2_like_to_es](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py:59:0-75:5): Converts CQL2 "LIKE" characters to Elasticsearch "wildcard" characters
+ - [_replace_like_patterns](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py:51:0-56:71): Helper function for pattern replacement
+
+- **transform.py**: Contains functions for transforming CQL2 queries to Elasticsearch/OpenSearch query DSL
+ - [to_es_field](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py:83:0-93:47): Maps field names using queryables mapping
+ - [to_es](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py:96:0-201:13): Transforms CQL2 query structures to Elasticsearch/OpenSearch query DSL
+
+- **client.py**: Contains the base filter client implementation
+ - [EsAsyncBaseFiltersClient](cci:2://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter.py:209:0-293:25): Base class for implementing the STAC filter extension
+
+## Usage
+
+Import the necessary components from the filter package:
+
+```python
+from stac_fastapi.sfeos_helpers.filter import cql2_like_to_es, to_es, EsAsyncBaseFiltersClient
\ No newline at end of file
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/__init__.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/__init__.py
new file mode 100644
index 00000000..9ea21fb9
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/__init__.py
@@ -0,0 +1,37 @@
+"""Shared filter extension methods for stac-fastapi elasticsearch and opensearch backends.
+
+This module provides shared functionality for implementing the STAC API Filter Extension
+with Elasticsearch and OpenSearch. It includes:
+
+1. Functions for converting CQL2 queries to Elasticsearch/OpenSearch query DSL
+2. Helper functions for field mapping and query transformation
+3. Base implementation of the AsyncBaseFiltersClient for Elasticsearch/OpenSearch
+
+The filter package is organized as follows:
+- cql2.py: CQL2 pattern conversion helpers
+- transform.py: Query transformation functions
+- client.py: Filter client implementation
+
+When adding new functionality to this package, consider:
+1. Will this code be used by both Elasticsearch and OpenSearch implementations?
+2. Is the functionality stable and unlikely to diverge between implementations?
+3. Is the function well-documented with clear input/output contracts?
+
+Function Naming Conventions:
+- Function names should be descriptive and indicate their purpose
+- Parameter names should be consistent across similar functions
+"""
+
+from .client import EsAsyncBaseFiltersClient
+
+# Re-export the main functions and classes for backward compatibility
+from .cql2 import _replace_like_patterns, cql2_like_to_es
+from .transform import to_es, to_es_field
+
+__all__ = [
+ "cql2_like_to_es",
+ "_replace_like_patterns",
+ "to_es_field",
+ "to_es",
+ "EsAsyncBaseFiltersClient",
+]
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/client.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/client.py
new file mode 100644
index 00000000..4b2a1a71
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/client.py
@@ -0,0 +1,98 @@
+"""Filter client implementation for Elasticsearch/OpenSearch."""
+
+from collections import deque
+from typing import Any, Dict, Optional
+
+import attr
+
+from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
+from stac_fastapi.core.extensions.filter import DEFAULT_QUERYABLES
+from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient
+from stac_fastapi.sfeos_helpers.mappings import ES_MAPPING_TYPE_TO_JSON
+
+
+@attr.s
+class EsAsyncBaseFiltersClient(AsyncBaseFiltersClient):
+ """Defines a pattern for implementing the STAC filter extension."""
+
+ database: BaseDatabaseLogic = attr.ib()
+
+ async def get_queryables(
+ self, collection_id: Optional[str] = None, **kwargs
+ ) -> Dict[str, Any]:
+ """Get the queryables available for the given collection_id.
+
+ If collection_id is None, returns the intersection of all
+ queryables over all collections.
+
+ This base implementation returns a blank queryable schema. This is not allowed
+ under OGC CQL but it is allowed by the STAC API Filter Extension
+
+ https://github.com/radiantearth/stac-api-spec/tree/master/fragments/filter#queryables
+
+ Args:
+ collection_id (str, optional): The id of the collection to get queryables for.
+ **kwargs: additional keyword arguments
+
+ Returns:
+ Dict[str, Any]: A dictionary containing the queryables for the given collection.
+ """
+ queryables: Dict[str, Any] = {
+ "$schema": "https://json-schema.org/draft/2019-09/schema",
+ "$id": "https://stac-api.example.com/queryables",
+ "type": "object",
+ "title": "Queryables for STAC API",
+ "description": "Queryable names for the STAC API Item Search filter.",
+ "properties": DEFAULT_QUERYABLES,
+ "additionalProperties": True,
+ }
+ if not collection_id:
+ return queryables
+
+ properties: Dict[str, Any] = queryables["properties"]
+ queryables.update(
+ {
+ "properties": properties,
+ "additionalProperties": False,
+ }
+ )
+
+ mapping_data = await self.database.get_items_mapping(collection_id)
+ mapping_properties = next(iter(mapping_data.values()))["mappings"]["properties"]
+ stack = deque(mapping_properties.items())
+
+ while stack:
+ field_name, field_def = stack.popleft()
+
+ # Iterate over nested fields
+ field_properties = field_def.get("properties")
+ if field_properties:
+ # Fields in Item Properties should be exposed with their un-prefixed names,
+ # and not require expressions to prefix them with properties,
+ # e.g., eo:cloud_cover instead of properties.eo:cloud_cover.
+ if field_name == "properties":
+ stack.extend(field_properties.items())
+ else:
+ stack.extend(
+ (f"{field_name}.{k}", v) for k, v in field_properties.items()
+ )
+
+ # Skip non-indexed or disabled fields
+ field_type = field_def.get("type")
+ if not field_type or not field_def.get("enabled", True):
+ continue
+
+ # Generate field properties
+ field_result = DEFAULT_QUERYABLES.get(field_name, {})
+ properties[field_name] = field_result
+
+ field_name_human = field_name.replace("_", " ").title()
+ field_result.setdefault("title", field_name_human)
+
+ field_type_json = ES_MAPPING_TYPE_TO_JSON.get(field_type, field_type)
+ field_result.setdefault("type", field_type_json)
+
+ if field_type in {"date", "date_nanos"}:
+ field_result.setdefault("format", "date-time")
+
+ return queryables
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/cql2.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/cql2.py
new file mode 100644
index 00000000..a66331ed
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/cql2.py
@@ -0,0 +1,35 @@
+"""CQL2 pattern conversion helpers for Elasticsearch/OpenSearch."""
+
+import re
+
+from stac_fastapi.core.extensions.filter import (
+ cql2_like_patterns,
+ valid_like_substitutions,
+)
+
+
+def _replace_like_patterns(match: re.Match) -> str:
+ pattern = match.group()
+ try:
+ return valid_like_substitutions[pattern]
+ except KeyError:
+ raise ValueError(f"'{pattern}' is not a valid escape sequence")
+
+
+def cql2_like_to_es(string: str) -> str:
+ """
+ Convert CQL2 "LIKE" characters to Elasticsearch "wildcard" characters.
+
+ Args:
+ string (str): The string containing CQL2 wildcard characters.
+
+ Returns:
+ str: The converted string with Elasticsearch compatible wildcards.
+
+ Raises:
+ ValueError: If an invalid escape sequence is encountered.
+ """
+ return cql2_like_patterns.sub(
+ repl=_replace_like_patterns,
+ string=string,
+ )
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/transform.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/transform.py
new file mode 100644
index 00000000..c78b19e5
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/transform.py
@@ -0,0 +1,133 @@
+"""Query transformation functions for Elasticsearch/OpenSearch."""
+
+from typing import Any, Dict
+
+from stac_fastapi.core.extensions.filter import (
+ AdvancedComparisonOp,
+ ComparisonOp,
+ LogicalOp,
+ SpatialOp,
+)
+
+from .cql2 import cql2_like_to_es
+
+
+def to_es_field(queryables_mapping: Dict[str, Any], field: str) -> str:
+ """
+ Map a given field to its corresponding Elasticsearch field according to a predefined mapping.
+
+ Args:
+ field (str): The field name from a user query or filter.
+
+ Returns:
+ str: The mapped field name suitable for Elasticsearch queries.
+ """
+ return queryables_mapping.get(field, field)
+
+
+def to_es(queryables_mapping: Dict[str, Any], query: Dict[str, Any]) -> Dict[str, Any]:
+ """
+ Transform a simplified CQL2 query structure to an Elasticsearch compatible query DSL.
+
+ Args:
+ query (Dict[str, Any]): The query dictionary containing 'op' and 'args'.
+
+ Returns:
+ Dict[str, Any]: The corresponding Elasticsearch query in the form of a dictionary.
+ """
+ if query["op"] in [LogicalOp.AND, LogicalOp.OR, LogicalOp.NOT]:
+ bool_type = {
+ LogicalOp.AND: "must",
+ LogicalOp.OR: "should",
+ LogicalOp.NOT: "must_not",
+ }[query["op"]]
+ return {
+ "bool": {
+ bool_type: [
+ to_es(queryables_mapping, sub_query) for sub_query in query["args"]
+ ]
+ }
+ }
+
+ elif query["op"] in [
+ ComparisonOp.EQ,
+ ComparisonOp.NEQ,
+ ComparisonOp.LT,
+ ComparisonOp.LTE,
+ ComparisonOp.GT,
+ ComparisonOp.GTE,
+ ]:
+ range_op = {
+ ComparisonOp.LT: "lt",
+ ComparisonOp.LTE: "lte",
+ ComparisonOp.GT: "gt",
+ ComparisonOp.GTE: "gte",
+ }
+
+ field = to_es_field(queryables_mapping, query["args"][0]["property"])
+ value = query["args"][1]
+ if isinstance(value, dict) and "timestamp" in value:
+ value = value["timestamp"]
+ if query["op"] == ComparisonOp.EQ:
+ return {"range": {field: {"gte": value, "lte": value}}}
+ elif query["op"] == ComparisonOp.NEQ:
+ return {
+ "bool": {
+ "must_not": [{"range": {field: {"gte": value, "lte": value}}}]
+ }
+ }
+ else:
+ return {"range": {field: {range_op[query["op"]]: value}}}
+ else:
+ if query["op"] == ComparisonOp.EQ:
+ return {"term": {field: value}}
+ elif query["op"] == ComparisonOp.NEQ:
+ return {"bool": {"must_not": [{"term": {field: value}}]}}
+ else:
+ return {"range": {field: {range_op[query["op"]]: value}}}
+
+ elif query["op"] == ComparisonOp.IS_NULL:
+ field = to_es_field(queryables_mapping, query["args"][0]["property"])
+ return {"bool": {"must_not": {"exists": {"field": field}}}}
+
+ elif query["op"] == AdvancedComparisonOp.BETWEEN:
+ field = to_es_field(queryables_mapping, query["args"][0]["property"])
+ gte, lte = query["args"][1], query["args"][2]
+ if isinstance(gte, dict) and "timestamp" in gte:
+ gte = gte["timestamp"]
+ if isinstance(lte, dict) and "timestamp" in lte:
+ lte = lte["timestamp"]
+ return {"range": {field: {"gte": gte, "lte": lte}}}
+
+ elif query["op"] == AdvancedComparisonOp.IN:
+ field = to_es_field(queryables_mapping, query["args"][0]["property"])
+ values = query["args"][1]
+ if not isinstance(values, list):
+ raise ValueError(f"Arg {values} is not a list")
+ return {"terms": {field: values}}
+
+ elif query["op"] == AdvancedComparisonOp.LIKE:
+ field = to_es_field(queryables_mapping, query["args"][0]["property"])
+ pattern = cql2_like_to_es(query["args"][1])
+ return {"wildcard": {field: {"value": pattern, "case_insensitive": True}}}
+
+ elif query["op"] in [
+ SpatialOp.S_INTERSECTS,
+ SpatialOp.S_CONTAINS,
+ SpatialOp.S_WITHIN,
+ SpatialOp.S_DISJOINT,
+ ]:
+ field = to_es_field(queryables_mapping, query["args"][0]["property"])
+ geometry = query["args"][1]
+
+ relation_mapping = {
+ SpatialOp.S_INTERSECTS: "intersects",
+ SpatialOp.S_CONTAINS: "contains",
+ SpatialOp.S_WITHIN: "within",
+ SpatialOp.S_DISJOINT: "disjoint",
+ }
+
+ relation = relation_mapping[query["op"]]
+ return {"geo_shape": {field: {"shape": geometry, "relation": relation}}}
+
+ return {}
From a432274968f521254e9615abffb5af4312a07aa7 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Mon, 19 May 2025 17:25:42 +0800
Subject: [PATCH 14/29] database package
---
.../elasticsearch/database_logic.py | 2 +-
.../stac_fastapi/opensearch/database_logic.py | 2 +-
.../sfeos_helpers/database/README.md | 35 +++
.../sfeos_helpers/database/__init__.py | 42 ++++
.../sfeos_helpers/database/index.py | 79 +++++++
.../sfeos_helpers/database/mapping.py | 38 +++
.../sfeos_helpers/database/query.py | 85 +++++++
.../sfeos_helpers/database_logic_helpers.py | 221 ------------------
8 files changed, 281 insertions(+), 223 deletions(-)
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/README.md
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/__init__.py
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/mapping.py
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py
delete mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
index 9781c677..b353b319 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
@@ -21,7 +21,7 @@
ElasticsearchSettings as SyncElasticsearchSettings,
)
from stac_fastapi.sfeos_helpers import filter
-from stac_fastapi.sfeos_helpers.database_logic_helpers import (
+from stac_fastapi.sfeos_helpers.database import (
apply_free_text_filter_shared,
apply_intersects_filter_shared,
create_index_templates_shared,
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
index b1703bf1..8e7441e0 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
@@ -21,7 +21,7 @@
)
from stac_fastapi.opensearch.config import OpensearchSettings as SyncSearchSettings
from stac_fastapi.sfeos_helpers import filter
-from stac_fastapi.sfeos_helpers.database_logic_helpers import (
+from stac_fastapi.sfeos_helpers.database import (
apply_free_text_filter_shared,
apply_intersects_filter_shared,
create_index_templates_shared,
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/README.md b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/README.md
new file mode 100644
index 00000000..84b17218
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/README.md
@@ -0,0 +1,35 @@
+# STAC FastAPI Database Package
+
+This package contains shared database operations used by both the Elasticsearch and OpenSearch
+implementations of STAC FastAPI. It helps reduce code duplication and ensures consistent behavior
+between the two implementations.
+
+## Package Structure
+
+The database package is organized into three main modules:
+
+- **index.py**: Contains functions for managing indices
+ - [create_index_templates_shared](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py:15:0-48:33): Creates index templates for Collections and Items
+ - [delete_item_index_shared](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py:128:0-153:30): Deletes an item index for a collection
+
+- **query.py**: Contains functions for building and manipulating queries
+ - [apply_free_text_filter_shared](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py:51:0-74:16): Applies a free text filter to a search
+ - [apply_intersects_filter_shared](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py:77:0-104:5): Creates a geo_shape filter for intersecting geometry
+ - [populate_sort_shared](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py:107:0-125:16): Creates a sort configuration for queries
+
+- **mapping.py**: Contains functions for working with mappings
+ - [get_queryables_mapping_shared](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py:156:0-185:27): Retrieves mapping of Queryables for search
+
+## Usage
+
+Import the necessary components from the database package:
+
+```python
+from stac_fastapi.sfeos_helpers.database import (
+ create_index_templates_shared,
+ delete_item_index_shared,
+ apply_free_text_filter_shared,
+ apply_intersects_filter_shared,
+ populate_sort_shared,
+ get_queryables_mapping_shared,
+)
\ No newline at end of file
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/__init__.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/__init__.py
new file mode 100644
index 00000000..0b464b74
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/__init__.py
@@ -0,0 +1,42 @@
+"""Shared database operations for stac-fastapi elasticsearch and opensearch backends.
+
+This module provides shared database functionality used by both the Elasticsearch and OpenSearch
+implementations of STAC FastAPI. It includes:
+
+1. Index management functions for creating and deleting indices
+2. Query building functions for constructing search queries
+3. Mapping functions for working with Elasticsearch/OpenSearch mappings
+
+The database package is organized as follows:
+- index.py: Index management functions
+- query.py: Query building functions
+- mapping.py: Mapping functions
+
+When adding new functionality to this package, consider:
+1. Will this code be used by both Elasticsearch and OpenSearch implementations?
+2. Is the functionality stable and unlikely to diverge between implementations?
+3. Is the function well-documented with clear input/output contracts?
+
+Function Naming Conventions:
+- All shared functions should end with `_shared` to clearly indicate they're meant to be used by both implementations
+- Function names should be descriptive and indicate their purpose
+- Parameter names should be consistent across similar functions
+"""
+
+# Re-export all functions for backward compatibility
+from .index import create_index_templates_shared, delete_item_index_shared
+from .mapping import get_queryables_mapping_shared
+from .query import (
+ apply_free_text_filter_shared,
+ apply_intersects_filter_shared,
+ populate_sort_shared,
+)
+
+__all__ = [
+ "create_index_templates_shared",
+ "delete_item_index_shared",
+ "apply_free_text_filter_shared",
+ "apply_intersects_filter_shared",
+ "populate_sort_shared",
+ "get_queryables_mapping_shared",
+]
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py
new file mode 100644
index 00000000..0bdead5d
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py
@@ -0,0 +1,79 @@
+"""Index management functions for Elasticsearch/OpenSearch.
+
+This module provides functions for creating and managing indices in Elasticsearch/OpenSearch.
+"""
+
+from typing import Any
+
+from stac_fastapi.sfeos_helpers.mappings import (
+ COLLECTIONS_INDEX,
+ ES_COLLECTIONS_MAPPINGS,
+ ES_ITEMS_MAPPINGS,
+ ES_ITEMS_SETTINGS,
+ ITEMS_INDEX_PREFIX,
+)
+from stac_fastapi.sfeos_helpers.utilities import index_alias_by_collection_id
+
+
+async def create_index_templates_shared(settings: Any) -> None:
+ """Create index templates for Elasticsearch/OpenSearch Collection and Item indices.
+
+ Args:
+ settings (Any): The settings object containing the client configuration.
+ Must have a create_client attribute that returns an Elasticsearch/OpenSearch client.
+
+ Returns:
+ None: This function doesn't return any value but creates index templates in the database.
+
+ Notes:
+ This function creates two index templates:
+ 1. A template for the Collections index with the appropriate mappings
+ 2. A template for the Items indices with both settings and mappings
+
+ These templates ensure that any new indices created with matching patterns
+ will automatically have the correct structure.
+ """
+ client = settings.create_client
+ await client.indices.put_index_template(
+ name=f"template_{COLLECTIONS_INDEX}",
+ body={
+ "index_patterns": [f"{COLLECTIONS_INDEX}*"],
+ "template": {"mappings": ES_COLLECTIONS_MAPPINGS},
+ },
+ )
+ await client.indices.put_index_template(
+ name=f"template_{ITEMS_INDEX_PREFIX}",
+ body={
+ "index_patterns": [f"{ITEMS_INDEX_PREFIX}*"],
+ "template": {"settings": ES_ITEMS_SETTINGS, "mappings": ES_ITEMS_MAPPINGS},
+ },
+ )
+ await client.close()
+
+
+async def delete_item_index_shared(settings: Any, collection_id: str) -> None:
+ """Delete the index for items in a collection.
+
+ Args:
+ settings (Any): The settings object containing the client configuration.
+ Must have a create_client attribute that returns an Elasticsearch/OpenSearch client.
+ collection_id (str): The ID of the collection whose items index will be deleted.
+
+ Returns:
+ None: This function doesn't return any value but deletes an item index in the database.
+
+ Notes:
+ This function deletes an item index and its alias. It first resolves the alias to find
+ the actual index name, then deletes both the alias and the index.
+ """
+ client = settings.create_client
+
+ name = index_alias_by_collection_id(collection_id)
+ resolved = await client.indices.resolve_index(name=name)
+ if "aliases" in resolved and resolved["aliases"]:
+ [alias] = resolved["aliases"]
+ await client.indices.delete_alias(index=alias["indices"], name=alias["name"])
+ await client.indices.delete(index=alias["indices"])
+ else:
+ await client.indices.delete(index=name)
+ await client.close()
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/mapping.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/mapping.py
new file mode 100644
index 00000000..8f664651
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/mapping.py
@@ -0,0 +1,38 @@
+"""Mapping functions for Elasticsearch/OpenSearch.
+
+This module provides functions for working with Elasticsearch/OpenSearch mappings.
+"""
+
+from typing import Any, Dict
+
+
+async def get_queryables_mapping_shared(
+ mappings: Dict[str, Dict[str, Any]], collection_id: str = "*"
+) -> Dict[str, str]:
+ """Retrieve mapping of Queryables for search.
+
+ Args:
+ mappings (Dict[str, Dict[str, Any]]): The mapping information returned from
+ Elasticsearch/OpenSearch client's indices.get_mapping() method.
+ Expected structure is {index_name: {"mappings": {...}}}.
+ collection_id (str, optional): The id of the Collection the Queryables
+ belongs to. Defaults to "*".
+
+ Returns:
+ Dict[str, str]: A dictionary containing the Queryables mappings, where keys are
+ field names and values are the corresponding paths in the Elasticsearch/OpenSearch
+ document structure.
+ """
+ queryables_mapping = {}
+
+ for mapping in mappings.values():
+ fields = mapping["mappings"].get("properties", {})
+ properties = fields.pop("properties", {}).get("properties", {}).keys()
+
+ for field_key in fields:
+ queryables_mapping[field_key] = field_key
+
+ for property_key in properties:
+ queryables_mapping[property_key] = f"properties.{property_key}"
+
+ return queryables_mapping
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py
new file mode 100644
index 00000000..dacbb590
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py
@@ -0,0 +1,85 @@
+"""Query building functions for Elasticsearch/OpenSearch.
+
+This module provides functions for building and manipulating Elasticsearch/OpenSearch queries.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from stac_fastapi.sfeos_helpers.mappings import Geometry
+
+
+def apply_free_text_filter_shared(
+ search: Any, free_text_queries: Optional[List[str]]
+) -> Any:
+ """Create a free text query for Elasticsearch/OpenSearch.
+
+ Args:
+ search (Any): The search object to apply the query to.
+ free_text_queries (Optional[List[str]]): A list of text strings to search for in the properties.
+
+ Returns:
+ Any: The search object with the free text query applied, or the original search
+ object if no free_text_queries were provided.
+
+ Notes:
+ This function creates a query_string query that searches for the specified text strings
+ in all properties of the documents. The query strings are joined with OR operators.
+ """
+ if free_text_queries is not None:
+ free_text_query_string = '" OR properties.\\*:"'.join(free_text_queries)
+ search = search.query(
+ "query_string", query=f'properties.\\*:"{free_text_query_string}"'
+ )
+
+ return search
+
+
+def apply_intersects_filter_shared(
+ intersects: Geometry,
+) -> Dict[str, Dict]:
+ """Create a geo_shape filter for intersecting geometry.
+
+ Args:
+ intersects (Geometry): The intersecting geometry, represented as a GeoJSON-like object.
+
+ Returns:
+ Dict[str, Dict]: A dictionary containing the geo_shape filter configuration
+ that can be used with Elasticsearch/OpenSearch Q objects.
+
+ Notes:
+ This function creates a geo_shape filter configuration to find documents that intersect
+ with the specified geometry. The returned dictionary should be wrapped in a Q object
+ when applied to a search.
+ """
+ return {
+ "geo_shape": {
+ "geometry": {
+ "shape": {
+ "type": intersects.type.lower(),
+ "coordinates": intersects.coordinates,
+ },
+ "relation": "intersects",
+ }
+ }
+ }
+
+
+def populate_sort_shared(sortby: List) -> Optional[Dict[str, Dict[str, str]]]:
+ """Create a sort configuration for Elasticsearch/OpenSearch queries.
+
+ Args:
+ sortby (List): A list of sort specifications, each containing a field and direction.
+
+ Returns:
+ Optional[Dict[str, Dict[str, str]]]: A dictionary mapping field names to sort direction
+ configurations, or None if no sort was specified.
+
+ Notes:
+ This function transforms a list of sort specifications into the format required by
+ Elasticsearch/OpenSearch for sorting query results. The returned dictionary can be
+ directly used in search requests.
+ """
+ if sortby:
+ return {s.field: {"order": s.direction} for s in sortby}
+ else:
+ return None
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
deleted file mode 100644
index 941b8fcf..00000000
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py
+++ /dev/null
@@ -1,221 +0,0 @@
-"""Shared code for elasticsearch/ opensearch database logic.
-
-This module contains shared functions used by both the Elasticsearch and OpenSearch
-implementations of STAC FastAPI for database operations. It helps reduce code duplication
-and ensures consistent behavior between the two implementations.
-
-The sfeos_helpers package is organized as follows:
-- database_logic_helpers.py: Shared database operations (this file)
-- filter.py: Shared filter extension implementation
-- mappings.py: Shared constants and mapping definitions
-- utilities.py: Shared utility functions
-
-When adding new functionality to this package, consider:
-1. Will this code be used by both Elasticsearch and OpenSearch implementations?
-2. Is the functionality stable and unlikely to diverge between implementations?
-3. Is the function well-documented with clear input/output contracts?
-
-Function Naming Conventions:
-- All shared functions should end with `_shared` to clearly indicate they're meant to be used by both implementations
-- Function names should be descriptive and indicate their purpose
-- Parameter names should be consistent across similar functions
-"""
-
-from typing import Any, Dict, List, Optional
-
-from stac_fastapi.sfeos_helpers.mappings import (
- COLLECTIONS_INDEX,
- ES_COLLECTIONS_MAPPINGS,
- ES_ITEMS_MAPPINGS,
- ES_ITEMS_SETTINGS,
- ITEMS_INDEX_PREFIX,
- Geometry,
-)
-from stac_fastapi.sfeos_helpers.utilities import index_alias_by_collection_id
-
-# ============================================================================
-# Index Management Functions
-# ============================================================================
-
-
-async def create_index_templates_shared(settings: Any) -> None:
- """Create index templates for Elasticsearch/OpenSearch Collection and Item indices.
-
- Args:
- settings (Any): The settings object containing the client configuration.
- Must have a create_client attribute that returns an Elasticsearch/OpenSearch client.
-
- Returns:
- None: This function doesn't return any value but creates index templates in the database.
-
- Notes:
- This function creates two index templates:
- 1. A template for the Collections index with the appropriate mappings
- 2. A template for the Items indices with both settings and mappings
-
- These templates ensure that any new indices created with matching patterns
- will automatically have the correct structure.
- """
- client = settings.create_client
- await client.indices.put_index_template(
- name=f"template_{COLLECTIONS_INDEX}",
- body={
- "index_patterns": [f"{COLLECTIONS_INDEX}*"],
- "template": {"mappings": ES_COLLECTIONS_MAPPINGS},
- },
- )
- await client.indices.put_index_template(
- name=f"template_{ITEMS_INDEX_PREFIX}",
- body={
- "index_patterns": [f"{ITEMS_INDEX_PREFIX}*"],
- "template": {"settings": ES_ITEMS_SETTINGS, "mappings": ES_ITEMS_MAPPINGS},
- },
- )
- await client.close()
-
-
-async def delete_item_index_shared(settings: Any, collection_id: str) -> None:
- """Delete the index for items in a collection.
-
- Args:
- settings (Any): The settings object containing the client configuration.
- Must have a create_client attribute that returns an Elasticsearch/OpenSearch client.
- collection_id (str): The ID of the collection whose items index will be deleted.
-
- Returns:
- None: This function doesn't return any value but deletes an item index in the database.
-
- Notes:
- This function deletes an item index and its alias. It first resolves the alias to find
- the actual index name, then deletes both the alias and the index.
- """
- client = settings.create_client
-
- name = index_alias_by_collection_id(collection_id)
- resolved = await client.indices.resolve_index(name=name)
- if "aliases" in resolved and resolved["aliases"]:
- [alias] = resolved["aliases"]
- await client.indices.delete_alias(index=alias["indices"], name=alias["name"])
- await client.indices.delete(index=alias["indices"])
- else:
- await client.indices.delete(index=name)
- await client.close()
-
-
-# ============================================================================
-# Query Building Functions
-# ============================================================================
-
-
-def apply_free_text_filter_shared(
- search: Any, free_text_queries: Optional[List[str]]
-) -> Any:
- """Create a free text query for Elasticsearch/OpenSearch.
-
- Args:
- search (Any): The search object to apply the query to.
- free_text_queries (Optional[List[str]]): A list of text strings to search for in the properties.
-
- Returns:
- Any: The search object with the free text query applied, or the original search
- object if no free_text_queries were provided.
-
- Notes:
- This function creates a query_string query that searches for the specified text strings
- in all properties of the documents. The query strings are joined with OR operators.
- """
- if free_text_queries is not None:
- free_text_query_string = '" OR properties.\\*:"'.join(free_text_queries)
- search = search.query(
- "query_string", query=f'properties.\\*:"{free_text_query_string}"'
- )
-
- return search
-
-
-def apply_intersects_filter_shared(
- intersects: Geometry,
-) -> Dict[str, Dict]:
- """Create a geo_shape filter for intersecting geometry.
-
- Args:
- intersects (Geometry): The intersecting geometry, represented as a GeoJSON-like object.
-
- Returns:
- Dict[str, Dict]: A dictionary containing the geo_shape filter configuration
- that can be used with Elasticsearch/OpenSearch Q objects.
-
- Notes:
- This function creates a geo_shape filter configuration to find documents that intersect
- with the specified geometry. The returned dictionary should be wrapped in a Q object
- when applied to a search.
- """
- return {
- "geo_shape": {
- "geometry": {
- "shape": {
- "type": intersects.type.lower(),
- "coordinates": intersects.coordinates,
- },
- "relation": "intersects",
- }
- }
- }
-
-
-def populate_sort_shared(sortby: List) -> Optional[Dict[str, Dict[str, str]]]:
- """Create a sort configuration for Elasticsearch/OpenSearch queries.
-
- Args:
- sortby (List): A list of sort specifications, each containing a field and direction.
-
- Returns:
- Optional[Dict[str, Dict[str, str]]]: A dictionary mapping field names to sort direction
- configurations, or None if no sort was specified.
-
- Notes:
- This function transforms a list of sort specifications into the format required by
- Elasticsearch/OpenSearch for sorting query results. The returned dictionary can be
- directly used in search requests.
- """
- if sortby:
- return {s.field: {"order": s.direction} for s in sortby}
- else:
- return None
-
-
-# ============================================================================
-# Mapping Functions
-# ============================================================================
-
-
-async def get_queryables_mapping_shared(
- mappings: Dict[str, Dict[str, Any]], collection_id: str = "*"
-) -> Dict[str, str]:
- """Retrieve mapping of Queryables for search.
-
- Args:
- mappings (Dict[str, Dict[str, Any]]): The mapping information returned from
- Elasticsearch/OpenSearch client's indices.get_mapping() method.
- Expected structure is {index_name: {"mappings": {...}}}.
- collection_id (str, optional): The id of the Collection the Queryables
- belongs to. Defaults to "*".
-
- Returns:
- Dict[str, str]: A dictionary containing the Queryables mappings, where keys are
- field names and values are the corresponding paths in the Elasticsearch/OpenSearch
- document structure.
- """
- queryables_mapping = {}
-
- for mapping in mappings.values():
- fields = mapping["mappings"].get("properties", {})
- properties = fields.pop("properties", {}).get("properties", {}).keys()
-
- for field_key in fields:
- queryables_mapping[field_key] = field_key
-
- for property_key in properties:
- queryables_mapping[property_key] = f"properties.{property_key}"
-
- return queryables_mapping
From 36504af864915e611ca3d99b46b33c4f1b3497c7 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Mon, 19 May 2025 17:42:07 +0800
Subject: [PATCH 15/29] move utility functions
---
.../stac_fastapi/elasticsearch/config.py | 3 +-
.../elasticsearch/database_logic.py | 14 +-
.../stac_fastapi/opensearch/config.py | 3 +-
.../stac_fastapi/opensearch/database_logic.py | 14 +-
.../sfeos_helpers/database/README.md | 30 ++-
.../sfeos_helpers/database/__init__.py | 25 ++-
.../sfeos_helpers/database/document.py | 48 +++++
.../sfeos_helpers/database/index.py | 55 +++++-
.../sfeos_helpers/database/utils.py | 50 +++++
.../stac_fastapi/sfeos_helpers/utilities.py | 176 ------------------
stac_fastapi/tests/database/test_database.py | 2 +-
11 files changed, 220 insertions(+), 200 deletions(-)
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/document.py
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/utils.py
delete mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py
index 1321a0f7..d371c6a5 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py
@@ -10,7 +10,8 @@
from elasticsearch import Elasticsearch # type: ignore[attr-defined]
from stac_fastapi.core.base_settings import ApiBaseSettings
-from stac_fastapi.sfeos_helpers.utilities import get_bool_env, validate_refresh
+from stac_fastapi.core.utilities import get_bool_env
+from stac_fastapi.sfeos_helpers.database import validate_refresh
from stac_fastapi.types.config import ApiSettings
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
index b353b319..be2247b3 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
@@ -27,7 +27,13 @@
create_index_templates_shared,
delete_item_index_shared,
get_queryables_mapping_shared,
+ index_alias_by_collection_id,
+ index_by_collection_id,
+ indices,
+ mk_actions,
+ mk_item_id,
populate_sort_shared,
+ validate_refresh,
)
from stac_fastapi.sfeos_helpers.mappings import (
AGGREGATION_MAPPING,
@@ -37,14 +43,6 @@
ITEMS_INDEX_PREFIX,
Geometry,
)
-from stac_fastapi.sfeos_helpers.utilities import (
- index_alias_by_collection_id,
- index_by_collection_id,
- indices,
- mk_actions,
- mk_item_id,
- validate_refresh,
-)
from stac_fastapi.types.errors import ConflictError, NotFoundError
from stac_fastapi.types.stac import Collection, Item
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py
index f707cf45..d3811376 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py
@@ -8,7 +8,8 @@
from opensearchpy import AsyncOpenSearch, OpenSearch
from stac_fastapi.core.base_settings import ApiBaseSettings
-from stac_fastapi.sfeos_helpers.utilities import get_bool_env, validate_refresh
+from stac_fastapi.core.utilities import get_bool_env
+from stac_fastapi.sfeos_helpers.database import validate_refresh
from stac_fastapi.types.config import ApiSettings
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
index 8e7441e0..dd19e6b4 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
@@ -27,7 +27,13 @@
create_index_templates_shared,
delete_item_index_shared,
get_queryables_mapping_shared,
+ index_alias_by_collection_id,
+ index_by_collection_id,
+ indices,
+ mk_actions,
+ mk_item_id,
populate_sort_shared,
+ validate_refresh,
)
from stac_fastapi.sfeos_helpers.mappings import (
AGGREGATION_MAPPING,
@@ -40,14 +46,6 @@
ITEMS_INDEX_PREFIX,
Geometry,
)
-from stac_fastapi.sfeos_helpers.utilities import (
- index_alias_by_collection_id,
- index_by_collection_id,
- indices,
- mk_actions,
- mk_item_id,
- validate_refresh,
-)
from stac_fastapi.types.errors import ConflictError, NotFoundError
from stac_fastapi.types.stac import Collection, Item
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/README.md b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/README.md
index 84b17218..5f4a6ada 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/README.md
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/README.md
@@ -6,11 +6,14 @@ between the two implementations.
## Package Structure
-The database package is organized into three main modules:
+The database package is organized into five main modules:
- **index.py**: Contains functions for managing indices
- [create_index_templates_shared](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py:15:0-48:33): Creates index templates for Collections and Items
- [delete_item_index_shared](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py:128:0-153:30): Deletes an item index for a collection
+ - [index_by_collection_id](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py:86:0-100:5): Translates a collection ID into an index name
+ - [index_alias_by_collection_id](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py:103:0-115:5): Translates a collection ID into an index alias
+ - [indices](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py:118:0-132:5): Gets a comma-separated string of index names
- **query.py**: Contains functions for building and manipulating queries
- [apply_free_text_filter_shared](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py:51:0-74:16): Applies a free text filter to a search
@@ -20,16 +23,39 @@ The database package is organized into three main modules:
- **mapping.py**: Contains functions for working with mappings
- [get_queryables_mapping_shared](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database_logic_helpers.py:156:0-185:27): Retrieves mapping of Queryables for search
+- **document.py**: Contains functions for working with documents
+ - [mk_item_id](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py:140:0-150:5): Creates a document ID for an Item
+ - [mk_actions](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py:153:0-175:5): Creates bulk actions for indexing items
+
+- **utils.py**: Contains utility functions for database operations
+ - [validate_refresh](cci:1://file:///home/computer/Code/stac-fastapi-elasticsearch-opensearch/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py:41:0-78:5): Validates the refresh parameter value
+
## Usage
Import the necessary components from the database package:
```python
from stac_fastapi.sfeos_helpers.database import (
+ # Index operations
create_index_templates_shared,
delete_item_index_shared,
+ index_alias_by_collection_id,
+ index_by_collection_id,
+ indices,
+
+ # Query operations
apply_free_text_filter_shared,
apply_intersects_filter_shared,
populate_sort_shared,
+
+ # Mapping operations
get_queryables_mapping_shared,
-)
\ No newline at end of file
+
+ # Document operations
+ mk_item_id,
+ mk_actions,
+
+ # Utility functions
+ validate_refresh,
+)
+```
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/__init__.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/__init__.py
index 0b464b74..7ff5076d 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/__init__.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/__init__.py
@@ -6,11 +6,15 @@
1. Index management functions for creating and deleting indices
2. Query building functions for constructing search queries
3. Mapping functions for working with Elasticsearch/OpenSearch mappings
+4. Document operations for working with documents
+5. Utility functions for database operations
The database package is organized as follows:
- index.py: Index management functions
- query.py: Query building functions
- mapping.py: Mapping functions
+- document.py: Document operations
+- utils.py: Utility functions
When adding new functionality to this package, consider:
1. Will this code be used by both Elasticsearch and OpenSearch implementations?
@@ -24,19 +28,38 @@
"""
# Re-export all functions for backward compatibility
-from .index import create_index_templates_shared, delete_item_index_shared
+from .document import mk_actions, mk_item_id
+from .index import (
+ create_index_templates_shared,
+ delete_item_index_shared,
+ index_alias_by_collection_id,
+ index_by_collection_id,
+ indices,
+)
from .mapping import get_queryables_mapping_shared
from .query import (
apply_free_text_filter_shared,
apply_intersects_filter_shared,
populate_sort_shared,
)
+from .utils import validate_refresh
__all__ = [
+ # Index operations
"create_index_templates_shared",
"delete_item_index_shared",
+ "index_alias_by_collection_id",
+ "index_by_collection_id",
+ "indices",
+ # Query operations
"apply_free_text_filter_shared",
"apply_intersects_filter_shared",
"populate_sort_shared",
+ # Mapping operations
"get_queryables_mapping_shared",
+ # Document operations
+ "mk_item_id",
+ "mk_actions",
+ # Utility functions
+ "validate_refresh",
]
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/document.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/document.py
new file mode 100644
index 00000000..0ba0e025
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/document.py
@@ -0,0 +1,48 @@
+"""Document operations for Elasticsearch/OpenSearch.
+
+This module provides functions for working with documents in Elasticsearch/OpenSearch,
+including document ID generation and bulk action creation.
+"""
+
+from typing import Any, Dict, List
+
+from stac_fastapi.sfeos_helpers.database.index import index_alias_by_collection_id
+from stac_fastapi.types.stac import Item
+
+
+def mk_item_id(item_id: str, collection_id: str) -> str:
+ """Create the document id for an Item in Elasticsearch.
+
+ Args:
+ item_id (str): The id of the Item.
+ collection_id (str): The id of the Collection that the Item belongs to.
+
+ Returns:
+ str: The document id for the Item, combining the Item id and the Collection id, separated by a `|` character.
+ """
+ return f"{item_id}|{collection_id}"
+
+
+def mk_actions(collection_id: str, processed_items: List[Item]) -> List[Dict[str, Any]]:
+ """Create Elasticsearch bulk actions for a list of processed items.
+
+ Args:
+ collection_id (str): The identifier for the collection the items belong to.
+ processed_items (List[Item]): The list of processed items to be bulk indexed.
+
+ Returns:
+ List[Dict[str, Union[str, Dict]]]: The list of bulk actions to be executed,
+ each action being a dictionary with the following keys:
+ - `_index`: the index to store the document in.
+ - `_id`: the document's identifier.
+ - `_source`: the source of the document.
+ """
+ index_alias = index_alias_by_collection_id(collection_id)
+ return [
+ {
+ "_index": index_alias,
+ "_id": mk_item_id(item["id"], item["collection"]),
+ "_source": item,
+ }
+ for item in processed_items
+ ]
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py
index 0bdead5d..3305f50f 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py
@@ -3,16 +3,67 @@
This module provides functions for creating and managing indices in Elasticsearch/OpenSearch.
"""
-from typing import Any
+from functools import lru_cache
+from typing import Any, List, Optional
from stac_fastapi.sfeos_helpers.mappings import (
+ _ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE,
COLLECTIONS_INDEX,
ES_COLLECTIONS_MAPPINGS,
ES_ITEMS_MAPPINGS,
ES_ITEMS_SETTINGS,
+ ITEM_INDICES,
ITEMS_INDEX_PREFIX,
)
-from stac_fastapi.sfeos_helpers.utilities import index_alias_by_collection_id
+
+
+@lru_cache(256)
+def index_by_collection_id(collection_id: str) -> str:
+ """
+ Translate a collection id into an Elasticsearch index name.
+
+ Args:
+ collection_id (str): The collection id to translate into an index name.
+
+ Returns:
+ str: The index name derived from the collection id.
+ """
+ cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
+ return (
+ f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{collection_id.encode('utf-8').hex()}"
+ )
+
+
+@lru_cache(256)
+def index_alias_by_collection_id(collection_id: str) -> str:
+ """
+ Translate a collection id into an Elasticsearch index alias.
+
+ Args:
+ collection_id (str): The collection id to translate into an index alias.
+
+ Returns:
+ str: The index alias derived from the collection id.
+ """
+ cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
+ return f"{ITEMS_INDEX_PREFIX}{cleaned}"
+
+
+def indices(collection_ids: Optional[List[str]]) -> str:
+ """
+ Get a comma-separated string of index names for a given list of collection ids.
+
+ Args:
+ collection_ids: A list of collection ids.
+
+ Returns:
+ A string of comma-separated index names. If `collection_ids` is empty, returns the default indices.
+ """
+ return (
+ ",".join(map(index_alias_by_collection_id, collection_ids))
+ if collection_ids
+ else ITEM_INDICES
+ )
async def create_index_templates_shared(settings: Any) -> None:
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/utils.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/utils.py
new file mode 100644
index 00000000..0c6b4c45
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/utils.py
@@ -0,0 +1,50 @@
+"""Utility functions for database operations in Elasticsearch/OpenSearch.
+
+This module provides utility functions for working with database operations
+in Elasticsearch/OpenSearch, such as parameter validation.
+"""
+
+import logging
+from typing import Union
+
+from stac_fastapi.core.utilities import get_bool_env
+
+
+def validate_refresh(value: Union[str, bool]) -> str:
+ """
+ Validate the `refresh` parameter value.
+
+ Args:
+ value (Union[str, bool]): The `refresh` parameter value, which can be a string or a boolean.
+
+ Returns:
+ str: The validated value of the `refresh` parameter, which can be "true", "false", or "wait_for".
+ """
+ logger = logging.getLogger(__name__)
+
+ # Handle boolean-like values using get_bool_env
+ if isinstance(value, bool) or value in {
+ "true",
+ "false",
+ "1",
+ "0",
+ "yes",
+ "no",
+ "y",
+ "n",
+ }:
+ is_true = get_bool_env("DATABASE_REFRESH", default=value)
+ return "true" if is_true else "false"
+
+ # Normalize to lowercase for case-insensitivity
+ value = value.lower()
+
+ # Handle "wait_for" explicitly
+ if value == "wait_for":
+ return "wait_for"
+
+ # Log a warning for invalid values and default to "false"
+ logger.warning(
+ f"Invalid value for `refresh`: '{value}'. Expected 'true', 'false', or 'wait_for'. Defaulting to 'false'."
+ )
+ return "false"
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py
deleted file mode 100644
index 7317d9e5..00000000
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/utilities.py
+++ /dev/null
@@ -1,176 +0,0 @@
-"""Shared utilities functions for stac-fastapi elasticsearch and opensearch backends.
-
-This module contains general utility functions used by both the Elasticsearch and OpenSearch
-implementations of STAC FastAPI. These functions handle common tasks like parameter validation,
-index naming, and document ID generation.
-
-The sfeos_helpers package is organized as follows:
-- database_logic_helpers.py: Shared database operations
-- filter.py: Shared filter extension implementation
-- mappings.py: Shared constants and mapping definitions
-- utilities.py: Shared utility functions (this file)
-
-When adding new functionality to this package, consider:
-1. Will this code be used by both Elasticsearch and OpenSearch implementations?
-2. Is the functionality stable and unlikely to diverge between implementations?
-3. Is the function well-documented with clear input/output contracts?
-
-Function Naming Conventions:
-- All shared functions should end with `_shared` to clearly indicate they're meant to be used by both implementations
-- Function names should be descriptive and indicate their purpose
-- Parameter names should be consistent across similar functions
-"""
-import logging
-from functools import lru_cache
-from typing import Any, Dict, List, Optional, Union
-
-from stac_fastapi.core.utilities import get_bool_env
-
-# Import constants from mappings
-from stac_fastapi.sfeos_helpers.mappings import (
- _ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE,
- ITEM_INDICES,
- ITEMS_INDEX_PREFIX,
-)
-from stac_fastapi.types.stac import Item
-
-# ============================================================================
-# Parameter Validation
-# ============================================================================
-
-
-def validate_refresh(value: Union[str, bool]) -> str:
- """
- Validate the `refresh` parameter value.
-
- Args:
- value (Union[str, bool]): The `refresh` parameter value, which can be a string or a boolean.
-
- Returns:
- str: The validated value of the `refresh` parameter, which can be "true", "false", or "wait_for".
- """
- logger = logging.getLogger(__name__)
-
- # Handle boolean-like values using get_bool_env
- if isinstance(value, bool) or value in {
- "true",
- "false",
- "1",
- "0",
- "yes",
- "no",
- "y",
- "n",
- }:
- is_true = get_bool_env("DATABASE_REFRESH", default=value)
- return "true" if is_true else "false"
-
- # Normalize to lowercase for case-insensitivity
- value = value.lower()
-
- # Handle "wait_for" explicitly
- if value == "wait_for":
- return "wait_for"
-
- # Log a warning for invalid values and default to "false"
- logger.warning(
- f"Invalid value for `refresh`: '{value}'. Expected 'true', 'false', or 'wait_for'. Defaulting to 'false'."
- )
- return "false"
-
-
-# ============================================================================
-# Index and Document ID Utilities
-# ============================================================================
-
-
-@lru_cache(256)
-def index_by_collection_id(collection_id: str) -> str:
- """
- Translate a collection id into an Elasticsearch index name.
-
- Args:
- collection_id (str): The collection id to translate into an index name.
-
- Returns:
- str: The index name derived from the collection id.
- """
- cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
- return (
- f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{collection_id.encode('utf-8').hex()}"
- )
-
-
-@lru_cache(256)
-def index_alias_by_collection_id(collection_id: str) -> str:
- """
- Translate a collection id into an Elasticsearch index alias.
-
- Args:
- collection_id (str): The collection id to translate into an index alias.
-
- Returns:
- str: The index alias derived from the collection id.
- """
- cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
- return f"{ITEMS_INDEX_PREFIX}{cleaned}"
-
-
-def indices(collection_ids: Optional[List[str]]) -> str:
- """
- Get a comma-separated string of index names for a given list of collection ids.
-
- Args:
- collection_ids: A list of collection ids.
-
- Returns:
- A string of comma-separated index names. If `collection_ids` is empty, returns the default indices.
- """
- return (
- ",".join(map(index_alias_by_collection_id, collection_ids))
- if collection_ids
- else ITEM_INDICES
- )
-
-
-# ============================================================================
-# Document ID and Action Generation
-# ============================================================================
-
-
-def mk_item_id(item_id: str, collection_id: str) -> str:
- """Create the document id for an Item in Elasticsearch.
-
- Args:
- item_id (str): The id of the Item.
- collection_id (str): The id of the Collection that the Item belongs to.
-
- Returns:
- str: The document id for the Item, combining the Item id and the Collection id, separated by a `|` character.
- """
- return f"{item_id}|{collection_id}"
-
-
-def mk_actions(collection_id: str, processed_items: List[Item]) -> List[Dict[str, Any]]:
- """Create Elasticsearch bulk actions for a list of processed items.
-
- Args:
- collection_id (str): The identifier for the collection the items belong to.
- processed_items (List[Item]): The list of processed items to be bulk indexed.
-
- Returns:
- List[Dict[str, Union[str, Dict]]]: The list of bulk actions to be executed,
- each action being a dictionary with the following keys:
- - `_index`: the index to store the document in.
- - `_id`: the document's identifier.
- - `_source`: the source of the document.
- """
- index_alias = index_alias_by_collection_id(collection_id)
- return [
- {
- "_index": index_alias,
- "_id": mk_item_id(item["id"], item["collection"]),
- "_source": item,
- }
- for item in processed_items
- ]
diff --git a/stac_fastapi/tests/database/test_database.py b/stac_fastapi/tests/database/test_database.py
index 12196186..86611235 100644
--- a/stac_fastapi/tests/database/test_database.py
+++ b/stac_fastapi/tests/database/test_database.py
@@ -3,12 +3,12 @@
import pytest
from stac_pydantic import api
+from stac_fastapi.sfeos_helpers.database import index_alias_by_collection_id
from stac_fastapi.sfeos_helpers.mappings import (
COLLECTIONS_INDEX,
ES_COLLECTIONS_MAPPINGS,
ES_ITEMS_MAPPINGS,
)
-from stac_fastapi.sfeos_helpers.utilities import index_alias_by_collection_id
from ..conftest import MockRequest, database
From c39a85dbd643117cb4c061b974f7a9ef99088652 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Mon, 19 May 2025 18:20:04 +0800
Subject: [PATCH 16/29] move return_date
---
stac_fastapi/core/stac_fastapi/core/core.py | 8 +--
.../core/stac_fastapi/core/datetime_utils.py | 57 +-----------------
.../core/extensions/aggregation.py | 56 +----------------
.../elasticsearch/database_logic.py | 11 +++-
.../stac_fastapi/opensearch/database_logic.py | 11 +++-
.../sfeos_helpers/database/__init__.py | 8 ++-
.../sfeos_helpers/database/datetime.py | 60 +++++++++++++++++++
7 files changed, 89 insertions(+), 122 deletions(-)
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py
diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py
index 124b58a8..866b429a 100644
--- a/stac_fastapi/core/stac_fastapi/core/core.py
+++ b/stac_fastapi/core/stac_fastapi/core/core.py
@@ -21,7 +21,7 @@
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
from stac_fastapi.core.base_settings import ApiBaseSettings
-from stac_fastapi.core.datetime_utils import format_datetime_range, return_date
+from stac_fastapi.core.datetime_utils import format_datetime_range
from stac_fastapi.core.models.links import PagingLinks
from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
from stac_fastapi.core.session import Session
@@ -316,9 +316,8 @@ async def item_collection(
)
if datetime:
- datetime_search = return_date(datetime)
search = self.database.apply_datetime_filter(
- search=search, datetime_search=datetime_search
+ search=search, interval=datetime
)
if bbox:
@@ -493,9 +492,8 @@ async def post_search(
)
if search_request.datetime:
- datetime_search = return_date(search_request.datetime)
search = self.database.apply_datetime_filter(
- search=search, datetime_search=datetime_search
+ search=search, interval=search_request.datetime
)
if search_request.bbox:
diff --git a/stac_fastapi/core/stac_fastapi/core/datetime_utils.py b/stac_fastapi/core/stac_fastapi/core/datetime_utils.py
index 8e965614..f9dbacf5 100644
--- a/stac_fastapi/core/stac_fastapi/core/datetime_utils.py
+++ b/stac_fastapi/core/stac_fastapi/core/datetime_utils.py
@@ -1,60 +1,7 @@
"""Utility functions to handle datetime parsing."""
-from datetime import datetime
-from datetime import datetime as datetime_type
-from datetime import timezone
-from typing import Dict, Optional, Union
+from datetime import datetime, timezone
-from stac_fastapi.types.rfc3339 import DateTimeType, rfc3339_str_to_datetime
-
-
-def return_date(
- interval: Optional[Union[DateTimeType, str]]
-) -> Dict[str, Optional[str]]:
- """
- Convert a date interval.
-
- (which may be a datetime, a tuple of one or two datetimes a string
- representing a datetime or range, or None) into a dictionary for filtering
- search results with Elasticsearch.
-
- This function ensures the output dictionary contains 'gte' and 'lte' keys,
- even if they are set to None, to prevent KeyError in the consuming logic.
-
- Args:
- interval (Optional[Union[DateTimeType, str]]): The date interval, which might be a single datetime,
- a tuple with one or two datetimes, a string, or None.
-
- Returns:
- dict: A dictionary representing the date interval for use in filtering search results,
- always containing 'gte' and 'lte' keys.
- """
- result: Dict[str, Optional[str]] = {"gte": None, "lte": None}
-
- if interval is None:
- return result
-
- if isinstance(interval, str):
- if "/" in interval:
- parts = interval.split("/")
- result["gte"] = parts[0] if parts[0] != ".." else None
- result["lte"] = parts[1] if len(parts) > 1 and parts[1] != ".." else None
- else:
- converted_time = interval if interval != ".." else None
- result["gte"] = result["lte"] = converted_time
- return result
-
- if isinstance(interval, datetime_type):
- datetime_iso = interval.isoformat()
- result["gte"] = result["lte"] = datetime_iso
- elif isinstance(interval, tuple):
- start, end = interval
- # Ensure datetimes are converted to UTC and formatted with 'Z'
- if start:
- result["gte"] = start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
- if end:
- result["lte"] = end.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
-
- return result
+from stac_fastapi.types.rfc3339 import rfc3339_str_to_datetime
def format_datetime_range(date_str: str) -> str:
diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py b/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py
index d41d763c..2f263055 100644
--- a/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py
+++ b/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py
@@ -1,7 +1,6 @@
"""Request model for the Aggregation extension."""
from datetime import datetime
-from datetime import datetime as datetime_type
from typing import Dict, List, Literal, Optional, Union
from urllib.parse import unquote_plus, urljoin
@@ -210,58 +209,6 @@ def extract_date_histogram_interval(self, value: Optional[str]) -> str:
else:
return self.DEFAULT_DATETIME_INTERVAL
- @staticmethod
- def _return_date(
- interval: Optional[Union[DateTimeType, str]]
- ) -> Dict[str, Optional[str]]:
- """
- Convert a date interval.
-
- (which may be a datetime, a tuple of one or two datetimes a string
- representing a datetime or range, or None) into a dictionary for filtering
- search results with Elasticsearch.
-
- This function ensures the output dictionary contains 'gte' and 'lte' keys,
- even if they are set to None, to prevent KeyError in the consuming logic.
-
- Args:
- interval (Optional[Union[DateTimeType, str]]): The date interval, which might be a single datetime,
- a tuple with one or two datetimes, a string, or None.
-
- Returns:
- dict: A dictionary representing the date interval for use in filtering search results,
- always containing 'gte' and 'lte' keys.
- """
- result: Dict[str, Optional[str]] = {"gte": None, "lte": None}
-
- if interval is None:
- return result
-
- if isinstance(interval, str):
- if "/" in interval:
- parts = interval.split("/")
- result["gte"] = parts[0] if parts[0] != ".." else None
- result["lte"] = (
- parts[1] if len(parts) > 1 and parts[1] != ".." else None
- )
- else:
- converted_time = interval if interval != ".." else None
- result["gte"] = result["lte"] = converted_time
- return result
-
- if isinstance(interval, datetime_type):
- datetime_iso = interval.isoformat()
- result["gte"] = result["lte"] = datetime_iso
- elif isinstance(interval, tuple):
- start, end = interval
- # Ensure datetimes are converted to UTC and formatted with 'Z'
- if start:
- result["gte"] = start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
- if end:
- result["lte"] = end.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
-
- return result
-
def frequency_agg(self, es_aggs, name, data_type):
"""Format an aggregation for a frequency distribution aggregation."""
buckets = []
@@ -418,9 +365,8 @@ async def aggregate(
)
if aggregate_request.datetime:
- datetime_search = self._return_date(aggregate_request.datetime)
search = self.database.apply_datetime_filter(
- search=search, datetime_search=datetime_search
+ search=search, interval=aggregate_request.datetime
)
if aggregate_request.bbox:
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
index be2247b3..d529ce01 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
@@ -5,7 +5,7 @@
import logging
from base64 import urlsafe_b64decode, urlsafe_b64encode
from copy import deepcopy
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Type
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
import attr
import elasticsearch.helpers as helpers
@@ -33,6 +33,7 @@
mk_actions,
mk_item_id,
populate_sort_shared,
+ return_date,
validate_refresh,
)
from stac_fastapi.sfeos_helpers.mappings import (
@@ -44,6 +45,7 @@
Geometry,
)
from stac_fastapi.types.errors import ConflictError, NotFoundError
+from stac_fastapi.types.rfc3339 import DateTimeType
from stac_fastapi.types.stac import Collection, Item
logger = logging.getLogger(__name__)
@@ -241,17 +243,20 @@ def apply_collections_filter(search: Search, collection_ids: List[str]):
return search.filter("terms", collection=collection_ids)
@staticmethod
- def apply_datetime_filter(search: Search, datetime_search: dict):
+ def apply_datetime_filter(
+ search: Search, interval: Optional[Union[DateTimeType, str]]
+ ):
"""Apply a filter to search on datetime, start_datetime, and end_datetime fields.
Args:
search (Search): The search object to filter.
- datetime_search (dict): The datetime filter criteria.
+ interval: Optional[Union[DateTimeType, str]]
Returns:
Search: The filtered search object.
"""
should = []
+ datetime_search = return_date(interval)
# If the request is a single datetime return
# items with datetimes equal to the requested datetime OR
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
index dd19e6b4..f93311f9 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
@@ -5,7 +5,7 @@
import logging
from base64 import urlsafe_b64decode, urlsafe_b64encode
from copy import deepcopy
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Type
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
import attr
from opensearchpy import exceptions, helpers
@@ -33,6 +33,7 @@
mk_actions,
mk_item_id,
populate_sort_shared,
+ return_date,
validate_refresh,
)
from stac_fastapi.sfeos_helpers.mappings import (
@@ -47,6 +48,7 @@
Geometry,
)
from stac_fastapi.types.errors import ConflictError, NotFoundError
+from stac_fastapi.types.rfc3339 import DateTimeType
from stac_fastapi.types.stac import Collection, Item
logger = logging.getLogger(__name__)
@@ -278,17 +280,20 @@ def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]
)
@staticmethod
- def apply_datetime_filter(search: Search, datetime_search):
+ def apply_datetime_filter(
+ search: Search, interval: Optional[Union[DateTimeType, str]]
+ ):
"""Apply a filter to search based on datetime field, start_datetime, and end_datetime fields.
Args:
search (Search): The search object to filter.
- datetime_search (dict): The datetime filter criteria.
+ interval: Optional[Union[DateTimeType, str]]
Returns:
Search: The filtered search object.
"""
should = []
+ datetime_search = return_date(interval)
# If the request is a single datetime return
# items with datetimes equal to the requested datetime OR
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/__init__.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/__init__.py
index 7ff5076d..31bf28d8 100644
--- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/__init__.py
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/__init__.py
@@ -8,6 +8,7 @@
3. Mapping functions for working with Elasticsearch/OpenSearch mappings
4. Document operations for working with documents
5. Utility functions for database operations
+6. Datetime utilities for query formatting
The database package is organized as follows:
- index.py: Index management functions
@@ -15,6 +16,7 @@
- mapping.py: Mapping functions
- document.py: Document operations
- utils.py: Utility functions
+- datetime.py: Datetime utilities for query formatting
When adding new functionality to this package, consider:
1. Will this code be used by both Elasticsearch and OpenSearch implementations?
@@ -28,6 +30,7 @@
"""
# Re-export all functions for backward compatibility
+from .datetime import return_date
from .document import mk_actions, mk_item_id
from .index import (
create_index_templates_shared,
@@ -42,7 +45,7 @@
apply_intersects_filter_shared,
populate_sort_shared,
)
-from .utils import validate_refresh
+from .utils import get_bool_env, validate_refresh
__all__ = [
# Index operations
@@ -62,4 +65,7 @@
"mk_actions",
# Utility functions
"validate_refresh",
+ "get_bool_env",
+ # Datetime utilities
+ "return_date",
]
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py
new file mode 100644
index 00000000..352ed4b5
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py
@@ -0,0 +1,60 @@
+"""Elasticsearch/OpenSearch-specific datetime utilities.
+
+This module provides datetime utility functions specifically designed for
+Elasticsearch and OpenSearch query formatting.
+"""
+
+from datetime import datetime as datetime_type
+from typing import Dict, Optional, Union
+
+from stac_fastapi.types.rfc3339 import DateTimeType
+
+
+def return_date(
+ interval: Optional[Union[DateTimeType, str]]
+) -> Dict[str, Optional[str]]:
+ """
+ Convert a date interval to an Elasticsearch/OpenSearch query format.
+
+ This function converts a date interval (which may be a datetime, a tuple of one or two datetimes,
+ a string representing a datetime or range, or None) into a dictionary for filtering
+ search results with Elasticsearch/OpenSearch.
+
+ This function ensures the output dictionary contains 'gte' and 'lte' keys,
+ even if they are set to None, to prevent KeyError in the consuming logic.
+
+ Args:
+ interval (Optional[Union[DateTimeType, str]]): The date interval, which might be a single datetime,
+ a tuple with one or two datetimes, a string, or None.
+
+ Returns:
+ dict: A dictionary representing the date interval for use in filtering search results,
+ always containing 'gte' and 'lte' keys.
+ """
+ result: Dict[str, Optional[str]] = {"gte": None, "lte": None}
+
+ if interval is None:
+ return result
+
+ if isinstance(interval, str):
+ if "/" in interval:
+ parts = interval.split("/")
+ result["gte"] = parts[0] if parts[0] != ".." else None
+ result["lte"] = parts[1] if len(parts) > 1 and parts[1] != ".." else None
+ else:
+ converted_time = interval if interval != ".." else None
+ result["gte"] = result["lte"] = converted_time
+ return result
+
+ if isinstance(interval, datetime_type):
+ datetime_iso = interval.isoformat()
+ result["gte"] = result["lte"] = datetime_iso
+ elif isinstance(interval, tuple):
+ start, end = interval
+ # Ensure datetimes are converted to UTC and formatted with 'Z'
+ if start:
+ result["gte"] = start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+ if end:
+ result["lte"] = end.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+
+ return result
From 690379198fd8659dc30a9921733f705deea296a7 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Mon, 19 May 2025 18:58:03 +0800
Subject: [PATCH 17/29] move format date range
---
.../core/extensions/aggregation.py | 22 ++-----------------
1 file changed, 2 insertions(+), 20 deletions(-)
diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py b/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py
index 2f263055..8fec0ba4 100644
--- a/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py
+++ b/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py
@@ -14,7 +14,7 @@
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
from stac_fastapi.core.base_settings import ApiBaseSettings
-from stac_fastapi.core.datetime_utils import datetime_to_str
+from stac_fastapi.core.datetime_utils import datetime_to_str, format_datetime_range
from stac_fastapi.core.session import Session
from stac_fastapi.extensions.core.aggregation.client import AsyncBaseAggregationClient
from stac_fastapi.extensions.core.aggregation.request import (
@@ -257,24 +257,6 @@ def get_filter(self, filter, filter_lang):
detail=f"Unknown filter-lang: {filter_lang}. Only cql2-json or cql2-text are supported.",
)
- def _format_datetime_range(self, date_tuple: DateTimeType) -> str:
- """
- Convert a tuple of datetime objects or None into a formatted string for API requests.
-
- Args:
- date_tuple (tuple): A tuple containing two elements, each can be a datetime object or None.
-
- Returns:
- str: A string formatted as 'YYYY-MM-DDTHH:MM:SS.sssZ/YYYY-MM-DDTHH:MM:SS.sssZ', with '..' used if any element is None.
- """
-
- def format_datetime(dt):
- """Format a single datetime object to the ISO8601 extended format with 'Z'."""
- return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" if dt else ".."
-
- start, end = date_tuple
- return f"{format_datetime(start)}/{format_datetime(end)}"
-
async def aggregate(
self,
aggregate_request: Optional[EsAggregationExtensionPostRequest] = None,
@@ -325,7 +307,7 @@ async def aggregate(
base_args["intersects"] = orjson.loads(unquote_plus(intersects))
if datetime:
- base_args["datetime"] = self._format_datetime_range(datetime)
+ base_args["datetime"] = format_datetime_range(datetime)
if filter_expr:
base_args["filter"] = self.get_filter(filter_expr, filter_lang)
From 45dee7db57ef6c30622ba40cb95425c22b935e16 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Tue, 20 May 2025 00:03:43 +0800
Subject: [PATCH 18/29] move refactor aggregations
---
.../core/extensions/aggregation.py | 460 +----------------
.../stac_fastapi/elasticsearch/app.py | 4 +-
.../opensearch/stac_fastapi/opensearch/app.py | 4 +-
.../sfeos_helpers/aggregation/README.md | 57 +++
.../sfeos_helpers/aggregation/__init__.py | 31 ++
.../sfeos_helpers/aggregation/client.py | 469 ++++++++++++++++++
.../sfeos_helpers/aggregation/format.py | 60 +++
stac_fastapi/tests/conftest.py | 10 +-
8 files changed, 628 insertions(+), 467 deletions(-)
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/README.md
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/__init__.py
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py
create mode 100644 stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/format.py
diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py b/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py
index 8fec0ba4..cdce486f 100644
--- a/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py
+++ b/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py
@@ -1,35 +1,19 @@
"""Request model for the Aggregation extension."""
-from datetime import datetime
-from typing import Dict, List, Literal, Optional, Union
-from urllib.parse import unquote_plus, urljoin
+from typing import Literal, Optional
import attr
-import orjson
-from fastapi import HTTPException, Path, Request
-from pygeofilter.backends.cql2_json import to_cql2
-from pygeofilter.parsers.cql2_text import parse as parse_cql2_text
-from stac_pydantic.shared import BBox
+from fastapi import Path
from typing_extensions import Annotated
-from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
-from stac_fastapi.core.base_settings import ApiBaseSettings
-from stac_fastapi.core.datetime_utils import datetime_to_str, format_datetime_range
-from stac_fastapi.core.session import Session
-from stac_fastapi.extensions.core.aggregation.client import AsyncBaseAggregationClient
from stac_fastapi.extensions.core.aggregation.request import (
AggregationExtensionGetRequest,
AggregationExtensionPostRequest,
)
-from stac_fastapi.extensions.core.aggregation.types import (
- Aggregation,
- AggregationCollection,
-)
from stac_fastapi.extensions.core.filter.request import (
FilterExtensionGetRequest,
FilterExtensionPostRequest,
)
-from stac_fastapi.types.rfc3339 import DateTimeType
FilterLang = Literal["cql-json", "cql2-json", "cql2-text"]
@@ -63,443 +47,3 @@ class EsAggregationExtensionPostRequest(
geometry_geohash_grid_frequency_precision: Optional[int] = None
geometry_geotile_grid_frequency_precision: Optional[int] = None
datetime_frequency_interval: Optional[str] = None
-
-
-@attr.s
-class EsAsyncAggregationClient(AsyncBaseAggregationClient):
- """Defines a pattern for implementing the STAC aggregation extension."""
-
- database: BaseDatabaseLogic = attr.ib()
- settings: ApiBaseSettings = attr.ib()
- session: Session = attr.ib(default=attr.Factory(Session.create_from_env))
-
- DEFAULT_AGGREGATIONS = [
- {"name": "total_count", "data_type": "integer"},
- {"name": "datetime_max", "data_type": "datetime"},
- {"name": "datetime_min", "data_type": "datetime"},
- {
- "name": "datetime_frequency",
- "data_type": "frequency_distribution",
- "frequency_distribution_data_type": "datetime",
- },
- {
- "name": "collection_frequency",
- "data_type": "frequency_distribution",
- "frequency_distribution_data_type": "string",
- },
- {
- "name": "geometry_geohash_grid_frequency",
- "data_type": "frequency_distribution",
- "frequency_distribution_data_type": "string",
- },
- {
- "name": "geometry_geotile_grid_frequency",
- "data_type": "frequency_distribution",
- "frequency_distribution_data_type": "string",
- },
- ]
-
- GEO_POINT_AGGREGATIONS = [
- {
- "name": "grid_code_frequency",
- "data_type": "frequency_distribution",
- "frequency_distribution_data_type": "string",
- },
- {
- "name": "centroid_geohash_grid_frequency",
- "data_type": "frequency_distribution",
- "frequency_distribution_data_type": "string",
- },
- {
- "name": "centroid_geohex_grid_frequency",
- "data_type": "frequency_distribution",
- "frequency_distribution_data_type": "string",
- },
- {
- "name": "centroid_geotile_grid_frequency",
- "data_type": "frequency_distribution",
- "frequency_distribution_data_type": "string",
- },
- ]
-
- MAX_GEOHASH_PRECISION = 12
- MAX_GEOHEX_PRECISION = 15
- MAX_GEOTILE_PRECISION = 29
- SUPPORTED_DATETIME_INTERVAL = {"day", "month", "year"}
- DEFAULT_DATETIME_INTERVAL = "month"
-
- async def get_aggregations(self, collection_id: Optional[str] = None, **kwargs):
- """Get the available aggregations for a catalog or collection defined in the STAC JSON. If no aggregations, default aggregations are used."""
- request: Request = kwargs["request"]
- base_url = str(request.base_url)
- links = [{"rel": "root", "type": "application/json", "href": base_url}]
-
- if collection_id is not None:
- collection_endpoint = urljoin(base_url, f"collections/{collection_id}")
- links.extend(
- [
- {
- "rel": "collection",
- "type": "application/json",
- "href": collection_endpoint,
- },
- {
- "rel": "self",
- "type": "application/json",
- "href": urljoin(collection_endpoint + "/", "aggregations"),
- },
- ]
- )
- if await self.database.check_collection_exists(collection_id) is None:
- collection = await self.database.find_collection(collection_id)
- aggregations = collection.get(
- "aggregations", self.DEFAULT_AGGREGATIONS.copy()
- )
- else:
- raise IndexError(f"Collection {collection_id} does not exist")
- else:
- links.append(
- {
- "rel": "self",
- "type": "application/json",
- "href": urljoin(base_url, "aggregations"),
- }
- )
-
- aggregations = self.DEFAULT_AGGREGATIONS
- return AggregationCollection(
- type="AggregationCollection", aggregations=aggregations, links=links
- )
-
- def extract_precision(
- self, precision: Union[int, None], min_value: int, max_value: int
- ) -> Optional[int]:
- """Ensure that the aggregation precision value is withing the a valid range, otherwise return the minumium value."""
- if precision is not None:
- if precision < min_value or precision > max_value:
- raise HTTPException(
- status_code=400,
- detail=f"Invalid precision. Must be a number between {min_value} and {max_value} inclusive",
- )
- return precision
- else:
- return min_value
-
- def extract_date_histogram_interval(self, value: Optional[str]) -> str:
- """
- Ensure that the interval for the date histogram is valid. If no value is provided, the default will be returned.
-
- Args:
- value: value entered by the user
-
- Returns:
- string containing the date histogram interval to use.
-
- Raises:
- HTTPException: if the supplied value is not in the supported intervals
- """
- if value is not None:
- if value not in self.SUPPORTED_DATETIME_INTERVAL:
- raise HTTPException(
- status_code=400,
- detail=f"Invalid datetime interval. Must be one of {self.SUPPORTED_DATETIME_INTERVAL}",
- )
- else:
- return value
- else:
- return self.DEFAULT_DATETIME_INTERVAL
-
- def frequency_agg(self, es_aggs, name, data_type):
- """Format an aggregation for a frequency distribution aggregation."""
- buckets = []
- for bucket in es_aggs.get(name, {}).get("buckets", []):
- bucket_data = {
- "key": bucket.get("key_as_string") or bucket.get("key"),
- "data_type": data_type,
- "frequency": bucket.get("doc_count"),
- "to": bucket.get("to"),
- "from": bucket.get("from"),
- }
- buckets.append(bucket_data)
- return Aggregation(
- name=name,
- data_type="frequency_distribution",
- overflow=es_aggs.get(name, {}).get("sum_other_doc_count", 0),
- buckets=buckets,
- )
-
- def metric_agg(self, es_aggs, name, data_type):
- """Format an aggregation for a metric aggregation."""
- value = es_aggs.get(name, {}).get("value_as_string") or es_aggs.get(
- name, {}
- ).get("value")
- # ES 7.x does not return datetimes with a 'value_as_string' field
- if "datetime" in name and isinstance(value, float):
- value = datetime_to_str(datetime.fromtimestamp(value / 1e3))
- return Aggregation(
- name=name,
- data_type=data_type,
- value=value,
- )
-
- def get_filter(self, filter, filter_lang):
- """Format the filter parameter in cql2-json or cql2-text."""
- if filter_lang == "cql2-text":
- return orjson.loads(to_cql2(parse_cql2_text(filter)))
- elif filter_lang == "cql2-json":
- if isinstance(filter, str):
- return orjson.loads(unquote_plus(filter))
- else:
- return filter
- else:
- raise HTTPException(
- status_code=400,
- detail=f"Unknown filter-lang: {filter_lang}. Only cql2-json or cql2-text are supported.",
- )
-
- async def aggregate(
- self,
- aggregate_request: Optional[EsAggregationExtensionPostRequest] = None,
- collection_id: Optional[
- Annotated[str, Path(description="Collection ID")]
- ] = None,
- collections: Optional[List[str]] = [],
- datetime: Optional[DateTimeType] = None,
- intersects: Optional[str] = None,
- filter_lang: Optional[str] = None,
- filter_expr: Optional[str] = None,
- aggregations: Optional[str] = None,
- ids: Optional[List[str]] = None,
- bbox: Optional[BBox] = None,
- centroid_geohash_grid_frequency_precision: Optional[int] = None,
- centroid_geohex_grid_frequency_precision: Optional[int] = None,
- centroid_geotile_grid_frequency_precision: Optional[int] = None,
- geometry_geohash_grid_frequency_precision: Optional[int] = None,
- geometry_geotile_grid_frequency_precision: Optional[int] = None,
- datetime_frequency_interval: Optional[str] = None,
- **kwargs,
- ) -> Union[Dict, Exception]:
- """Get aggregations from the database."""
- request: Request = kwargs["request"]
- base_url = str(request.base_url)
- path = request.url.path
- search = self.database.make_search()
-
- if aggregate_request is None:
-
- base_args = {
- "collections": collections,
- "ids": ids,
- "bbox": bbox,
- "aggregations": aggregations,
- "centroid_geohash_grid_frequency_precision": centroid_geohash_grid_frequency_precision,
- "centroid_geohex_grid_frequency_precision": centroid_geohex_grid_frequency_precision,
- "centroid_geotile_grid_frequency_precision": centroid_geotile_grid_frequency_precision,
- "geometry_geohash_grid_frequency_precision": geometry_geohash_grid_frequency_precision,
- "geometry_geotile_grid_frequency_precision": geometry_geotile_grid_frequency_precision,
- "datetime_frequency_interval": datetime_frequency_interval,
- }
-
- if collection_id:
- collections = [str(collection_id)]
-
- if intersects:
- base_args["intersects"] = orjson.loads(unquote_plus(intersects))
-
- if datetime:
- base_args["datetime"] = format_datetime_range(datetime)
-
- if filter_expr:
- base_args["filter"] = self.get_filter(filter_expr, filter_lang)
- aggregate_request = EsAggregationExtensionPostRequest(**base_args)
- else:
- # Workaround for optional path param in POST requests
- if "collections" in path:
- collection_id = path.split("/")[2]
-
- filter_lang = "cql2-json"
- if aggregate_request.filter_expr:
- aggregate_request.filter_expr = self.get_filter(
- aggregate_request.filter_expr, filter_lang
- )
-
- if collection_id:
- if aggregate_request.collections:
- raise HTTPException(
- status_code=400,
- detail="Cannot query multiple collections when executing '/collections//aggregate'. Use '/aggregate' and the collections field instead",
- )
- else:
- aggregate_request.collections = [collection_id]
-
- if (
- aggregate_request.aggregations is None
- or aggregate_request.aggregations == []
- ):
- raise HTTPException(
- status_code=400,
- detail="No 'aggregations' found. Use '/aggregations' to return available aggregations",
- )
-
- if aggregate_request.ids:
- search = self.database.apply_ids_filter(
- search=search, item_ids=aggregate_request.ids
- )
-
- if aggregate_request.datetime:
- search = self.database.apply_datetime_filter(
- search=search, interval=aggregate_request.datetime
- )
-
- if aggregate_request.bbox:
- bbox = aggregate_request.bbox
- if len(bbox) == 6:
- bbox = [bbox[0], bbox[1], bbox[3], bbox[4]]
-
- search = self.database.apply_bbox_filter(search=search, bbox=bbox)
-
- if aggregate_request.intersects:
- search = self.database.apply_intersects_filter(
- search=search, intersects=aggregate_request.intersects
- )
-
- if aggregate_request.collections:
- search = self.database.apply_collections_filter(
- search=search, collection_ids=aggregate_request.collections
- )
- # validate that aggregations are supported for all collections
- for collection_id in aggregate_request.collections:
- aggs = await self.get_aggregations(
- collection_id=collection_id, request=request
- )
- supported_aggregations = (
- aggs["aggregations"] + self.DEFAULT_AGGREGATIONS
- )
-
- for agg_name in aggregate_request.aggregations:
- if agg_name not in set([x["name"] for x in supported_aggregations]):
- raise HTTPException(
- status_code=400,
- detail=f"Aggregation {agg_name} not supported by collection {collection_id}",
- )
- else:
- # Validate that the aggregations requested are supported by the catalog
- aggs = await self.get_aggregations(request=request)
- supported_aggregations = aggs["aggregations"]
- for agg_name in aggregate_request.aggregations:
- if agg_name not in [x["name"] for x in supported_aggregations]:
- raise HTTPException(
- status_code=400,
- detail=f"Aggregation {agg_name} not supported at catalog level",
- )
-
- if aggregate_request.filter_expr:
- try:
- search = await self.database.apply_cql2_filter(
- search, aggregate_request.filter_expr
- )
- except Exception as e:
- raise HTTPException(
- status_code=400, detail=f"Error with cql2 filter: {e}"
- )
-
- centroid_geohash_grid_precision = self.extract_precision(
- aggregate_request.centroid_geohash_grid_frequency_precision,
- 1,
- self.MAX_GEOHASH_PRECISION,
- )
-
- centroid_geohex_grid_precision = self.extract_precision(
- aggregate_request.centroid_geohex_grid_frequency_precision,
- 0,
- self.MAX_GEOHEX_PRECISION,
- )
-
- centroid_geotile_grid_precision = self.extract_precision(
- aggregate_request.centroid_geotile_grid_frequency_precision,
- 0,
- self.MAX_GEOTILE_PRECISION,
- )
-
- geometry_geohash_grid_precision = self.extract_precision(
- aggregate_request.geometry_geohash_grid_frequency_precision,
- 1,
- self.MAX_GEOHASH_PRECISION,
- )
-
- geometry_geotile_grid_precision = self.extract_precision(
- aggregate_request.geometry_geotile_grid_frequency_precision,
- 0,
- self.MAX_GEOTILE_PRECISION,
- )
-
- datetime_frequency_interval = self.extract_date_histogram_interval(
- aggregate_request.datetime_frequency_interval,
- )
-
- try:
- db_response = await self.database.aggregate(
- collections,
- aggregate_request.aggregations,
- search,
- centroid_geohash_grid_precision,
- centroid_geohex_grid_precision,
- centroid_geotile_grid_precision,
- geometry_geohash_grid_precision,
- geometry_geotile_grid_precision,
- datetime_frequency_interval,
- )
- except Exception as error:
- if not isinstance(error, IndexError):
- raise error
- aggs = []
- if db_response:
- result_aggs = db_response.get("aggregations", {})
- for agg in {
- frozenset(item.items()): item
- for item in supported_aggregations + self.GEO_POINT_AGGREGATIONS
- }.values():
- if agg["name"] in aggregate_request.aggregations:
- if agg["name"].endswith("_frequency"):
- aggs.append(
- self.frequency_agg(
- result_aggs, agg["name"], agg["data_type"]
- )
- )
- else:
- aggs.append(
- self.metric_agg(result_aggs, agg["name"], agg["data_type"])
- )
- links = [
- {"rel": "root", "type": "application/json", "href": base_url},
- ]
-
- if collection_id:
- collection_endpoint = urljoin(base_url, f"collections/{collection_id}")
- links.extend(
- [
- {
- "rel": "collection",
- "type": "application/json",
- "href": collection_endpoint,
- },
- {
- "rel": "self",
- "type": "application/json",
- "href": urljoin(collection_endpoint, "aggregate"),
- },
- ]
- )
- else:
- links.append(
- {
- "rel": "self",
- "type": "application/json",
- "href": urljoin(base_url, "aggregate"),
- }
- )
- results = AggregationCollection(
- type="AggregationCollection", aggregations=aggs, links=links
- )
-
- return results
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py
index 21495586..7ebb94a0 100644
--- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py
+++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py
@@ -17,7 +17,6 @@
from stac_fastapi.core.extensions.aggregation import (
EsAggregationExtensionGetRequest,
EsAggregationExtensionPostRequest,
- EsAsyncAggregationClient,
)
from stac_fastapi.core.extensions.fields import FieldsExtension
from stac_fastapi.core.rate_limit import setup_rate_limit
@@ -39,6 +38,7 @@
TransactionExtension,
)
from stac_fastapi.extensions.third_party import BulkTransactionExtension
+from stac_fastapi.sfeos_helpers.aggregation import EsAsyncBaseAggregationClient
from stac_fastapi.sfeos_helpers.filter import EsAsyncBaseFiltersClient
logging.basicConfig(level=logging.INFO)
@@ -60,7 +60,7 @@
)
aggregation_extension = AggregationExtension(
- client=EsAsyncAggregationClient(
+ client=EsAsyncBaseAggregationClient(
database=database_logic, session=session, settings=settings
)
)
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py
index 50ab45b6..9e1ec38c 100644
--- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py
+++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py
@@ -17,7 +17,6 @@
from stac_fastapi.core.extensions.aggregation import (
EsAggregationExtensionGetRequest,
EsAggregationExtensionPostRequest,
- EsAsyncAggregationClient,
)
from stac_fastapi.core.extensions.fields import FieldsExtension
from stac_fastapi.core.rate_limit import setup_rate_limit
@@ -39,6 +38,7 @@
create_collection_index,
create_index_templates,
)
+from stac_fastapi.sfeos_helpers.aggregation import EsAsyncBaseAggregationClient
from stac_fastapi.sfeos_helpers.filter import EsAsyncBaseFiltersClient
logging.basicConfig(level=logging.INFO)
@@ -60,7 +60,7 @@
)
aggregation_extension = AggregationExtension(
- client=EsAsyncAggregationClient(
+ client=EsAsyncBaseAggregationClient(
database=database_logic, session=session, settings=settings
)
)
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/README.md b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/README.md
new file mode 100644
index 00000000..253855b4
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/README.md
@@ -0,0 +1,57 @@
+# STAC FastAPI Aggregation Package
+
+This package contains shared aggregation functionality used by both the Elasticsearch and OpenSearch implementations of STAC FastAPI. It helps reduce code duplication and ensures consistent behavior between the two implementations.
+
+## Package Structure
+
+The aggregation package is organized into three main modules:
+
+- **client.py**: Contains the base aggregation client implementation
+ - `EsAsyncBaseAggregationClient`: The main class that implements the STAC aggregation extension for Elasticsearch/OpenSearch
+ - Methods for handling aggregation requests, validating parameters, and formatting responses
+
+- **format.py**: Contains functions for formatting aggregation responses
+ - `frequency_agg`: Formats frequency distribution aggregation responses
+ - `metric_agg`: Formats metric aggregation responses
+
+- **__init__.py**: Package initialization and exports
+ - Exports the main classes and functions for use by other modules
+
+## Features
+
+The aggregation package provides the following features:
+
+- Support for various aggregation types:
+ - Datetime frequency
+ - Collection frequency
+ - Property frequency
+ - Geospatial grid aggregations (geohash, geohex, geotile)
+ - Metric aggregations (min, max, etc.)
+
+- Parameter validation:
+ - Precision validation for geospatial aggregations
+ - Interval validation for datetime aggregations
+
+- Response formatting:
+ - Consistent response structure
+ - Proper typing and documentation
+
+## Usage
+
+The aggregation package is used by the Elasticsearch and OpenSearch implementations to provide aggregation functionality for STAC API. The main entry point is the `EsAsyncBaseAggregationClient` class, which is instantiated in the respective app.py files.
+
+Example:
+```python
+from stac_fastapi.sfeos_helpers.aggregation import EsAsyncBaseAggregationClient
+
+# Create an instance of the aggregation client
+aggregation_client = EsAsyncBaseAggregationClient(database)
+
+# Register the aggregation extension with the API
+api = StacApi(
+ ...,
+ extensions=[
+ ...,
+ AggregationExtension(client=aggregation_client),
+ ],
+)
\ No newline at end of file
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/__init__.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/__init__.py
new file mode 100644
index 00000000..2beeff67
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/__init__.py
@@ -0,0 +1,31 @@
+"""Shared aggregation extension methods for stac-fastapi elasticsearch and opensearch backends.
+
+This module provides shared functionality for implementing the STAC API Aggregation Extension
+with Elasticsearch and OpenSearch. It includes:
+
+1. Functions for formatting aggregation responses
+2. Helper functions for handling aggregation parameters
+3. Base implementation of the AsyncBaseAggregationClient for Elasticsearch/OpenSearch
+
+The aggregation package is organized as follows:
+- client.py: Aggregation client implementation
+- format.py: Response formatting functions
+
+When adding new functionality to this package, consider:
+1. Will this code be used by both Elasticsearch and OpenSearch implementations?
+2. Is the functionality stable and unlikely to diverge between implementations?
+3. Is the function well-documented with clear input/output contracts?
+
+Function Naming Conventions:
+- Function names should be descriptive and indicate their purpose
+- Parameter names should be consistent across similar functions
+"""
+
+from .client import EsAsyncBaseAggregationClient
+from .format import frequency_agg, metric_agg
+
+__all__ = [
+ "EsAsyncBaseAggregationClient",
+ "frequency_agg",
+ "metric_agg",
+]
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py
new file mode 100644
index 00000000..bb34c05b
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py
@@ -0,0 +1,469 @@
+"""Client implementation for the STAC API Aggregation Extension."""
+
+from pathlib import Path
+from typing import Annotated, Any, Dict, List, Optional, Union
+from urllib.parse import unquote_plus, urljoin
+
+import attr
+import orjson
+from fastapi import HTTPException, Request
+from pygeofilter.backends.cql2_json import to_cql2
+from pygeofilter.parsers.cql2_text import parse as parse_cql2_text
+from stac_pydantic.shared import BBox
+
+from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
+from stac_fastapi.core.base_settings import ApiBaseSettings
+from stac_fastapi.core.datetime_utils import format_datetime_range
+from stac_fastapi.core.extensions.aggregation import EsAggregationExtensionPostRequest
+from stac_fastapi.core.session import Session
+from stac_fastapi.extensions.core.aggregation.client import AsyncBaseAggregationClient
+from stac_fastapi.extensions.core.aggregation.types import (
+ Aggregation,
+ AggregationCollection,
+)
+from stac_fastapi.types.rfc3339 import DateTimeType
+
+from .format import frequency_agg, metric_agg
+
+
+@attr.s
+class EsAsyncBaseAggregationClient(AsyncBaseAggregationClient):
+ """Defines a pattern for implementing the STAC aggregation extension with Elasticsearch/OpenSearch."""
+
+ database: BaseDatabaseLogic = attr.ib()
+ settings: ApiBaseSettings = attr.ib()
+ session: Session = attr.ib(default=attr.Factory(Session.create_from_env))
+
+ # Default aggregations to use if none are specified
+ DEFAULT_AGGREGATIONS = [
+ {"name": "total_count", "data_type": "integer"},
+ {"name": "datetime_max", "data_type": "datetime"},
+ {"name": "datetime_min", "data_type": "datetime"},
+ {
+ "name": "datetime_frequency",
+ "data_type": "frequency_distribution",
+ "frequency_distribution_data_type": "datetime",
+ },
+ {
+ "name": "collection_frequency",
+ "data_type": "frequency_distribution",
+ "frequency_distribution_data_type": "string",
+ },
+ {
+ "name": "geometry_geohash_grid_frequency",
+ "data_type": "frequency_distribution",
+ "frequency_distribution_data_type": "string",
+ },
+ {
+ "name": "geometry_geotile_grid_frequency",
+ "data_type": "frequency_distribution",
+ "frequency_distribution_data_type": "string",
+ },
+ ]
+
+ # Geo point aggregations
+ GEO_POINT_AGGREGATIONS = [
+ {
+ "name": "grid_code_frequency",
+ "data_type": "frequency_distribution",
+ "frequency_distribution_data_type": "string",
+ },
+ ]
+
+ # Supported datetime intervals
+ SUPPORTED_DATETIME_INTERVAL = [
+ "year",
+ "quarter",
+ "month",
+ "week",
+ "day",
+ "hour",
+ "minute",
+ "second",
+ ]
+
+ # Default datetime interval
+ DEFAULT_DATETIME_INTERVAL = "month"
+
+ # Maximum precision values
+ MAX_GEOHASH_PRECISION = 12
+ MAX_GEOHEX_PRECISION = 15
+ MAX_GEOTILE_PRECISION = 29
+
+ async def get_aggregations(
+ self, collection_id: Optional[str] = None, **kwargs
+ ) -> Dict[str, Any]:
+ """Get the available aggregations for a catalog or collection defined in the STAC JSON.
+
+ If no aggregations are defined, default aggregations are used.
+
+ Args:
+ collection_id: Optional collection ID to get aggregations for
+ **kwargs: Additional keyword arguments
+
+ Returns:
+ Dict[str, Any]: A dictionary containing the available aggregations
+ """
+ request: Request = kwargs.get("request")
+ base_url = str(request.base_url) if request else ""
+ links = [{"rel": "root", "type": "application/json", "href": base_url}]
+
+ if collection_id is not None:
+ collection_endpoint = urljoin(base_url, f"collections/{collection_id}")
+ links.extend(
+ [
+ {
+ "rel": "collection",
+ "type": "application/json",
+ "href": collection_endpoint,
+ },
+ {
+ "rel": "self",
+ "type": "application/json",
+ "href": urljoin(collection_endpoint + "/", "aggregations"),
+ },
+ ]
+ )
+ if await self.database.check_collection_exists(collection_id) is None:
+ collection = await self.database.find_collection(collection_id)
+ aggregations = collection.get(
+ "aggregations", self.DEFAULT_AGGREGATIONS.copy()
+ )
+ else:
+ raise IndexError(f"Collection {collection_id} does not exist")
+ else:
+ links.append(
+ {
+ "rel": "self",
+ "type": "application/json",
+ "href": urljoin(base_url, "aggregations"),
+ }
+ )
+ aggregations = self.DEFAULT_AGGREGATIONS
+
+ return {
+ "type": "AggregationCollection",
+ "aggregations": aggregations,
+ "links": links,
+ }
+
+ def extract_precision(
+ self, precision: Union[int, None], min_value: int, max_value: int
+ ) -> int:
+ """Ensure that the aggregation precision value is within a valid range.
+
+ Args:
+ precision: The precision value to validate
+ min_value: The minimum allowed precision value
+ max_value: The maximum allowed precision value
+
+ Returns:
+ int: A validated precision value
+
+ Raises:
+ HTTPException: If the precision is outside the valid range
+ """
+ if precision is None:
+ return min_value
+ if precision < min_value or precision > max_value:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Invalid precision value. Must be between {min_value} and {max_value}",
+ )
+ return precision
+
+ def extract_date_histogram_interval(self, value: Optional[str]) -> str:
+ """Ensure that the interval for the date histogram is valid.
+
+ If no value is provided, the default will be returned.
+
+ Args:
+ value: The interval value to validate
+
+ Returns:
+ str: A validated date histogram interval
+
+ Raises:
+ HTTPException: If the supplied value is not in the supported intervals
+ """
+ if value is not None:
+ if value not in self.SUPPORTED_DATETIME_INTERVAL:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Invalid datetime interval. Must be one of {self.SUPPORTED_DATETIME_INTERVAL}",
+ )
+ else:
+ return value
+ else:
+ return self.DEFAULT_DATETIME_INTERVAL
+
+ def get_filter(self, filter, filter_lang):
+ """Format the filter parameter in cql2-json or cql2-text.
+
+ Args:
+ filter: The filter expression
+ filter_lang: The filter language (cql2-json or cql2-text)
+
+ Returns:
+ dict: A formatted filter expression
+
+ Raises:
+ HTTPException: If the filter language is not supported
+ """
+ if filter_lang == "cql2-text":
+ return orjson.loads(to_cql2(parse_cql2_text(filter)))
+ elif filter_lang == "cql2-json":
+ if isinstance(filter, str):
+ return orjson.loads(unquote_plus(filter))
+ else:
+ return filter
+ else:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Unknown filter-lang: {filter_lang}. Only cql2-json or cql2-text are supported.",
+ )
+
+ async def aggregate(
+ self,
+ aggregate_request: Optional[EsAggregationExtensionPostRequest] = None,
+ collection_id: Optional[
+ Annotated[str, Path(description="Collection ID")]
+ ] = None,
+ collections: Optional[List[str]] = [],
+ datetime: Optional[DateTimeType] = None,
+ intersects: Optional[str] = None,
+ filter_lang: Optional[str] = None,
+ filter_expr: Optional[str] = None,
+ aggregations: Optional[str] = None,
+ ids: Optional[List[str]] = None,
+ bbox: Optional[BBox] = None,
+ centroid_geohash_grid_frequency_precision: Optional[int] = None,
+ centroid_geohex_grid_frequency_precision: Optional[int] = None,
+ centroid_geotile_grid_frequency_precision: Optional[int] = None,
+ geometry_geohash_grid_frequency_precision: Optional[int] = None,
+ geometry_geotile_grid_frequency_precision: Optional[int] = None,
+ datetime_frequency_interval: Optional[str] = None,
+ **kwargs,
+ ) -> Union[Dict, Exception]:
+ """Get aggregations from the database."""
+ request: Request = kwargs["request"]
+ base_url = str(request.base_url)
+ path = request.url.path
+ search = self.database.make_search()
+
+ if aggregate_request is None:
+
+ base_args = {
+ "collections": collections,
+ "ids": ids,
+ "bbox": bbox,
+ "aggregations": aggregations,
+ "centroid_geohash_grid_frequency_precision": centroid_geohash_grid_frequency_precision,
+ "centroid_geohex_grid_frequency_precision": centroid_geohex_grid_frequency_precision,
+ "centroid_geotile_grid_frequency_precision": centroid_geotile_grid_frequency_precision,
+ "geometry_geohash_grid_frequency_precision": geometry_geohash_grid_frequency_precision,
+ "geometry_geotile_grid_frequency_precision": geometry_geotile_grid_frequency_precision,
+ "datetime_frequency_interval": datetime_frequency_interval,
+ }
+
+ if collection_id:
+ collections = [str(collection_id)]
+
+ if intersects:
+ base_args["intersects"] = orjson.loads(unquote_plus(intersects))
+
+ if datetime:
+ base_args["datetime"] = format_datetime_range(datetime)
+
+ if filter_expr:
+ base_args["filter"] = self.get_filter(filter_expr, filter_lang)
+ aggregate_request = EsAggregationExtensionPostRequest(**base_args)
+ else:
+ # Workaround for optional path param in POST requests
+ if "collections" in path:
+ collection_id = path.split("/")[2]
+
+ filter_lang = "cql2-json"
+ if aggregate_request.filter_expr:
+ aggregate_request.filter_expr = self.get_filter(
+ aggregate_request.filter_expr, filter_lang
+ )
+
+ if collection_id:
+ if aggregate_request.collections:
+ raise HTTPException(
+ status_code=400,
+ detail="Cannot query multiple collections when executing '/collections//aggregate'. Use '/aggregate' and the collections field instead",
+ )
+ else:
+ aggregate_request.collections = [collection_id]
+
+ if (
+ aggregate_request.aggregations is None
+ or aggregate_request.aggregations == []
+ ):
+ raise HTTPException(
+ status_code=400,
+ detail="No 'aggregations' found. Use '/aggregations' to return available aggregations",
+ )
+
+ if aggregate_request.ids:
+ search = self.database.apply_ids_filter(
+ search=search, item_ids=aggregate_request.ids
+ )
+
+ if aggregate_request.datetime:
+ search = self.database.apply_datetime_filter(
+ search=search, interval=aggregate_request.datetime
+ )
+
+ if aggregate_request.bbox:
+ bbox = aggregate_request.bbox
+ if len(bbox) == 6:
+ bbox = [bbox[0], bbox[1], bbox[3], bbox[4]]
+
+ search = self.database.apply_bbox_filter(search=search, bbox=bbox)
+
+ if aggregate_request.intersects:
+ search = self.database.apply_intersects_filter(
+ search=search, intersects=aggregate_request.intersects
+ )
+
+ if aggregate_request.collections:
+ search = self.database.apply_collections_filter(
+ search=search, collection_ids=aggregate_request.collections
+ )
+ # validate that aggregations are supported for all collections
+ for collection_id in aggregate_request.collections:
+ aggregation_info = await self.get_aggregations(
+ collection_id=collection_id, request=request
+ )
+ supported_aggregations = (
+ aggregation_info["aggregations"] + self.DEFAULT_AGGREGATIONS
+ )
+
+ for agg_name in aggregate_request.aggregations:
+ if agg_name not in set([x["name"] for x in supported_aggregations]):
+ raise HTTPException(
+ status_code=400,
+ detail=f"Aggregation {agg_name} not supported by collection {collection_id}",
+ )
+ else:
+ # Validate that the aggregations requested are supported by the catalog
+ aggregation_info = await self.get_aggregations(request=request)
+ supported_aggregations = aggregation_info["aggregations"]
+ for agg_name in aggregate_request.aggregations:
+ if agg_name not in [x["name"] for x in supported_aggregations]:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Aggregation {agg_name} not supported at catalog level",
+ )
+
+ if aggregate_request.filter_expr:
+ try:
+ search = await self.database.apply_cql2_filter(
+ search, aggregate_request.filter_expr
+ )
+ except Exception as e:
+ raise HTTPException(
+ status_code=400, detail=f"Error with cql2 filter: {e}"
+ )
+
+ centroid_geohash_grid_precision = self.extract_precision(
+ aggregate_request.centroid_geohash_grid_frequency_precision,
+ 1,
+ self.MAX_GEOHASH_PRECISION,
+ )
+
+ centroid_geohex_grid_precision = self.extract_precision(
+ aggregate_request.centroid_geohex_grid_frequency_precision,
+ 0,
+ self.MAX_GEOHEX_PRECISION,
+ )
+
+ centroid_geotile_grid_precision = self.extract_precision(
+ aggregate_request.centroid_geotile_grid_frequency_precision,
+ 0,
+ self.MAX_GEOTILE_PRECISION,
+ )
+
+ geometry_geohash_grid_precision = self.extract_precision(
+ aggregate_request.geometry_geohash_grid_frequency_precision,
+ 1,
+ self.MAX_GEOHASH_PRECISION,
+ )
+
+ geometry_geotile_grid_precision = self.extract_precision(
+ aggregate_request.geometry_geotile_grid_frequency_precision,
+ 0,
+ self.MAX_GEOTILE_PRECISION,
+ )
+
+ datetime_frequency_interval = self.extract_date_histogram_interval(
+ aggregate_request.datetime_frequency_interval,
+ )
+
+ try:
+ db_response = await self.database.aggregate(
+ collections,
+ aggregate_request.aggregations,
+ search,
+ centroid_geohash_grid_precision,
+ centroid_geohex_grid_precision,
+ centroid_geotile_grid_precision,
+ geometry_geohash_grid_precision,
+ geometry_geotile_grid_precision,
+ datetime_frequency_interval,
+ )
+ except Exception as error:
+ if not isinstance(error, IndexError):
+ raise error
+ aggs: List[Aggregation] = []
+ if db_response:
+ result_aggs = db_response.get("aggregations", {})
+ for agg in {
+ frozenset(item.items()): item
+ for item in supported_aggregations + self.GEO_POINT_AGGREGATIONS
+ }.values():
+ if agg["name"] in aggregate_request.aggregations:
+ if agg["name"].endswith("_frequency"):
+ aggs.append(
+ frequency_agg(result_aggs, agg["name"], agg["data_type"])
+ )
+ else:
+ aggs.append(
+ metric_agg(result_aggs, agg["name"], agg["data_type"])
+ )
+ links = [
+ {"rel": "root", "type": "application/json", "href": base_url},
+ ]
+
+ if collection_id:
+ collection_endpoint = urljoin(base_url, f"collections/{collection_id}")
+ links.extend(
+ [
+ {
+ "rel": "collection",
+ "type": "application/json",
+ "href": collection_endpoint,
+ },
+ {
+ "rel": "self",
+ "type": "application/json",
+ "href": urljoin(collection_endpoint, "aggregate"),
+ },
+ ]
+ )
+ else:
+ links.append(
+ {
+ "rel": "self",
+ "type": "application/json",
+ "href": urljoin(base_url, "aggregate"),
+ }
+ )
+ results = AggregationCollection(
+ type="AggregationCollection", aggregations=aggs, links=links
+ )
+
+ return results
diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/format.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/format.py
new file mode 100644
index 00000000..9553ede4
--- /dev/null
+++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/format.py
@@ -0,0 +1,60 @@
+"""Formatting functions for aggregation responses."""
+
+from datetime import datetime
+from typing import Any, Dict
+
+from stac_fastapi.core.datetime_utils import datetime_to_str
+from stac_fastapi.extensions.core.aggregation.types import Aggregation
+
+
+def frequency_agg(es_aggs: Dict[str, Any], name: str, data_type: str) -> Aggregation:
+ """Format an aggregation for a frequency distribution aggregation.
+
+ Args:
+ es_aggs: The Elasticsearch/OpenSearch aggregation response
+ name: The name of the aggregation
+ data_type: The data type of the aggregation
+
+ Returns:
+ Aggregation: A formatted aggregation response
+ """
+ buckets = []
+ for bucket in es_aggs.get(name, {}).get("buckets", []):
+ bucket_data = {
+ "key": bucket.get("key_as_string") or bucket.get("key"),
+ "data_type": data_type,
+ "frequency": bucket.get("doc_count"),
+ "to": bucket.get("to"),
+ "from": bucket.get("from"),
+ }
+ buckets.append(bucket_data)
+ return Aggregation(
+ name=name,
+ data_type="frequency_distribution",
+ overflow=es_aggs.get(name, {}).get("sum_other_doc_count", 0),
+ buckets=buckets,
+ )
+
+
+def metric_agg(es_aggs: Dict[str, Any], name: str, data_type: str) -> Aggregation:
+ """Format an aggregation for a metric aggregation.
+
+ Args:
+ es_aggs: The Elasticsearch/OpenSearch aggregation response
+ name: The name of the aggregation
+ data_type: The data type of the aggregation
+
+ Returns:
+ Aggregation: A formatted aggregation response
+ """
+ value = es_aggs.get(name, {}).get("value_as_string") or es_aggs.get(name, {}).get(
+ "value"
+ )
+ # ES 7.x does not return datetimes with a 'value_as_string' field
+ if "datetime" in name and isinstance(value, float):
+ value = datetime_to_str(datetime.fromtimestamp(value / 1e3))
+ return Aggregation(
+ name=name,
+ data_type=data_type,
+ value=value,
+ )
diff --git a/stac_fastapi/tests/conftest.py b/stac_fastapi/tests/conftest.py
index 066b014d..afb9ac9b 100644
--- a/stac_fastapi/tests/conftest.py
+++ b/stac_fastapi/tests/conftest.py
@@ -23,11 +23,11 @@
from stac_fastapi.core.extensions.aggregation import (
EsAggregationExtensionGetRequest,
EsAggregationExtensionPostRequest,
- EsAsyncAggregationClient,
)
from stac_fastapi.core.rate_limit import setup_rate_limit
from stac_fastapi.core.route_dependencies import get_route_dependencies
from stac_fastapi.core.utilities import get_bool_env
+from stac_fastapi.sfeos_helpers.aggregation import EsAsyncBaseAggregationClient
if os.getenv("BACKEND", "elasticsearch").lower() == "opensearch":
from stac_fastapi.opensearch.config import AsyncOpensearchSettings as AsyncSettings
@@ -199,7 +199,7 @@ async def app():
settings = AsyncSettings()
aggregation_extension = AggregationExtension(
- client=EsAsyncAggregationClient(
+ client=EsAsyncBaseAggregationClient(
database=database, session=None, settings=settings
)
)
@@ -255,7 +255,7 @@ async def app_rate_limit():
settings = AsyncSettings()
aggregation_extension = AggregationExtension(
- client=EsAsyncAggregationClient(
+ client=EsAsyncBaseAggregationClient(
database=database, session=None, settings=settings
)
)
@@ -349,7 +349,7 @@ async def app_basic_auth():
settings = AsyncSettings()
aggregation_extension = AggregationExtension(
- client=EsAsyncAggregationClient(
+ client=EsAsyncBaseAggregationClient(
database=database, session=None, settings=settings
)
)
@@ -488,7 +488,7 @@ def build_test_app():
)
settings = AsyncSettings()
aggregation_extension = AggregationExtension(
- client=EsAsyncAggregationClient(
+ client=EsAsyncBaseAggregationClient(
database=database, session=None, settings=settings
)
)
From fc1488ea09454561c7be3a6e6165fcef0d94bfb7 Mon Sep 17 00:00:00 2001
From: jonhealy1
Date: Tue, 20 May 2025 00:52:07 +0800
Subject: [PATCH 19/29] update banner
---
README.md | 4 +++-
assets/sfeos-banner.png | Bin 0 -> 295397 bytes
2 files changed, 3 insertions(+), 1 deletion(-)
create mode 100644 assets/sfeos-banner.png
diff --git a/README.md b/README.md
index 4b01e982..fad1e9d1 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,9 @@
-
+
+
Elasticsearch and Opensearch backends for the stac-fastapi project.
diff --git a/assets/sfeos-banner.png b/assets/sfeos-banner.png
new file mode 100644
index 0000000000000000000000000000000000000000..04edf9e5222829c4f92c661c72c8f44b736ec8f3
GIT binary patch
literal 295397
zcmeFa30RY7+BQtbPMv98YOCS~IAGlc6*u-e6|rtqDbe5%idwIoG>wLO-!Mu6+m~U+N@{fMLDDBwi^A4=uhOdbK#AG}w`KLwuJX_~)
zKEkOa$TCi;kJtL6rn~-iw>#X+%Zn&eD*d;7_Sx9?^X8$!wURiXRFV^a;#bsEhutKuE?&Rr
z!L{WVGn<;0EwkR@@70~`@PY|KOc?QD)<_#kIvu2
z4miVfbTaFp|9!$vyKM+oROkDBdrw4X54RO3ubQj#>65O%f4ian);C)h+1{MD_|mU8
zOST)V{t5lnc$OL8!S)Z7!CcVL|Lo^Y63s>-m2%Ag@+aenr&y3DgyOQ4KY355dxRFM
zlp~6a{_Mr*st0ra?45z<<{x?Gl{w>+sb9VH(VM!SeG%-=oqiDbXu3q&gK2%!viP5vH$6-GpGAso?|WxuUTws+t%rwn%Rfy+L9ck
z*}J&Q{}p?fYB)aSKWipl?DpXmn`%j?dq3FeZY$2mY;f|;inRZ%%gOgY^7B6(`A7cw
zenV3|a_2G$ix6&dfyMumKz4)^w()q89YDQQ8^UcKThD|!3YFzQ`Js+{S
z=uc)_&Yl$ZN8xO0ivRNyoEi$a_}@F4cvef9Rs`>F5~L
z|NkdA&YbxF`5@&{|Iu*fadHOf#r3((4i3}HWHQr;rXSC``Y&^FsI~aKaF7Uute#=B
zvpSi)0q+5N^XKpVx5d}Vfz0^%|Ba+<#(@93xW~*cG}r%+*QTOyGU0l+wKwOd0a8bY
zKlm}Do%mBD`aj4x6#fDx+{lbRp+nw0Zu~E|y%F08i8{`!DJ_3JaCFzo;o8dcza@3;
zxfZ(j%ABQ@e+iAJKFh!K^O645xYM-fA-nQ>l3s@$UhuTXAXIV2k?|WQtAT%7H!Ci;
zHbJ^TAQs^;pIaYXF=Hp<8`fXY%8{XWtNj*#_c277YRk*?@^61CusmE|^M{t~T;15K
zq|*aa&Ca|CuBFZT*YQu5e0W@5rV+DRsP6u6AKR>P?%#H9=KTk+cjYhp(?G40b!^_p
zq^7AxKe&a}*Pg$>8{CcW?{)RHSIgfU$3yH#c3`Sga1mU)P~BYCFPZ0&Vj7L+#hO3!
zg?Fzb)qCGv>D!;9K)w6hnfL3}`x?)^n)#jG#f#skWc~WrnZJMMI-bCtt1^Um?yuk<
zTt7xHEh#be`s$t6X5I2{sizpGw;eX{w~RV!(euC0dSUiWHBYk>t!bI2ymGP<{T-$9
zS1jv?T{GW{UyEifn)v`+b2q402~#($nEOcngq~MJT$ew;5U(#p8O?k*%!&8jX2)-F
zQa^J{%ghnx5%(lDjyQKBDhv{5Ud(1*
zXiDA4XoP}!8-W#mx57{U{}c6z$u=mnTPQe(1snwf21w(>7gJ+H-~=uTwJSv>85&W
z03oGB8!UOHsg32~0Y48HVUI52P|i*;8JmeL4tM|15x=9ZdH&iI?P^Q)K6vF)t4jNy
z^c=H{jx2qfna?g6&(A|g{yfhq2vbMBgAzRQ*EWoAGt|TQB{S#ja)Ld@NkN*6OX+=T
zhk!lW(cw^3iEAg<_6P0Jx*Q#M)cf*aNHdNmrp&~X0&Mbm(td8exxb&E&%#sEJ=(B5
zeyWi`@nAGjPOmMNoqPG?4as)@jtJSc%~k!Tz#P1?hTk*jX)6h-FwmGbv?0-NhR*O{
zQF+NmhP|wbr)1W3IyIfv(dNugZGK=tK65D3yA8(4DW*ZV@oFtbzB@JqoAfs&0jei`
z=<{u!8uF_*@~t`qyazIJ%-0JSq#BwQo^=qJUb9*UYcjTg>v8$eVlA5;SYCalsQYnN
zqbEJoh|a%8%?@x9fdMbm%M)D-0LEw&H({txLUnXx-oZ@J>JK^+Jw@_a|5zt%rNkm&
z*2a3pXdH9nc2j74@=jpS6K_TmU!lb)sg={&dLQzGUCi588Tg}iNv2uzwR=2|ywdtH+>Ng_Ba7cTUT-pcl&ZVIwS6dm?+~`
zPyR|r2kWU`!jo%Cq$`aDR?E-6wh-GhDK=6Z`SJtg(ibW8&UR{TXdteERo*Sr&Bj09
zh?xH8nyYwj>C8TK`0;9&_~RQ>Bh!^I0=;&nOp%Zf%J#HEnZb!m&PEZ(2AMW8+0up#
z7JdN1;kjOi{cQDKyE7gHSf|bnECBU!sgUNndM)E-W}o`b=)_;Q<1
z)(=;i2>a^=IsWO0DjICo4$o*1y7VHbDO7jJb831JK%sDDAD3jbrH|sN_bfFpxsXi_
zE;Y_<*j0iW3E&S}hc?2guyh2GrvFYuvu+>L)gic=mysFNf_gh{3(~G((BwY9TC!p;
zI@GUdlYmA$iYXvDW~ILP{pj$^Kv;hg7B9sY4G!i;$$cH$wE1~;lsb4-^R8tE@zCSa
zvjDh&pSQ>h8{s2w-kkO6KOE#*nrjx2j8C+eC1GI%Dxn6`DT~b0oi)&RZg#t>Z6O3l
zbt9m}gB7?vL2WiM254#D?h%%!66LqWbmduQEG(pH1xz=uMph{pta3%R_&*%dz8>Fc
z+ARd_Dtfr(Bi${>JkweqL+1OoJ`@1{Qs52;>#0t#D~=hB*=JZ;W6MP)l~j!9>-(a2
zMmg&?I6-3-)|qXGvR#S7UuWg#i0aiopv;b_q;aA{NEX
z8reHHua2srrNU}^Nrseqv5UjDHgwODUlET`lsrb7*$dZ<%;eTwOlk+>emJk{3~Da}
z8ARU_yBKM;?wz(~-9EP03^f4^)!MXIQLh5QMO+toZ-anMSB#Y}+V-L=f*FNY
zxbTt^=~YM`ktaWkwE7U(Of~=bkh*DpE;ZZK2G6%P2;JHcs}P2g63G)Sdwu-!Z4aaD
z+t#wBE`%r#a=XEzw8knp4`F7Jn}=k=1~A&JB0j579r5<<^Erwo*xPM0w_dr6x
zB3=f~D=%$EgJ1QR9;s~!&2|ssbNNoJjAJWHhbk$;lJX!ESWE2GNI!59a!q3yTQc!s
zNO4>Aci&4z0y1g|S_Eh92UfAKRO+ya;TO~|*wRIs>fky5=l`=Orh6(R6-(%7%rYu^NR%X9%pWlAJ
zmOM8M;+L|lMr;<(iRGHh!Vc=w=`1Xf>Y#NV%QUg`v8Nx_UD&9LK-MgOIo=gB(V;ZE
z49qeuvze8nn|JX#ZFg%)un6ymL}EKld3H!r49=e-6q*P(Gwx}df!9g3#lH&4#77)4
zvobeJ1dK+!Jm?2DoHJsND%O>)6BwNymDMAD(an>_dCPlqPq2gEav8D6n5dnsDozQz
z4AuL)Rti{wSMSC{;&xav(LtGSG8TIPR+7ZZq1Yhb8g9>TeG~{LF=Xw?@I%6n>>4`M
zmnmwOu=&FnOGYG9TL{zWjbSHqgt$`kRU
zR~j5!cL%yM`rSmtFiN#gRg*^yU?X5`52u%b%oTYhu6L0ZTIJC+_LPl=O1
zrw+=WZ+1OvTvs->v!Cj;yV%{;2It;k#ix;mVIJ^>A29i;M_E6r*aAqvdbs(~(c@7A
z%NTr|SQ$F;n(G8McIl!t=f{bX(dJ{j@qU~#Gbdzd?XYwr96lD+wf#?Gk4ALxBKY*X
zM2ZHx3IUFt{aN{j%NV(W^R$H=E7HDz{d}+@Fubv{^%0&Nm0Djiq*;`H?TgYaRnw;K
zu@_V`p@R~D88rOwE`?mk;}b9C#mP5rFUha5B`l-7_DdH(s&w4h#dUDMXNe(MqdRku
zOjCECbgeYPO+zDsQ>nwe2LOecwDz@VWY}WQIQUl}2pJmIxn^f>lOG1iu+$SHXt?_s
zkF&=Dta_bLAk5Laqe#f$<1hXVt+PX?+|Ex-#3uUmn43D;irJC7QobbaI
zmzyz>)*hw}Q5JiwQpwS=UjipfGsBnrCa$M!b5FeZ{0!C}+qi;GaAf5in{SLLaIfc9
zX>(tld8$5ipqY5Ml*rC426@r^+xWHxN9pW58^zjM0n^UUtu(mwVxYC5VLk#G(QpXc
z0{1_0UTW&vFLA#U(xr7L-fG(44FE_r{9R!9yzsDCePu6t!Y>wWn%_$2O9nhlIM7BI
z)9>NRWjw0}NefHs+FY14ekh^-V^K)<`DWpGMk@O2%$UYot$C#Y(d_p5T!l4=B0@&W
zr2NrrX)J-oY|hRr*-FulnFAWK9P*gzhR0foUI-zd&7qJHx=Q}w-hGGp-Z8AT@@!@Y
zb;#P^C%?dLa_j*?HuMO7NYu&Tx$EjQmT
zymh1>6TZ`iXSXYGtvt|E?h?qb-N(+xwg-0R5Z%k4I|NSIr=j75gWVJ$Okc>=VpDZP
zI=<6JzY2r3-l+lRCEQl&+nTKLbAls2{kDmT*P0P-G<(^xJQG<K8@bNijecm;bh-L(r@n+FM9Q>1CP`%QG8SaNg?s`}w@&V66Jw
z0vby;(g*Hm>EXuNOJBd=;`EbAH8afA*UyAw8HZfFa3O0fqSbvwQXa}45wAR$rSY4dG+PqdpJypE1kt$D;}
ztpgDyW-6lOz<|-)#465$uw^fZ1VO#D_6mM*&!b{%{C-Mj@R1GqD>e(WJFaz=)V0lQ
z@7?PMnurBl7FH!YXLqREAL|lVuWhv{t8Lp;N?AxMs;H{dS%`!4|>SU~9p<|W-IMe^?SnLD=J~ASN
zMotfC3bigDbE^$Twt2?oH2i_G@l%;Q&Jn46cd2LIPs$LF?fi@#N@Q_WKkb0iIdkns
z2ANr4&?VR3dh^}(?W-MjX%kdlD~L=h!$%zv=RWpj=3rh5>(@+I3jhmHDBST
z@zn2tr#7_4jmA~|whzvrd!TfoXiLqu&iLM@pJuzyhl)*?2O6)0T|pfdJHZ+fY(PL{
ziu8+qoPx6tjJ~=>jbvWYnQirE1bZ>UETJB$;B`b#$iMgLe=9j6-3G_L9ELaaQ9I9V
zsiY6;!~H$SkX$bD(+@N^`RL|xHM!wkZPWJ$W41he-C_V(xwT@*!BU}Ev)y#G_ou_s
z#_L&BPpU1^xG>1UorZ4>H1r?hq!NVX+Yt*e??SnpiW
zpOEVLc2UBnJs`TKl4F;!n}yG*wXnLRK{uGSk%=Pyw$#PhlLiNYigqT~xtuDC(9@g{
z>In6pRrQ$es9Ns_%~W=)F?gXK`=~kSVIgT%&G=i(?XdQPB7B6o+E-C8r9y0%Sp@U<
ze_3VlT(vt=jhb9;&8E8DE3gtl^xpRLUm+xWl97M$`SSkau?fa={j&2j@NccKi1G
z5gm5&@YJNL@hd&`WwE^8AYSM7E(YJzudN-E{rApo79TDXX%^_!0(JhdY`-720%xXp
z-ByvX<89A~fLTMKd)hvGEpaT0Wr_e!-cM-vaE&8c{wT9@E%k4MN3_7TW%uzHHC}Z%
zGbW$`$legeoD*=eRpYW3uUkI7??5lL1{n6Z6;p-{3UJYN&I)3j6F5`03)W3<;QLpT
zq&`OFu;W+!+4@a?luvr=qAULHWkYQY&cWqvE~x=zR1VKcm<-g7HLw{N5EMkvvo$
zmg_dv#QPWV9e34PK45vUIAz}{fL|B>{m#P|k_(Sl=?mvN1?C;IV(`|2LDu4EEl4qF?D7bVP}<)(weoVVWQP7M$mU|q6u
zf;?769?RqM8QhL*F}m85*os6=mgdg4T3$0S9z3wj?a7nDn7B86>uU=+X_+9KDx+Sf
z?c|tl2ZZA&m%9`2fzhO~Si)m(Zl|d(K2RG8u6EbNr3Ros2Dbm0XK5f_a%sK>18jj0
zwY-Y}E$|Hgmy8biSZ@DBpMYn~Cq(tsCE}eFXyCtG_=lCEi!k0;5^7{?
z3dr?am%FP5-NIx$u-=uB6*dSlZY3pc%YyNWuRW`Eqx|eDpPc@YlgQO=^Q_bT<4Dxk
z-1(br6Yq>IIcF-C)t77cki7~Dm4_Fat#J*G=$FO+
z(;>&)0ru&+omC#sXB9vnl$;)b%0kOL{Gy)S5X$IdA$M0;mkLm3$vh%k1&%|`cPR(W
z>T#{!*4E_E+-(N_;tFEna?eK(pZN;4h16E}ZReoh$DPcK;h8R=F)3H%o`|~_I(kPG
z;lTnuyJv5g-*XXW22i#?D)Z>eVL6d=8s&!{=e78z@_q9Lt@-c!(Dak)%JmnF!EO90
zUq8DCW@Kfs;KKc$wN^}<6dsY)FQjId1-jK>{J$}B<+dndS*=S#IK~}_MiNQ;pdB=W
z-UAJqWD~p6KJ_Rx=z$JKVKFSCHQI5z>Gpb(erifpHZHwbDm@+CN;&P3A=K}ww`L(b
zwXuMB0(My?G=-21YX3xIV&m4I!;BVCu0+2XJu;k~ZR@wcRX?73ol-Y$xwNke@5pPQ
zk+7EeL?(^*s&a?eFrz_R>nC^D`~itLZ%+C2m%%hcmu?#Z=AZh!r&cdzU9o6l
z^?o)WF;%vQ^Ae+iSJ0n75A3WnVRs=#9evA1qOl4hr?TB&8=ePWR-55~r=N!f=Y`^!
zrIk4kvxXo4bY#urT!VdT=RdieMtu(A^tn}KBacW@9^|5kPG)nG=LA(;tZiw!v76M!
zl9iFESKL3*!yf>~vde6h{b_E3)1tIA=g?-+o#MOtH%BFP4^lHkx{Q}7j||>mEY<`u
zt9}mc-i%LmrKVw&Z~v099US|y)ZXqcS?nCZ6O-Glf-@vV_5?rQS_q>#OLRTYBe+{6^0=*+d7<6qwI(|
zHC^Ip<(|jl6r^8g9g1nG
z*mUiSVu{PSyk;vhHE$>m!`$+It5Z+n2?ZM8Pf)f=$iE%xUTU$*zV}J}?rkgV4M8St
zWa!7fPPsq6d8w!>@a(o$K89sXzl$?zE{ki_7JlN799LV(Q;R(a1N$+wUgf+HyxMV8
zE}ulr#`)G-I@WTE*K-;Gr7#{0GE4hH!W)TkY#{Kpw%wrtEb9l<;fp&t;yhm04?8;}
z!IA`9*<8GsM7^KhUQBh~{HWN&bJI+ZhXTaZ+&^&faPQ^cpmP;oCs_u>XXb)eS8h;aUp+t(
zTFfK#Vkn}SB{qzyosu&ZD))GY7p-E@1G6|aqg6GMCCV_4)|5OiBtBSQYOTOzu_YdeE06#w<=}bNOxb>D>F0K=+A&y;oCM;ys;4Pm{=Yl`Fs5Mvy|?M)Rbfrpc4n6o**zB|oS@iwW8|AJ
zoT%CYWks@P*!@0!mhGxx^TDu-ICG&OR(9Otnpq;?uv`g8+!CrH#~PnmB7A44vu;DO
zH1XF?MFSITh?TzuzB&iFx%|yWbp-QBO$W@!Md+fCfWR&?9cqW)X4jqT-%=8&4MM8V
zYT#)%r;&eV;Hy`wEy&76p5$mf$X#~1d#=dpb24jbm&i}gZ~NSsMhFV!{ArUK+t972
z9>SG>2T)1m{MCck1ka;W(W`DSsbLf&+E@-k%>6M2J+%LvAb>Eybl;`T$GdlFNTtxX
zzl^>|jiYsJbbc1D%y3*QMao(AH@vzB%CP4`+2fJ9`*+{um_P?Qi!
z2miGxrV72P13>}N%Ma5UccG&==|OEtS%jMnMn5aXn?y!Mb@i&8yvKJGYVg9_MitGQ
z-MHrNyoQd1Kr)jCS&7+yWsLB+&A?rPo)#6Fnk@ArDxij|H=
zdCH?|dr4#ChM4}b7@W4)gT<&8XMkk~@Z8{p3Io>3>;T@%!7QB0g))N?Q6fXdlO2v)
zT+2*LD-s39q0r$+cotF2T&yfQ$MR#V1D?k()U1BN_D$F70Ph!J?A)e0&J-s6rI*YM?AGLN{s>6u$W)`cN;M3;^1+~9lQ!Y>%{4fT*}SZ?C;BT_`a)G7yZGPw
zQN0_Y_wRRPQ)5t8g&5o>`pL&0xO@vzJCPJ<{eeMGKB|t?V6j9Ms9EU8@b6lx8&nCQ
zWbZy7AIG8-sp&nSuoe8ko^ouZ30KX=%PFAhI{xd=-xz0UizCpf=D$AN5cd!qW^9*A
zAIh6OS+&ji%^tLWb@W5=+Q0)6w1`I-8MS^&i&D2u>FhobRhK8qR3vvI0$zoKYvV7~
zF<_CrevI1zrQ~7jB9Y@Hz1Z<)kka)|_4DxwBCg0|F=%IV
z8h3^W0%|ihZw_zMc1SC}UTu+HUyDTBWQAlB#L+<{DQ|<5vunqg;i|?X^M;C#(?FcP
zo=O*F`BN!vR#59EaF#6AW^%1R^z)pClZLn(5JD656*~am@QR*HKMTb9feo^z6kjmx
zItwxmW#K(y-3sgQ_q+;EtM8wqQ|&&gB?#&SSE!wJ*AY*Hhv-Sh`vw29DvFCc;&L-f
z9FsB8kt|4g;^D++-Fu?=P?W7_%1O*6U`0E|*QrwP$-&3FR(p=k*pT>SM-4v>KnoQ;
z{4>}8>E*>DStsXQDS6(0=Q;KHW?8JI`Dg?QU7!2y}&_1?#5x{SJL
zIEX1R`;kaV6+scjwAJne4-cz#hOE;q3tP&C?H&4xw;LBQi2}^#ijsHeu(&`kt9B~x
zPWCd)D;HdN9csn7--R+J%gl
z>TvS#?^5X=`O`cPQN)4=SD>^S6}@DVx+V^JT!Zf@vG!<%Gs2lxBSq
z6Nn$CmwD#Z_%CXN^SaMvi=ezIzpHV+K^oXVE9-whCaf5EH@>rfQD#)jNhWbNo6QBG@&=dT65&9k9zPj1(f__mHp)L7VOC
z;p34ibqY#^TSEO$T%q!T&S*;V=~?BZ^-OQ@W3@%Lwrw={&GQ=$Aj;NtKPA7J6p>=d
z_gkw%c_#txx4D?{hcK`}_L*Fk3fN3g25+@Wfv?=uVJrBNufbhh0$Hu_hTb(ixlrT2
z7q2r*?mGyNF9(I>yRCH-AZ^v906~x>s2|&mFmdY-p0yUUZ{mH`TYJ*UN&Y4d(E*62
z{{5dCAY}uJjhxpH7SSPW$YwQfccg=OSnQUo+r>FYH<7J#<=+>qn%&&2JFgOT(0X{!
zQL9c1XGmY4O%a
zxjt7vhxvo;Ly`sxfKNGWD)Y`PIqBm~1gJSvoCZdtjlXtZ)-3vNB=R}{FMbD#HH%xfS*
zh>k7zV|137)YTCC;Oe9Z+rL~?787@h=`PzSNWr|;HtMW9nrw$88CeY=*PwFVN!OWD
zlHqn!!&?@+G*unX2mc`aD_P__A{1*|X1CkwkQ7;5H+O(f*qqA)j>cFV~{g8LTh{
z$UTwv7T!>Dpp+n~Vg|Y-$gKt|_IK;c(NaLy3C`8T>za)QIm&*@`IUD+y?AkaOcVgB
zA`JpR(5{g5*c%jzaic@X4oj08{_tcVmlP+mwCTk<;P@C6Qy(tCN_!+@kHK*D?|4ed)yJ}U$gUrN?+QRuYjtPL
z?eF+aUH7|LuV&Yf{u;QbSys72RfW2Lnn$|!bwk-06VgZ-7X9Yh?7BKu9eBEpx$LJS
z8VSPcL`kM%B*SK*`(J{3r}=No!9(6@$)dqMfoxwO|M1{-Dy4&Ei>xYvM){FXJVBjK
z#vf@`&?Z!sDCaq7c{69EMK=#p@_c5hy#?Gu-pAaBsWl^`sB7guXs8$v#oUhRSDSk0
zQiJ)knBH`!4>Hw@#J}%gy%c?nrbgTV&u`9VlhH9IeB@|&oJazc>)%&M_!-Twm=-BO
z5s$@007*P48^qNX7bnl0om``!;;5tGjFSWNP{s@OzE{kjuvt0X(~PS>z@c1dXLfxr
z>ljt3*FN(nc=d>I!IjhmYpXZ=CN*UJH=MKJIT%?i#~vKg1Hlb+MyVEs9ZISRrI#5nKW
zY7mN{&cxP|Jz(EPzHNC(n@)rqyfe*$S}I%;TdQ1f2GJJIzHlPz6
znS=^S?UqZO>w~cABYaTKOahsA(I{5(rM8{RP$fQRHQg=2#C;07(}*3i^#SZo*=zPV
zH_uw&CKH%g6M6z)th+
zk>II8HZgUbjJ&{!swW2SR&-U|!pTE*AG|q=%d
z%t$b(G>JFYo+goVAjbxj2CJRYl!GtiK{MyxRU8=?q%t8nrROr94dGQf4A_DSiwz#G
zIQdqsb!X94v3*IZq$uPd{0@
z`ik+*PxF8H8*P4kCO_b9{2Z^X{_zs|eejhQI0+^-NwY3-E#s<>U4w6*r(1yefMZX}
zr4uE)J7WIT`AMoH;n=zM3>RUTujE40wa*z((%FCM3=2-7d+&1RLu9*0}6a@GEx4j)8e}0?Vn4S
zn~aGCEceQv)`S&n`h1*v)bP#@%aund+a9EgN?cCJ_H#ZH707Oy7+yfUaKl)s;g;+&k43hH$iYZ4X`=VDrfLpvRn9Xu!?hTB
z?yC)V(o7BSENk7*-EjPNM0V2wC72^_#qhQe(v5t)LSiZ~2ja0#hKSHP%L!(VpnE}3
z-vaPpYiNP!6S0Bya7{n+$%!$SeOzMiLGI9Q>t!WtJ_SN3O
z4qoTBkRr}Pz9%9T;fdPz^O`U#w1l&_Dq6YC@c8W$#G~{}7nj|e4|QdIft@JGN6e$#
zi*fTbbulpAR_m*EZgJ|lb*?>P0K;YED&(A3QeO1_OZ;oMnp5mBUH68Y!Q}q2$c+y6
zwl-00VAT6|Vm+==c-!MpTAlpR6RRrY{
zyJ&hnt+9%3#mlkSibs}Z#BOtv^367nDkbBM*`0Gy>Fh`PeX!f@ad=WJE@&eILATlM
zZy-(yH}24uR#QDTLCSx8$F0FRd{I-L^6}369dLUxVX-{D?;d}yu(!`-?#NJc&vUb1
zk6*s}nK;mDg|sOwJ{Tmn1yRjOxb%~<32x36AcRKIe$ohE4xKT*r(
z_YvkIPH?#FM^Qy+>K9^d-+a;7`Y+9E3{K^}HhP1e;K~AQUG@NMZuE2}6%&_-dJ9(P
z+EnCO)c)QE)|G-QH#D?8*up)B3f|K6d}46VMs_%`!t+-5g`yGLwLAiU1+-ot`AkB`
zb6*z&Pu8oB)CS`Dd&6ZHTM;$Mye?Ibf`27<$)ii#lgn04nBHuoHGXfY7~!bA+C0a=
zHF30aMfRx;4iq*&kAA}DYf2HmuFb4SsO^Mv1xdY$X5iZaq5@LvU|SY1V4uqYbp+b|
z7~L4a8c>vXwf2PX+xI|x`6PEqX7lD;C#AyPGKR2n(T3yqJ$&tn*N$I4*J=}u9rK8@
z;U)@$YJpGWy`4wYI%6-X#>xFwB4ovQ_%9ZH
zaiQ1C`W$)mh{Ai{zS~zfG6scbh;A8%E&bkmNaG$@1HMC7#R%REK@9!M$yuAHA9hmv
zhjZ53@3=>hq9zb>n=h9B(i&^KIA<)G%;~8cO(wH3W
z7QvtBM15_=DvI(S&Fu(;!O?q+&3Vy^2B$`WvG3v1>ly%Z+LxVc)3-BLza?nHaWm_z
zl}TCrKI?66o?VF2!k8oO%`pae?Qp;ZaIYrl=2
zeyXKzPL!pxf0L5AJz7$~w@P|$Pu$TBIqrQG*_-Gki~BlnqPJq1^i=N(QUFN;B9#NJ
zH6WMr5xa&SSHISTs;q)k9OIBp+^fVQR*L`C0lu?fi7C>POxV#Z?po`bKq)RJ*c{&k
zCNJSJ00qiCzNKKA{H!udP;sB!3$XJpWw=$lTbjm6XPAW&$wuY5h8r|jEko$`D^s(O
z^Z?WkauB@k@ygBs@3j%atwBtdScG0lCZ30tf$);SLgoCU;dF-@E4^g{YmOg$3gezz
z_0+}L24h=>%eL@bNlblfTNX7E)3(E^!Bne+xxBIeYnSJOXeK
zeX!#SCJG+VFQ_`Q3oc&CwJp@cDc{JJ7oGxusZVBJRsm*m_Mr_m4|m;N7_==~KW7u8
zpcc{Ml`5TBF}S
z2j4^wyEO**%V4fP7Iy1K#zl_%ZXdW;^%^582Gb9HGr@#SIkg~Wjtvhr(mtniSk=X5KP
z%fSsz&dP3lJ7R?RD0D|&j;JJ}bCJbho3<68YT|H5Zz>v@H@yOg&uTk5yOhE7SlmSr
zn$PbzA@9qZJL0L7;Z#6=8N}s%NiRQZaz>jOJRPYKH(K3?J(c1?@E4I1-3n$E2j8*x
z$-Ukk(6q2{x5(iTaHx@|37&;Zf_lKJb}s5=W_Z3%*~&nfTjt&s5L!Y&bl#nPp;D=!
z6soWlTI=LoPmzl`7e||4tIMMMk9WqHhr~9)mc7&2%-ie9>g~qwME;xTA!U2A=<6H0
zglEcD0&>&2?bbkPKFhsM&pYI-o<60Puke=vt@bv;go$0BlJA=86aIg1q7Y`=EQfRP1a@^bV%J;H&<~s8>9uuJE
z<6p0p{2*Opl||-LK$f$^aLa9xN1Da&7Xd8?Kip)4Z^nr_zI6ji#F7ZyUfeAAcFSt
zPWvY=Hue-zvT1-e0ve`4)`GLYrsmr>2{rL_|LvbknVTZ)&P;B6_|Ie^m^|H}ll;>T
zHy;hKUR=B(o$)R4T9Y$G*1WKlSc%8rrAEB89ZJ5|_`mU$&RpwvuH#>5!bw@U!BJcW
z6iWBrAOK7O2ImMU16Xz7=H#YIkgMPoFft}8S6*9>@k*Nsc5aIqK4qCVTg(kr)z@WY)l7bPd2WA}A;Ku+Zbnrg|NXd(f&u)o>9C<%SKt5sC_-Sr1B(Eii4O_4-NTP_Xbe
zYHzA(ETO(O78K?{-Nc!p#5M*MO?2T)w}x
zwP)nS*akIoQ@GP#X)p#1U_XeTZKvSoDQWEiR)O@Eo!gFwlAK}ZW#!g2-Oy+f0
zNU%Z9zJmd;=N;Z~p{z0X^5T1|3U@Tw+}#b77D(MmMdbyL2HruN`&RZzgvG$?nCgr*B6&f`ca;DP@YiLFD4Z=m9%wvkJVgE*%SPAfI9b9F{E=v{jxUwWBg+|xYT-N
zme4+;*0bOPV_7Q+-8Us)cRyCA-(gqQO2#F%OjIymkGi^{O9i(Tzd|AJ6jpvN+EQd?
zP!#3OyQVRJl~;c}m_}20ev>6NA!P`ThNIk(6J}S&Jp8mp_>(?r$Hxxc
zH`oarf6Na7rTt(DfXVZhIkgIm`%<=+YbfQA46Djie`IAO5~X5vuYD-$(hTUid?L$)
zqRX<$yN9~qS?bp$RNUA47eXH%nB*7VTSt?M%MG#+b0Op_5(7?!?dz$F;QtMFmVH4{
zQq0_Rns-gvg-=$D($A0MlyWCEi{$v0?TWKjL!PYqp@|ytV0u@wAtm!;pNG%Y9lO;y
z*!XyOjsfX(ant4km2{e8QCkLZMzirNJ7k0wELypILTqDQ1LLLGZwxew)rnAf+%gld*m{i=Lz{h77qL-bXaf4
zB+b+0un$sm2_k%8k+o+ljSq;{1o)dugS7%}e?MoRD&~e<50(98-!sX=0_B@iCXZ1T
zw6FUYoD?j<8y#){X#^AmYMD=x`B3kG=;edn-&!TL&rsab=WM^Tc*A^ti`1DnjIMY<
z2Y+9Yvr=~YEz*ILNB75x<7iq{(Qd3t)9!r80<>?d`<1wdalhskhL#sD^4Dlz_l7{^
zv%4OYMUd@Qq^NUQBwKLmS>y+Et{*rs6FCv;IFAyccTsy9b-W_hTu>*R}8be
zUJ2$50s}k=Uk2pCl?0h-j1!QsUXZV{im*
zLRg2UR)8&7n9eu^$->L;;1NK|$tOPo{*a8Q&`~Baii_b}g>WE%Lxc?QQ9lLWRxD3Q
z9qE;mFVHvKDZ#AD5RI-n3W^nd(jg-Do|{0gd<@1d=z2d2Pw#!XyP6Cn9%j#ZtyWF9
z{=6eGK$YlzE$1JH0uc;uQ;*OvQ4k)liA^45kXJp!*e4iq!VYdYzM>N&Ix}$SDSmH}
zIhqs|l`2FYF_y+u42M#7pn|45d+%y8{%<`i5~SN8N^;gc*Lwq!eA_}0j1G5Bc|WC%
z>8=Jr-F`uD)$F3f{Z-fxX<2c^@>7Gan152c>~vH|q}t5a`RqJyn_4D^WO-FY$^`sV
z#r?0ls#TXMwUAD|uw(%oY>
z2>wvI`-{r!RrEX0;-CBWS-S9yhgpr;Gk70Jcqd-DTul&m45>GdL<6^1W?#*DW!G9{S-(#c91Vzu{$2ZaT
zuvn2m`7Dn)@s+ARQ7H5d@|U0I{1nRr*{%r}
z10>^sXU+$Jia^)<2L=IXCvY`yI4C&%rpj{-;YmLWGtm0a&~gD;RS3ci2Rz6TWe9x*
zHR1?lr1HLy-q5yABEMd%tqyeEUv1)_t(9Dl{?VTJmP0;U(;7+vCFCE#WC6~=D9?v9
zoD)=_;Qw9(3I}R^t>+!~SGW>Pyf1DZAiLfbsGABzm;%x^$UB=)YMuFg^~|50voTbF
zu<~-uWrup+gy~eHa}gx-XwWUf&Urf9ZZ@p~jPEUAeD@nNkVnKY))(s9sd+ntiur4iUxWHinHM>YW+mw1?x2Q#v
z?XM_a+N7#5LV776yc$)DuE)++Q!G6Q0qgz=RRJ~C)wEa@&^EuBXV40cY19F%t+XfZ
z%1#s_*?G88dq!{*OWU6bxbCF3f`UF}l@yIP>U06%u1fc=uUsf6RQyrP+{^!4G6Xg#
zS)|6-rwXjL2R9h@2G6WlU?#KO<{fgpwj~JD)yN$efOI$B^Av~o?sN%)=sq5lbwQHF
z0CftO-7?HTDO+TKno;~bttylNL6LT6@zeG*R<=Ow_;JOU6XJ#-J@)`d%4H`WZij1M
ze-rik#~p2+Tf@C$qY%lHnmVn3umei2Ku^Wah%tC0x4U2rppY>GYN2xn9&=jSZ)VR>
z*EIEIX$8rz)Y@+v>(c+wYEF1*?s
z=lf?foAXlIXksyyjM1$--16S
z?}Tcis4Q%gwmOMW&E_6nUn|+8f;-jmpoYAa3Y1h8Q%P+Zou{W&&2S(($wEM30dSpV
zUvOj#4cIlZR8W@9&0q{jJaBgo^o<9HfchkcYcJrKCULhwjz79m+bQUB2NkRG8ad3J
z(+d?RRi*cx87O#32ar&+kWaP%S>@gRzodr0P3YXo8V0Thc5rZbBgL0Efjkh70;AwN
z>>f@WL2~kN;|t+?Kg8lEtxyB|@ip5A=RPr!#!I{uP{gTm*`snk*PfY(e+h`K8lkWA
z*iu@(=Yn2vqN4|&3zbkts&9htYV}UIMasS0RAtIt(yR!DB#!?*8aLV92Yz-xR_EB8
zQ@*!Z@qzP10E9lDW|;7QAJT|@piGO;-+XntOlt`z6`qUjUx*DX8eX1*Y0csr$?VJ-
zHed*2+=&n^sYP{_=75nikXfZsAhNqcl}ga_idhzr;i#sUFd*{IQu(57_uXF1231mP
zo840ipJ(muAnc>vADHswqt}0^a=DxL2fynPH8qo!tH3cT`ufJM25<iGZ31&hg~T1OV*qfl@i=ALlid6gBFBI;DRZIJ(xNNK}pTgR#&Z8vIv
zw*5FKtL|P%wdX(AVk}N>I<6;i`+eg}=A7jx%MM0;v(C=IK69bG^(4`2ShcrQ(O;z%UDLog5JaVhxIAox1f$*a_n*A0&RzwK_VyZKzNXMd$w;*U4l
zT$;Bq6=qlQYs=mqf1~pmWcea=jNDia<&45skf7#9@Onq$E}YnuOAe)Z2(tWIwTa~w
zAvsXFKR#;wkDpjaxS}Sw4JpU}w&U`#jMLHLqx@yNk2i8_ACf_a`g^;jWlC9&ymK4Y
zC-R49(6I6&6$-(5P+>4pji4^Kw%r+x_ZZui+()S6U<7%?0jL80cZhb1wk@rW9C{65
zO(#mYX8<95vnMxn!aQLIuXOwR2)p>7$9NGo7vy(3x4CcthwcbaaTd4(MZz$=!BYY}f>P0_
zZwx5I$usDVagd*E+7#OrE$v3llO4k6m(mulM2rmcJoaVt=l1QyTmQ9e&E#1Co0rfr
z7aP}=4H&`w)}x$lD3X+aUnp*9^KWBYe?YCC9vt>N)t+~c-C#Un1`%Y%B*fMXcv<5R
zwDq+snZzp=zyH&lSGEV3_LnWD9dwIX!_
zL>ZJSlMI5)gy6N6Rs`AtiXcRbfHFmd2nZx70xClUgvb~WAwqx%AqgZT`JR1Fz<`?X
z?VpDB$C>tihPBqSwyEPkq{D7YDi}+O5+4}#2gUj+8x1Q?seOuwsyt$*$4j50;n(~I
zgJsm(XE%QThgqf3*)Sp+h;NxQuc>JOidWAg!w&{~8Kh^h>VXS>Rg+frQV)gCkxN@!
zH35$d`U{a2*O?+?es|Wk@?GMWyNzkh3z4G_Ii=DHHewxxhIXC&`bdG{x9j@ASM^N}EO-@{4p%!%68(9;g7B5B16CA9
zg>Vbot%;Rct1O1|8t|83)$Qjl6uRB-7G&N(x)5Do4=u7s--}1zZuR-q^i_uki*aZ=
zu<(O#H0eJ;9reulmZf?$5G2b?(#^%%^4D%nCOt_n;_r@Mtjpe6s1%fObs*&+d8!xk2Cpt8E34TbxPEWMQ{OBpYy0i(^DgIAec
zdDnV}388$o>~D7>1iq4My<+ArhEBhidCz-2W4qp%Ii>-XZvy5&k^1n{J)n|zcYw<}
zQ`K+f!W&KlG%$}!qXn!0apwK|0e|X0T9FNxKK1kK@ESdmOt@oAD3h+hBc5T3r8bVI$`9+;vQ~1OsU4Vt;
zHlUCTYo7&sz9UlPozLZNpDyE(xSfs7MYu3p80F;hEi~+$4GHFOn#a&+GLU55ec63}
zS9_F{kHKkg@Mx=QzU@YV^i8DhG4p+O)~0k3+H?JFY;srl-RD6@1|TesT>X*Qrwg%9
z_Mvzey6>1YMl!fMK$%NcWh(aGx7XXq@~;8y^+sZO0GwM?O
zl>wrc&tbwkSK3jvF4x|)s|kgTvBtN|EYXV3dBpVLzJ|wd@K*CM9(gT~IS};*R%YCj
zhaI^33PB`j(sdT4!GyuBvo>BCf!-cPS=!$bD30LE5a&fMRN6WPUfz!_!{sc=$QGYp
z)tz73T)#e1lD94K?dKXp%KbOW6Bmg4w4vtCH`sehD?)&L$~&H86=%0QBxe#7ZoMathw>`
zH&;Kfw^pjgK3#(c9myiKnUrXnc(8wSQa%w9uN_u(WE>cNu%S=8D?ID&7PSaF7kg-6
zoex0(M!U1zAs%FgzQazNuu5`yeba5Me?BQWw{nt)!a}T(ZAb;*9e_1+uL6m#GS7;>Dg6JpUbwC4qG+p(Q#o#8t{>jB2
z@wTP(CJ5qt0(R=xJ;NHR)>!4B8+(L3g#
zT(f)4*F~IJCSirABpqX*Td1C_#Ud#Z8bW(IZG
zzTk~lXU_#Owhn9c2xt0AvvXw&St(J~nI6^+6~xuMQZueH%*!7ndldOxDSB%sQaT-Cd&$FIg_%@Ll(2I5RZ^pvs7&Ycjq$pr4@%jI=sv@Z7j**L=a^Z
zo0B|Y=eBR1$dGnIDxk9oUWl$L;*k}C%>8gQQbt?0UwBK-hp?&A14VvXgn;4jUVoxB
zsXKLD0pY!B9e(%A
zOV&(=D_&;RpT=~}yHlwLwgDAi`&iUtu$t$n@XV3?RWs-3RVvPkzPm2|E9VgY=1<2k
zKWSpgI9KxvQ^Fv+EI@6=^sl7`>+Yla>d_-#h&^aM?&g37X>y1=FyNOc(N5|OY#Qm<
z$CsRI+H|$ihfOtC^3d)DUU;IVi2RxX9~nBm#x&-bEvllnLr&R^x(h72L#wqi?ZP9*
z+3DIO@$x0~MqK++$G=0(mexr26nO_3z>o%GAKuTFo=Q*x58SZAi^@>9D?798dq!o}
zQW@{}y1gaRgR{`izA#QuzP$$j5Cw!F^NE1or
zO`HvTaMsk9tMGR}p3)pb@Pz6Ny))LkpI@CsXUNuhY=|aGD^Rmu#tLu1?v-144k%P3}E;wUwb{;pN54UjS2U
zhW*qRP}$A_WC7lQbI^=^sE>RVr|l^(W5(+CpV2^)zd~|+wcYVG9DC|*L9{d5nJ+J7
z>U7pJ?g^EY62-gZYrf~+3`Yp?L;lXHj!llIe*OXNv4^u1D~XvIkksnQvZ(Q0ZXTEA
zg+^HaAZjNqVD9FhUAtQt>D-m=_{^f`k8_!HBl?q5f;SZdoT*=2;qMkBd4{>uQ{3m@
zGg{`F&RudG0-QanCWZa+W0{01M7^iTT#nOUrTx{
z;!V{n7_R=t5Y|uMlb#jx3k3@5MumsQf4bhsuipPNL0=3HenuK}eMRNkyS;bC>g@54
z4xq;JmM;L*FdG}QDVPM>JxlOkn>4p4opjEozsNVeP}fP_pSWLIg
zs`RiGXc?l%Hok7=FRBNDoBO{|bb+Fk2vzg1>UI+$4wY{sT$#ia*Z7I8sayU=VQv9K
zd&vsG3)Gp6R+xi3WkdU@zzscgt~{RzMmYpsJ4m3=x7`{~pn1OrkV#iC(BL`u*NDM)
z*d0mAC1UlPJlLJJDY{ZEN^^$2o^C|aOJJ?mL9c2{;{p>74h6<^=+
zN(kbkF!KSZ(iJM1uoSfcv3~SJ9J*^8F+yXpynEIIp{uGXVcqpU^r
zKb2ixAa9))XD_}iVUk>}cmCb%INrRue{Lh@6+FklBHg`skWEB^&oO>JSM^nwxMYxa
z6)_&8Pi|w3hb4csrce)cZd>x%33LKE;0jS0)h>Vbj(Z_?XqTUs?{Zk_Un>@nuGis_
z=d%JAVh~{(%H^G8Z*+Z6vEAO3fbKV`4bKvuQk{w%ultg2c}nWI412N{f!;?`QbiV5
z;f3=Pw<{%?>S~>tSkqRxj9Pt{J{QAH+b>Pp=h&Kn+qfo=G;j|mvBt4;&8u^=ILpu)
z$tZQAj1rZ{@UK-AeoS2vysNzhB#-S*_&e~c;{3^x_pQg<>UXaJyzsMb$TCxlNDRIH
zQ1{xoX4$f&80&X8p?^fmP^m9^Kyx!~F0*fCvH6DYZFc#82<8miV91s3acmbZC8sU^
zpOrRI*xBs&gHsX|l6Yz5IYcE?%*^Ot@;cnz4pNSRq&IZDmoI8U!v)dIJ@`b&V}Gg)
zH8SrS*Dj$lR})tX7IwBT;TgQ}ckv-KXjqdR5o-Ebw*#u9p@HG{vBTXHAqkdWVl8C4
zA^<_siOT#k{w=w$ZPR1EUilMUCg|Rx6G1E
z{}?aH;0j7^<}hsw1Xcvm#CFi{1&R7SxjY6+cv#AJ?}Sdn0st1foazde^mlq;(0p9#
z0^PH{|>8`$-@$NH#sWPp9qm1WVqY-&xRC=;a3
zOFDt4$<%`vFWDI&R?5@3^W^XRP~9g?el0-7*c~92rv0l$3M55G17u#rnN03s55O4B
z-oy>@X-CA*{0^;+PYYxIGirz`ytuZ2Xk{#w>lJzD_A`MaQpmFS!vjbO*8`HTq5CFF
zgLfFOuO#sWrx8{?k`nJbDqRoDy?V>DI=_7{_G1-R(x7GI)#S^V)FXwZ%cclTHaROx
zxw^)98|7c+VauPIa%T_lAD{{w(PML5qwxUzUgY~qY_9;B{qe4e6y=B=yOqta6<0mS
zVGE<|#fJ@lVgPWu5H<134|3og8yttxo`;(!(G|I<%>Q`gyB&7_C9)3bDr_L6>PA#;
zTh~O_Zfde`6cUs%Rl%il_+$xy#}6ih$Dd_Bg+)JsS>-}D78yojm6ktsA?)|n;G<8(pgk-6v&b+G}<2rsvsfw*k&7QlcD)ojLZtZfSm$ND2iDmXs!mJ-Tacxoq3
z*BH)Hh<7oL^YxVk7!_SPE7(0UP@W4Fq{T!N>nxxWc=F7W_Aa)vWW+CJ61og;pnpd&
z@eeM6Bp^wzI-7Jp$-~Dug1T4taz8dq0WZG28?SP-GJ6*`0Q)i+*vSk$;kq@XDz{D%
zo^B8$R^r5DRssb$=ZU&9t#j0GR8GwLvvwIN8wU<9#rtO{FfOFh5g`0+e*6R={EPdf
zgu+4Tl<1D}!Ux{*!uy4dL$NJB94hc|W#Z%ke
zm{hhI-o)*ot=DZqgw*+c(6?edG{l*Xv+4biU(b%;2mCLes0P2mfu+_o!s2m=&@PaW
zdW4xu75N|4R|iX!VR41D&V;n$qpMp*B{T!et#^HU*+&R|MbRsstbOaA&vhTYB32|L
zGk%r_Iim-Yg4`)j6BJ{!5D~jbc!c@PegzXIHimeq)hsf@nmhVugfg-EQZgQAQ#MuH
ztkdj03VGFQMvYAq_WYHKeIx?tKitJ~6DD=Bk)1!-9CnTrOAKhsk5iQd*kA*^wj-CF
zQoEZ8v`ky9`K<;ZBC(A6XJZkAP=QihAP~{tr@==5AWWAfOP<@=^b1n+RUcX;1T|k-
zYK?w!hPkH>lQ2enwOScAe4LIy{uq>(J#u7toQ6ODIlS_zl!O_E)k4z+g^ggv$)Ho?
zS^(%S$79}KPA-S{12?Oa6{HvOEU?KUNbp9Zo-e|%@=lt0B`4quB@2SQ8}IaJrwxxk
zxI+P=ph`|s!;<`bh!3<8$~&zW0Lf(oL(9E03uA`~@)0rwqwV>MvpjKhcv5|NhNZ0(
ztuE&GU^HPRBRym<(v~uGy-~C0_t+`v$c!h4uvY;@nUBEJ*deKY$DB1m=2jo-ZC4$1
zDY|ZBVAgglz_<=QH1R%nq0*6wc7J)DQ3oXkaIslKiI(D8n4OE?f6=e+FH=^Z!Rr1X|)y
zN-0mS{=txv^Plr4edsi$%tT^g#NlJwWJM-;3%6#7jw~niBLoqVqP4zjCm~pO
z<e2Q8GsKlqL(Um9wHT@~s~j_@@0$*|bb&Ocuo0ld*A}o3^lpi=$M(VCF=scPRQu
zrH6W4O2X70cUas#rYTbOs8=>J)}4lTS6;lW>;0#Qp;Tf*;ezHQ>7j8MjI~Qy1M-b3
zGyma-y=ODMElvM|>fTazyUpMV+0ftEizt#r%MPfsl!#i#xBBratr@#?`7eF~TH4$C
zFBDD)7ZD%+a5m&1U`ls{im=F!{EW`qTwi=ZLrQBTSTtvr92XWhz_)OD`U8C`~!;+HO=C4LyejW|iM&c5NoO{Cz5X106IooKiSf^N#&FRt{rBH66N=T86Ye|qty;9V?5O3qH@n;4{
zsW57^5!4e%d;+`-e$}ze0`Do(lX0<~wSiee#L#c6HZ#%~FY3jo&t6k{$?JHI
zE05!^e}f#fo^mn}Y9VJ}r|VWKN+51LeAmayM(M(LO>;n?yp0{h2-q$YzW4`8L1_-<
z+w36o{9a{*R3QG;#VIxCEY4xVp9YQ9a`?jah9XztfXbZVhgc^KFxq%0y|uNu!r6mD
zVS?Ra(ETm3x?x=bt9dbjBs{-7d-VA_o-(FJGX>jXZ_jr7sSGPBp4@jrT{+P}!~iRa
z2wv}b9BnhTLRL6mc%6ac&gGHL4}iTFv5{W@OJsRWkSntrVSp;oWY~B>o9n9{{FOgcEhD#my?aR(yj?K
zo)rHty3cZNc;>u+o^I}mp7(9#f4`XSHO(VAAlC53?~Ai8*3mVe-|nT*21aN8dGYfn
zW)t+p-QV$ADw}z3JG|k}xT2S$lJi4XsWdY{3ukY61HpWJ{2a2{cc@+W=6d7p`aLus
zVBKHF6NcFFk<4Xr-5a*-u37&Ywh2`yUptmn7bwf3-+LM>*f{x7U%yS6>~I-udINjY
z>x+&q5sgVI`mYtGxg@mYN9%v8+tNBdgbd<3#1M5h7B!_XWh(CE
z`Hae=4xUYi+22s%GLwqz7W(ANPx@;-q%_8MukWFcxvaS_-3k(`&=k0k)CnHiHE#z9
zjbIQgzW_FwCSFieP&t8Q%5P7~4POMC2uTa-FY*tpy(sJiUIb+oS|ZJ;iq>f-$`mPG
zX1K5}ruv9#Kqf&n(k1x1E{7SZpX*_4`$^y+((_hQb3Xgwe{{P>uM=+S+%zQ%gx
zn?MYJx+sg^+jJKT=7GjuMMf0DCTsebwbGC#PvsQ!)JR?Q+~(3lnE79z6%5G(J@mx1
zalYO9-^6JW-#l0rHG2Ahmb9~N=S>xc5qm-y(>XPU~q
zBa1`um}+#UX=eh8spZIy~rf2B&C0LC})S3Y&wlFYBsLbH+ieJSJ9WtZO7jTSB|F-2XhjX^j)@xhiS}4^6rLQ*ZW|$MQ9jzOT
zhSNsseoHe^iWYd__Yhgt`!lMygQi@%8tbU~
zH?UjH%&60YEjIo7cjHY@KSwY)^0H!w7ylQ5WCcFWKJ3lNCeN7S-8m{QscBn^F+n#^
zWE7f4v!7Xd6B5Z{~NvaRv3`j%|T{biRJ}h9K7+qG14!WYzS-EP2Ces^XL!EFoLQL+vxNs
z>B@|iV$$)zdW_E_L;C|Wy))Z8EHM$)^qJ{EgrLVCH!uS=4aSXuum`D^(eTH^P<;5~
z;H_t6OBDhfPCIw#_a1cdJjhDh*BkdslQLvB7T@dDaos9xNB^L}?D=3elKuOOy&Pv?
z&{adMrQse12d#&{D23Dl~k1vD_Q;S@@B2m
zM6V^^8Vt1sn{OT6r^WVfVd
zrGbYr5gaWP7ub~F3vl#vfluSkI8KCQ?yDTy!++48FGTK9G~Qcrn^
z7lE_N3Ln?5@a>_ny0L)rZqoC@LJONUuR1J|m}iU+hJC}66@!THIPMI5M|{{{tEv8`
zyiHzpeSS)Z6oL=o3%>T0no}l00Pf9U8H8~AKW`IZZ4XcY_*Ajj!^#w3D=*P2$d|V+
zfhOm9y4n&Un_ybO?)t&8!a$m1(I~v{oPflNftyoavf`gjx8?Oi^3Lm=Bp5OZ$|suq
zF^_>I(Hp;^%fcR$@w$aAb7KC2aJRySwD1BnX)8fo6X%0H_SV)lQOptPKXPS@yEW0A
z>a7=11KJkzco5jM<;e2)DgmapH{)x3e^sx0H&-U*>}#vr4cC}u25v?-ejB3|yevbh
zrSfOM1|lhRTs!BLAo2c^0mBiGduWM;&%_=XEpfN@*omV$&J|v%uHWqpUM@pG^E(QE
z%x*GOvdp#a;h(;%p5d{7RcYtdX1_;T!6Df8IxN>wVPl@2#{cpsQ!|W%$6C*NZFvKnt1wB
z8X~x3oWZ#Ssu(-ykQoEuNR7vsnzU9JFk9Gs473$V9WlF|6D@D;WsSTmXNa|uZ?gxK
zK;RCE__Lom14fJyZb2}@T;`98mF*RiDv1>zsov(UN~*EkyCHQpC(&{imqD4$L3+Uw
zQ0c}Uzk9C3^XmIE0`DH*Ar8CvbMLyvk6K`p>_V>2&evX+hswxh0(taQiSFu_yn{J8Gm
zZrt`h4=PwkV#X7VPoOtJFCI@XUp;`FA3_VIw4w@fWR!_bhRile-?iu%CDRH)0a#Lz
z{plD$=#GE)jPLz&!Fi^pzI!vYL1ZviA(&O0Qzxm%sJsPV&Y6QY*B747oVQ!IcnhIH
z{>pM9&`d7m@JP42*Li+~X7yo~rqys|PQaatf&BNb`R+f=JAi|o_Z(GEwc%zN&U>?SN<0
z_U;Pr{c6HHl$`ogYR4d~ja=F?srLzyzR#66tKgtI9RUoPr?xE3b#^9Kw
z=&EBwQ*%E+rP#z-*b)6XE6?nMdW5Q3r@Zs2r@9Kt+Cn?D-wsUZ(+o;V(djIQVq{YA~
zEe8w&idc5|CtndKnnaqo388MKx;9lNm#5F^7e^C8Mu_=};?51T`77>Amk!KA~NvxMAG*Ze-$BMo#G|
zHB%G=x=_%IoK~B^O0jkP60O|Lh#r0aJ(kQVxRic*?
z<|(1`_(eYOZ(aPM+Yi@0e1;}(i)hI@q4wkL`ub>XYziY^AOODOvyJZp?c|D1xW2yW
zXRs$_+Ud}J7n8~z5@OkW`!dc~^S&E-sRU^bj!VXhtDJ{gs{7iIStM)B=z>oF>z~iL
z{{4RE_-9RS8j-Yn<48wU0T%qO54b#Y4x}7XoWrOJxFcHPTziFMCp?+`vOr1jsZ7JN
z>vrCWcaP2nUN8dd@0Ok=9iFg)vnw{05{I&qBjAz6sNBdnb
zVHdyq=JeS;L&z9X?}elml1j^*_QcuHn&a-2EpwhgC-r{MS}xcMRKDX@GfvI(tGk^`
zEWIRw{bw%oh>_g5Y*HWX%slZ&Ph5dfYd?JUW-BEqd8pIoKJT-mDnfIy>JIj9kz&ti
ztBMD`X5pA(F$W!E>Y3vQFCB2P(5ye{OQ4w<#VsmN?a+^HV1SIp&7ZqV>A;h18@}Op
zZK-K<=
z)k?ssclT2l`R2}~h>PlUs65@fq^`p12;nwsYVYs#2
zNTBqAO|UY`;)kpHI(}o)a$PW8E+4%7A1dR)h3ca!klc|p6j{yO1kj%yu6s_@ky*iaERp^mLf;G!v00(uq~;^L9e)3FyKQhG+AE7y9JQ;YR$!PiF7
zSP_E%5Gb?wT|hHvZj49)HU%VVDTU99OsQLK&_KXzjyBdf!$1bsI7{L1jDZM80BO
z?_P)5D=*m7H68={pC|@s?nre7)gvP6pX2HRH8%9_2D=%$8M^3Qc-#0WCAU2A3Rkx^
zXNJ1moNCM?U|yDG55grr4gAlPR>C$MiF!U%OunZ)z&NSH~=`2Vt1#k=^uGWMvf0$ddq4PQNZ7g~zxf&cN5w9#;f
zs>*x_fA&IGw!FxCQ;+*GuR5qFG2U+aAu2*PSd-kPW~dMYicQ!KJK?&)TLvkL(vIsm
zdbAZ;eF8je2;P$%{8enkUj?#4^!dyc!So=Uj+TRk7v{@P8QXJbEJgFZ$UnE|xa?Nj
zcXa|Q{=)HxegEIV92_cbx-s~+%d)3{Y;AKWw$zgZs~@d}_Q+Ai+=FP(R$mXcnY`9j
zvp$2E&S%cIt$*rCpXYd{=)snB-H0xGHOyv!!??^`p*3;QnU6^;-q)Aqm|{U`=}9{`
z!WN+__-th185a$iFBk@??1T%V{&d~P5R(mtJv*DwIco~>(8*;b48oOhP0M=&Yb}Dj
z>-2j;+R@365F7Uq-VORb>7P>K;Wu}@0B`Ac^LY`PvU^|%`Dq?>&L}EPGs-WRu*Zg$
zYX|BvabAQWlaNPv+H{QUHE)E1Cu&(dUyYkqG(RF*xZ8zyB)}K|%c3?VIEs5Do_6ET
z#O{nByvRm|U8)ZB9IqMCb@GiM|N1eApb#ze=$DEnSuzJC!?V9nPC$jr-B>j7?%)5x
z#1&sEbIhCI2U?68L#ADTT(TvbdRne_FiD?~+>%vaOYhK_4%28#~iLmMTd
zrJbX*G@HszEE7)4lsxx(E|&k+FNfWD!|Jrxlfei#!AV7qAaBbDn$3$yST)4IDm`=R
ztsKXzAtJLX_Nk(lfTXSTEg&f>gJ5XCrK+N-J}4}^NgIbx*OSIlEZ=_%;4X6x%MR_y
z*96aBscFQ($T({DR-17Qj)OaTu(OX3e(**Xob2?e?gzoVaKS#zc0g^mGYp|tQ)VNP
zO(N~KQL_Oe=Hgtn@%>)>YDjmtvYYtsn!vWxQFX7BE*mjK81k-kGf0Q%Hgpvchl}!_
z6M+%nK=6*4Weeblqk)u*=2SDmWy);ldcMC)&OEmGJHeJSvdnuQ*nbnZ^8{ZIIS2Nx
zqLK2!vnh;^V5>PxLnrkcu6I8i3LVcO)2c}44TYLE`Zj43K|p&WM+s37%)zZz=9$!;
zC>h<*Q&DX7kCuYqdUEr-22}rg)66unax9ZZc7EBmSzxl!
zMM=k9?1dL7E|CW>0Wk@3-7LJ-{l#n2_dgEm<)e|4o7{0mw!@O2-(aRi`uGz5?KT
zKJ7=PT+zr2Y!ehObJQ!&0|g(KZVkbA(;MwlTUc_lo;WGO3|K@$1tJbJs0B$k%k7=g
zs5g;h+m4gMyIr{=jJ@fF(1Q&vhi?BT6lD%CZ&%^5=f2;V2b?^hUU9SY%fV9Q^Rb@U
zwgg5pi~k2~!C1@F$*nDd(?WIpjrX$Ik>%O6e=$(j2W(0C>y=yTxL#7z*xQn0jiaW!
zoubF-*zMl$Z+F`r>6EVr-2v7FLn(-Q3fLNcnh|(PWo`yfsp_&l(UwIiqt_m&_0K2V
zSJGj$AHqMgE;P5x6`R`|XH`5#rWSp2HwEFTiB6iL0Oo;!srn>Ii7MTFAzpa4nRtGI
z6`sJvqWB8*gdJ95RE7a+f#5t@A*B7!=IxO)1&F1JY8ZXjQB$Gf;~p8W_$ahAOa1tA
zRP&|b<)>_9194&_zyBlq1vVwn-I%VbDaTK*6{dw@dDb=)5fZD052T
z-)=Q0|B>a-!=_M4z?4YS`c^2x6O5weE&UdO066HSP+U1K}n?->U)FcG8aK#Kwmj8O#;P9LUH
zI})_uMI78l;j3|~Ive%>!AJvbRx=#0$@Nt>p%Ywzk0l6eH2<~qe;UG93O
zVwW+m1=a%jh+>x;&HS0VMGG^xKSmMQOyVl+^EH)>{@%u$E!_ik9VPr)R9A4+W@Okj
zJSZvYxPT*faLmiuGpO5q_`hSoQ`{t*GY39|-{B4_$lIW(1Gz^nR~}Ha@OW-;t?T?F
z7QH2*v7o6NQQOOl!`v0K$XWoiAD4g<;&KBF%c{MiONBQ0ILH7S9K$5YEDr+P&P19#
zuBF9vwauN_P4E4%#evEeow~(a$K>5D^Uu?R6G>F}(GBDqCGRY+K%nC??54{k5yE#2
zu+aI#QLfR{dg^?Jtze9xsN0n5%8Co~o_+I(Z>&jgp`DBs_@hSQ3wOIolBVbXfjIv`
z&f@AC%3zVzJI(@|LQ9;OLbDSjm^Y?O;lhtskufugi8o?&9^dW`(xF_aUEaxkio>9!
z-GFMSciy+NERmA$eeKCOgt(wp%~`=iN_SLxkN)7PGSy37I(GT>YRQIM^FWpFk2>4G
zW3cRc*VndeRqQG)5q?!|A4T%k4`Fh@;A?ktXPm}qwAWkuPdqds*4M(m
z8yo2s9xZi>$HA`U@3;&QygmOOUJ59=S0h&`pmSzu-^cx!?E-QYxQs-plc@riaZX&L
z^jze_@*eBP;P%cB^tih}&}gnj(}M4_whA$63fxFKU1)UT#Q&74=@;O^a9dSxaQjqA
zR9?+EvlU2)M93pxZv#`i+&`#LZ*ybG>)wh{@8Uqzlo@dY=iGAE#dv=x?{JUw!o0{!DAv
zf0ZmMeDe9JeFr{c&$dkt@b`Zny!-ljDV+RX-NQC$<@u;w&!?JC>h2t_%F}GD>X$I2
zGnVJM({UidxI%*Ex^^>lXyjq+H!dIqKJrQDHM`O?v9aG!mh3WJGVCm;Oi@{f^j_QO
zf$4?0f^bakh0MA#o0JrF_)7P<#=Kb^N7pFpme!!*Z)a86MWLW)g(i55Ja1b}?gn*0
zM#kK%UT$lcxSSsGs*!U6$25iKA@V(VgE1w63QUQ4b4s(o*oSp^Q;Sf0~W7J$|71%NI=SABJNNpu%pIAiB&+=
zAhl6IVJe%5Yf8d6vX`Q%
zvQDmlK`wP%=Y}F}#g{($_jBBw)V8LOv%MXpS41kd*ZJB;6&jxmq{E&^v%a!3k}LY3
zUS0)b67x5`golQv8YzgoHXqKIU4cek%NiJnGXYx_oHJpP7gB6!rtwV08G)9|5t9=<
z+{+CrUS^qGV}3(74H`SOj_a6+`bZ(SEBq6mD`ZLq2#LGHRlZuXR%qvygxW{!=?Nr!
z0+t7}Z!3k7B}J!zKFJwuz1AC0@5nfAZ>moru2*9U0+
zaSx`#;p^FuEeqagwyD)SM|SSA*=Mu~JaGf+68Kh`?++;yM{iw=UEj82WPU0t`D1XB
zw&mBOs)G0=w#;u(R<(Tp8el0mnsW&@>R4|ltTazr+o*&0pp%ee7WLRPYqj@x){O*la$yp9
zTZm6d)yR;v2z~!D>4AYS*4Hz^?f_x94)HQ6J$JB=zXOZ(yKaS6+1mC01AYfJV?!V;
zR!izl&K`spP~8C`-o_7rY_#s;=PSCQ419>GJzsHVAu<91H(68!G2dhqe0d~XA
z`(phOXzN5JGqA$1R*&VP@i?0~W6oKcwiXCvPRd8|lF$C5UB)tOL-r0@;+K8~3ZVUz
zM^|alz6NzCBOwnw22)
zpR3)r#Q5>3^e(9WkhEBv8PYWrR7eqp|Ni^jD$LO;rQcrLGhPXi6kLE0A_wka8W!hD
zU)GVmyZ=m%hjp&eFkANwh8MkyJMMR&n`YY(2e|Cn&GNW2-z}oN7o(|@Yos2BLl5_?
zVP2&6f?;Hh>ZpPcn*E$s)H2Djd<~WdyzkqmcE(^`$x{SnE
zuSN2^u`~v1s#_zoM+EF5Jp4FNe2pRwWTe}?_@~Jnyh3~B(I>^xGU1f6g!|Gs)
zinBy;jMUd-s_(g{ZhzVuT$c4*4=ZhFx_ENegG}c~`1loc>3`*67b1`y`?c1=y>G$B);*ITEX3#F1RZI1*3G|362vUXEV5T-iyq@9`UQvR`u6(h*O$4T^FZ=toJL-L5^pg@)0%ON@5f;fmSKr
zj7hPAcMS_kF69Ev6fh#+3x!?86`+l`bBLm+(6?yi=#~eSd+HO5S|Sj52YtOO2WF3r
zeuo5quuG4wnTidiWo@}`Uf#=Ncm8tGu}GPl*LoZ2E_b+k`xZHXqEoShhA5gkbA8^W
z7kV6u^n+Y}er)qNgzrJZC$a^SgqfjfrcJdV!|GHs_
z$NIwKLKPPH_u2GUf`4BIbPmf^zE<4ndH3-^w*Pxs@noHo8hUS)k6~Kt97ABUY
zS*{5vW2W+x&-O|F`c9O>GCfW9q6D@YDvHe%uMNN51Mn{&SLF
zHsx8z-FicJBEi5Vz<@*ZTOtN^T5zrLWHp@P-F%I6X5Z;(yf~&!gdD66hiw>G4wU0>
zX1a~2QNAwTwgJx9Kp)Y17lh?yASLPC0hqP0?xzP%h;~pHeX1i?36@>lkH3dDrh#gE
z>r29AlaP3u=BB-%99Un9fn47=x9C@DeA4}@#)DucF{GU>SYbanLdKlwQ<ap>OeIf1!e
zUoxyU@v8EN2QP!SG^|4=f>mjquP4MI%G{J@$pfxR7{2Shsyv>%t~OP1l_%M%4Mz&F
z9#3VHZw_v2#b|JYmhbonx?2eRZ1MAxV`Pv9!3VzhYhv#W@lSZC
zg~C3%K11GW!SzqdORYMEQVVI2$GcodVO?c`5>UrN3y(3YPjB1jb0vW;bJmA?Bi{H{
z1JVGX1u!k|pYdX|PqQmrMH88De5l?_M5cZV(q`vUP~V9RrYzi+#CaaSuS+}AY*3LP
zTCFVLyuum7Y=833bDRhxB9)im*o1fcDw!P|%%d4EI7e=PM>Fd!UT#(N%NZ1|X&SX7
zXtM%?QmVk9kQyXHabMl}f?}>>fSvq1&jtSuTHSt623Rtxgm%^v8{dVR%T9#}Wj0=d
zq`+>NA#}*sgn}6H!~q`jB4neO6++>~RO$&Zi|%>VK9VejW-t5DJ|lB7y~04)QEbIO
zpm_ep-_o~l^D1PwV8IsGji`A90^Ex
z_;FY-&?i~cHhRy#&4+;GSReUE>k3xpx<}GRJG=(I$ZbI4G_la5T&1n#YS2Y(Y8EL<
z|I?P=Ki1X$04E2YOyBZ=j#^vAu@9cnhdi$G*2ueh>~^|wHYIRRy{C6>glVTZgEjP9
ze%IpPXXmmMGmL8`vw}$fY9n~32B<5BzC1JMojq7@3pSqB73{c%IOwXG;&qUnx09AMN1J{x&}jp&0Hm>wyKu0isE1MjcK*fc|m^l1C)wRQntQ
z^FnB_uX9gSI0u>@!|39ZE?Y6Wn2gcI+M8Vbt{4ivl#G7_$Kk++QB-f^1H8S9(ym+!
z9~S0MDtFXy^j=8+GtptRpjl#mh#EF_Iw
zr9xK$6{jcEy~yhZL~4-Y)gjn_xVJS8nAF>rm1ogzH^j2*r4Nn@-f-)bguFltyuXS)
zs`}Wa_s-P>Y0f6fo{a1V2eGR9lggSDav%&leIMW%7_0y3FMw)(u)urSwgpCa%Qyg~
zdnDA}rZ?5;n9XJ#u%QU|H~-DJXHFt3k^i@s;QCCHZm%x~gvVffnWTfvuJ97e{sV=k
z*2gM5Wk8<`!-!%yIs3VrNNpsF`y+TJ_C9X7HcWGm6P2kBn5;@DY`SKi-f$GPyedY2
zPKy7HP=Gd1`{k}e)bo$0m}XdwIUg*c-s}hzVcKmrJJvJ7gl~V8>4vE)qDgEFL)eU;tj9+v~gAgO8~#fE3Qcakc37+LeUvO
zE4j%a2Y_z^30on@=H1{Ae0w|
z-bN|LgRlArUrKc$%g-)%GF&98b#Spr%QS6RR~FOf?D4cViBXlPM1JN=z_)%Iz{pQE
zIvOxcN5%EYqz|oyOsfIUd%w-ae6Ae8=jwe&5^JuD?>Q%Kfl`ybJQ~hh4`-IRsH=5k
zL_^-?8!V?jo5;JaNSidZ(KrF6u7RwVhh|&lQP_KbHJpm^0C=s3hd>KRKCa#n2gODR
z=B3HE?`@;fNi1a_!5=Giwm%M3rM5yx!_5tf+ZpSPzYmppNR9r{5-;X(?W=?
zC};xW06@hjwkZ9B)0G{9H7)kQon<+3-8(-3>ou1Y`Am=+qlnaC5GY(F8nxnE*q6^?
zriU=Mz%&4~c-P?AEB_q)yrF4Nz=20HW^)nh}zs$M=1AFvb<}+zZ77^E?&LR+@uX@5UXD4F{XT5#Xqm&?^wUe-`3@jP0{dh
zN<6Z-s*hgw?~sJBWiaWaE{e{BG1h@vCFMi|a8g?*<%ZxV}HqT;DUmycyfWUle~j
zxw2gIv6$B6MrPO@Uc;G>ox>4Hg=zvDW_66c!X5=A@1y_3;Gh0>Tvu_kyp1h7uj7Lb
z^-F~AaY6B?ikx{dW}*5a-0o$N-lPDJ9mRpM7N;9%LZTArY9*G~ZWy+)w>OCr0Y@#g
zj}=(TUco4N@aMbhZoZ0x_7%(@s)~~OIBR&6~mU7;4Qbp0C=e*{(BID`WL9$I!
zK^Lk@b{823kr2s-K&c}y5dx}
z-@!mTA6EK|=Uezkeu92&P}1PWyHu~IdTsi;DWAOIBP?wgw}3-ES7T!Z;F%qNly`Px
zNlqI;mg01ll2Rs^gEtU!szMqYeqxQi6JfI6HljzSG_1gM@{6=wpP1&;Oo^pSJKsOQx(X#hclOvo$6mFB>N0K+wPLOgeuJ
z)u_M|VU0q&89F=|bSZ-WDz>i)FMU$Z(dKJf*IPDjqM2prhS_l=bZQ8app5}_`f-&_
zlO$J)*Huxuj)nK)*3A{?v$KcKNWD=;<$>N978ihZYX>Lqa&*|vhwwTKDueScIwY@~
zChX-Tt4|ZV^|&7oFv0in=l0@naw3oAMXq?Fad*6FZ9#(1DJrPoV~}M!yKH2Cqggg!
zYrwrG%hO{X8XZ1MlYv3%r0e7McI12RRg)XWNpHXbk25VM*^4*o|HP1Yt`T6og&`L$
z8*SP^AMoh==C`{4>IS&N4iShrG3^WU57?2BJ3)UoA8G9BF@$WqXf5M#r33EDbk1@g
zrkuK*X%654Pw{qLTGg6^m`IhD?;|U#wJQ$fqpI;Ht4blE>i*g7D$gj#
zQsM2QtPZF4jZ2{1QvkaE5H)+s6qA8y2gH>PS)o!6D`#{ognuYwCAK$X>Ac-Ff8fMD
zetCji{1oy-KROH)7e5tOeuSLJr?{o#A=SM6so&)UH%mA0MKR(C!#P2Ok8kea0=$mt
zuRZttOc8KM0ED`(tZB{0oATTil+58}lkNv0Arv}o2Y@lGyh-vJFCkwAYFb-&1_=Gv
z?pn>B?_A=?x2kU%1%AYwWwjX%3-%P0ij~kY+mGQ7J^+XI-Fr{_)Z!5oaoiQ_67nJu
zDx+zH4bLZe$@M1@pj1L7)K^p|n{95XNq;mstVO<^SOL3~C(2YT<79*YpY*xq(E_y*
z{K;F>6Ic-`GjE4n)UZh2U9G+)G(cQ1Bqd9;ub_$EJ8
z$)YMQ#KLzmU-#*bEF?1UA5TOG5`yR=m*wa%DLW1yLh)o>Q(jv_w;KjSE@^Y03X4ES9lkyrR-T*z@`_|Qr
zC4}pDXu_?WYd;EC&2LrGlNNhn^Vh4?K4pz-0fI-jJbjqutPcm!X8U~Z*j<-;fqXvV
zb#;|Gdj8EM?}yPxtyZ5b^+{MGjF1hKvZRt@xO|$$1ExvxBR)Jbtz>+7q$1`Q#>byt
zg@>PX%w+CT`!$0gcvIRv7p%RW9mGY<>r<|XXQB)9(XzjBgl5$zE{b}{lJSfZ#*b_2
z3c6vbzGnr#c3%IO%m>|1CeHF6H?7nAOwdtLbq5Tu&=-bP2LPjL6rlu2
z_jDI-clalmxORl2E$2H@=y3S2P+S6h$5Y@tL~nv_Lfry`!j<>&vISO6^YsXu@EaNIk1E9#>2s?xh#aF6?_i<0_;A=nu9FcEMslm6p)iHel_d$-4>fC8Yhx7++fwKWaK_LJdzOg
zFdI#46pr8>IetVXc8mwvaN-yb2T5Q+TyrmNzQcNL~z6^Ec?V`QVjiG9XzLy@kWShiK^RzHdfw}TsX!!8_WdVAZoe$zhx
z{sUBDUkZBd?Ym%)
z*A?UvLmr%z2N;6@x+YvW+GNMx4e7IcODty%7|W+Me2wcwJ#co03D>Z~PdufXRJgva
z8CEpg-Z$~^<%IY*z=IR*-7NpI#PiHjsFYEXv#U%8A!h35WW6$@np2@)^$csCK2D=NpLBa11{I7{Ys=EO7Z2wQyil3YUco|IKeYi=AHNEK$7fRjK?856w`G
zmG3U3_-p-IEXTxHMJ|3R0-LM0zQ-bPfrJp(i>k}w=dk+F2OzP=fK+~t9QOy!$?_0?
zRyMj@v9{`Y{iNE7v=@(X`!|TT9jSZSxsRj7Vde=wDZ@0cJCVt+Ex8(J9(7uZLfCSZ
z%N8_wfVWKbq|8PgEAeR0=+y0o$BL;MrcuSod|-D{*~vfIv(h
zygFYl6&w`3Z2D2w9(2Q6-!Ngg^iB7z9_$3((EdNFm3dtqr{Wh+_)*m!`Uof}9AQEM
ztC&CCo_GvRoo{kMwd7~*vmn#U!#P1(FrOqAy2+1)(`ncb>bZNuzO13ZDDCut6TRYV
zXPe`G6d2a|ejJEV$vlV!x6>8hhTPi%N4Ch*>edWwEC^4l+Y|LsVp(4Y$I>Ady}cR2
zk8nPIViKEZ;5TkOFWF_+e+x!5I>xXd*Ja|ReTc=p-288od&aS4N`O!IBX~m7Pxf2D
zNxT|&2+(!|De#1IRLVr&klLY_VZk>0WCVE{eW<1@6t1My6u~K|aB58?^c#FI36;!$zQtJL9zW#jsuZ_-VO{8~5{aYGnDK`l1_kFjqLggR~C
zpSJg{ecy_0J7@`Q?bdceQRFnWl@6O!Dv?miu|g`xnX&8Lv@6uA$Z6Vy$|*4!{@CYpc%J*d?$dQ$Yz)_gHlhU}l?0=>_D*~V
zC`n7vpaP(}q1`X*TU5Qe+Jsav^Q)1f$AULl(Zg5t1Qb1@K+%H|q}~CrepiUJguDJj
zyahgLw!Dvr^
z`@YV451~FT&z39V+GmrwM(gWt3s#T2?{Qz4D~N-yI^5GX40>4)B|es4aU6=l!~e6D+e@c>|@gJs6~N9(l%L6rDlak$26+NVj5
zd{T}PF|ft_idbD4dWgDm!!3bPwIO@2Q6kC&e3)fmac5xO}h@Ixs;R=u%c^g|*@HXF~_8poK&q~1LG&EYDb~=Y8(4T=QWzjT!(UY~1YlYdC8|
zQaO>4b6H~TnMe*E!bd_NZpA3TBiDr7zPdZ&U
z-4MV(4QLx|zV%$ycD#
zvem9!#!;Y)MiDF~u&Ud8B)Ozyt<j
zKWQD%>r9yk-oh;z1?0R&XDJ*v_VO%w;*U*qDFS
z)bYY
hWwiHO2~6S6U`W-XED{?*?`$r)tY1(&UF@i4lp)62KZ6OUw_;7))y_N-TjH%%ibcwPctE34Kr=^VIFp
z%qSs@V>ITeVw80*Q~%+7`~JYIyGsg!mQk#uP#wU6t6*Cy&B75BALfy=iV@YA$f|*3
z4@6dA+=v*CwI5Q$7*B=&e7=C-
zr#I&Ro%Pm3%3MhOTH-CQi*>PTg=iI2dn%XTwqNG={Zvw6kdvGMZ@B1l*)$N`kYwtj
z;w08}Y^LtE*8}@DDLoAJgmvxjU%KHZ2I`UOD5xxh9x48O0TH#6l=H@g5uLLq3SQx!
z8z{Rx0y-R9u@rB@EG)%a(6c}vN%2A~*B0DXR9q&d&M8MkP(hAn({CrWx8*}9Giq&>
zi@J#{DbCUR1R=`aF9O*mAKVd|ocZ2kuJS2rg8&tQMvR@)*;9L*
zl)%KS8T2^76S%}mqqGVp3Izo&H#Qq{u#W`+FXUqvie%DQkpQjXfI9--fhJYciE&-&
zKcb~E3o54Zj;@}@kE@)9#V=5z-mthj?1&9pN^m1bfGY1WnO_n^sP3tAI5N%I_aPk*
zu5M-5*cyfNzp<+W7a%q-bSE^J^kU@)cbfXc&!YFiOm8QU0;N=)7@Y8N;~(oc6VIGd
zrc2_X95E=?5pXsM#!kiniz{CK6!uhgoE#(z7vgc>UsD*{mvG7%sQ@6S<9dklT6?Yr
z-*fWy(oAq5*KMc0h+#4KzG(N!q4!oD{$}S`YhJBIIjY%Xa#ZG5P))*HF7r$Y>Rajl
z5Hgw-JhNoKrHvd8ACYw^w|ZVoz$)A+A6$B4NPaXJy)>
z{m)Q&3N;;%1#kr%YsY266Gj;o8Zt8liw`*PqMB?~tiuFSY-%;pCu8FD*XF&$gCzkF
ztatvmw2%&P!OLG*vxbyZ164Rl1Gl7Kys}jy?yumdU}J
z&pjNc?J7ATqn$_|gw*ekAjL|T*=1y;Gxy{
z>L(?jVIFE68RE$igeJvhmVF2&oQ6IDIWj14GS#--QG<>QX^cgU_o-4h-LlH4FlpAB#gS16c_
z`BM4d4y?EVxjMs!o2rX15Q-bc`Yoc4TjSdM$8Y|$!{W>Z6d$2is@kkW_1S}|b-!Le
zj09?%X=p?Z`nVAc{AWZF`D)-
z<0A@C0XDQg@aJo{K=sr1rCD`bCreaXY3pM~5v@49C(Q%$H=AI&;qfVAOqBU~lKMDG
z9rhDU3*yuvZfKcWUVRe?ZRgq_7rHB5g9MtmCJI9UuLnbaUsEhauyLb+Xf-$!d+uV(
zU@|I4YF>ju5F%?>$P|YbMfCJno(Kng5HwtSheBt<>fN(ImSAs3#yvsu6Yn8tmh`LJ
z3thToZnl$|V9WTIKlWRbG2RIJ*|D22dS}iI^
z+e5eTf$4LnUSjg9VAf7tjV7rdJ9qcuJob;tHRGV&nPH&1xg9Jzqj#Yvwcxee%;viu
z7BrhQ(R*R-iNQ(J)jk7_`8)oPZ
znU#>j^wZQqZy(tNTr5xv9_dKvu~i)@7d%5I8dyt-I!XG7P0FgVj2Xi&6@^-Do?CF&
zPG|~P-hv<8h_~^8XlN@~)AIhFtmSEoO<@CsK~hUy_?G$*m!o~YLo@DQlw+3&;*dV=
zf_R?Yr(bdzn-U*UADv&D-;xb^`l^EVf&58czgo_aV+>xM*!e>$o*V6jDJZDVSN1dB
z{$@83)T?tOxdcIv+ybH-<;&FFk{FE#l{=STU~Hls@_!=yWHxZpVPX6}G0LO6Ik~^i
znxMxjSz!BNq!(ahWA+e9^fJxo?_;1frvFS2xW_ngRJUt_l#*4x*iL`mpOTAbVJ+c$
zS{Trv0h9NgKxL}ca2@IwM{xaOWKI09*>iH4WTkO@;&Jy?aUrt!0OKu28bo`W<&&
z5xHd&$rYs&2Q)34h#E19EQuPi`>acekwZDJ;~kf8J)wVk62
z7p@?zq4;X_lL$v$`GeozkmZ1|cI8eDni!f?J{=c(sKIbxrCae#adXTl6w-J=)djYd
z8n8vEYBM;rN=hCPE!;%#yDimB^xduu;Vfzl7)_#9{>BcmJ&ejlQ$mj-3yAzBE{wYj
z$I%J7TzAUJ)i1KDt=QN1ozp*!P?)}Rjk;Ww>TbuN$)_?{Ji__}#Zyn)YkKiT~wyF8w=Qfw08AlW1eiA}X^ZsXN^k|FV)xAZTd84S%+E
z5}}NnjYD%P;f|f#iQ??q;D!3yJ4%Li1v#jAAweW8#+i6DoDoaJ7=_GH^V?E!8kY_N
z`KFBrIv(F+{0f!_rO~VhhB}WhgtbI?HzTfVk}-+ZeLL_JGV27shWb_ndt{xcMT!m8
zXIN^if!eYC5YDr_AHNMkDMgT77lNkaN1oGc(4DzwUcis$?F7wQY<3sulhqC>fJ{hM
zo#r3@QU)YgJfdbG8~%~?IyTA^rHqY$WdU^>J;<$|Qrr$@5|m5Boe=f%s6Yrn40|K!
z@552HXf;&o%p2D|nvVGV&QKq$4@DZ}$NIf_wb6P-C2tHL1Vu>cV$-D_>An1V&^pNC>sXg4Lvu8p5x2Qn6WK4k+FCSHx)&96;9`gFo0Ebba
z&=g5>T(W1b7ZV-?a0y9{k8zIYUsEQjQKG|ENgOs6+rOr6N405+Ef*O%BD`~Km-Igw
zOl?Nx8Yq^CGiRt$f-`5c_(vg?Q1Jo~=sikfbxZXptx}+xsS@PGHty)Vgch0jFO8t~
z$bbCl*W0`TrL4CRKf}nG=?c{4Q$S)?5I0Y>K>0W##?*-n-0__2`(`@&4&I0O@-?Ho
z>W$XHS#^6Y>1RvauVgiDYWmMS~=Ka=-$Ux#+etOSV6HZvKrHQ
z=c$-%;y4wm^lLn6x(AyZ^1D;R>-OT_7rGODLn3AQxLaop+j2$M-f5KzU2^>YmGqnc
zi<@?H@82%GwEoKf-VDT|&QjIy=Llp@ZcVtx|2FXF0i}Of_YQox7O_xtcBh8@6^j>J
zuiMY}-$a}GZRnG4ZZ7)vm~UwF-LTfEboO`N5qs<+$e1fnye8|UeIrl}uzKH=ETZ(;
zXs#dDEe30vprx_E%0a0v`x?qmztaPV&-@!A`a^wLyy~p}hUEUHaxwoqkfQ;fNEL~Y
z=21#_-JMfI4&0x(+h7bs;v2`^Ao}u3Hxab~XgU4myKNZ=9`PzPK4_G-3Q_X2@S24i
z5zeN9#=Ns{;;MNmnaFu)kwxdf86IK}6MR`2mhlyXrN9QQ
zlX=Vq_=Nv`O$WZy^WDf!`Awj|XkD{?Yn)=x{Kr5SQFu)U>yGd8^7_O+S?`c-)@G3W
z__B^c!w_e{MhWUWiw9+2ue3^i|(Yuu+aMHcnfqTGf<@2U`g$CE#I>l{?B=}BMj3m_{G
zmVN?}ETQI60%{%=bWroChhj)zSO_CtsJZ)r(d4mweUlwtyx_)1pVwS!UFL-!dhS*p
zxmeeJ7~OS@pD5eH?i7C!6hbbn)NVgSaaM)p-LB?)h&mQoWRe%QRqoqqYqEjMse~+e
zge8Xzoi)=adnxC8q2A}*o-6oe%rpM%W0TQJgj;up=p9cJy$!BBY`nc}*GtsKM{S
z>5lxqz_(|1Ka$OAqt5$m)zPm$Rb_~m(#JS>p}h9E6@q7m8D@I`n$e`+jzOJRQyFUx~p0vclnopqUk&6{QURNbQApaj5khN8G9nN0oABH{7
zuNbw$L-elP6kE6g+{(RiN$(?~n>JF2+Dm)vQWRchb1WXWM+dtPam6fAXS2*Wi0xk^
z4GWRiU?Fl35%p}}4*uP;8YvYy=PWZ+2h%7&Ap7q`vgl)Z#kIQ1<8bcxo~?#Mpz(D4
zCfE!Gm{qFKO{h`qPZs^@WRXSa`IOeRdT!)-bnD)gO$3{S@|uAYQVt(@T1
zuYZ@+K|P}5$-0;uWO%T^K57}^Qur^W)#p(Gk`N>J*>K=&b;@A91$r3K8pX0seIHu#
zEkM+*>xH9xi3uU`kw_Lxe1Ppk5+6auqQCW83UK`mFK<45Six}og}^sCM{EGyqhH_(sLSRScb12Q&tiU#o%_-3JB}#f2*umF~%h-N9Up(SJuw~-1uE`
ztSchyv$u14OP8rE&8W6oUCa;2QI4d%@tTx03$9?J1>ORo=EA~Pwod1*drBH~k2bNy
zdw*W!!>JXj_dPzb&)1i&&~A4ZHOh(RTc*P1@9ztVbnik+A6JUkyT|n`xEEOwecqC?
z^g?Sis*JcoWn^1(GJotF`QrXb19$`2-P$1|in!e@s61SDv21%hA5}wM*|{xDqW;Yw
z&Yj8G>aL-es?hQjAzu8gw)P)aOJ8NH#w>VqLCl=8hnA!7xr+RF7$Rh*^vZ~M?}yp^
zHI)qBzN5M5?NE=q{1tidc@3|Rp~RJem%9M;5TF>3e9+A)QZ3+~#GG}z#DX8f!A~w{
z1O59Vl7EuvMd=D>>0qJgC{tslwS9~z@M;~+e?06F}bC#Z-q3*M*T>z!6cI5
znj+tgIRcNKdoJ?e{lqRVG7S5
z3MK+^!K6q}<;yod@=IkrzVA8aJkl%Af#;mWcms!3)BC?F3?z^i1Kg6MjsR@ntOpPF
z_a{X>8E5c_D(s*8534p4vg9Eay@dz(k4FCYU;g{?HTHNxJv<`?;3qkh)$udoV0_eN
zg@)O$syjA8J8Ld9_f3fGFhSLw^-MD8^S)&4wPxQ`-4}C^`<<}xz#I1DiGhv!_~gu0AiM6xUl0^YtZ`b&K}9xd$C7EeX)mhuYcb
zAtga($`rD;R~C1%a@pj(OsqHwy67K(8?L;3Twf1hBLOY$Fnb1omIkt+eg)uEXYHTF
z!pXjLbk8|7nT6ATPVJK>Sm!N|Z+$_sr>MEQgPOoq`Z&Wi7mzb7xySfywamU8R?~Zz
zgmyq#Q#AIxwl8S)^+IkBtZa|l^sW!W_w`~?awT5@HSrNKr%|;45Ek3AFX!}-&g5X8
zN!YaoiPVnfV3WfhzZL$v&HspB?^lI){L-C3FbjGqWudW8{vI6nfe_1nB}x(K2Yq;F
zdHzG5AQ!rqk3v}78ieG?OK&DWI;gLcssBLr0y(*LNI1~Oi;^9XCdnKdhjuqBGyG#`
zu`Dm4y`$ifZ<6XMT$k`!<5`QUfz^Y>ke=A+J1_FQ?r^ui7}8`5Pb_;F2tr00Z&jm;
zt@o_t*kbyEX2;xyJ(l|&a|Lhg+fF4*QCDD{#YQ#MHbn31Gz-se2S>9j^0pq3j0U$c
z)n`qIw+Pq(p?@3cH*L&?|&$)La;xs-b_g#{Y
zALK4bwR1qvN;&&&kmYF