Skip to content

Commit 6b25e56

Browse files
Zaczerojonhealy1
Zaczero
and
jonhealy1
authored
Zero-config dynamically-generated queryables, Performance fixes (#351)
**Related Issue(s):** - Fixes #345 - Fixes #344 - Fixes #336 **Description:** This PR consists of self-contained commits (except the first commit that provides database_logic deduplication), making it easy to change or remove individual patches. It addresses several small issues, improves the performance of certain methods, and adds support for dynamically-generated queryables. This enhancement doesn't require any new configuration as queryables are generated on the fly based on the os/es mappings. The implementation is designed for extensibility, with built-in logic for augmenting fields metadata with additional information. Currently, it only includes the _DEFAULT_QUERYABLES configuration, which was simply copied from the pre-PR code. Example queryables response: ```json {"$schema":"https://json-schema.org/draft/2019-09/schema","$id":"https://stac-api.example.com/queryables","type":"object","title":"Queryables for STAC API","description":"Queryable names for the STAC API Item Search filter.","properties":{"bbox":{"title":"Bbox","type":"number"},"collection":{"description":"Collection","$ref":"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/then/properties/collection","title":"Collection","type":"string"},"geometry":{"description":"Geometry","$ref":"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/1/oneOf/0/properties/geometry","title":"Geometry","type":"object"},"id":{"description":"ID","$ref":"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id","title":"Id","type":"string"},"stac_extensions":{"title":"Stac Extensions","type":"string"},"stac_version":{"title":"Stac Version","type":"string"},"type":{"title":"Type","type":"string"},"constellation":{"title":"Constellation","type":"string"},"created":{"description":"Creation Timestamp","$ref":"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/created","title":"Created","type":"string","format":"date-time"},"datetime":{"description":"Acquisition Timestamp","$ref":"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/datetime","title":"Datetime","type":"string","format":"date-time"},"end_datetime":{"title":"End Datetime","type":"string","format":"date-time"},"eopf:datatake_id":{"title":"Eopf:Datatake Id","type":"string"},"eopf:instrument_configuration_id":{"title":"Eopf:Instrument Configuration Id","type":"number"},"instruments":{"title":"Instruments","type":"string"},"platform":{"title":"Platform","type":"string"},"processing:datetime":{"title":"Processing:Datetime","type":"string","format":"date-time"},"processing:facility":{"title":"Processing:Facility","type":"string"},"processing:level":{"title":"Processing:Level","type":"string"},"processing:version":{"title":"Processing:Version","type":"string"},"product:timeliness":{"title":"Product:Timeliness","type":"string"},"product:timeliness_category":{"title":"Product:Timeliness Category","type":"string"},"product:type":{"title":"Product:Type","type":"string"},"published":{"title":"Published","type":"string","format":"date-time"},"sar:center_frequency":{"title":"Sar:Center Frequency","type":"number"},"sar:frequency_band":{"title":"Sar:Frequency Band","type":"string"},"sar:instrument_mode":{"title":"Sar:Instrument Mode","type":"string"},"sar:observation_direction":{"title":"Sar:Observation Direction","type":"string"},"sar:pixel_spacing_azimuth":{"title":"Sar:Pixel Spacing Azimuth","type":"number"},"sar:pixel_spacing_range":{"title":"Sar:Pixel Spacing Range","type":"number"},"sar:polarizations":{"title":"Sar:Polarizations","type":"string"},"sar:resolution_azimuth":{"title":"Sar:Resolution Azimuth","type":"number"},"sar:resolution_range":{"title":"Sar:Resolution Range","type":"number"},"sat:absolute_orbit":{"title":"Sat:Absolute Orbit","type":"integer"},"sat:orbit_cycle":{"title":"Sat:Orbit Cycle","type":"number"},"sat:orbit_state":{"title":"Sat:Orbit State","type":"string"},"sat:platform_international_designator":{"title":"Sat:Platform International Designator","type":"string"},"sat:relative_orbit":{"title":"Sat:Relative Orbit","type":"integer"},"start_datetime":{"title":"Start Datetime","type":"string","format":"date-time"},"updated":{"description":"Creation Timestamp","$ref":"https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/updated","title":"Updated","type":"string","format":"date-time"},"view:azimuth":{"title":"View:Azimuth","type":"number"},"view:incidence_angle":{"title":"View:Incidence Angle","type":"number"},"auth:schemes.oidc.openIdConnectUrl":{"title":"Auth:Schemes.Oidc.Openidconnecturl","type":"string"},"auth:schemes.oidc.type":{"title":"Auth:Schemes.Oidc.Type","type":"string"},"auth:schemes.s3.type":{"title":"Auth:Schemes.S3.Type","type":"string"},"storage:schemes.cdse-s3.description":{"title":"Storage:Schemes.Cdse-S3.Description","type":"string"},"storage:schemes.cdse-s3.platform":{"title":"Storage:Schemes.Cdse-S3.Platform","type":"string"},"storage:schemes.cdse-s3.requester_pays":{"title":"Storage:Schemes.Cdse-S3.Requester Pays","type":"boolean"},"storage:schemes.cdse-s3.title":{"title":"Storage:Schemes.Cdse-S3.Title","type":"string"},"storage:schemes.cdse-s3.type":{"title":"Storage:Schemes.Cdse-S3.Type","type":"string"},"storage:schemes.creodias-s3.description":{"title":"Storage:Schemes.Creodias-S3.Description","type":"string"},"storage:schemes.creodias-s3.platform":{"title":"Storage:Schemes.Creodias-S3.Platform","type":"string"},"storage:schemes.creodias-s3.requester_pays":{"title":"Storage:Schemes.Creodias-S3.Requester Pays","type":"boolean"},"storage:schemes.creodias-s3.title":{"title":"Storage:Schemes.Creodias-S3.Title","type":"string"},"storage:schemes.creodias-s3.type":{"title":"Storage:Schemes.Creodias-S3.Type","type":"string"}},"additionalProperties":false} ``` PS. I think the auto-generated "title" should be removed completely, but I included it because I found it to be common practice in some STAC projects. I'm not sure how you feel about it. **PR Checklist:** - [ ] Code is formatted and linted (run `pre-commit run --all-files`) - [ ] Tests pass (run `make test`) - [ ] Documentation has been updated to reflect changes, if applicable - [ ] Changes are added to the changelog --------- Co-authored-by: Jonathan Healy <jonathan.d.healy@gmail.com>
1 parent 353210d commit 6b25e56

File tree

11 files changed

+447
-486
lines changed

11 files changed

+447
-486
lines changed

‎CHANGELOG.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
88
## [Unreleased]
99

1010
### Added
11+
- Added support for dynamically-generated queryables based on Elasticsearch/OpenSearch mappings, with extensible metadata augmentation [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351)
12+
- Included default queryables configuration for seamless integration. [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351)
1113

1214
### Changed
15+
- Refactored database logic to reduce duplication [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351)
16+
- Replaced `fastapi-slim` with `fastapi` dependency [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351)
17+
18+
### Fixed
19+
- Improved performance of `mk_actions` and `filter-links` methods [#351](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/351)
1320

1421
## [v3.2.5] - 2025-04-07
1522

‎stac_fastapi/core/setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
desc = f.read()
77

88
install_requires = [
9-
"fastapi-slim",
9+
"fastapi",
1010
"attrs>=23.2.0",
1111
"pydantic",
1212
"stac_pydantic>=3",

‎stac_fastapi/core/stac_fastapi/core/core.py

Lines changed: 127 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
"""Core client."""
22

33
import logging
4+
from collections import deque
45
from datetime import datetime as datetime_type
56
from datetime import timezone
67
from enum import Enum
7-
from typing import Any, Dict, List, Optional, Set, Type, Union
8+
from typing import Any, Dict, List, Literal, Optional, Set, Type, Union
89
from urllib.parse import unquote_plus, urljoin
910

1011
import attr
@@ -41,8 +42,6 @@
4142

4243
logger = logging.getLogger(__name__)
4344

44-
NumType = Union[float, int]
45-
4645

4746
@attr.s
4847
class CoreClient(AsyncBaseCoreClient):
@@ -907,11 +906,81 @@ def bulk_item_insert(
907906
return f"Successfully added {len(processed_items)} Items."
908907

909908

909+
_DEFAULT_QUERYABLES: Dict[str, Dict[str, Any]] = {
910+
"id": {
911+
"description": "ID",
912+
"$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id",
913+
},
914+
"collection": {
915+
"description": "Collection",
916+
"$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/then/properties/collection",
917+
},
918+
"geometry": {
919+
"description": "Geometry",
920+
"$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/1/oneOf/0/properties/geometry",
921+
},
922+
"datetime": {
923+
"description": "Acquisition Timestamp",
924+
"$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/datetime",
925+
},
926+
"created": {
927+
"description": "Creation Timestamp",
928+
"$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/created",
929+
},
930+
"updated": {
931+
"description": "Creation Timestamp",
932+
"$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/updated",
933+
},
934+
"cloud_cover": {
935+
"description": "Cloud Cover",
936+
"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fields/properties/eo:cloud_cover",
937+
},
938+
"cloud_shadow_percentage": {
939+
"title": "Cloud Shadow Percentage",
940+
"description": "Cloud Shadow Percentage",
941+
"type": "number",
942+
"minimum": 0,
943+
"maximum": 100,
944+
},
945+
"nodata_pixel_percentage": {
946+
"title": "No Data Pixel Percentage",
947+
"description": "No Data Pixel Percentage",
948+
"type": "number",
949+
"minimum": 0,
950+
"maximum": 100,
951+
},
952+
}
953+
954+
_ES_MAPPING_TYPE_TO_JSON: Dict[
955+
str, Literal["string", "number", "boolean", "object", "array", "null"]
956+
] = {
957+
"date": "string",
958+
"date_nanos": "string",
959+
"keyword": "string",
960+
"match_only_text": "string",
961+
"text": "string",
962+
"wildcard": "string",
963+
"byte": "number",
964+
"double": "number",
965+
"float": "number",
966+
"half_float": "number",
967+
"long": "number",
968+
"scaled_float": "number",
969+
"short": "number",
970+
"token_count": "number",
971+
"unsigned_long": "number",
972+
"geo_point": "object",
973+
"geo_shape": "object",
974+
"nested": "array",
975+
}
976+
977+
910978
@attr.s
911979
class EsAsyncBaseFiltersClient(AsyncBaseFiltersClient):
912980
"""Defines a pattern for implementing the STAC filter extension."""
913981

914-
# todo: use the ES _mapping endpoint to dynamically find what fields exist
982+
database: BaseDatabaseLogic = attr.ib()
983+
915984
async def get_queryables(
916985
self, collection_id: Optional[str] = None, **kwargs
917986
) -> Dict[str, Any]:
@@ -932,55 +1001,62 @@ async def get_queryables(
9321001
Returns:
9331002
Dict[str, Any]: A dictionary containing the queryables for the given collection.
9341003
"""
935-
return {
1004+
queryables: Dict[str, Any] = {
9361005
"$schema": "https://json-schema.org/draft/2019-09/schema",
9371006
"$id": "https://stac-api.example.com/queryables",
9381007
"type": "object",
939-
"title": "Queryables for Example STAC API",
940-
"description": "Queryable names for the example STAC API Item Search filter.",
941-
"properties": {
942-
"id": {
943-
"description": "ID",
944-
"$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id",
945-
},
946-
"collection": {
947-
"description": "Collection",
948-
"$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/then/properties/collection",
949-
},
950-
"geometry": {
951-
"description": "Geometry",
952-
"$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/1/oneOf/0/properties/geometry",
953-
},
954-
"datetime": {
955-
"description": "Acquisition Timestamp",
956-
"$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/datetime",
957-
},
958-
"created": {
959-
"description": "Creation Timestamp",
960-
"$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/created",
961-
},
962-
"updated": {
963-
"description": "Creation Timestamp",
964-
"$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/updated",
965-
},
966-
"cloud_cover": {
967-
"description": "Cloud Cover",
968-
"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fields/properties/eo:cloud_cover",
969-
},
970-
"cloud_shadow_percentage": {
971-
"description": "Cloud Shadow Percentage",
972-
"title": "Cloud Shadow Percentage",
973-
"type": "number",
974-
"minimum": 0,
975-
"maximum": 100,
976-
},
977-
"nodata_pixel_percentage": {
978-
"description": "No Data Pixel Percentage",
979-
"title": "No Data Pixel Percentage",
980-
"type": "number",
981-
"minimum": 0,
982-
"maximum": 100,
983-
},
984-
},
1008+
"title": "Queryables for STAC API",
1009+
"description": "Queryable names for the STAC API Item Search filter.",
1010+
"properties": _DEFAULT_QUERYABLES,
9851011
"additionalProperties": True,
9861012
}
1013+
if not collection_id:
1014+
return queryables
1015+
1016+
properties: Dict[str, Any] = queryables["properties"]
1017+
queryables.update(
1018+
{
1019+
"properties": properties,
1020+
"additionalProperties": False,
1021+
}
1022+
)
1023+
1024+
mapping_data = await self.database.get_items_mapping(collection_id)
1025+
mapping_properties = next(iter(mapping_data.values()))["mappings"]["properties"]
1026+
stack = deque(mapping_properties.items())
1027+
1028+
while stack:
1029+
field_name, field_def = stack.popleft()
1030+
1031+
# Iterate over nested fields
1032+
field_properties = field_def.get("properties")
1033+
if field_properties:
1034+
# Fields in Item Properties should be exposed with their un-prefixed names,
1035+
# and not require expressions to prefix them with properties,
1036+
# e.g., eo:cloud_cover instead of properties.eo:cloud_cover.
1037+
if field_name == "properties":
1038+
stack.extend(field_properties.items())
1039+
else:
1040+
stack.extend(
1041+
(f"{field_name}.{k}", v) for k, v in field_properties.items()
1042+
)
1043+
1044+
# Skip non-indexed or disabled fields
1045+
field_type = field_def.get("type")
1046+
if not field_type or not field_def.get("enabled", True):
1047+
continue
1048+
1049+
# Generate field properties
1050+
field_result = _DEFAULT_QUERYABLES.get(field_name, {})
1051+
properties[field_name] = field_result
1052+
1053+
field_name_human = field_name.replace("_", " ").title()
1054+
field_result.setdefault("title", field_name_human)
1055+
1056+
field_type_json = _ES_MAPPING_TYPE_TO_JSON.get(field_type, field_type)
1057+
field_result.setdefault("type", field_type_json)
1058+
1059+
if field_type in {"date", "date_nanos"}:
1060+
field_result.setdefault("format", "date-time")
1061+
1062+
return queryables

0 commit comments

Comments
 (0)