Update to stac-fastapi v3.0.0a3, remove deprecated filter fields (#269)

jonhealy1 · web-flow · commit c5891e1ef9ae · 2024-06-16T18:54:57.000+08:00
**Related Issue(s):** - #217 - stac-utils/stac-fastapi#642 - stac-utils/stac-fastapi@d8528ae **Description:** - Update to stac-fastapi v3.0.0a3 - Remove deprecated filter_fields - Default to returning all properties, copy stac-fastapi-pgstac behaviour for now **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog
diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
@@ -1,4 +1,4 @@
-name: stac-fastapi-elasticsearch
+name: sfeos
 
 on:
   push:
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,11 +13,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Changed
 
 - Updated stac-fastapi libraries to v3.0.0a1 [#265](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/265)
+- Updated stac-fastapi libraries to v3.0.0a3 [#269](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/269)
 
 ### Fixed
 
 - API sort extension tests [#264](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/264)
 - Basic auth permission fix for checking route path instead of absolute path [#266](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/266)
+- Remove deprecated filter_fields property, return all properties as default [#269](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/269)
 
 ## [v3.0.0a1]
 
diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py
@@ -10,9 +10,9 @@
     "attrs>=23.2.0",
     "pydantic[dotenv]",
     "stac_pydantic>=3",
-    "stac-fastapi.types==3.0.0a1",
-    "stac-fastapi.api==3.0.0a1",
-    "stac-fastapi.extensions==3.0.0a1",
+    "stac-fastapi.types==3.0.0a3",
+    "stac-fastapi.api==3.0.0a3",
+    "stac-fastapi.extensions==3.0.0a3",
     "orjson",
     "overrides",
     "geojson-pydantic",
diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py
@@ -9,7 +9,6 @@
 
 import attr
 import orjson
-import stac_pydantic
 from fastapi import HTTPException, Request
 from overrides import overrides
 from pydantic import ValidationError
@@ -25,19 +24,16 @@
 from stac_fastapi.core.models.links import PagingLinks
 from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
 from stac_fastapi.core.session import Session
+from stac_fastapi.core.utilities import filter_fields
+from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient
 from stac_fastapi.extensions.third_party.bulk_transactions import (
     BaseBulkTransactionsClient,
     BulkTransactionMethod,
     Items,
 )
 from stac_fastapi.types import stac as stac_types
-from stac_fastapi.types.config import Settings
 from stac_fastapi.types.conformance import BASE_CONFORMANCE_CLASSES
-from stac_fastapi.types.core import (
-    AsyncBaseCoreClient,
-    AsyncBaseFiltersClient,
-    AsyncBaseTransactionsClient,
-)
+from stac_fastapi.types.core import AsyncBaseCoreClient, AsyncBaseTransactionsClient
 from stac_fastapi.types.extension import ApiExtension
 from stac_fastapi.types.requests import get_base_url
 from stac_fastapi.types.rfc3339 import DateTimeType
@@ -491,34 +487,26 @@ async def get_search(
             base_args["intersects"] = orjson.loads(unquote_plus(intersects))
 
         if sortby:
-            sort_param = []
-            for sort in sortby:
-                sort_param.append(
-                    {
-                        "field": sort[1:],
-                        "direction": "desc" if sort[0] == "-" else "asc",
-                    }
-                )
-            base_args["sortby"] = sort_param
+            base_args["sortby"] = [
+                {"field": sort[1:], "direction": "desc" if sort[0] == "-" else "asc"}
+                for sort in sortby
+            ]
 
         if filter:
-            if filter_lang == "cql2-json":
-                base_args["filter-lang"] = "cql2-json"
-                base_args["filter"] = orjson.loads(unquote_plus(filter))
-            else:
-                base_args["filter-lang"] = "cql2-json"
-                base_args["filter"] = orjson.loads(to_cql2(parse_cql2_text(filter)))
+            base_args["filter-lang"] = "cql2-json"
+            base_args["filter"] = orjson.loads(
+                unquote_plus(filter)
+                if filter_lang == "cql2-json"
+                else to_cql2(parse_cql2_text(filter))
+            )
 
         if fields:
-            includes = set()
-            excludes = set()
+            includes, excludes = set(), set()
             for field in fields:
                 if field[0] == "-":
                     excludes.add(field[1:])
-                elif field[0] == "+":
-                    includes.add(field[1:])
                 else:
-                    includes.add(field)
+                    includes.add(field[1:] if field[0] in "+ " else field)
             base_args["fields"] = {"include": includes, "exclude": excludes}
 
         # Do the request
@@ -614,32 +602,22 @@ async def post_search(
             collection_ids=search_request.collections,
         )
 
+        fields = (
+            getattr(search_request, "fields", None)
+            if self.extension_is_enabled("FieldsExtension")
+            else None
+        )
+        include: Set[str] = fields.include if fields and fields.include else set()
+        exclude: Set[str] = fields.exclude if fields and fields.exclude else set()
+
         items = [
-            self.item_serializer.db_to_stac(item, base_url=base_url) for item in items
+            filter_fields(
+                self.item_serializer.db_to_stac(item, base_url=base_url),
+                include,
+                exclude,
+            )
+            for item in items
         ]
-
-        if self.extension_is_enabled("FieldsExtension"):
-            if search_request.query is not None:
-                query_include: Set[str] = set(
-                    [
-                        k if k in Settings.get().indexed_fields else f"properties.{k}"
-                        for k in search_request.query.keys()
-                    ]
-                )
-                if not search_request.fields.include:
-                    search_request.fields.include = query_include
-                else:
-                    search_request.fields.include.union(query_include)
-
-            filter_kwargs = search_request.fields.filter_fields
-
-            items = [
-                orjson.loads(
-                    stac_pydantic.Item(**feat).json(**filter_kwargs, exclude_unset=True)
-                )
-                for feat in items
-            ]
-
         links = await PagingLinks(request=request, next=next_token).get_links()
 
         return stac_types.ItemCollection(
diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/fields.py b/stac_fastapi/core/stac_fastapi/core/extensions/fields.py
@@ -0,0 +1,41 @@
+"""Fields extension."""
+
+from typing import Optional, Set
+
+from pydantic import BaseModel, Field
+
+from stac_fastapi.extensions.core import FieldsExtension as FieldsExtensionBase
+from stac_fastapi.extensions.core.fields import request
+
+
+class PostFieldsExtension(request.PostFieldsExtension):
+    """PostFieldsExtension."""
+
+    # Set defaults if needed
+    # include : Optional[Set[str]] = Field(
+    #    default_factory=lambda: {
+    #         "id",
+    #         "type",
+    #         "stac_version",
+    #         "geometry",
+    #         "bbox",
+    #         "links",
+    #         "assets",
+    #         "properties.datetime",
+    #         "collection",
+    #     }
+    # )
+    include: Optional[Set[str]] = set()
+    exclude: Optional[Set[str]] = set()
+
+
+class FieldsExtensionPostRequest(BaseModel):
+    """Additional fields and schema for the POST request."""
+
+    fields: Optional[PostFieldsExtension] = Field(PostFieldsExtension())
+
+
+class FieldsExtension(FieldsExtensionBase):
+    """Override the POST model."""
+
+    POST = FieldsExtensionPostRequest
diff --git a/stac_fastapi/core/stac_fastapi/core/utilities.py b/stac_fastapi/core/stac_fastapi/core/utilities.py
@@ -3,7 +3,9 @@
 This module contains functions for transforming geospatial coordinates,
 such as converting bounding boxes to polygon representations.
 """
-from typing import List
+from typing import Any, Dict, List, Optional, Set, Union
+
+from stac_fastapi.types.stac import Item
 
 MAX_LIMIT = 10000
 
@@ -21,3 +23,113 @@ def bbox2polygon(b0: float, b1: float, b2: float, b3: float) -> List[List[List[f
         List[List[List[float]]]: A polygon represented as a list of lists of coordinates.
     """
     return [[[b0, b1], [b2, b1], [b2, b3], [b0, b3], [b0, b1]]]
+
+
+# copied from stac-fastapi-pgstac
+# https://github.com/stac-utils/stac-fastapi-pgstac/blob/26f6d918eb933a90833f30e69e21ba3b4e8a7151/stac_fastapi/pgstac/utils.py#L10-L116
+def filter_fields(  # noqa: C901
+    item: Union[Item, Dict[str, Any]],
+    include: Optional[Set[str]] = None,
+    exclude: Optional[Set[str]] = None,
+) -> Item:
+    """Preserve and remove fields as indicated by the fields extension include/exclude sets.
+
+    Returns a shallow copy of the Item with the fields filtered.
+
+    This will not perform a deep copy; values of the original item will be referenced
+    in the return item.
+    """
+    if not include and not exclude:
+        return item
+
+    # Build a shallow copy of included fields on an item, or a sub-tree of an item
+    def include_fields(
+        source: Dict[str, Any], fields: Optional[Set[str]]
+    ) -> Dict[str, Any]:
+        if not fields:
+            return source
+
+        clean_item: Dict[str, Any] = {}
+        for key_path in fields or []:
+            key_path_parts = key_path.split(".")
+            key_root = key_path_parts[0]
+            if key_root in source:
+                if isinstance(source[key_root], dict) and len(key_path_parts) > 1:
+                    # The root of this key path on the item is a dict, and the
+                    # key path indicates a sub-key to be included. Walk the dict
+                    # from the root key and get the full nested value to include.
+                    value = include_fields(
+                        source[key_root], fields={".".join(key_path_parts[1:])}
+                    )
+
+                    if isinstance(clean_item.get(key_root), dict):
+                        # A previously specified key and sub-keys may have been included
+                        # already, so do a deep merge update if the root key already exists.
+                        dict_deep_update(clean_item[key_root], value)
+                    else:
+                        # The root key does not exist, so add it. Fields
+                        # extension only allows nested referencing on dicts, so
+                        # this won't overwrite anything.
+                        clean_item[key_root] = value
+                else:
+                    # The item value to include is not a dict, or, it is a dict but the
+                    # key path is for the whole value, not a sub-key. Include the entire
+                    # value in the cleaned item.
+                    clean_item[key_root] = source[key_root]
+            else:
+                # The key, or root key of a multi-part key, is not present in the item,
+                # so it is ignored
+                pass
+        return clean_item
+
+    # For an item built up for included fields, remove excluded fields. This
+    # modifies `source` in place.
+    def exclude_fields(source: Dict[str, Any], fields: Optional[Set[str]]) -> None:
+        for key_path in fields or []:
+            key_path_part = key_path.split(".")
+            key_root = key_path_part[0]
+            if key_root in source:
+                if isinstance(source[key_root], dict) and len(key_path_part) > 1:
+                    # Walk the nested path of this key to remove the leaf-key
+                    exclude_fields(
+                        source[key_root], fields={".".join(key_path_part[1:])}
+                    )
+                    # If, after removing the leaf-key, the root is now an empty
+                    # dict, remove it entirely
+                    if not source[key_root]:
+                        del source[key_root]
+                else:
+                    # The key's value is not a dict, or there is no sub-key to remove. The
+                    # entire key can be removed from the source.
+                    source.pop(key_root, None)
+
+    # Coalesce incoming type to a dict
+    item = dict(item)
+
+    clean_item = include_fields(item, include)
+
+    # If, after including all the specified fields, there are no included properties,
+    # return just id and collection.
+    if not clean_item:
+        return Item({"id": item["id"], "collection": item["collection"]})
+
+    exclude_fields(clean_item, exclude)
+
+    return Item(**clean_item)
+
+
+def dict_deep_update(merge_to: Dict[str, Any], merge_from: Dict[str, Any]) -> None:
+    """Perform a deep update of two dicts.
+
+    merge_to is updated in-place with the values from merge_from.
+    merge_from values take precedence over existing values in merge_to.
+    """
+    for k, v in merge_from.items():
+        if (
+            k in merge_to
+            and isinstance(merge_to[k], dict)
+            and isinstance(merge_from[k], dict)
+        ):
+            dict_deep_update(merge_to[k], merge_from[k])
+        else:
+            merge_to[k] = v
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py
@@ -12,6 +12,7 @@
     TransactionsClient,
 )
 from stac_fastapi.core.extensions import QueryExtension
+from stac_fastapi.core.extensions.fields import FieldsExtension
 from stac_fastapi.core.session import Session
 from stac_fastapi.elasticsearch.config import ElasticsearchSettings
 from stac_fastapi.elasticsearch.database_logic import (
@@ -20,7 +21,6 @@
     create_index_templates,
 )
 from stac_fastapi.extensions.core import (
-    FieldsExtension,
     FilterExtension,
     SortExtension,
     TokenPaginationExtension,
diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py
@@ -12,9 +12,9 @@
     TransactionsClient,
 )
 from stac_fastapi.core.extensions import QueryExtension
+from stac_fastapi.core.extensions.fields import FieldsExtension
 from stac_fastapi.core.session import Session
 from stac_fastapi.extensions.core import (
-    FieldsExtension,
     FilterExtension,
     SortExtension,
     TokenPaginationExtension,
diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py
@@ -118,7 +118,10 @@ async def test_app_context_results(app_client, txn_client, ctx, load_test_data):
 
 @pytest.mark.asyncio
 async def test_app_fields_extension(app_client, ctx, txn_client):
-    resp = await app_client.get("/search", params={"collections": ["test-collection"]})
+    resp = await app_client.get(
+        "/search",
+        params={"collections": ["test-collection"], "fields": "+properties.datetime"},
+    )
     assert resp.status_code == 200
     resp_json = resp.json()
     assert list(resp_json["features"][0]["properties"]) == ["datetime"]
@@ -132,11 +135,12 @@ async def test_app_fields_extension_query(app_client, ctx, txn_client):
         json={
             "query": {"proj:epsg": {"gte": item["properties"]["proj:epsg"]}},
             "collections": ["test-collection"],
+            "fields": {"include": ["properties.datetime", "properties.proj:epsg"]},
         },
     )
     assert resp.status_code == 200
     resp_json = resp.json()
-    assert list(resp_json["features"][0]["properties"]) == ["datetime", "proj:epsg"]
+    assert set(resp_json["features"][0]["properties"]) == set(["datetime", "proj:epsg"])
 
 
 @pytest.mark.asyncio
diff --git a/stac_fastapi/tests/resources/test_item.py b/stac_fastapi/tests/resources/test_item.py
@@ -754,7 +754,11 @@ async def test_field_extension_post(app_client, ctx):
         "ids": [test_item["id"]],
         "fields": {
             "exclude": ["assets.B1"],
-            "include": ["properties.eo:cloud_cover", "properties.orientation"],
+            "include": [
+                "properties.eo:cloud_cover",
+                "properties.orientation",
+                "assets",
+            ],
         },
     }
 
@@ -782,7 +786,7 @@ async def test_field_extension_exclude_and_include(app_client, ctx):
 
     resp = await app_client.post("/search", json=body)
     resp_json = resp.json()
-    assert "eo:cloud_cover" not in resp_json["features"][0]["properties"]
+    assert "properties" not in resp_json["features"][0]
 
 
 @pytest.mark.asyncio

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-name: stac-fastapi-elasticsearch`
	`1`	`+name: sfeos`
`2`	`2`
`3`	`3`	`on:`
`4`	`4`	`push:`