From 321d08c16681846a930aaa535379d2de792a1554 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 15 Jan 2023 17:03:32 +0300 Subject: [PATCH 01/15] add fields extension to app --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 32054977..a93b4e4f 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -15,7 +15,8 @@ from stac_fastapi.elasticsearch.database_logic import create_collection_index from stac_fastapi.elasticsearch.extensions import QueryExtension from stac_fastapi.elasticsearch.session import Session -from stac_fastapi.extensions.core import ( # FieldsExtension, +from stac_fastapi.extensions.core import ( + FieldsExtension, ContextExtension, FilterExtension, SortExtension, @@ -67,7 +68,7 @@ class FixedQueryExtension(QueryExtension): extensions = [ TransactionExtension(client=TransactionsClient(session=session), settings=settings), BulkTransactionExtension(client=BulkTransactionsClient(session=session)), - # FieldsExtension(), + FieldsExtension(), FixedQueryExtension(), FixedSortExtension(), TokenPaginationExtension(), From b55ef7d23df7cf7687a2629c95b4de94cda2fe7e Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 15 Jan 2023 17:03:55 +0300 Subject: [PATCH 02/15] enable fields extension in core logic --- .../stac_fastapi/elasticsearch/core.py | 64 ++++++++++--------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 10bc20df..7385d3d6 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -1,9 +1,10 @@ """Item crud client.""" import json import logging +import stac_pydantic from datetime import datetime as datetime_type from datetime import timezone -from typing import Any, Dict, List, Optional, Type, Union +from typing import Any, Dict, List, Optional, Type, Union, Set from urllib.parse import urljoin import attr @@ -13,6 +14,7 @@ from stac_pydantic.links import Relations from stac_pydantic.shared import MimeTypes from starlette.requests import Request +from stac_fastapi.types.config import Settings from stac_fastapi.elasticsearch import serializers from stac_fastapi.elasticsearch.config import ElasticsearchSettings @@ -242,17 +244,17 @@ async def get_search( # base_args["filter"] = orjson.loads(to_cql2(parse_cql2_text(filter))) # print(f'>>> {base_args["filter"]}') - # if fields: - # includes = set() - # excludes = set() - # for field in fields: - # if field[0] == "-": - # excludes.add(field[1:]) - # elif field[0] == "+": - # includes.add(field[1:]) - # else: - # includes.add(field) - # base_args["fields"] = {"include": includes, "exclude": excludes} + if fields: + includes = set() + excludes = set() + for field in fields: + if field[0] == "-": + excludes.add(field[1:]) + elif field[0] == "+": + includes.add(field[1:]) + else: + includes.add(field) + base_args["fields"] = {"include": includes, "exclude": excludes} # Do the request try: @@ -338,25 +340,25 @@ async def post_search( self.item_serializer.db_to_stac(item, base_url=base_url) for item in items ] - # if self.extension_is_enabled("FieldsExtension"): - # if search_request.query is not None: - # query_include: Set[str] = set( - # [ - # k if k in Settings.get().indexed_fields else f"properties.{k}" - # for k in search_request.query.keys() - # ] - # ) - # if not search_request.fields.include: - # search_request.fields.include = query_include - # else: - # search_request.fields.include.union(query_include) - - # filter_kwargs = search_request.fields.filter_fields - - # response_features = [ - # json.loads(stac_pydantic.Item(**feat).json(**filter_kwargs)) - # for feat in response_features - # ] + if self.extension_is_enabled("FieldsExtension"): + if search_request.query is not None: + query_include: Set[str] = set( + [ + k if k in Settings.get().indexed_fields else f"properties.{k}" + for k in search_request.query.keys() + ] + ) + if not search_request.fields.include: + search_request.fields.include = query_include + else: + search_request.fields.include.union(query_include) + + filter_kwargs = search_request.fields.filter_fields + + items = [ + json.loads(stac_pydantic.Item(**feat).json(**filter_kwargs)) + for feat in items + ] context_obj = None if self.extension_is_enabled("ContextExtension"): From e4a44ec61386b7c7f95af4d320bc5fbfc1c644ab Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 15 Jan 2023 17:04:18 +0300 Subject: [PATCH 03/15] add indexed fields tto settings --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index 8fda8032..e1630d67 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -37,6 +37,7 @@ class ElasticsearchSettings(ApiSettings): # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields + indexed_fields: Set[str] = {"datetime"} @property def create_client(self): @@ -49,6 +50,7 @@ class AsyncElasticsearchSettings(ApiSettings): # Fields which are defined by STAC but not included in the database model forbidden_fields: Set[str] = _forbidden_fields + indexed_fields: Set[str] = {"datetime"} @property def create_client(self): From f7573f6650bc1374e9c5987bac97fdecec57ca23 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 15 Jan 2023 17:04:48 +0300 Subject: [PATCH 04/15] enable fields extension for conftest --- stac_fastapi/elasticsearch/tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index 72f67451..de704728 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -26,6 +26,7 @@ ContextExtension, TokenPaginationExtension, TransactionExtension, + FieldsExtension, ) from stac_fastapi.types.config import Settings @@ -160,7 +161,7 @@ async def app(): ), ContextExtension(), FixedSortExtension(), - # FieldsExtension(), + FieldsExtension(), FixedQueryExtension(), TokenPaginationExtension(), FixedFilterExtension(), From c9e6b0b34267f1b151f36e3a28477e017a489afc Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 15 Jan 2023 17:05:06 +0300 Subject: [PATCH 05/15] add fields extension tests --- .../elasticsearch/tests/api/test_api.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index 40b06423..c1694f50 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -110,17 +110,27 @@ async def test_app_context_extension(app_client, ctx, txn_client): assert matched == 1 -@pytest.mark.skip(reason="fields not implemented yet") -async def test_app_fields_extension(load_test_data, app_client, txn_client): - item = load_test_data("test_item.json") - txn_client.create_item(item, request=MockRequest, refresh=True) - +async def test_app_fields_extension(app_client, ctx, txn_client): resp = await app_client.get("/search", params={"collections": ["test-collection"]}) assert resp.status_code == 200 resp_json = resp.json() assert list(resp_json["features"][0]["properties"]) == ["datetime"] - txn_client.delete_item(item["id"], item["collection"]) + +async def test_app_fields_extension_return_all_properties(app_client, ctx, txn_client): + item = ctx.item + resp = await app_client.get( + "/search", params={"collections": ["test-collection"], "fields": "properties"} + ) + assert resp.status_code == 200 + resp_json = resp.json() + feature = resp_json["features"][0] + assert len(feature["properties"]) >= len(item["properties"]) + for expected_prop, expected_value in item["properties"].items(): + if expected_prop in ("datetime", "created", "updated"): + assert feature["properties"][expected_prop][0:19] == expected_value[0:19] + else: + assert feature["properties"][expected_prop] == expected_value async def test_app_query_extension_gt(app_client, ctx): From 8579a65bd2191fe690bfd0bd395225946787e0a8 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 15 Jan 2023 17:08:21 +0300 Subject: [PATCH 06/15] pre-commit --- .../elasticsearch/stac_fastapi/elasticsearch/app.py | 2 +- .../elasticsearch/stac_fastapi/elasticsearch/core.py | 6 +++--- stac_fastapi/elasticsearch/tests/api/test_api.py | 4 +--- stac_fastapi/elasticsearch/tests/conftest.py | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index a93b4e4f..84eece90 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -16,8 +16,8 @@ from stac_fastapi.elasticsearch.extensions import QueryExtension from stac_fastapi.elasticsearch.session import Session from stac_fastapi.extensions.core import ( - FieldsExtension, ContextExtension, + FieldsExtension, FilterExtension, SortExtension, TokenPaginationExtension, diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 7385d3d6..ec79470f 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -1,20 +1,19 @@ """Item crud client.""" import json import logging -import stac_pydantic from datetime import datetime as datetime_type from datetime import timezone -from typing import Any, Dict, List, Optional, Type, Union, Set +from typing import Any, Dict, List, Optional, Set, Type, Union from urllib.parse import urljoin import attr +import stac_pydantic from fastapi import HTTPException from overrides import overrides from pydantic import ValidationError from stac_pydantic.links import Relations from stac_pydantic.shared import MimeTypes from starlette.requests import Request -from stac_fastapi.types.config import Settings from stac_fastapi.elasticsearch import serializers from stac_fastapi.elasticsearch.config import ElasticsearchSettings @@ -28,6 +27,7 @@ Items, ) from stac_fastapi.types import stac as stac_types +from stac_fastapi.types.config import Settings from stac_fastapi.types.core import ( AsyncBaseCoreClient, AsyncBaseFiltersClient, diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index c1694f50..572e9e9b 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -2,9 +2,7 @@ import uuid from datetime import datetime, timedelta -import pytest - -from ..conftest import MockRequest, create_collection, create_item +from ..conftest import create_collection, create_item ROUTES = { "GET /_mgmt/ping", diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index de704728..b755425c 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -24,9 +24,9 @@ from stac_fastapi.elasticsearch.database_logic import create_collection_index from stac_fastapi.extensions.core import ( # FieldsExtension, ContextExtension, + FieldsExtension, TokenPaginationExtension, TransactionExtension, - FieldsExtension, ) from stac_fastapi.types.config import Settings From 1a64dc98e2006ba97a078b14a9b2eccd61b7e5a9 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 15 Jan 2023 17:25:09 +0300 Subject: [PATCH 07/15] add two more fields tests --- .../elasticsearch/tests/api/test_api.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index 572e9e9b..204dfc40 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -115,6 +115,28 @@ async def test_app_fields_extension(app_client, ctx, txn_client): assert list(resp_json["features"][0]["properties"]) == ["datetime"] +async def test_app_fields_extension_no_properties_get(app_client, ctx, txn_client): + resp = await app_client.get( + "/search", params={"collections": ["test-collection"], "fields": "-properties"} + ) + assert resp.status_code == 200 + resp_json = resp.json() + assert "properties" not in resp_json["features"][0] + + +async def test_app_fields_extension_no_properties_post(app_client, ctx, txn_client): + resp = await app_client.post( + "/search", + json={ + "collections": ["test-collection"], + "fields": {"exclude": ["properties"]}, + }, + ) + assert resp.status_code == 200 + resp_json = resp.json() + assert "properties" not in resp_json["features"][0] + + async def test_app_fields_extension_return_all_properties(app_client, ctx, txn_client): item = ctx.item resp = await app_client.get( From eb6402ff68a25265c58462b1072cd808974528bc Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 15 Jan 2023 17:27:41 +0300 Subject: [PATCH 08/15] update changelog --- CHANGELOG.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff957c6f..93f78872 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,16 +9,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added -- Added bbox and datetime parameters and functionality to item_collection https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127 -- Added collection_id parameter to create_item function https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127 -- Added item_id and collection_id to update_item https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127 -- The default Collection objects index can be overridden by the `STAC_COLLECTIONS_INDEX` environment variable. -- The default Item objects index prefix can be overridden by the `STAC_ITEMS_INDEX_PREFIX` environment variable. +- Added bbox and datetime parameters and functionality to item_collection [#127](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127) +- Added collection_id parameter to create_item function [#127](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127) +- Added item_id and collection_id to update_item [#127](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127) +- The default Collection objects index can be overridden by the `STAC_COLLECTIONS_INDEX` environment variable [#128](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/128) +- The default Item objects index prefix can be overridden by the `STAC_ITEMS_INDEX_PREFIX` environment variable [#128](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/128) +- Fields Extension [#129](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/129) ### Changed -- Updated core stac-fastapi libraries to 2.4.3 from 2.3.0 https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127 +- Updated core stac-fastapi libraries to 2.4.3 from 2.3.0 [#127](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127) ## [v0.2.0] From dee1c0fad7c866228c39fcb24889ea1f725aa104 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 15 Jan 2023 17:34:07 +0300 Subject: [PATCH 09/15] remove irrelevant comments --- stac_fastapi/elasticsearch/tests/extensions/test_filter.py | 1 - stac_fastapi/elasticsearch/tests/resources/test_item.py | 1 - 2 files changed, 2 deletions(-) diff --git a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py index 5ea062cc..462f40bd 100644 --- a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py +++ b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py @@ -43,7 +43,6 @@ async def test_search_filter_extension_gte(app_client, ctx): assert resp.status_code == 200 assert len(resp.json()["features"]) == 1 - # this part fails params = { "filter": { "op": ">", diff --git a/stac_fastapi/elasticsearch/tests/resources/test_item.py b/stac_fastapi/elasticsearch/tests/resources/test_item.py index 21f7d87e..fc53eab1 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_item.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_item.py @@ -99,7 +99,6 @@ async def test_update_new_item(app_client, ctx): test_item = ctx.item test_item["id"] = "a" - # note: this endpoint is wrong in stac-fastapi -- should be /collections/{c_id}/items/{item_id} resp = await app_client.put( f"/collections/{test_item['collection']}/items/{test_item['id']}", json=test_item, From 115db6ead9502b8baa170b7df52390b8a071b1df Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 15 Jan 2023 18:03:16 +0300 Subject: [PATCH 10/15] update postman collection --- ...tapi-elasticsearch.postman_collection.json | 170 ++++++++++++++++-- 1 file changed, 155 insertions(+), 15 deletions(-) diff --git a/postman_collections/stac-fastapi-elasticsearch.postman_collection.json b/postman_collections/stac-fastapi-elasticsearch.postman_collection.json index a49d5994..cb141e0c 100644 --- a/postman_collections/stac-fastapi-elasticsearch.postman_collection.json +++ b/postman_collections/stac-fastapi-elasticsearch.postman_collection.json @@ -2,7 +2,8 @@ "info": { "_postman_id": "ca9d0979-4035-45ad-bfba-582a680a05ab", "name": "stac-fastapi-elasticsearch", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "12888943" }, "item": [ { @@ -11,7 +12,7 @@ "method": "GET", "header": [], "url": { - "raw": "http://localhost:8080/", + "raw": "http://localhost:8083/", "protocol": "http", "host": [ "localhost" @@ -35,7 +36,7 @@ "host": [ "localhost" ], - "port": "8083", + "port": "8080", "path": [ "collections" ] @@ -49,7 +50,7 @@ "method": "DELETE", "header": [], "url": { - "raw": "http://localhost:8080/collections/test-collection", + "raw": "http://localhost:8083/collections/test-collection", "protocol": "http", "host": [ "localhost" @@ -69,7 +70,7 @@ "method": "DELETE", "header": [], "url": { - "raw": "http://localhost:8080/collections/test-collection/items/test-item", + "raw": "http://localhost:8083/collections/test-collection/items/test-item", "protocol": "http", "host": [ "localhost" @@ -96,7 +97,7 @@ "host": [ "localhost" ], - "port": "8083", + "port": "8080", "path": [ "collections", "test-collection" @@ -116,7 +117,7 @@ "host": [ "localhost" ], - "port": "8083", + "port": "8080", "path": [ "collections", "test-collection", @@ -137,7 +138,7 @@ "host": [ "localhost" ], - "port": "8083", + "port": "8080", "path": [ "collections", "test-collection", @@ -174,7 +175,7 @@ "host": [ "localhost" ], - "port": "8083", + "port": "8080", "path": [ "collections" ] @@ -208,7 +209,7 @@ "host": [ "localhost" ], - "port": "8083", + "port": "8080", "path": [ "collections", "test-collection", @@ -218,6 +219,43 @@ }, "response": [] }, + { + "name": "UPDATE item", + "protocolProfileBehavior": { + "disabledSystemHeaders": { + "content-type": true + } + }, + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "default" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"type\": \"Feature\",\n \"id\": \"test-item\",\n \"stac_version\": \"1.0.1\",\n \"stac_extensions\": [\n \"https://stac-extensions.github.io/eo/v1.0.0/schema.json\",\n \"https://stac-extensions.github.io/projection/v1.0.0/schema.json\"\n ],\n \"geometry\": {\n \"coordinates\": [\n [\n [\n 152.15052873427666,\n -33.82243006904891\n ],\n [\n 150.1000346138806,\n -34.257132625788756\n ],\n [\n 149.5776607193635,\n -32.514709769700254\n ],\n [\n 151.6262528041627,\n -32.08081674221862\n ],\n [\n 152.15052873427666,\n -33.82243006904891\n ]\n ]\n ],\n \"type\": \"Polygon\"\n },\n \"properties\": {\n \"datetime\": \"2018-02-12T12:30:22Z\",\n \"landsat:scene_id\": \"LC82081612020043LGN00\",\n \"landsat:row\": \"161\",\n \"gsd\": 15,\n \"landsat:revision\": \"00\",\n \"view:sun_azimuth\": -148.83296771,\n \"instrument\": \"OLI_TIRS\",\n \"landsat:product_id\": \"LC08_L1GT_208161_20200212_20200212_01_RT\",\n \"eo:cloud_cover\": 0,\n \"landsat:tier\": \"RT\",\n \"landsat:processing_level\": \"L1GT\",\n \"landsat:column\": \"208\",\n \"platform\": \"landsat-8\",\n \"proj:epsg\": 32756,\n \"view:sun_elevation\": -37.30791534,\n \"view:off_nadir\": 0,\n \"height\": 2500,\n \"width\": 2500\n },\n \"bbox\": [\n 149.57574,\n -34.25796,\n 152.15194,\n -32.07915\n ],\n \"collection\": \"test-collection\",\n \"assets\": {},\n \"links\": [\n {\n \"href\": \"http://localhost:8081/collections/landsat-8-l1/items/LC82081612020043\",\n \"rel\": \"self\",\n \"type\": \"application/geo+json\"\n },\n {\n \"href\": \"http://localhost:8081/collections/landsat-8-l1\",\n \"rel\": \"parent\",\n \"type\": \"application/json\"\n },\n {\n \"href\": \"http://localhost:8081/collections/landsat-8-l1\",\n \"rel\": \"collection\",\n \"type\": \"application/json\"\n },\n {\n \"href\": \"http://localhost:8081/\",\n \"rel\": \"root\",\n \"type\": \"application/json\"\n }\n ]\n}" + }, + "url": { + "raw": "http://localhost:8080/collections/test-collection/items/test-item", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "8080", + "path": [ + "collections", + "test-collection", + "items", + "test-item" + ] + } + }, + "response": [] + }, { "name": "POST search ", "protocolProfileBehavior": { @@ -236,7 +274,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"collections\":[\"test-collection\"],\n \"intersects\":{\"type\": \"Point\", \"coordinates\": [150.04, -33.14]}\n}" + "raw": "{\n \"collections\":[\"test-collection\"],\n \"limit\": 2,\n \"intersects\":{\"type\": \"Point\", \"coordinates\": [150.04, -33.14]},\n \"query\": {\n \"gsd\":{\"gt\":10}\n }\n}" }, "url": { "raw": "http://localhost:8080/search", @@ -244,7 +282,41 @@ "host": [ "localhost" ], - "port": "8083", + "port": "8080", + "path": [ + "search" + ] + } + }, + "response": [] + }, + { + "name": "POST search ", + "protocolProfileBehavior": { + "disabledSystemHeaders": { + "content-type": true + } + }, + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "default" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"collections\":[\"test-collection\"],\n \"intersects\":{\n \"type\": \"Polygon\", \n \"coordinates\": [\n [\n [\n 98.0859375,\n -7.362466865535738\n ],\n [\n 95.2734375,\n -44.33956524809713\n ],\n [\n 188.4375,\n -50.28933925329178\n ],\n [\n 168.75,\n 10.487811882056695\n ],\n [\n 98.0859375,\n -7.362466865535738\n ]\n ]\n ]\n }\n}" + }, + "url": { + "raw": "http://localhost:8080/search", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "8080", "path": [ "search" ] @@ -270,7 +342,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"collections\":[\"sentinel-s2-l2a\"],\n \"bbox\": [-69.433594,-10.660608,-47.285156,3.513421]\n}" + "raw": "{\n \"bbox\": [97.504892,-45.254738,174.321298,-2.431580]\n}" }, "url": { "raw": "http://localhost:8080/search", @@ -278,7 +350,75 @@ "host": [ "localhost" ], - "port": "8083", + "port": "8080", + "path": [ + "search" + ] + } + }, + "response": [] + }, + { + "name": "POST search Fields Include", + "protocolProfileBehavior": { + "disabledSystemHeaders": { + "content-type": true + } + }, + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "default" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"bbox\": [97.504892,-45.254738,174.321298,-2.431580],\n \"fields\": {\n \"include\": [\"properties.gsd\"]\n }\n}" + }, + "url": { + "raw": "http://localhost:8080/search", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "8080", + "path": [ + "search" + ] + } + }, + "response": [] + }, + { + "name": "POST search Fields Exclude", + "protocolProfileBehavior": { + "disabledSystemHeaders": { + "content-type": true + } + }, + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "default" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"bbox\": [97.504892,-45.254738,174.321298,-2.431580],\n \"fields\": {\n \"exclude\": [\"properties\"]\n }\n}" + }, + "url": { + "raw": "http://localhost:8080/search", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "8080", "path": [ "search" ] @@ -303,7 +443,7 @@ "host": [ "localhost" ], - "port": "8083", + "port": "8080", "path": [ "search" ], From b1f76afc0a06b1edcf502c547c743e73d2c3bf6c Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 15 Jan 2023 18:14:18 +0300 Subject: [PATCH 11/15] update readme --- README.md | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 10d0be8b..d1a10df6 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,14 @@ # STAC FastAPI Elasticsearch -Elasticsearch backend for stac-fastapi. +## Elasticsearch backend for stac-fastapi +----- -Join our [Gitter](https://gitter.im/stac-fastapi-elasticsearch/community) page +### Join our [Gitter](https://gitter.im/stac-fastapi-elasticsearch/community) page -For changes, see the [Changelog](CHANGELOG.md). +### Check out the public Postman documentation [Postman](https://documenter.getpostman.com/view/12888943/2s8ZDSdRHA) + +### For changes, see the [Changelog](CHANGELOG.md) +----- ## Development Environment Setup @@ -23,14 +27,14 @@ Prior to commit, run: ```shell pre-commit run --all-files ``` - +----- ## Building ```shell docker-compose build ``` - +----- ## Running API on localhost:8080 ```shell @@ -55,24 +59,26 @@ curl -X "POST" "http://localhost:8080/collections" \ }' ``` -Note: this "Collections Transaction" behavior is not part of the STAC API, but may be soon. - +Note: this "Collections Transaction" behavior is not part of the STAC API, but may be soon. + +------ ## Testing ```shell make test ``` - +----- ## Ingest sample data ```shell make ingest ``` - +----- ## Elasticsearch Mappings -Mappings apply to search index, not source. - +Mappings apply to search index, not source. + +----- ## Managing Elasticsearch Indices This section covers how to create a snapshot repository and then create and restore snapshots with this. From d024cf67338e11b7e8136a1e6253bd18cfb62f6b Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 15 Jan 2023 18:18:15 +0300 Subject: [PATCH 12/15] docs not implemented --- .github/pull_request_template.md | 2 +- Makefile | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8210bd36..688fc57c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,5 +9,5 @@ - [ ] Code is formatted and linted (run `pre-commit run --all-files`) - [ ] Tests pass (run `make test`) -- [ ] Documentation has been updated to reflect changes, if applicable, and docs build successfully (run `make docs`) +- [ ] Documentation has been updated to reflect changes, if applicable - [ ] Changes are added to the changelog \ No newline at end of file diff --git a/Makefile b/Makefile index 0690fde5..ce2609bc 100644 --- a/Makefile +++ b/Makefile @@ -60,16 +60,6 @@ pybase-install: install: pybase-install pip install -e ./stac_fastapi/elasticsearch[dev,server] -.PHONY: docs-image -docs-image: - docker-compose -f docker-compose.docs.yml \ - build - -.PHONY: docs -docs: docs-image - docker-compose -f docker-compose.docs.yml \ - run docs - .PHONY: ingest ingest: python3 data_loader/data_loader.py From 1db11877ee1119b75fefad392d49a102769e1db5 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 18 Jan 2023 17:32:23 +0300 Subject: [PATCH 13/15] unskip 4 more tests --- .../tests/resources/test_item.py | 34 ++----------------- 1 file changed, 3 insertions(+), 31 deletions(-) diff --git a/stac_fastapi/elasticsearch/tests/resources/test_item.py b/stac_fastapi/elasticsearch/tests/resources/test_item.py index 7537a97f..6e0d9b94 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_item.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_item.py @@ -649,7 +649,7 @@ async def test_pagination_token_idempotent(app_client, ctx, txn_client): ] -async def test_field_extension_get_includes(app_client, ctx): +async def test_field_extension_get_includes(app_client): """Test GET search with included fields (fields extension)""" params = {"fields": "+properties.proj:epsg,+properties.gsd"} resp = await app_client.get("/search", params=params) @@ -657,15 +657,8 @@ async def test_field_extension_get_includes(app_client, ctx): assert not set(feat_properties) - {"proj:epsg", "gsd", "datetime"} -@pytest.mark.skip(reason="fields not implemented") -async def test_field_extension_get_excludes(app_client, load_test_data): +async def test_field_extension_get_excludes(app_client): """Test GET search with included fields (fields extension)""" - test_item = load_test_data("test_item.json") - resp = await app_client.post( - f"/collections/{test_item['collection']}/items", json=test_item - ) - assert resp.status_code == 200 - params = {"fields": "-properties.proj:epsg,-properties.gsd"} resp = await app_client.get("/search", params=params) resp_json = resp.json() @@ -673,15 +666,8 @@ async def test_field_extension_get_excludes(app_client, load_test_data): assert "gsd" not in resp_json["features"][0]["properties"].keys() -@pytest.mark.skip(reason="fields not implemented") -async def test_field_extension_post(app_client, load_test_data): +async def test_field_extension_post(app_client): """Test POST search with included and excluded fields (fields extension)""" - test_item = load_test_data("test_item.json") - resp = await app_client.post( - f"/collections/{test_item['collection']}/items", json=test_item - ) - assert resp.status_code == 200 - body = { "fields": { "exclude": ["assets.B1"], @@ -699,15 +685,8 @@ async def test_field_extension_post(app_client, load_test_data): } -@pytest.mark.skip(reason="fields not implemented") async def test_field_extension_exclude_and_include(app_client, load_test_data): """Test POST search including/excluding same field (fields extension)""" - test_item = load_test_data("test_item.json") - resp = await app_client.post( - f"/collections/{test_item['collection']}/items", json=test_item - ) - assert resp.status_code == 200 - body = { "fields": { "exclude": ["properties.eo:cloud_cover"], @@ -720,15 +699,8 @@ async def test_field_extension_exclude_and_include(app_client, load_test_data): assert "eo:cloud_cover" not in resp_json["features"][0]["properties"] -@pytest.mark.skip(reason="fields not implemented") async def test_field_extension_exclude_default_includes(app_client, load_test_data): """Test POST search excluding a forbidden field (fields extension)""" - test_item = load_test_data("test_item.json") - resp = await app_client.post( - f"/collections/{test_item['collection']}/items", json=test_item - ) - assert resp.status_code == 200 - body = {"fields": {"exclude": ["gsd"]}} resp = await app_client.post("/search", json=body) From 0845f2d1e0b9afda4a1fe87f79e789549a81dce3 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 18 Jan 2023 17:41:03 +0300 Subject: [PATCH 14/15] remove hyphens in readme --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d1a10df6..1397fb14 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,13 @@ # STAC FastAPI Elasticsearch ## Elasticsearch backend for stac-fastapi ------ - + ### Join our [Gitter](https://gitter.im/stac-fastapi-elasticsearch/community) page ### Check out the public Postman documentation [Postman](https://documenter.getpostman.com/view/12888943/2s8ZDSdRHA) ### For changes, see the [Changelog](CHANGELOG.md) ------ + ## Development Environment Setup @@ -27,14 +26,14 @@ Prior to commit, run: ```shell pre-commit run --all-files ``` ------ + ## Building ```shell docker-compose build ``` ------ + ## Running API on localhost:8080 ```shell @@ -61,24 +60,24 @@ curl -X "POST" "http://localhost:8080/collections" \ Note: this "Collections Transaction" behavior is not part of the STAC API, but may be soon. ------- + ## Testing ```shell make test ``` ------ + ## Ingest sample data ```shell make ingest ``` ------ + ## Elasticsearch Mappings Mappings apply to search index, not source. ------ + ## Managing Elasticsearch Indices This section covers how to create a snapshot repository and then create and restore snapshots with this. From 7e0ca91c73eb1450a70891358e1a9317408dd346 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 18 Jan 2023 17:59:37 +0300 Subject: [PATCH 15/15] fix port number in postman collection --- ...tac-fastapi-elasticsearch.postman_collection.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/postman_collections/stac-fastapi-elasticsearch.postman_collection.json b/postman_collections/stac-fastapi-elasticsearch.postman_collection.json index cb141e0c..eb02cbd2 100644 --- a/postman_collections/stac-fastapi-elasticsearch.postman_collection.json +++ b/postman_collections/stac-fastapi-elasticsearch.postman_collection.json @@ -12,12 +12,12 @@ "method": "GET", "header": [], "url": { - "raw": "http://localhost:8083/", + "raw": "http://localhost:8080/", "protocol": "http", "host": [ "localhost" ], - "port": "8083", + "port": "8080", "path": [ "" ] @@ -50,12 +50,12 @@ "method": "DELETE", "header": [], "url": { - "raw": "http://localhost:8083/collections/test-collection", + "raw": "http://localhost:8080/collections/test-collection", "protocol": "http", "host": [ "localhost" ], - "port": "8083", + "port": "8080", "path": [ "collections", "test-collection" @@ -70,12 +70,12 @@ "method": "DELETE", "header": [], "url": { - "raw": "http://localhost:8083/collections/test-collection/items/test-item", + "raw": "http://localhost:8080/collections/test-collection/items/test-item", "protocol": "http", "host": [ "localhost" ], - "port": "8083", + "port": "8080", "path": [ "collections", "test-collection",