From d8dde4310ec9c476a8e0467683215043cf8f45e6 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 4 Feb 2022 02:53:21 -0800 Subject: [PATCH 1/9] create version to test es on test_pypi --- .../elasticsearch/stac_fastapi/elasticsearch/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index 3c7bccdd..04a6346d 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "2.3.0" +__version__ = "0.1.0" From 57b5c2f2782e7738f3bfc658bc4e7fe6cf0bd567 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 4 Feb 2022 02:55:10 -0800 Subject: [PATCH 2/9] push es to test pypi --- .github/workflows/publish-to-test-pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index 18b1af8b..5a4e5ec7 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -30,7 +30,7 @@ jobs: --sdist --wheel --outdir dist/ - stac_fastapi/mongo + stac_fastapi/elasticsearch - name: Publish distribution to Test PyPI # if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@v1.4.2 From 1e424df68c57d4dc7365f10c3034e35d151b5dad Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 14 Feb 2022 07:45:08 -0800 Subject: [PATCH 3/9] add limited and count --- stac_fastapi/mongo/stac_fastapi/mongo/core.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/mongo/stac_fastapi/mongo/core.py b/stac_fastapi/mongo/stac_fastapi/mongo/core.py index b9927ed1..5bb93ac5 100644 --- a/stac_fastapi/mongo/stac_fastapi/mongo/core.py +++ b/stac_fastapi/mongo/stac_fastapi/mongo/core.py @@ -105,10 +105,12 @@ def item_collection( with self.client.start_session() as session: collection_children = self.item_table.find( {"collection": collection_id}, session=session - ).sort( + ).limit(limit).sort( [("properties.datetime", pymongo.ASCENDING), ("id", pymongo.ASCENDING)] ) + matched = self.item_table.count_documents({"collection": collection_id}) + for item in collection_children: response_features.append( self.item_serializer.db_to_stac(item, base_url=base_url) @@ -120,7 +122,7 @@ def item_collection( context_obj = { "returned": count if count <= 10 else limit, "limit": limit, - "matched": len(response_features) or None, + "matched": matched or None, } return ItemCollection( @@ -296,6 +298,8 @@ def post_search( .sort(sort_list) ) + matched = self.item_table.count_documents(queries) + results = [] links = [] @@ -329,7 +333,7 @@ def post_search( context_obj = { "returned": count if count <= 10 else search_request.limit, "limit": search_request.limit, - "matched": len(results) or None, + "matched": matched or None, } return ItemCollection( From 7a7cd13412557a5350d0978f1e3fa2954a3892d0 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 15 Feb 2022 00:12:38 -0800 Subject: [PATCH 4/9] run pre-commit --- stac_fastapi/mongo/stac_fastapi/mongo/core.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/stac_fastapi/mongo/stac_fastapi/mongo/core.py b/stac_fastapi/mongo/stac_fastapi/mongo/core.py index 5bb93ac5..e7c11fec 100644 --- a/stac_fastapi/mongo/stac_fastapi/mongo/core.py +++ b/stac_fastapi/mongo/stac_fastapi/mongo/core.py @@ -103,10 +103,15 @@ def item_collection( base_url = str(kwargs["request"].base_url) with self.client.start_session() as session: - collection_children = self.item_table.find( - {"collection": collection_id}, session=session - ).limit(limit).sort( - [("properties.datetime", pymongo.ASCENDING), ("id", pymongo.ASCENDING)] + collection_children = ( + self.item_table.find({"collection": collection_id}, session=session) + .limit(limit) + .sort( + [ + ("properties.datetime", pymongo.ASCENDING), + ("id", pymongo.ASCENDING), + ] + ) ) matched = self.item_table.count_documents({"collection": collection_id}) From 81a32945c5dc904f140f0aedf735d8ca2ea2bf34 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 15 Feb 2022 11:40:30 -0800 Subject: [PATCH 5/9] save es data locally --- .gitignore | 2 ++ docker-compose.elasticsearch.yml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f00589ff..0bf0e27b 100644 --- a/.gitignore +++ b/.gitignore @@ -131,6 +131,8 @@ dmypy.json # mongo db data /mongo_data +/esdata + # Virtualenv venv diff --git a/docker-compose.elasticsearch.yml b/docker-compose.elasticsearch.yml index 9696bc94..9165064d 100644 --- a/docker-compose.elasticsearch.yml +++ b/docker-compose.elasticsearch.yml @@ -24,6 +24,7 @@ services: volumes: - ./stac_fastapi:/app/stac_fastapi - ./scripts:/app/scripts + - ./esdata:/usr/share/elasticsearch/data depends_on: - elasticsearch command: @@ -44,4 +45,4 @@ services: networks: default: - name: stac-fastapi-network + name: stac-fastapi-es-network From bfe51542a7434c12df22ea5ba2e0f6be0f538b5f Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 15 Feb 2022 11:57:54 -0800 Subject: [PATCH 6/9] add count and limit to post search --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index c17b2dce..c11bd97e 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -308,8 +308,11 @@ def post_search( search = search.sort({field: {"order": sort.direction}}) # search = search.sort({"id.keyword" : {"order" : "asc"}}) + search = search.query()[0:search_request.limit] response = search.execute().to_dict() + count = search.count() + if len(response["hits"]["hits"]) > 0: response_features = [ self.item_serializer.db_to_stac(item["_source"], base_url=base_url) @@ -347,7 +350,6 @@ def post_search( limit = 10 context_obj = None if self.extension_is_enabled("ContextExtension"): - count = len(response_features) context_obj = { "returned": count if count <= 10 else limit, "limit": limit, From c7296ed4940de27e8cf898f66c097ce245567777 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 15 Feb 2022 23:08:20 -0800 Subject: [PATCH 7/9] fix count in es item collection --- .../stac_fastapi/elasticsearch/core.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index c11bd97e..b83f0798 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -100,11 +100,15 @@ def item_collection( links = [] base_url = str(kwargs["request"].base_url) - collection_children = self.client.search( - index="stac_items", - doc_type="_doc", - query={"match_phrase": {"collection": collection_id}}, - ) + search = Search(using=self.client, index="stac_items") + + collection_filter = Q("bool", should=[Q("match_phrase", **{"collection": collection_id})]) + search = search.query(collection_filter) + + count = search.count() + # search = search.sort({"id.keyword" : {"order" : "asc"}}) + search = search.query()[0:limit] + collection_children = search.execute().to_dict() serialized_children = [ self.item_serializer.db_to_stac(item["_source"], base_url=base_url) @@ -113,8 +117,7 @@ def item_collection( context_obj = None if self.extension_is_enabled("ContextExtension"): - count = len(serialized_children) - context_obj = {"returned": count, "limit": limit, "matched": count} + context_obj = {"returned": count if count < limit else limit, "limit": limit, "matched": count} return ItemCollection( type="FeatureCollection", @@ -307,11 +310,11 @@ def post_search( field = sort.field + ".keyword" search = search.sort({field: {"order": sort.direction}}) + count = search.count() # search = search.sort({"id.keyword" : {"order" : "asc"}}) search = search.query()[0:search_request.limit] response = search.execute().to_dict() - count = search.count() if len(response["hits"]["hits"]) > 0: response_features = [ @@ -351,7 +354,7 @@ def post_search( context_obj = None if self.extension_is_enabled("ContextExtension"): context_obj = { - "returned": count if count <= 10 else limit, + "returned": count if count < limit else limit, "limit": limit, "matched": count, } From 5c17e5fab75072231cd56206120fa65993a0da2a Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 15 Feb 2022 23:14:39 -0800 Subject: [PATCH 8/9] run pre-commit --- .../stac_fastapi/elasticsearch/core.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index b83f0798..89ce4989 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -102,7 +102,9 @@ def item_collection( search = Search(using=self.client, index="stac_items") - collection_filter = Q("bool", should=[Q("match_phrase", **{"collection": collection_id})]) + collection_filter = Q( + "bool", should=[Q("match_phrase", **{"collection": collection_id})] + ) search = search.query(collection_filter) count = search.count() @@ -117,7 +119,11 @@ def item_collection( context_obj = None if self.extension_is_enabled("ContextExtension"): - context_obj = {"returned": count if count < limit else limit, "limit": limit, "matched": count} + context_obj = { + "returned": count if count < limit else limit, + "limit": limit, + "matched": count, + } return ItemCollection( type="FeatureCollection", @@ -312,10 +318,9 @@ def post_search( count = search.count() # search = search.sort({"id.keyword" : {"order" : "asc"}}) - search = search.query()[0:search_request.limit] + search = search.query()[0 : search_request.limit] response = search.execute().to_dict() - if len(response["hits"]["hits"]) > 0: response_features = [ self.item_serializer.db_to_stac(item["_source"], base_url=base_url) From b6bd236b2432fd0513bd877ba869a0e9b38c08ab Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 22 Feb 2022 06:47:53 -0800 Subject: [PATCH 9/9] mongo 2.3.0 --- .github/workflows/publish-to-test-pypi.yml | 2 +- stac_fastapi/elasticsearch/setup.py | 2 +- .../elasticsearch/stac_fastapi/elasticsearch/version.py | 2 +- stac_fastapi/mongo/setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index 5a4e5ec7..18b1af8b 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -30,7 +30,7 @@ jobs: --sdist --wheel --outdir dist/ - stac_fastapi/elasticsearch + stac_fastapi/mongo - name: Publish distribution to Test PyPI # if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@v1.4.2 diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 196cd26a..93653197 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -33,7 +33,7 @@ setup( - name="stac-fastapi.elasticsearch", + name="stac-fastapi.nosql.elasticsearch", description="An implementation of STAC API based on the FastAPI framework.", long_description=desc, long_description_content_type="text/markdown", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index 04a6346d..3c7bccdd 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "0.1.0" +__version__ = "2.3.0" diff --git a/stac_fastapi/mongo/setup.py b/stac_fastapi/mongo/setup.py index f0737ca1..847e32b6 100644 --- a/stac_fastapi/mongo/setup.py +++ b/stac_fastapi/mongo/setup.py @@ -32,7 +32,7 @@ setup( - name="stac-fastapi.mongo", + name="stac-fastapi.nosql.mongo", description="An implementation of STAC API based on the FastAPI framework.", long_description=desc, long_description_content_type="text/markdown",