From 6e8fe9237003974b7d0395d06dfd6e4cb516e094 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 3 Aug 2024 11:48:28 +0300 Subject: [PATCH 01/14] update to sfapi v3.0.0 --- stac_fastapi/core/setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index bb894b9b..ac16bfc8 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -10,9 +10,9 @@ "attrs>=23.2.0", "pydantic[dotenv]", "stac_pydantic>=3", - "stac-fastapi.types==3.0.0a4", - "stac-fastapi.api==3.0.0a4", - "stac-fastapi.extensions==3.0.0a4", + "stac-fastapi.types==3.0.0", + "stac-fastapi.api==3.0.0", + "stac-fastapi.extensions==3.0.0", "orjson", "overrides", "geojson-pydantic", From ad8e861b4e3deee8fd01dfd2c94e02163dbd3cbf Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 3 Aug 2024 12:14:42 +0300 Subject: [PATCH 02/14] change test --- stac_fastapi/tests/extensions/test_aggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/tests/extensions/test_aggregation.py b/stac_fastapi/tests/extensions/test_aggregation.py index dd471159..752563d4 100644 --- a/stac_fastapi/tests/extensions/test_aggregation.py +++ b/stac_fastapi/tests/extensions/test_aggregation.py @@ -186,7 +186,7 @@ async def test_aggregate_filter_extension_neq_post(app_client, ctx): } resp = await app_client.post("/aggregate", json=params) assert resp.status_code == 200 - assert resp.json()["aggregations"][0]["value"] == 0 + assert resp.json()["aggregations"][0]["value"] == 1 @pytest.mark.asyncio From 5024eea6f3ba79564d2f2756e7221e04162deef9 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 3 Aug 2024 12:49:17 +0300 Subject: [PATCH 03/14] increase timeout minutes --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index b15494c4..f28f385d 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -12,7 +12,7 @@ on: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 20 services: From 97055da30499560bd7f1a03f3ed74be356a5d255 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 3 Aug 2024 13:31:18 +0300 Subject: [PATCH 04/14] revert test --- stac_fastapi/tests/extensions/test_aggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/tests/extensions/test_aggregation.py b/stac_fastapi/tests/extensions/test_aggregation.py index 752563d4..dd471159 100644 --- a/stac_fastapi/tests/extensions/test_aggregation.py +++ b/stac_fastapi/tests/extensions/test_aggregation.py @@ -186,7 +186,7 @@ async def test_aggregate_filter_extension_neq_post(app_client, ctx): } resp = await app_client.post("/aggregate", json=params) assert resp.status_code == 200 - assert resp.json()["aggregations"][0]["value"] == 1 + assert resp.json()["aggregations"][0]["value"] == 0 @pytest.mark.asyncio From aa8a97da4c61928087cea36cda9aef615473173f Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 3 Aug 2024 13:31:30 +0300 Subject: [PATCH 05/14] skip broken tests for now --- stac_fastapi/tests/resources/test_item.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stac_fastapi/tests/resources/test_item.py b/stac_fastapi/tests/resources/test_item.py index d84577d0..0a0bd420 100644 --- a/stac_fastapi/tests/resources/test_item.py +++ b/stac_fastapi/tests/resources/test_item.py @@ -577,6 +577,7 @@ async def test_pagination_base_links(app_client, ctx): @pytest.mark.asyncio +@pytest.mark.skip(reason="Not working in stac-fastapi v3.0.0?") async def test_pagination_links_behavior(app_client, ctx, txn_client): """Test the links in pagination specifically look for last page behavior.""" @@ -619,6 +620,7 @@ async def test_pagination_links_behavior(app_client, ctx, txn_client): @pytest.mark.asyncio +@pytest.mark.skip(reason="Not working in stac-fastapi v3.0.0?") async def test_pagination_item_collection(app_client, ctx, txn_client): """Test item collection pagination links (paging extension)""" ids = [ctx.item["id"]] From 372d241a7fffdce698d020a0d90ab5a4c9f291fd Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 3 Aug 2024 13:33:22 +0300 Subject: [PATCH 06/14] update gh workflow --- .github/workflows/cicd.yml | 44 +++++++++++++++----------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index f28f385d..85a7ca60 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -15,7 +15,6 @@ jobs: timeout-minutes: 20 services: - elasticsearch_8_svc: image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0 env: @@ -63,22 +62,31 @@ jobs: OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m ports: - 9202:9202 + strategy: matrix: python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12"] + backend: [ "elasticsearch7", "elasticsearch8", "opensearch"] - name: Python ${{ matrix.python-version }} testing + name: Python ${{ matrix.python-version }} testing with ${{ matrix.backend }} steps: - name: Check out repository code uses: actions/checkout@v4 - # Setup Python (faster than using Python container) - name: Setup Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Cache pipenv packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile.lock') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Lint code if: ${{ matrix.python-version == 3.11 }} run: | @@ -101,35 +109,17 @@ jobs: run: | pip install ./stac_fastapi/opensearch[dev,server] - - name: Run test suite against Elasticsearch 7.x - run: | - pipenv run pytest -svvv - env: - ENVIRONMENT: testing - ES_PORT: 9400 - ES_HOST: 172.17.0.1 - ES_USE_SSL: false - ES_VERIFY_CERTS: false - BACKEND: elasticsearch - - - name: Run test suite against Elasticsearch 8.x + - name: Install pytest-timeout run: | - pipenv run pytest -svvv - env: - ENVIRONMENT: testing - ES_PORT: 9200 - ES_HOST: 172.17.0.1 - ES_USE_SSL: false - ES_VERIFY_CERTS: false - BACKEND: elasticsearch + pip install pytest-timeout - - name: Run test suite against OpenSearch 2.11.1 + - name: Run test suite run: | - pipenv run pytest -svvv + pipenv run pytest -svvv --timeout=300 env: ENVIRONMENT: testing - ES_PORT: 9202 + ES_PORT: ${{ matrix.backend == 'elasticsearch7' && '9400' || matrix.backend == 'elasticsearch8' && '9200' || '9202' }} ES_HOST: 172.17.0.1 ES_USE_SSL: false ES_VERIFY_CERTS: false - BACKEND: opensearch \ No newline at end of file + BACKEND: ${{ matrix.backend == 'elasticsearch7' && 'elasticsearch' || matrix.backend == 'elasticsearch8' && 'elasticsearch' || 'opensearch' }} From f7d6884c16c30c64f8b820e6d2dc1d1346034160 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 4 Aug 2024 02:33:14 +0300 Subject: [PATCH 07/14] unskip pagination tests --- stac_fastapi/tests/resources/test_item.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stac_fastapi/tests/resources/test_item.py b/stac_fastapi/tests/resources/test_item.py index 0a0bd420..d84577d0 100644 --- a/stac_fastapi/tests/resources/test_item.py +++ b/stac_fastapi/tests/resources/test_item.py @@ -577,7 +577,6 @@ async def test_pagination_base_links(app_client, ctx): @pytest.mark.asyncio -@pytest.mark.skip(reason="Not working in stac-fastapi v3.0.0?") async def test_pagination_links_behavior(app_client, ctx, txn_client): """Test the links in pagination specifically look for last page behavior.""" @@ -620,7 +619,6 @@ async def test_pagination_links_behavior(app_client, ctx, txn_client): @pytest.mark.asyncio -@pytest.mark.skip(reason="Not working in stac-fastapi v3.0.0?") async def test_pagination_item_collection(app_client, ctx, txn_client): """Test item collection pagination links (paging extension)""" ids = [ctx.item["id"]] From 6024840323d2b063905e061e550939ae074a1a9f Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 4 Aug 2024 02:35:25 +0300 Subject: [PATCH 08/14] get token from kwargs --- stac_fastapi/core/stac_fastapi/core/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 6cc8f7a5..3290c052 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -280,7 +280,7 @@ async def item_collection( bbox: Optional[BBox] = None, datetime: Optional[DateTimeType] = None, limit: int = 10, - token: str = None, + token: Optional[str] = None, **kwargs, ) -> stac_types.ItemCollection: """Read items from a specific collection in the database. @@ -302,6 +302,8 @@ async def item_collection( Exception: If any error occurs while reading the items from the database. """ request: Request = kwargs["request"] + token = request.query_params.get('token') + base_url = str(request.base_url) collection = await self.get_collection( From 992a352a506962e71900e9c80593b209e4abce38 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 4 Aug 2024 02:39:06 +0300 Subject: [PATCH 09/14] lint --- stac_fastapi/core/stac_fastapi/core/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 3290c052..d96a2e47 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -302,8 +302,8 @@ async def item_collection( Exception: If any error occurs while reading the items from the database. """ request: Request = kwargs["request"] - token = request.query_params.get('token') - + token = request.query_params.get("token") + base_url = str(request.base_url) collection = await self.get_collection( From cee78090a2d321e472db5a85a009937c84c6708e Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 4 Aug 2024 02:44:59 +0300 Subject: [PATCH 10/14] ues Optional --- stac_fastapi/core/stac_fastapi/core/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index d96a2e47..71ae6899 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -279,7 +279,7 @@ async def item_collection( collection_id: str, bbox: Optional[BBox] = None, datetime: Optional[DateTimeType] = None, - limit: int = 10, + limit: Optional[int] = 10, token: Optional[str] = None, **kwargs, ) -> stac_types.ItemCollection: From 2f603d34ab60911f1b28055120e4b19a41891fa2 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 4 Aug 2024 02:49:27 +0300 Subject: [PATCH 11/14] update versions --- stac_fastapi/core/stac_fastapi/core/version.py | 2 +- stac_fastapi/elasticsearch/setup.py | 2 +- .../elasticsearch/stac_fastapi/elasticsearch/version.py | 2 +- stac_fastapi/opensearch/setup.py | 2 +- stac_fastapi/opensearch/stac_fastapi/opensearch/version.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index ad0a12bd..e0db94fc 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.0.0a3" +__version__ = "3.0.0" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index eee387e0..37fc81b3 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==3.0.0a3", + "stac-fastapi.core==3.0.0", "elasticsearch[async]==8.11.0", "elasticsearch-dsl==8.11.0", "uvicorn", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index ad0a12bd..e0db94fc 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.0.0a3" +__version__ = "3.0.0" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index 689b3894..c79f669e 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==3.0.0a3", + "stac-fastapi.core==3.0.0", "opensearch-py==2.4.2", "opensearch-py[async]==2.4.2", "uvicorn", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index ad0a12bd..e0db94fc 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.0.0a3" +__version__ = "3.0.0" From d08b09cc86e6dc95d8fd08dbb344f4a1c0d79856 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 4 Aug 2024 02:52:55 +0300 Subject: [PATCH 12/14] update changelog --- CHANGELOG.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 870366c9..1c2dbd1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [v3.0.0] - 2024-08-14 ### Changed - Aggregation bug fixes [#281](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/281) +- Updated stac-fastapi libraries to v3.0.0 [#282](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/282) ## [v3.0.0a3] - 2024-07-17 @@ -238,11 +239,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: -[v3.0.0a3]: -[v3.0.0a2]: -[v3.0.0a1]: -[v3.0.0a0]: +[Unreleased]: +[v3.0.0]: [v2.4.1]: [v2.4.0]: [v2.3.0]: From 80525a7455064d3aa360080bd680c4d3d3c8828a Mon Sep 17 00:00:00 2001 From: James Date: Mon, 5 Aug 2024 20:40:19 -0400 Subject: [PATCH 13/14] remove filter-lang regex --- stac_fastapi/core/stac_fastapi/core/core.py | 8 -------- .../core/stac_fastapi/core/extensions/aggregation.py | 12 ------------ 2 files changed, 20 deletions(-) diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 71ae6899..8e360ef4 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -492,14 +492,6 @@ async def get_search( "query": orjson.loads(query) if query else query, } - # this is borrowed from stac-fastapi-pgstac - # Kludgy fix because using factory does not allow alias for filter-lan - query_params = str(request.query_params) - if filter_lang is None: - match = re.search(r"filter-lang=([a-z0-9-]+)", query_params, re.IGNORECASE) - if match: - filter_lang = match.group(1) - if datetime: base_args["datetime"] = self._format_datetime_range(datetime) diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py b/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py index facc8bcc..ba331771 100644 --- a/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py +++ b/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py @@ -351,18 +351,6 @@ async def aggregate( if datetime: base_args["datetime"] = self._format_datetime_range(datetime) - # this is borrowed from stac-fastapi-pgstac - # Kludgy fix because using factory does not allow alias for filter-lang - # If the value is the default, check if the request is different. - query_params = str(request.query_params) - if filter_lang is None: - match = re.search( - r"filter-lang=([a-z0-9-]+)", query_params, re.IGNORECASE - ) - if match: - filter_lang = match.group(1) - else: - filter_lang = "cql2-text" if filter: base_args["filter"] = self.get_filter(filter, filter_lang) aggregate_request = EsAggregationExtensionPostRequest(**base_args) From 218db9dd730dd6df405c81627a728a7e66736af9 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 5 Aug 2024 20:44:16 -0400 Subject: [PATCH 14/14] linting --- stac_fastapi/core/stac_fastapi/core/core.py | 1 - stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py | 1 - 2 files changed, 2 deletions(-) diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 8e360ef4..df20c103 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -1,6 +1,5 @@ """Core client.""" import logging -import re from datetime import datetime as datetime_type from datetime import timezone from enum import Enum diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py b/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py index ba331771..241373e7 100644 --- a/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py +++ b/stac_fastapi/core/stac_fastapi/core/extensions/aggregation.py @@ -1,6 +1,5 @@ """Request model for the Aggregation extension.""" -import re from datetime import datetime from datetime import datetime as datetime_type from typing import Dict, List, Literal, Optional, Union