From a10c5a20c2308ef33bd24bf50308a882748b970b Mon Sep 17 00:00:00 2001 From: John Murner Date: Thu, 16 Jan 2025 11:29:05 -0500 Subject: [PATCH 1/9] Updating mapping to double for long --- .../elasticsearch/database_logic.py | 2 +- stac_fastapi/tests/api/test_api.py | 73 +++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index c404b5e5..ff19f182 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -103,7 +103,7 @@ }, # Default all other strings not otherwise specified to keyword {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, - {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, + {"numerics": {"match_mapping_type": "long", "mapping": {"type": "double"}}}, ] ES_ITEMS_MAPPINGS = { diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 64545807..4ce0c919 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -1,4 +1,6 @@ +import random import uuid +from copy import deepcopy from datetime import datetime, timedelta, timezone import pytest @@ -459,3 +461,74 @@ async def test_search_line_string_intersects(app_client, ctx): resp_json = resp.json() assert len(resp_json["features"]) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "value, expected", + [ + (32767, 1), # Short Limit, + (2147483647, 1), # Int Limit + (2147483647 + 5000, 1), # Above int Limit + # All below fail, return three values + (21474836470, 1), # Above int Limit + # This value still fails to return 1 + # Commenting out + (9223372036854775807, 2), + ], +) +async def test_big_int_eo_search( + app_client, txn_client, test_item, test_collection, value, expected +): + + random_str = "".join(random.choice("abcdef") for i in range(random.randint(1, 5))) + collection_id = f"test-collection-eo-{random_str}" + + test_big_int_item = test_item + del test_big_int_item["properties"]["eo:bands"] + test_big_int_item["collection"] = collection_id + test_big_int_collection = test_collection + test_big_int_collection["id"] = collection_id + + # type number + attr = "eo:full_width_half_max" + + stac_extensions = [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + ] + + test_collection["stac_extensions"] = stac_extensions + + test_item["stac_extensions"] = stac_extensions + + await create_collection(txn_client, test_collection) + + for val in [ + value, + value + random.randint(10, 1010), + value - random.randint(10, 1010), + ]: + item = deepcopy(test_item) + item["id"] = str(uuid.uuid4()) + item["properties"][attr] = val + await create_item(txn_client, item) + + params = { + "collections": [item["collection"]], + "filter": { + "args": [ + { + "args": [ + {"property": f"properties.{attr}"}, + value, + ], + "op": "=", + } + ], + "op": "and", + }, + } + resp = await app_client.post("/search", json=params) + resp_json = resp.json() + results = set([x["properties"][attr] for x in resp_json["features"]]) + assert len(results) == expected From 00aa14e18c7a051a59e6289b4102c74aeea5c42d Mon Sep 17 00:00:00 2001 From: John Murner Date: Thu, 16 Jan 2025 11:48:53 -0500 Subject: [PATCH 2/9] Commenting out superduper large number --- stac_fastapi/tests/api/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 4ce0c919..3c84a697 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -474,7 +474,7 @@ async def test_search_line_string_intersects(app_client, ctx): (21474836470, 1), # Above int Limit # This value still fails to return 1 # Commenting out - (9223372036854775807, 2), + # (9223372036854775807, 1), ], ) async def test_big_int_eo_search( From 612b27aaed6041e74b0a8f7a0fb4831b3af37e90 Mon Sep 17 00:00:00 2001 From: John Murner Date: Thu, 16 Jan 2025 12:03:25 -0500 Subject: [PATCH 3/9] Removing comment updating changelog --- CHANGELOG.md | 2 ++ stac_fastapi/tests/api/test_api.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e41a1462..52b3d33a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +- Update dynamic mapping for items to map long values to double versus float [#326](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/326) + ## [v3.2.2] - 2024-12-15 ### Changed diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 3c84a697..7f60d360 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -470,7 +470,6 @@ async def test_search_line_string_intersects(app_client, ctx): (32767, 1), # Short Limit, (2147483647, 1), # Int Limit (2147483647 + 5000, 1), # Above int Limit - # All below fail, return three values (21474836470, 1), # Above int Limit # This value still fails to return 1 # Commenting out From 648c9be87d3f918c798173f81ccc15beac0edd80 Mon Sep 17 00:00:00 2001 From: John Murner Date: Fri, 17 Jan 2025 14:43:52 -0500 Subject: [PATCH 4/9] Updating Open Search --- .../stac_fastapi/opensearch/database_logic.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 63a42427..05419879 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -15,8 +15,8 @@ from opensearchpy.helpers.search import Search from starlette.requests import Request -from stac_fastapi.core import serializers from stac_fastapi.core.extensions import filter +from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon from stac_fastapi.opensearch.config import ( AsyncOpensearchSettings as AsyncSearchSettings, @@ -105,7 +105,7 @@ }, # Default all other strings not otherwise specified to keyword {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, - {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, + {"numerics": {"match_mapping_type": "long", "mapping": {"type": "double"}}}, ] ES_ITEMS_MAPPINGS = { @@ -330,11 +330,9 @@ class DatabaseLogic: client = AsyncSearchSettings().create_client sync_client = SyncSearchSettings().create_client - item_serializer: Type[serializers.ItemSerializer] = attr.ib( - default=serializers.ItemSerializer - ) - collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( - default=serializers.CollectionSerializer + item_serializer: Type[ItemSerializer] = attr.ib(default=ItemSerializer) + collection_serializer: Type[CollectionSerializer] = attr.ib( + default=CollectionSerializer ) extensions: List[str] = attr.ib(default=attr.Factory(list)) From ba1ea92486f7a7ab9b13b3f30c7f659e528b2aa6 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Thu, 24 Apr 2025 22:57:36 +0800 Subject: [PATCH 5/9] Update database_logic.py --- stac_fastapi/core/stac_fastapi/core/database_logic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/core/stac_fastapi/core/database_logic.py b/stac_fastapi/core/stac_fastapi/core/database_logic.py index 7ddd8af7..fbc78015 100644 --- a/stac_fastapi/core/stac_fastapi/core/database_logic.py +++ b/stac_fastapi/core/stac_fastapi/core/database_logic.py @@ -96,7 +96,7 @@ class Geometry(Protocol): # noqa }, # Default all other strings not otherwise specified to keyword {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, - {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, + {"numerics": {"match_mapping_type": "long", "mapping": {"type": "double"}}}, ] ES_ITEMS_MAPPINGS = { From bf3e5b8a4e031c7c4ec411b9587d2e3d1d6f980d Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Thu, 24 Apr 2025 23:00:30 +0800 Subject: [PATCH 6/9] Remove serializers import --- .../opensearch/stac_fastapi/opensearch/database_logic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index a3a66469..998e4851 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -14,7 +14,6 @@ from starlette.requests import Request -from stac_fastapi.core import serializers from stac_fastapi.core.base_database_logic import BaseDatabaseLogic from stac_fastapi.core.database_logic import ( COLLECTIONS_INDEX, From f4dc677741ce5927b6d9c8df74a4a3e02e235c1e Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Thu, 24 Apr 2025 23:08:06 +0800 Subject: [PATCH 7/9] Remove unused import timezone --- stac_fastapi/tests/api/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 47f46cb9..f92cc034 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -1,7 +1,7 @@ import random import uuid from copy import deepcopy -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta import pytest From 6cd5988274a1ee468d916a40567476a4fb85c135 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 25 Apr 2025 15:32:04 +0800 Subject: [PATCH 8/9] lint --- .../opensearch/stac_fastapi/opensearch/database_logic.py | 1 - stac_fastapi/tests/api/test_api.py | 1 - 2 files changed, 2 deletions(-) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 998e4851..29bd6030 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -13,7 +13,6 @@ from opensearchpy.helpers.search import Search from starlette.requests import Request - from stac_fastapi.core.base_database_logic import BaseDatabaseLogic from stac_fastapi.core.database_logic import ( COLLECTIONS_INDEX, diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index f92cc034..91c0e811 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -3,7 +3,6 @@ from copy import deepcopy from datetime import datetime, timedelta - import pytest from ..conftest import create_collection, create_item From 80a53626ce423d8ae17b8d5f78fcfa5697cc1b0e Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 25 Apr 2025 16:15:09 +0800 Subject: [PATCH 9/9] double mapping --- stac_fastapi/core/stac_fastapi/core/database_logic.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/core/stac_fastapi/core/database_logic.py b/stac_fastapi/core/stac_fastapi/core/database_logic.py index fbc78015..85ebcf21 100644 --- a/stac_fastapi/core/stac_fastapi/core/database_logic.py +++ b/stac_fastapi/core/stac_fastapi/core/database_logic.py @@ -96,7 +96,13 @@ class Geometry(Protocol): # noqa }, # Default all other strings not otherwise specified to keyword {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, - {"numerics": {"match_mapping_type": "long", "mapping": {"type": "double"}}}, + {"long_to_double": {"match_mapping_type": "long", "mapping": {"type": "double"}}}, + { + "double_to_double": { + "match_mapping_type": "double", + "mapping": {"type": "double"}, + } + }, ] ES_ITEMS_MAPPINGS = {