From 25ebdc410dde9726913ccd27dd6c348aea7da798 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Fri, 4 Mar 2022 16:40:13 -0500 Subject: [PATCH 01/10] improve elasticsearch mappings --- .pre-commit-config.yaml | 106 +++++++++--------- README.md | 9 ++ stac_fastapi/elasticsearch/setup.py | 2 + .../stac_fastapi/elasticsearch/serializers.py | 2 +- .../elasticsearch/transactions.py | 96 +++++++++++++--- 5 files changed, 145 insertions(+), 70 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77f70226..b1512306 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,57 +1,53 @@ repos: - - repo: https://github.com/PyCQA/isort - rev: 5.8.0 - hooks: - - id: isort - language_version: python3.8 - - - repo: https://github.com/psf/black - rev: 20.8b1 - hooks: - - id: black - args: ['--safe'] - language_version: python3.8 - - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.0 - hooks: - - id: flake8 - language_version: python3.8 - args: [ - # E501 let black handle all line length decisions - # W503 black conflicts with "line break before operator" rule - # E203 black conflicts with "whitespace before ':'" rule - '--ignore=E501,W503,E203,C901'] - - - repo: https://github.com/chewse/pre-commit-mirrors-pydocstyle - # 2.1.1 - rev: v2.1.1 - hooks: - - id: pydocstyle - language_version: python3.8 - exclude: '.*(test|alembic|scripts).*' - args: [ - # Check for docstring presence only - '--select=D1', + - repo: https://github.com/PyCQA/isort + rev: 5.8.0 + hooks: + - id: isort + language_version: python3.8 + - repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + args: [ '--safe' ] + language_version: python3.8 + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.0 + hooks: + - id: flake8 + language_version: python3.8 + args: [ + # E501 let black handle all line length decisions + # W503 black conflicts with "line break before operator" rule + # E203 black conflicts with "whitespace before ':'" rule + '--ignore=E501,W503,E203,C901' ] + - repo: https://github.com/chewse/pre-commit-mirrors-pydocstyle + # 2.1.1 + rev: v2.1.1 + hooks: + - id: pydocstyle + language_version: python3.8 + exclude: '.*(test|alembic|scripts).*' + args: [ + # Check for docstring presence only + '--select=D1', - ] - # Don't require docstrings for tests - # '--match=(?!test).*\.py'] -# - -# repo: https://github.com/pre-commit/mirrors-mypy -# rev: v0.770 -# hooks: -# - id: mypy -# language_version: python3.8 -# args: [--no-strict-optional, --ignore-missing-imports] - - - repo: https://github.com/PyCQA/pydocstyle - rev: 6.0.0 - hooks: - - id: pydocstyle - language_version: python3.8 - exclude: '.*(test|alembic|scripts).*' - #args: [ - # Don't require docstrings for tests - #'--match=(?!test|alembic|scripts).*\.py', - #] \ No newline at end of file + ] + # Don't require docstrings for tests + # '--match=(?!test).*\.py'] + # - + # repo: https://github.com/pre-commit/mirrors-mypy + # rev: v0.770 + # hooks: + # - id: mypy + # language_version: python3.8 + # args: [--no-strict-optional, --ignore-missing-imports] + - repo: https://github.com/PyCQA/pydocstyle + rev: 6.0.0 + hooks: + - id: pydocstyle + language_version: python3.8 + exclude: '.*(test|alembic|scripts).*' + #args: [ + # Don't require docstrings for tests + #'--match=(?!test|alembic|scripts).*\.py', + #] \ No newline at end of file diff --git a/README.md b/README.md index fa248c89..58b8e503 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,11 @@ Prior to commit, run: pre-commit run --all-files` ``` +```shell +cd stac_fastapi/elasticsearch +pip install .[dev] +``` + ## Building ``` @@ -37,3 +42,7 @@ make test ``` make ingest ``` + +## Elasticsearch Mappings + +Mappings apply to search index, not source. \ No newline at end of file diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index e78bad36..596d71fd 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -27,6 +27,8 @@ "pre-commit", "requests", "ciso8601", + "overrides", + "black", ], "docs": ["mkdocs", "mkdocs-material", "pdocs"], "server": ["uvicorn[standard]>=0.12.0,<0.14.0"], diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/serializers.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/serializers.py index a8c19413..f196ff7e 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/serializers.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/serializers.py @@ -24,7 +24,7 @@ class ItemSerializer(Serializer): """Serialization methods for STAC items.""" @classmethod - def stac_to_db(cls, stac_data: TypedDict, base_url: str) -> stac_types.Item: + def stac_to_db(cls, stac_data: stac_types.Item, base_url: str) -> stac_types.Item: """Transform STAC Item to database-ready STAC Item.""" item_links = ItemLinks( collection_id=stac_data["collection"], diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py index 1746b8f9..86fc4d67 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py @@ -6,6 +6,7 @@ import attr import elasticsearch from elasticsearch import helpers +from overrides import overrides from stac_fastapi.elasticsearch.config import ElasticsearchSettings from stac_fastapi.elasticsearch.serializers import CollectionSerializer, ItemSerializer @@ -30,26 +31,87 @@ class TransactionsClient(BaseTransactionsClient): settings = ElasticsearchSettings() client = settings.create_client - def _create_item_index(self): - mapping = { - "mappings": { - "properties": { - "geometry": {"type": "geo_shape"}, - "id": {"type": "text", "fields": {"keyword": {"type": "keyword"}}}, - "properties__datetime": { - "type": "text", - "fields": {"keyword": {"type": "keyword"}}, - }, - } + dynamicTemplates = [ + # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md + { + "descriptions": { + "match_mapping_type": "string", + "match": "description", + "mapping": {"type": "text"}, + } + }, + { + "titles": { + "match_mapping_type": "string", + "match": "title", + "mapping": {"type": "text"}, + } + }, + # Projection Extension https://github.com/stac-extensions/projection + {"proj_epsg": {"match": "proj:epsg", "mapping": {"type": "integer"}}}, + { + "proj_projjson": { + "match": "proj:projjson", + "mapping": {"type": "object", "enabled": False}, + } + }, + { + "proj_centroid": { + "match_mapping_type": "string", + "match": "proj:centroid", + "mapping": {"type": "geo_point"}, } - } + }, + { + "proj_geometry": { + "match_mapping_type": "string", + "match": "proj:geometry", + "mapping": {"type": "geo_shape"}, + } + }, + { + "no_index_href": { + "match": "href", + "mapping": {"type": "text", "index": False}, + } + }, + # Default all other strings not otherwise specified to keyword + {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, + {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, + ] + + mappings = { + "numeric_detection": False, + "dynamic_templates": dynamicTemplates, + "properties": { + "geometry": {"type": "geo_shape"}, + "assets": {"type": "object", "enabled": False}, + "links": {"type": "object", "enabled": False}, + "properties": { + "type": "object", + "properties": { + # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md + "datetime": {"type": "date"}, + "start_datetime": {"type": "date"}, + "end_datetime": {"type": "date"}, + "created": {"type": "date"}, + "updated": {"type": "date"}, + # Satellite Extension https://github.com/stac-extensions/sat + "sat:absolute_orbit": {"type": "integer"}, + "sat:relative_orbit": {"type": "integer"}, + }, + }, + }, + } - _ = self.client.indices.create( + def _create_item_index(self): + self.client.indices.create( index="stac_items", - body=mapping, + body=self.mappings, ignore=400, # ignore 400 already exists code ) + @overrides def create_item(self, model: stac_types.Item, **kwargs): """Create item.""" base_url = str(kwargs["request"].base_url) @@ -83,6 +145,7 @@ def create_item(self, model: stac_types.Item, **kwargs): ) return ItemSerializer.db_to_stac(model, base_url) + @overrides def create_collection(self, model: stac_types.Collection, **kwargs): """Create collection.""" base_url = str(kwargs["request"].base_url) @@ -100,6 +163,7 @@ def create_collection(self, model: stac_types.Collection, **kwargs): ) return CollectionSerializer.db_to_stac(model, base_url) + @overrides def update_item(self, model: stac_types.Item, **kwargs): """Update item.""" base_url = str(kwargs["request"].base_url) @@ -118,6 +182,7 @@ def update_item(self, model: stac_types.Item, **kwargs): # body=model) return ItemSerializer.db_to_stac(model, base_url) + @overrides def update_collection(self, model: stac_types.Collection, **kwargs): """Update collection.""" base_url = str(kwargs["request"].base_url) @@ -130,6 +195,7 @@ def update_collection(self, model: stac_types.Collection, **kwargs): return CollectionSerializer.db_to_stac(model, base_url) + @overrides def delete_item(self, item_id: str, collection_id: str, **kwargs): """Delete item.""" try: @@ -138,6 +204,7 @@ def delete_item(self, item_id: str, collection_id: str, **kwargs): raise NotFoundError(f"Item {item_id} not found") self.client.delete(index="stac_items", doc_type="_doc", id=item_id) + @overrides def delete_collection(self, collection_id: str, **kwargs): """Delete collection.""" try: @@ -178,6 +245,7 @@ def bulk_sync(self, processed_items): ] helpers.bulk(self.client, actions) + @overrides def bulk_item_insert(self, items: Items, **kwargs) -> str: """Bulk item insertion using es.""" transactions_client = TransactionsClient() From a4992a4971fcdfe934356ff68e0b99873fd3234d Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Fri, 4 Mar 2022 16:49:31 -0500 Subject: [PATCH 02/10] remove black --- Dockerfile | 7 +++++-- stac_fastapi/elasticsearch/setup.py | 1 - 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c43f980f..53a0d003 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,10 @@ FROM python:3.8-slim as base FROM base as builder # Any python libraries that require system libraries to be installed will likely # need the following packages in order to build -RUN apt-get update && apt-get install -y build-essential git +RUN apt-get update && \ + apt-get install -y build-essential git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt @@ -17,4 +20,4 @@ COPY . /app ENV PATH=$PATH:/install/bin RUN mkdir -p /install && \ - pip install -e ./stac_fastapi/elasticsearch[dev,server] + pip install --no-cache-dir -e ./stac_fastapi/elasticsearch[dev,server] diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 596d71fd..3088351a 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -28,7 +28,6 @@ "requests", "ciso8601", "overrides", - "black", ], "docs": ["mkdocs", "mkdocs-material", "pdocs"], "server": ["uvicorn[standard]>=0.12.0,<0.14.0"], From 2cef40076bfbca1204c668defcb7146a4792a171 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Mon, 14 Mar 2022 16:48:18 -0400 Subject: [PATCH 03/10] apply mappings, fix type problem with transactions methods --- Dockerfile | 5 +- README.md | 10 +- .../stac_fastapi/elasticsearch/core.py | 3 +- .../elasticsearch/transactions.py | 125 ++++++++++-------- 4 files changed, 78 insertions(+), 65 deletions(-) diff --git a/Dockerfile b/Dockerfile index 53a0d003..f03ba0e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,10 +14,9 @@ ARG install_dev_dependencies=true WORKDIR /app -# Install stac_fastapi.types COPY . /app ENV PATH=$PATH:/install/bin -RUN mkdir -p /install && \ - pip install --no-cache-dir -e ./stac_fastapi/elasticsearch[dev,server] +RUN mkdir -p /install +RUN pip install --no-cache-dir -e ./stac_fastapi/elasticsearch[dev,server] diff --git a/README.md b/README.md index 58b8e503..f44b6c1c 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Install [pre-commit](https://pre-commit.com/#install). Prior to commit, run: -``` +```shell pre-commit run --all-files` ``` @@ -21,25 +21,25 @@ pip install .[dev] ## Building -``` +```shell docker-compose build ``` ## Running API on localhost:8083 -``` +```shell docker-compose up ``` ## Testing -``` +```shell make test ``` ## Ingest sample data -``` +```shell make ingest ``` diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 8c3468fe..5db2007e 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -227,7 +227,8 @@ def get_search( return resp - def bbox2poly(self, b0, b1, b2, b3): + @staticmethod + def bbox2poly(b0, b1, b2, b3): """Transform bbox to polygon.""" poly = [[[b0, b1], [b2, b1], [b2, b3], [b0, b3], [b0, b1]]] return poly diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py index 86fc4d67..954541b3 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py @@ -2,6 +2,7 @@ import logging from datetime import datetime, timezone +from typing import Optional import attr import elasticsearch @@ -104,25 +105,26 @@ class TransactionsClient(BaseTransactionsClient): }, } - def _create_item_index(self): + def create_item_index(self): + """Create the index for Items.""" self.client.indices.create( index="stac_items", - body=self.mappings, + body={"mappings": self.mappings}, ignore=400, # ignore 400 already exists code ) @overrides - def create_item(self, model: stac_types.Item, **kwargs): + def create_item(self, item: stac_types.Item, **kwargs) -> stac_types.Item: """Create item.""" base_url = str(kwargs["request"].base_url) - self._create_item_index() + self.create_item_index() # If a feature collection is posted - if model["type"] == "FeatureCollection": + if item["type"] == "FeatureCollection": bulk_client = BulkTransactionsClient() processed_items = [ bulk_client._preprocess_item(item, base_url) - for item in model["features"] + for item in item["features"] ] return_msg = f"Successfully added {len(processed_items)} items." bulk_client.bulk_sync(processed_items) @@ -130,82 +132,91 @@ def create_item(self, model: stac_types.Item, **kwargs): return return_msg # If a single item is posted - if not self.client.exists(index="stac_collections", id=model["collection"]): - raise ForeignKeyError(f"Collection {model['collection']} does not exist") + if not self.client.exists(index="stac_collections", id=item["collection"]): + raise ForeignKeyError(f"Collection {item['collection']} does not exist") - if self.client.exists(index="stac_items", id=model["id"]): + if self.client.exists(index="stac_items", id=item["id"]): raise ConflictError( - f"Item {model['id']} in collection {model['collection']} already exists" + f"Item {item['id']} in collection {item['collection']} already exists" ) - data = ItemSerializer.stac_to_db(model, base_url) + data = ItemSerializer.stac_to_db(item, base_url) self.client.index( - index="stac_items", doc_type="_doc", id=model["id"], document=data + index="stac_items", doc_type="_doc", id=item["id"], document=data ) - return ItemSerializer.db_to_stac(model, base_url) + return ItemSerializer.db_to_stac(item, base_url) @overrides - def create_collection(self, model: stac_types.Collection, **kwargs): - """Create collection.""" - base_url = str(kwargs["request"].base_url) - collection_links = CollectionLinks( - collection_id=model["id"], base_url=base_url - ).create_links() - model["links"] = collection_links - - self._create_item_index() - - if self.client.exists(index="stac_collections", id=model["id"]): - raise ConflictError(f"Collection {model['id']} already exists") - self.client.index( - index="stac_collections", doc_type="_doc", id=model["id"], document=model - ) - return CollectionSerializer.db_to_stac(model, base_url) - - @overrides - def update_item(self, model: stac_types.Item, **kwargs): + def update_item(self, item: stac_types.Item, **kwargs) -> stac_types.Item: """Update item.""" base_url = str(kwargs["request"].base_url) now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - model["properties"]["updated"] = str(now) + item["properties"]["updated"] = str(now) - if not self.client.exists(index="stac_collections", id=model["collection"]): - raise ForeignKeyError(f"Collection {model['collection']} does not exist") - if not self.client.exists(index="stac_items", id=model["id"]): + if not self.client.exists(index="stac_collections", id=item["collection"]): + raise ForeignKeyError(f"Collection {item['collection']} does not exist") + if not self.client.exists(index="stac_items", id=item["id"]): raise NotFoundError( - f"Item {model['id']} in collection {model['collection']} doesn't exist" + f"Item {item['id']} in collection {item['collection']} doesn't exist" ) - self.delete_item(model["id"], model["collection"]) - self.create_item(model, **kwargs) + self.delete_item(item["id"], item["collection"]) + self.create_item(item, **kwargs) # self.client.update(index="stac_items",doc_type='_doc',id=model["id"], # body=model) - return ItemSerializer.db_to_stac(model, base_url) + return ItemSerializer.db_to_stac(item, base_url) @overrides - def update_collection(self, model: stac_types.Collection, **kwargs): - """Update collection.""" - base_url = str(kwargs["request"].base_url) + def delete_item( + self, item_id: str, collection_id: str, **kwargs + ) -> stac_types.Item: + """Delete item.""" try: - _ = self.client.get(index="stac_collections", id=model["id"]) + _ = self.client.get(index="stac_items", id=item_id) except elasticsearch.exceptions.NotFoundError: - raise NotFoundError(f"Collection {model['id']} not found") - self.delete_collection(model["id"]) - self.create_collection(model, **kwargs) + raise NotFoundError(f"Item {item_id} not found") + self.client.delete(index="stac_items", doc_type="_doc", id=item_id) + + @overrides + def create_collection( + self, collection: stac_types.Collection, **kwargs + ) -> stac_types.Collection: + """Create collection.""" + base_url = str(kwargs["request"].base_url) + collection_links = CollectionLinks( + collection_id=collection["id"], base_url=base_url + ).create_links() + collection["links"] = collection_links - return CollectionSerializer.db_to_stac(model, base_url) + self.create_item_index() + + if self.client.exists(index="stac_collections", id=collection["id"]): + raise ConflictError(f"Collection {collection['id']} already exists") + self.client.index( + index="stac_collections", + doc_type="_doc", + id=collection["id"], + document=collection, + ) + return CollectionSerializer.db_to_stac(collection, base_url) @overrides - def delete_item(self, item_id: str, collection_id: str, **kwargs): - """Delete item.""" + def update_collection( + self, collection: stac_types.Collection, **kwargs + ) -> stac_types.Collection: + """Update collection.""" + base_url = str(kwargs["request"].base_url) try: - _ = self.client.get(index="stac_items", id=item_id) + _ = self.client.get(index="stac_collections", id=collection["id"]) except elasticsearch.exceptions.NotFoundError: - raise NotFoundError(f"Item {item_id} not found") - self.client.delete(index="stac_items", doc_type="_doc", id=item_id) + raise NotFoundError(f"Collection {collection['id']} not found") + self.delete_collection(collection["id"]) + self.create_collection(collection, **kwargs) + + return CollectionSerializer.db_to_stac(collection, base_url) @overrides - def delete_collection(self, collection_id: str, **kwargs): + def delete_collection(self, collection_id: str, **kwargs) -> stac_types.Collection: """Delete collection.""" try: _ = self.client.get(index="stac_collections", id=collection_id) @@ -246,10 +257,12 @@ def bulk_sync(self, processed_items): helpers.bulk(self.client, actions) @overrides - def bulk_item_insert(self, items: Items, **kwargs) -> str: + def bulk_item_insert( + self, items: Items, chunk_size: Optional[int] = None, **kwargs + ) -> str: """Bulk item insertion using es.""" transactions_client = TransactionsClient() - transactions_client._create_item_index() + transactions_client.create_item_index() try: base_url = str(kwargs["request"].base_url) except Exception: From f63db8579fcbcea0e00e24162bb49593083216a0 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Mon, 14 Mar 2022 17:16:15 -0400 Subject: [PATCH 04/10] add creating collection --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index f44b6c1c..457de3ae 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,18 @@ docker-compose build docker-compose up ``` +To create a new Collection: + +```shell +curl -X "POST" "http://localhost:8083/collections" \ + -H 'Content-Type: application/json; charset=utf-8' \ + -d $'{ + "id": "my_collection" +}' +``` + +Note: this "Collections Transaction" behavior is not part of the STAC API, but may be soon. + ## Testing ```shell From 317a99c682f5b7db07fabf0e0393777b763e84ce Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Mon, 14 Mar 2022 17:17:21 -0400 Subject: [PATCH 05/10] revert mappings back to original ones --- .../elasticsearch/transactions.py | 72 ++----------------- 1 file changed, 5 insertions(+), 67 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py index 954541b3..52ed2e00 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py @@ -32,77 +32,15 @@ class TransactionsClient(BaseTransactionsClient): settings = ElasticsearchSettings() client = settings.create_client - dynamicTemplates = [ - # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md - { - "descriptions": { - "match_mapping_type": "string", - "match": "description", - "mapping": {"type": "text"}, - } - }, - { - "titles": { - "match_mapping_type": "string", - "match": "title", - "mapping": {"type": "text"}, - } - }, - # Projection Extension https://github.com/stac-extensions/projection - {"proj_epsg": {"match": "proj:epsg", "mapping": {"type": "integer"}}}, - { - "proj_projjson": { - "match": "proj:projjson", - "mapping": {"type": "object", "enabled": False}, - } - }, - { - "proj_centroid": { - "match_mapping_type": "string", - "match": "proj:centroid", - "mapping": {"type": "geo_point"}, - } - }, - { - "proj_geometry": { - "match_mapping_type": "string", - "match": "proj:geometry", - "mapping": {"type": "geo_shape"}, - } - }, - { - "no_index_href": { - "match": "href", - "mapping": {"type": "text", "index": False}, - } - }, - # Default all other strings not otherwise specified to keyword - {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, - {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, - ] - mappings = { - "numeric_detection": False, - "dynamic_templates": dynamicTemplates, "properties": { "geometry": {"type": "geo_shape"}, - "assets": {"type": "object", "enabled": False}, - "links": {"type": "object", "enabled": False}, - "properties": { - "type": "object", - "properties": { - # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md - "datetime": {"type": "date"}, - "start_datetime": {"type": "date"}, - "end_datetime": {"type": "date"}, - "created": {"type": "date"}, - "updated": {"type": "date"}, - # Satellite Extension https://github.com/stac-extensions/sat - "sat:absolute_orbit": {"type": "integer"}, - "sat:relative_orbit": {"type": "integer"}, - }, + "id": {"type": "text", "fields": {"keyword": {"type": "keyword"}}}, + "properties__datetime": { + "type": "text", + "fields": {"keyword": {"type": "keyword"}}, }, - }, + } } def create_item_index(self): From 26b06ad40bf0e61f262f306565a0e72915e98fe0 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Mon, 14 Mar 2022 17:19:22 -0400 Subject: [PATCH 06/10] Add better elasticsearch mappings Revert "revert mappings back to original ones" This reverts commit 317a99c682f5b7db07fabf0e0393777b763e84ce. --- .../elasticsearch/transactions.py | 72 +++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py index 52ed2e00..954541b3 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py @@ -32,15 +32,77 @@ class TransactionsClient(BaseTransactionsClient): settings = ElasticsearchSettings() client = settings.create_client + dynamicTemplates = [ + # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md + { + "descriptions": { + "match_mapping_type": "string", + "match": "description", + "mapping": {"type": "text"}, + } + }, + { + "titles": { + "match_mapping_type": "string", + "match": "title", + "mapping": {"type": "text"}, + } + }, + # Projection Extension https://github.com/stac-extensions/projection + {"proj_epsg": {"match": "proj:epsg", "mapping": {"type": "integer"}}}, + { + "proj_projjson": { + "match": "proj:projjson", + "mapping": {"type": "object", "enabled": False}, + } + }, + { + "proj_centroid": { + "match_mapping_type": "string", + "match": "proj:centroid", + "mapping": {"type": "geo_point"}, + } + }, + { + "proj_geometry": { + "match_mapping_type": "string", + "match": "proj:geometry", + "mapping": {"type": "geo_shape"}, + } + }, + { + "no_index_href": { + "match": "href", + "mapping": {"type": "text", "index": False}, + } + }, + # Default all other strings not otherwise specified to keyword + {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, + {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, + ] + mappings = { + "numeric_detection": False, + "dynamic_templates": dynamicTemplates, "properties": { "geometry": {"type": "geo_shape"}, - "id": {"type": "text", "fields": {"keyword": {"type": "keyword"}}}, - "properties__datetime": { - "type": "text", - "fields": {"keyword": {"type": "keyword"}}, + "assets": {"type": "object", "enabled": False}, + "links": {"type": "object", "enabled": False}, + "properties": { + "type": "object", + "properties": { + # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md + "datetime": {"type": "date"}, + "start_datetime": {"type": "date"}, + "end_datetime": {"type": "date"}, + "created": {"type": "date"}, + "updated": {"type": "date"}, + # Satellite Extension https://github.com/stac-extensions/sat + "sat:absolute_orbit": {"type": "integer"}, + "sat:relative_orbit": {"type": "integer"}, + }, }, - } + }, } def create_item_index(self): From 889cb8bc8524b6155fef6ce64105b7649ab39e02 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 15 Mar 2022 09:59:00 -0400 Subject: [PATCH 07/10] add CHANGELOG --- CHANGELOG.md | 20 ++++++++++++++++++++ README.md | 2 ++ 2 files changed, 22 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..20675e19 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,20 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Deprecated + +### Added + +### Fixed + +### Changed + +- Elasticsearch index mappings updated to be more thorough. + +### Removed diff --git a/README.md b/README.md index 457de3ae..132d283e 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ Elasticsearch backend for stac-fastapi. **WIP** This backend is not yet stable (notice no releases yet), so use the pgstac backend instead. +For changes, see the [Changelog](CHANGELOG.md). + ## Development Environment Setup Install [pre-commit](https://pre-commit.com/#install). From e29b500d6209f8214c27959c9d8f5c5aa482f550 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 15 Mar 2022 10:00:54 -0400 Subject: [PATCH 08/10] update PR template --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9459f2df..8210bd36 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -10,4 +10,4 @@ - [ ] Code is formatted and linted (run `pre-commit run --all-files`) - [ ] Tests pass (run `make test`) - [ ] Documentation has been updated to reflect changes, if applicable, and docs build successfully (run `make docs`) -- [ ] Changes are added to the [CHANGELOG](https://github.com/stac-utils/pystac/blob/master/CHANGES.md), if applicable \ No newline at end of file +- [ ] Changes are added to the changelog \ No newline at end of file From 807bcf2708cadafb8a5801589a9c991188df9f38 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 15 Mar 2022 10:05:15 -0400 Subject: [PATCH 09/10] remove string match type on proj_centroid and proj_geometry --- .../elasticsearch/stac_fastapi/elasticsearch/transactions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py index 44738855..236f31fa 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py @@ -58,14 +58,12 @@ class TransactionsClient(BaseTransactionsClient): }, { "proj_centroid": { - "match_mapping_type": "string", "match": "proj:centroid", "mapping": {"type": "geo_point"}, } }, { "proj_geometry": { - "match_mapping_type": "string", "match": "proj:geometry", "mapping": {"type": "geo_shape"}, } From 4fc5202c328f8cfa59b3f03c00d4b74e5880f44f Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 15 Mar 2022 11:54:20 -0400 Subject: [PATCH 10/10] add unreleased to changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20675e19..2e5ad4d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,3 +18,5 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Elasticsearch index mappings updated to be more thorough. ### Removed + +[Unreleased]: