improve elasticsearch mappings

Phil Varner · Phil Varner · commit 25ebdc410dde · 2022-03-04T16:40:13.000-05:00
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,57 +1,53 @@
 repos:
-    - repo: https://github.com/PyCQA/isort
-      rev: 5.8.0
-      hooks:
-        - id: isort
-          language_version: python3.8
-    -
-        repo: https://github.com/psf/black
-        rev: 20.8b1
-        hooks:
-            - id: black
-              args: ['--safe']
-              language_version: python3.8
-    -
-        repo: https://gitlab.com/pycqa/flake8
-        rev: 3.9.0
-        hooks:
-            - id: flake8
-              language_version: python3.8
-              args: [
-                  # E501 let black handle all line length decisions
-                  # W503 black conflicts with "line break before operator" rule
-                  # E203 black conflicts with "whitespace before ':'" rule
-                  '--ignore=E501,W503,E203,C901']
-    -
-        repo: https://github.com/chewse/pre-commit-mirrors-pydocstyle
-        # 2.1.1
-        rev: v2.1.1
-        hooks:
-            - id: pydocstyle
-              language_version: python3.8
-              exclude: '.*(test|alembic|scripts).*'
-              args: [
-                 # Check for docstring presence only
-                 '--select=D1',
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.8.0
+    hooks:
+      - id: isort
+        language_version: python3.8
+  - repo: https://github.com/psf/black
+    rev: 20.8b1
+    hooks:
+      - id: black
+        args: [ '--safe' ]
+        language_version: python3.8
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.0
+    hooks:
+      - id: flake8
+        language_version: python3.8
+        args: [
+          # E501 let black handle all line length decisions
+          # W503 black conflicts with "line break before operator" rule
+          # E203 black conflicts with "whitespace before ':'" rule
+          '--ignore=E501,W503,E203,C901' ]
+  - repo: https://github.com/chewse/pre-commit-mirrors-pydocstyle
+    # 2.1.1
+    rev: v2.1.1
+    hooks:
+      - id: pydocstyle
+        language_version: python3.8
+        exclude: '.*(test|alembic|scripts).*'
+        args: [
+          # Check for docstring presence only
+          '--select=D1',
 
-              ]
-                 # Don't require docstrings for tests
-                 # '--match=(?!test).*\.py']
-#    -
-#        repo: https://github.com/pre-commit/mirrors-mypy
-#        rev: v0.770
-#        hooks:
-#            -   id: mypy
-#                language_version: python3.8
-#                args: [--no-strict-optional, --ignore-missing-imports]
-    -
-        repo: https://github.com/PyCQA/pydocstyle
-        rev: 6.0.0
-        hooks:
-          - id: pydocstyle
-            language_version: python3.8
-            exclude: '.*(test|alembic|scripts).*'
-            #args: [
-                # Don't require docstrings for tests
-                #'--match=(?!test|alembic|scripts).*\.py',
-            #]
+        ]
+          # Don't require docstrings for tests
+        # '--match=(?!test).*\.py']
+  #    -
+  #        repo: https://github.com/pre-commit/mirrors-mypy
+  #        rev: v0.770
+  #        hooks:
+  #            -   id: mypy
+  #                language_version: python3.8
+  #                args: [--no-strict-optional, --ignore-missing-imports]
+  - repo: https://github.com/PyCQA/pydocstyle
+    rev: 6.0.0
+    hooks:
+      - id: pydocstyle
+        language_version: python3.8
+        exclude: '.*(test|alembic|scripts).*'
+          #args: [
+        # Don't require docstrings for tests
+        #'--match=(?!test|alembic|scripts).*\.py',
+        #]
diff --git a/README.md b/README.md
@@ -14,6 +14,11 @@ Prior to commit, run:
 pre-commit run --all-files`
 ```
 
+```shell
+cd stac_fastapi/elasticsearch
+pip install .[dev]
+```
+
 ## Building
 
 ```
@@ -37,3 +42,7 @@ make test
 ```
 make ingest
 ```
+
+## Elasticsearch Mappings
+
+Mappings apply to search index, not source. 
diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py
@@ -27,6 +27,8 @@
         "pre-commit",
         "requests",
         "ciso8601",
+        "overrides",
+        "black",
     ],
     "docs": ["mkdocs", "mkdocs-material", "pdocs"],
     "server": ["uvicorn[standard]>=0.12.0,<0.14.0"],
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/serializers.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/serializers.py
@@ -24,7 +24,7 @@ class ItemSerializer(Serializer):
     """Serialization methods for STAC items."""
 
     @classmethod
-    def stac_to_db(cls, stac_data: TypedDict, base_url: str) -> stac_types.Item:
+    def stac_to_db(cls, stac_data: stac_types.Item, base_url: str) -> stac_types.Item:
         """Transform STAC Item to database-ready STAC Item."""
         item_links = ItemLinks(
             collection_id=stac_data["collection"],
diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/transactions.py
@@ -6,6 +6,7 @@
 import attr
 import elasticsearch
 from elasticsearch import helpers
+from overrides import overrides
 
 from stac_fastapi.elasticsearch.config import ElasticsearchSettings
 from stac_fastapi.elasticsearch.serializers import CollectionSerializer, ItemSerializer
@@ -30,26 +31,87 @@ class TransactionsClient(BaseTransactionsClient):
     settings = ElasticsearchSettings()
     client = settings.create_client
 
-    def _create_item_index(self):
-        mapping = {
-            "mappings": {
-                "properties": {
-                    "geometry": {"type": "geo_shape"},
-                    "id": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
-                    "properties__datetime": {
-                        "type": "text",
-                        "fields": {"keyword": {"type": "keyword"}},
-                    },
-                }
+    dynamicTemplates = [
+        # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md
+        {
+            "descriptions": {
+                "match_mapping_type": "string",
+                "match": "description",
+                "mapping": {"type": "text"},
+            }
+        },
+        {
+            "titles": {
+                "match_mapping_type": "string",
+                "match": "title",
+                "mapping": {"type": "text"},
+            }
+        },
+        # Projection Extension https://github.com/stac-extensions/projection
+        {"proj_epsg": {"match": "proj:epsg", "mapping": {"type": "integer"}}},
+        {
+            "proj_projjson": {
+                "match": "proj:projjson",
+                "mapping": {"type": "object", "enabled": False},
+            }
+        },
+        {
+            "proj_centroid": {
+                "match_mapping_type": "string",
+                "match": "proj:centroid",
+                "mapping": {"type": "geo_point"},
             }
-        }
+        },
+        {
+            "proj_geometry": {
+                "match_mapping_type": "string",
+                "match": "proj:geometry",
+                "mapping": {"type": "geo_shape"},
+            }
+        },
+        {
+            "no_index_href": {
+                "match": "href",
+                "mapping": {"type": "text", "index": False},
+            }
+        },
+        # Default all other strings not otherwise specified to keyword
+        {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}},
+        {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}},
+    ]
+
+    mappings = {
+        "numeric_detection": False,
+        "dynamic_templates": dynamicTemplates,
+        "properties": {
+            "geometry": {"type": "geo_shape"},
+            "assets": {"type": "object", "enabled": False},
+            "links": {"type": "object", "enabled": False},
+            "properties": {
+                "type": "object",
+                "properties": {
+                    # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md
+                    "datetime": {"type": "date"},
+                    "start_datetime": {"type": "date"},
+                    "end_datetime": {"type": "date"},
+                    "created": {"type": "date"},
+                    "updated": {"type": "date"},
+                    # Satellite Extension https://github.com/stac-extensions/sat
+                    "sat:absolute_orbit": {"type": "integer"},
+                    "sat:relative_orbit": {"type": "integer"},
+                },
+            },
+        },
+    }
 
-        _ = self.client.indices.create(
+    def _create_item_index(self):
+        self.client.indices.create(
             index="stac_items",
-            body=mapping,
+            body=self.mappings,
             ignore=400,  # ignore 400 already exists code
         )
 
+    @overrides
     def create_item(self, model: stac_types.Item, **kwargs):
         """Create item."""
         base_url = str(kwargs["request"].base_url)
@@ -83,6 +145,7 @@ def create_item(self, model: stac_types.Item, **kwargs):
         )
         return ItemSerializer.db_to_stac(model, base_url)
 
+    @overrides
     def create_collection(self, model: stac_types.Collection, **kwargs):
         """Create collection."""
         base_url = str(kwargs["request"].base_url)
@@ -100,6 +163,7 @@ def create_collection(self, model: stac_types.Collection, **kwargs):
         )
         return CollectionSerializer.db_to_stac(model, base_url)
 
+    @overrides
     def update_item(self, model: stac_types.Item, **kwargs):
         """Update item."""
         base_url = str(kwargs["request"].base_url)
@@ -118,6 +182,7 @@ def update_item(self, model: stac_types.Item, **kwargs):
         #         body=model)
         return ItemSerializer.db_to_stac(model, base_url)
 
+    @overrides
     def update_collection(self, model: stac_types.Collection, **kwargs):
         """Update collection."""
         base_url = str(kwargs["request"].base_url)
@@ -130,6 +195,7 @@ def update_collection(self, model: stac_types.Collection, **kwargs):
 
         return CollectionSerializer.db_to_stac(model, base_url)
 
+    @overrides
     def delete_item(self, item_id: str, collection_id: str, **kwargs):
         """Delete item."""
         try:
@@ -138,6 +204,7 @@ def delete_item(self, item_id: str, collection_id: str, **kwargs):
             raise NotFoundError(f"Item {item_id} not found")
         self.client.delete(index="stac_items", doc_type="_doc", id=item_id)
 
+    @overrides
     def delete_collection(self, collection_id: str, **kwargs):
         """Delete collection."""
         try:
@@ -178,6 +245,7 @@ def bulk_sync(self, processed_items):
         ]
         helpers.bulk(self.client, actions)
 
+    @overrides
     def bulk_item_insert(self, items: Items, **kwargs) -> str:
         """Bulk item insertion using es."""
         transactions_client = TransactionsClient()
