update to python 3.10 and ES 8.x

Author: Phil Varner

.github/workflows/cicd.yml

@@ -15,30 +15,15 @@ jobs:
 
     services:
 
-      elasticsearch_7_svc:
-        image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
+      elasticsearch_8_svc:
+        image: docker.elastic.co/elasticsearch/elasticsearch:8.1.3
         env:
-          node.name: es01
-          cluster.name: stac-cluster
-          discovery.type: single-node
-          network.host: 0.0.0.0
-          http.port: 9200
-          ES_JAVA_OPTS: -Xms512m -Xmx512m
+          ES_JAVA_OPTS: -Xms512m -Xmx1g
+        volumes:
+          - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
         ports:
           - 9200:9200
 
-#      elasticsearch_8_svc:
-#        image: docker.elastic.co/elasticsearch/elasticsearch:8.1.0
-#        env:
-#          node.name: es01
-#          cluster.name: stac-cluster
-#          discovery.type: single-node
-#          network.host: 0.0.0.0
-#          http.port: 9200
-#          ES_JAVA_OPTS: -Xms512m -Xmx512m
-#        ports:
-#          - 9200:9200
-
     steps:
       - name: Check out repository code
         uses: actions/checkout@v3
@@ -47,7 +32,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v3
         with:
-          python-version: "3.8"
+          python-version: "3.10"
 
       - name: Lint code
         uses: pre-commit/action@v2.0.3
@@ -60,24 +45,12 @@ jobs:
         run: |
           pip install ./stac_fastapi/elasticsearch[dev,server]
 
-      - name: Run test suite against Elasticsearch 7.x
+      - name: Run test suite against Elasticsearch 8.x
         run: |
           cd stac_fastapi/elasticsearch && pipenv run pytest -svvv
         env:
           ENVIRONMENT: testing
-#          ES_USER: dev
-#          ES_PASS: stac
           ES_PORT: 9200
           ES_HOST: 172.17.0.1
           ES_USE_SSL: false
-          ES_VERIFY_CERTS: false
-
-#      - name: Run test suite against Elasticsearch 8.x
-#        run: |
-#          cd stac_fastapi/elasticsearch && pipenv run pytest -svvv
-#        env:
-#          ENVIRONMENT: testing
-#          ES_USER: dev
-#          ES_PASS: stac
-#          ES_PORT: 9400
-#          ES_HOST: 172.17.0.1
+          ES_VERIFY_CERTS: false

.pre-commit-config.yaml

@@ -1,51 +1,32 @@
 repos:
   - repo: https://github.com/PyCQA/isort
-    rev: 5.8.0
+    rev: 5.10.1
     hooks:
       - id: isort
-        language_version: python3.8
   - repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 22.3.0
     hooks:
       - id: black
         args: [ '--safe' ]
-        language_version: python3.8
   - repo: https://gitlab.com/pycqa/flake8
     rev: 3.9.0
     hooks:
       - id: flake8
-        language_version: python3.8
         args: [
           # E501 let black handle all line length decisions
           # W503 black conflicts with "line break before operator" rule
           # E203 black conflicts with "whitespace before ':'" rule
           '--ignore=E501,W503,E203,C901' ]
-  - repo: https://github.com/chewse/pre-commit-mirrors-pydocstyle
-    # 2.1.1
-    rev: v2.1.1
-    hooks:
-      - id: pydocstyle
-        language_version: python3.8
-        exclude: '.*(test|alembic|scripts).*'
-        args: [
-          # Check for docstring presence only
-          '--select=D1',
-
-        ]
-          # Don't require docstrings for tests
-        # '--match=(?!test).*\.py']
   #    -
   #        repo: https://github.com/pre-commit/mirrors-mypy
   #        rev: v0.770
   #        hooks:
   #            -   id: mypy
-  #                language_version: python3.8
   #                args: [--no-strict-optional, --ignore-missing-imports]
   - repo: https://github.com/PyCQA/pydocstyle
-    rev: 6.0.0
+    rev: 6.1.1
     hooks:
       - id: pydocstyle
-        language_version: python3.8
         exclude: '.*(test|alembic|scripts).*'
           #args: [
         # Don't require docstrings for tests

Dockerfile.deploy

@@ -1,4 +1,4 @@
-FROM python:3.8-slim
+FROM python:3.10-slim
 
 RUN apt-get update && \
     apt-get -y upgrade && \

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM python:3.8-slim
+FROM python:3.10-slim
 
 
 # update apt pkgs, and install build-essential for ciso8601

README.md

@@ -44,7 +44,14 @@ file named `.env` in the same directory you run docker-compose from:
 ELASTICSEARCH_VERSION=7.12.0
 ```
 
-TBD: how to run this with 8.x with a password enabled and TLS.
+### Generate a keystore and cert
+
+```shell
+docker-compose -f docker-compose.setup.yml run --rm keystore
+docker-compose -f docker-compose.setup.yml run --rm certs
+```
+
+
 
 To create a new Collection:
 

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.9'
 
 services:
   app-elasticsearch:
@@ -16,8 +16,6 @@ services:
       - WEB_CONCURRENCY=10
       - ES_HOST=172.17.0.1
       - ES_PORT=9200
-#      - ES_USER=dev
-#      - ES_PASS=stac
       - ES_USE_SSL=false
       - ES_VERIFY_CERTS=false
     ports:
@@ -33,13 +31,10 @@ services:
 
   elasticsearch:
     container_name: es-container
-    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-7.17.1}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.1.3}
     environment:
-      node.name: es01
-      cluster.name: stac-cluster
-      discovery.type: single-node
-      network.host: 0.0.0.0
-      http.port: 9200
       ES_JAVA_OPTS: -Xms512m -Xmx1g
+    volumes:
+      - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
     ports:
       - "9200:9200"

elasticsearch/config/elasticsearch.yml

@@ -0,0 +1,14 @@
+## Cluster Settings
+cluster.name: stac-cluster
+node.name: es01
+network.host: 0.0.0.0
+transport.host: 0.0.0.0
+discovery.type: single-node
+http.port: 9200
+
+## License
+xpack.license.self_generated.type: basic
+
+# Security
+xpack.security.enabled: false
+xpack.security.transport.ssl.enabled: false

stac_fastapi/elasticsearch/setup.py

@@ -13,7 +13,6 @@
     "stac-fastapi.types==2.3.0",
     "stac-fastapi.api==2.3.0",
     "stac-fastapi.extensions==2.3.0",
-    "fastapi-utils",
     "elasticsearch[async]==7.17.2",
     "elasticsearch-dsl==7.4.0",
     "pystac[validation]",
@@ -47,6 +46,8 @@
         "Intended Audience :: Information Technology",
         "Intended Audience :: Science/Research",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "License :: OSI Approved :: MIT License",
     ],
     url="https://github.com/stac-utils/stac-fastapi-elasticsearch",

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py

@@ -3,7 +3,6 @@
 from typing import Any, Dict, Set
 
 from elasticsearch import AsyncElasticsearch, Elasticsearch
-
 from stac_fastapi.types.config import ApiSettings
 
 

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py

@@ -5,8 +5,6 @@
 from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
 
 import attr
-import elasticsearch
-from elasticsearch import helpers
 from elasticsearch_dsl import Q, Search
 from geojson_pydantic.geometries import (
     GeometryCollection,
@@ -18,6 +16,8 @@
     Polygon,
 )
 
+import elasticsearch
+from elasticsearch import helpers
 from stac_fastapi.elasticsearch import serializers
 from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
 from stac_fastapi.elasticsearch.config import (

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/session.py

@@ -1,22 +1,11 @@
 """database session management."""
 import logging
-from contextlib import contextmanager
 
 import attr
-from fastapi_utils.session import FastAPISessionMaker as _FastAPISessionMaker
 
 logger = logging.getLogger(__name__)
 
 
-class FastAPISessionMaker(_FastAPISessionMaker):
-    """FastAPISessionMaker."""
-
-    @contextmanager
-    def context_session(self):
-        """Override base method to include exception handling."""
-        ...
-
-
 @attr.s
 class Session:
     """Database session management."""