Merge pull request #186 from stac-utils/common_core

Create Common opensearch/ elasticsearch core package

Author: jonhealy1

.github/workflows/cicd.yml

@@ -88,29 +88,39 @@ jobs:
         run: |
           pip install ./stac_fastapi/elasticsearch[dev,server]
 
+      - name: Install opensearch stac-fastapi
+        run: |
+          pip install ./stac_fastapi/opensearch[dev,server]
+
+      - name: Install core library stac-fastapi
+        run: |
+          pip install ./stac_fastapi/core
+
       - name: Run test suite against Elasticsearch 7.x
         run: |
-          cd stac_fastapi/elasticsearch && pipenv run pytest -svvv
+          pipenv run pytest -svvv
         env:
           ENVIRONMENT: testing
           ES_PORT: 9200
           ES_HOST: 172.17.0.1
           ES_USE_SSL: false
           ES_VERIFY_CERTS: false
+          BACKEND: elasticsearch
 
       - name: Run test suite against Elasticsearch 8.x
         run: |
-          cd stac_fastapi/elasticsearch && pipenv run pytest -svvv
+          pipenv run pytest -svvv
         env:
           ENVIRONMENT: testing
           ES_PORT: 9400
           ES_HOST: 172.17.0.1
           ES_USE_SSL: false
           ES_VERIFY_CERTS: false
+          BACKEND: elasticsearch
 
       - name: Run test suite against OpenSearch 2.11.1
         run: |
-          cd stac_fastapi/elasticsearch && pipenv run pytest -svvv
+          pipenv run pytest -svvv
         env:
           ENVIRONMENT: testing
           ES_PORT: 9202

CHANGELOG.md

@@ -9,10 +9,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- Added core library package for common logic [#186](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/186)
+
 ### Changed
 
+- Moved Elasticsearch and Opensearch backends into separate packages [#186](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/186)
+
 ### Fixed
 
+- Allow additional top-level properties on collections [#191](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/191)
+
 ## [v1.1.0]
 
 ### Added
@@ -28,7 +34,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Fixed
 
-- Allow additional top-level properties on collections [#191](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/191)
 - Exclude unset fields in search response [#166](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/166)
 - Upgrade stac-fastapi to v2.4.9 [#172](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/172)
 - Set correct default filter-lang for GET /search requests [#179](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/179)
@@ -115,4 +120,4 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 [v1.0.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.3.0...v1.0.0>
 [v0.3.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.2.0...v0.3.0>
 [v0.2.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0...v0.2.0>
-[v0.1.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0>
+[v0.1.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0>

Dockerfile.deploy renamed to Dockerfile.deploy.es

@@ -12,6 +12,7 @@ WORKDIR /app
 
 COPY . /app
 
+RUN pip install --no-cache-dir -e ./stac_fastapi/core
 RUN pip install --no-cache-dir ./stac_fastapi/elasticsearch[server]
 
 EXPOSE 8080

Dockerfile.deploy.os

@@ -0,0 +1,20 @@
+FROM python:3.10-slim
+
+RUN apt-get update && \
+    apt-get -y upgrade && \
+    apt-get -y install gcc && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+
+WORKDIR /app
+
+COPY . /app
+
+RUN pip install --no-cache-dir -e ./stac_fastapi/core
+RUN pip install --no-cache-dir ./stac_fastapi/opensearch[server]
+
+EXPOSE 8080
+
+CMD ["uvicorn", "stac_fastapi.opensearch.app:app", "--host", "0.0.0.0", "--port", "8080"]

Dockerfile.dev renamed to Dockerfile.dev.es

@@ -15,4 +15,5 @@ WORKDIR /app
 
 COPY . /app
 
+RUN pip install --no-cache-dir -e ./stac_fastapi/core
 RUN pip install --no-cache-dir -e ./stac_fastapi/elasticsearch[dev,server]

Dockerfile.dev.os

@@ -0,0 +1,19 @@
+FROM python:3.10-slim
+
+
+# update apt pkgs, and install build-essential for ciso8601
+RUN apt-get update && \
+    apt-get -y upgrade && \
+    apt-get install -y build-essential && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# update certs used by Requests
+ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+
+WORKDIR /app
+
+COPY . /app
+
+RUN pip install --no-cache-dir -e ./stac_fastapi/core
+RUN pip install --no-cache-dir -e ./stac_fastapi/opensearch[dev,server]

Makefile

@@ -1,14 +1,13 @@
 #!make
 APP_HOST ?= 0.0.0.0
-ES_APP_PORT ?= 8080
 EXTERNAL_APP_PORT ?= ${APP_PORT}
 
 ES_APP_PORT ?= 8080
 ES_HOST ?= docker.for.mac.localhost
 ES_PORT ?= 9200
 
 OS_APP_PORT ?= 8082
-ES_HOST ?= docker.for.mac.localhost
+OS_HOST ?= docker.for.mac.localhost
 OS_PORT ?= 9202
 
 run_es = docker-compose \
@@ -27,9 +26,13 @@ run_os = docker-compose \
 	-e APP_PORT=${OS_APP_PORT} \
 	app-opensearch
 
-.PHONY: image-deploy
-image-deploy:
-	docker build -f Dockerfile.deploy -t stac-fastapi-elasticsearch:latest .
+.PHONY: image-deploy-es
+image-deploy-es:
+	docker build -f Dockerfile.dev.es -t stac-fastapi-elasticsearch:latest .
+	
+.PHONY: image-deploy-os
+image-deploy-os:
+	docker build -f Dockerfile.dev.os -t stac-fastapi-opensearch:latest .
 
 .PHONY: run-deploy-locally
 run-deploy-locally:
@@ -44,30 +47,38 @@ run-deploy-locally:
 image-dev:
 	docker-compose build
 
-.PHONY: docker-run
-docker-run: image-dev
+.PHONY: docker-run-es
+docker-run-es: image-dev
 	$(run_es)
 
-.PHONY: docker-shell
-docker-shell:
+.PHONY: docker-run-os
+docker-run-os: image-dev
+	$(run_os)
+
+.PHONY: docker-shell-es
+docker-shell-es:
 	$(run_es) /bin/bash
 
+.PHONY: docker-shell-os
+docker-shell-os:
+	$(run_os) /bin/bash
+
 .PHONY: test-elasticsearch
-test:
-	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+test-elasticsearch:
+	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest'
 	docker-compose down
 
 .PHONY: test-opensearch
 test-opensearch:
-	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest'
 	docker-compose down
 
 .PHONY: test
 test:
-	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest'
 	docker-compose down
 
-	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest'
 	docker-compose down
 
 .PHONY: run-database-es
@@ -83,12 +94,17 @@ pybase-install:
 	pip install wheel && \
 	pip install -e ./stac_fastapi/api[dev] && \
 	pip install -e ./stac_fastapi/types[dev] && \
-	pip install -e ./stac_fastapi/extensions[dev]
+	pip install -e ./stac_fastapi/extensions[dev] && \
+	pip install -e ./stac_fastapi/core
 
-.PHONY: install
-install: pybase-install
+.PHONY: install-es
+install-es: pybase-install
 	pip install -e ./stac_fastapi/elasticsearch[dev,server]
 
+.PHONY: install-os
+install-os: pybase-install
+	pip install -e ./stac_fastapi/opensearch[dev,server]
+
 .PHONY: ingest
 ingest:
 	python3 data_loader/data_loader.py

README.md

@@ -1,12 +1,18 @@
-# STAC FastAPI Elasticsearch (sfes)
+# stac-fastapi-elasticsearch-opensearch (sfeos)
 
-## Elasticsearch backend for stac-fastapi with Opensearch support
+## Elasticsearch and Opensearch backends for the stac-fastapi project
 
-#### Join our [Gitter](https://gitter.im/stac-fastapi-elasticsearch/community) page
+[![PyPI version](https://badge.fury.io/py/stac-fastapi.elasticsearch.svg)](https://badge.fury.io/py/stac-fastapi.elasticsearch)
 
-#### Check out the public Postman documentation [Postman](https://documenter.getpostman.com/view/12888943/2s8ZDSdRHA)
+To install from PyPI:
 
-#### Check out the examples folder for deployment options, ex. running sfes from pip in docker
+```shell
+pip install stac_fastapi.elasticsearch
+```
+or   
+```
+pip install stac_fastapi.opensearch
+```
 
 #### For changes, see the [Changelog](CHANGELOG.md)
 
@@ -19,6 +25,13 @@ To install the classes in your local Python env, run:
 pip install -e 'stac_fastapi/elasticsearch[dev]'
 ```
 
+or
+
+```shell
+pip install -e 'stac_fastapi/opensearch[dev]'
+```
+
+
 ### Pre-commit
 
 Install [pre-commit](https://pre-commit.com/#install).
@@ -29,17 +42,17 @@ Prior to commit, run:
 pre-commit run --all-files
 ```
 
-
-## Building
+## Build Elasticsearh API backend
 
 ```shell
-docker-compose build
+docker-compose up elasticsearch
+docker-compose build app-elasticsearch
 ```
 
-## Running API on localhost:8080
+## Running Elasticsearh API on localhost:8080
 
 ```shell
-docker-compose up
+docker-compose up app-elasticsearch
 ```
 
 By default, docker-compose uses Elasticsearch 8.x and OpenSearch 2.11.1. 

docker-compose.yml

@@ -7,7 +7,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.dev
+      dockerfile: Dockerfile.dev.es
     environment:
       - APP_HOST=0.0.0.0
       - APP_PORT=8080
@@ -36,7 +36,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.dev
+      dockerfile: Dockerfile.dev.os
     environment:
       - APP_HOST=0.0.0.0
       - APP_PORT=8082
@@ -57,7 +57,7 @@ services:
     depends_on:
       - opensearch
     command:
-      bash -c "./scripts/wait-for-it-es.sh os-container:9202 && python -m stac_fastapi.elasticsearch.app"
+      bash -c "./scripts/wait-for-it-es.sh os-container:9202 && python -m stac_fastapi.opensearch.app"
 
   elasticsearch:
     container_name: es-container

examples/pip_docker/Dockerfile

@@ -15,4 +15,4 @@ WORKDIR /app
 
 COPY . /app
 
-RUN pip install stac-fastapi.elasticsearch==0.3.0
+RUN pip install stac-fastapi.elasticsearch==1.1.0

postman_collections/stac-fastapi-elasticsearch.postman_collection.json renamed to examples/postman_collections/stac-fastapi-elasticsearch.postman_collection.json

@@ -0,0 +1 @@
+# stac-fastapi core library for Elasticsearch and Opensearch backends

stac_fastapi/core/README.md

@@ -0,0 +1,2 @@
+[metadata]
+version = attr: stac_fastapi.core.version.__version__

stac_fastapi/core/setup.cfg

@@ -0,0 +1,44 @@
+"""stac_fastapi: core elasticsearch/ opensearch module."""
+
+from setuptools import find_namespace_packages, setup
+
+with open("README.md") as f:
+    desc = f.read()
+
+install_requires = [
+    "fastapi",
+    "attrs",
+    "pydantic[dotenv]<2",
+    "stac_pydantic==2.0.*",
+    "stac-fastapi.types==2.4.9",
+    "stac-fastapi.api==2.4.9",
+    "stac-fastapi.extensions==2.4.9",
+    "pystac[validation]",
+    "orjson",
+    "overrides",
+    "geojson-pydantic",
+    "pygeofilter==0.2.1",
+]
+
+setup(
+    name="stac-fastapi.core",
+    description="Core library for the Elasticsearch and Opensearch stac-fastapi backends.",
+    long_description=desc,
+    long_description_content_type="text/markdown",
+    python_requires=">=3.8",
+    classifiers=[
+        "Intended Audience :: Developers",
+        "Intended Audience :: Information Technology",
+        "Intended Audience :: Science/Research",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "License :: OSI Approved :: MIT License",
+    ],
+    url="https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch",
+    license="MIT",
+    packages=find_namespace_packages(),
+    zip_safe=False,
+    install_requires=install_requires,
+)

stac_fastapi/core/setup.py

@@ -0,0 +1 @@
+"""Core library."""