From cf6573bb1846ece5b8f46b16fc33184f9c0e202e Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 19 Mar 2023 11:17:13 +0300 Subject: [PATCH 01/11] Dockerfile --- examples/pip_docker/Dockerfile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 examples/pip_docker/Dockerfile diff --git a/examples/pip_docker/Dockerfile b/examples/pip_docker/Dockerfile new file mode 100644 index 00000000..609ada8c --- /dev/null +++ b/examples/pip_docker/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.10-slim + + +# update apt pkgs, and install build-essential for ciso8601 +RUN apt-get update && \ + apt-get -y upgrade && \ + apt-get install -y build-essential && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# update certs used by Requests +ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + +WORKDIR /app + +COPY . /app + +RUN pip install stac-fastapi.elasticsearch==0.3.0 \ No newline at end of file From 9511d577c8bbf0ee97876d628cb308b6999820f9 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 19 Mar 2023 11:17:22 +0300 Subject: [PATCH 02/11] docker compose --- examples/pip_docker/docker-compose.yml | 45 ++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 examples/pip_docker/docker-compose.yml diff --git a/examples/pip_docker/docker-compose.yml b/examples/pip_docker/docker-compose.yml new file mode 100644 index 00000000..2c2d289e --- /dev/null +++ b/examples/pip_docker/docker-compose.yml @@ -0,0 +1,45 @@ +version: '3' + +services: + app-elasticsearch: + container_name: stac-fastapi-es + image: stac-utils/stac-fastapi-es + restart: always + build: + context: . + dockerfile: Dockerfile + platform: linux/amd64 + environment: + - APP_HOST=0.0.0.0 + - APP_PORT=8080 + - RELOAD=true + - ENVIRONMENT=local + - WEB_CONCURRENCY=10 + - ES_HOST=172.17.0.1 + - ES_PORT=9200 + - ES_USE_SSL=false + - ES_VERIFY_CERTS=false + ports: + - "8080:8080" + volumes: + - ./stac_fastapi:/app/stac_fastapi + - ./scripts:/app/scripts + depends_on: + - elasticsearch + command: + bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app" + + elasticsearch: + container_name: es-container + image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.1.3} + environment: + ES_JAVA_OPTS: -Xms512m -Xmx1g + volumes: + - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml + - ./elasticsearch/snapshots:/usr/share/elasticsearch/snapshots + ports: + - "9200:9200" + +networks: + default: + name: stac-fastapi-network \ No newline at end of file From a254de27f3b16594cc70bef1da8fd08ed4954d69 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 19 Mar 2023 11:17:36 +0300 Subject: [PATCH 03/11] read me --- examples/README.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 examples/README.md diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 00000000..38621e6b --- /dev/null +++ b/examples/README.md @@ -0,0 +1,6 @@ +# stac-fastapi-elasticsearch deployment examples + +## Deploy sfes from pip in docker + +```$ cd pip_docker``` +```$ docker-compose up``` \ No newline at end of file From dd52d6e3e357d741107c27b8f69dd33741f15eb1 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 19 Mar 2023 11:17:45 +0300 Subject: [PATCH 04/11] copy es config --- .../elasticsearch/config/elasticsearch.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 examples/pip_docker/elasticsearch/config/elasticsearch.yml diff --git a/examples/pip_docker/elasticsearch/config/elasticsearch.yml b/examples/pip_docker/elasticsearch/config/elasticsearch.yml new file mode 100644 index 00000000..0bf5b680 --- /dev/null +++ b/examples/pip_docker/elasticsearch/config/elasticsearch.yml @@ -0,0 +1,18 @@ +## Cluster Settings +cluster.name: stac-cluster +node.name: es01 +network.host: 0.0.0.0 +transport.host: 0.0.0.0 +discovery.type: single-node +http.port: 9200 + +path: + repo: + - /usr/share/elasticsearch/snapshots + +## License +xpack.license.self_generated.type: basic + +# Security +xpack.security.enabled: false +xpack.security.transport.ssl.enabled: false \ No newline at end of file From d6394e1314bc398874193bd5dc47e1ec94df7339 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 19 Mar 2023 11:17:55 +0300 Subject: [PATCH 05/11] use wait for it sh --- examples/pip_docker/scripts/wait-for-it-es.sh | 186 ++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100755 examples/pip_docker/scripts/wait-for-it-es.sh diff --git a/examples/pip_docker/scripts/wait-for-it-es.sh b/examples/pip_docker/scripts/wait-for-it-es.sh new file mode 100755 index 00000000..c622ce22 --- /dev/null +++ b/examples/pip_docker/scripts/wait-for-it-es.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +###################################################### +# Copied from https://github.com/vishnubob/wait-for-it +###################################################### + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-45} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi \ No newline at end of file From 2e8aaa1de9f7b35a7f119c785725d4fa9f280709 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 19 Mar 2023 11:23:51 +0300 Subject: [PATCH 06/11] changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf007a35..402fbe9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added + +- Examples folder with example docker setup for running sfes from pip + ### Changed ### Fixed From f5ea1e8d7b93bf5e55a2dfe9734249bc09b55a7c Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 19 Mar 2023 11:23:57 +0300 Subject: [PATCH 07/11] readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bd87b112..a38ce14a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# STAC FastAPI Elasticsearch +# STAC FastAPI Elasticsearch (sfes) ## Elasticsearch backend for stac-fastapi @@ -6,6 +6,8 @@ #### Check out the public Postman documentation [Postman](https://documenter.getpostman.com/view/12888943/2s8ZDSdRHA) +#### Check out the examples folder for deployment options, ex. running sfes from pip in docker + #### For changes, see the [Changelog](CHANGELOG.md) From dc3f2a4f96cb29953cb8a9952b3af8c61e5344b6 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 19 Mar 2023 14:19:28 +0300 Subject: [PATCH 08/11] update changelog --- CHANGELOG.md | 2 +- examples/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 402fbe9c..255f27e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added -- Examples folder with example docker setup for running sfes from pip +- Examples folder with example docker setup for running sfes from pip [#147](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/147) ### Changed ### Fixed diff --git a/examples/README.md b/examples/README.md index 38621e6b..def74db1 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,4 +1,4 @@ -# stac-fastapi-elasticsearch deployment examples +# stac-fastapi-elasticsearch (sfes) deployment examples ## Deploy sfes from pip in docker From 829829bfffef4e90ee0d8df4db020c9ef7e08bf4 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 19 Mar 2023 14:23:51 +0300 Subject: [PATCH 09/11] update isort --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 33353102..bef7f1c2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/psf/black From 97d30cf71c6fbe06fa77d68ef51ddf8a97f57b54 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Mon, 20 Mar 2023 17:18:49 +0300 Subject: [PATCH 10/11] Update examples/README.md Co-authored-by: Pete Gadomski --- examples/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/README.md b/examples/README.md index def74db1..196ed0c0 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,5 +2,6 @@ ## Deploy sfes from pip in docker -```$ cd pip_docker``` -```$ docker-compose up``` \ No newline at end of file +```shell +cd pip_docker +docker-compose up \ No newline at end of file From 1f0fddac9721a7b6be2c8ce2d0d7c2b6d6403ee3 Mon Sep 17 00:00:00 2001 From: Pete Gadomski Date: Mon, 20 Mar 2023 09:00:04 -0600 Subject: [PATCH 11/11] fix, docs: code fence --- examples/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/README.md b/examples/README.md index 196ed0c0..56dede98 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,7 +1,8 @@ -# stac-fastapi-elasticsearch (sfes) deployment examples +# stac-fastapi-elasticsearch (sfes) deployment examples ## Deploy sfes from pip in docker ```shell cd pip_docker -docker-compose up \ No newline at end of file +docker-compose up +```