From bf102f950dae0a2f98bf305a81bd7a2f1d453b4b Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Wed, 23 Mar 2022 12:05:05 -0400 Subject: [PATCH 1/3] await index creation --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 256413d2..74dfb216 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -46,7 +46,7 @@ @app.on_event("startup") async def _startup_event(): - IndexesClient().create_indexes() + await IndexesClient().create_indexes() def run(): From 20ebd6d28baca1ebccb2269e65ad4fa96acdcce6 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Wed, 23 Mar 2022 12:49:50 -0400 Subject: [PATCH 2/3] add dockerfile for deployment, use port 8080 insteaf of 8083 --- Dockerfile | 22 ------ Dockerfile.deploy | 18 +++++ Dockerfile.dev | 18 +++++ Makefile | 16 +++- README.md | 4 +- data_loader/data_loader.py | 2 +- docker-compose.yml | 6 +- ...tapi-elasticsearch.postman_collection.json | 24 +++--- stac_fastapi/elasticsearch/setup.py | 2 +- .../elasticsearch/tests/api/test_api.py | 2 +- .../tests/clients/test_elasticsearch.py | 10 +-- .../elasticsearch/tests/features/__init__.py | 0 .../tests/features/test_custom_models.py | 75 ------------------- 13 files changed, 76 insertions(+), 123 deletions(-) delete mode 100644 Dockerfile create mode 100644 Dockerfile.deploy create mode 100644 Dockerfile.dev delete mode 100644 stac_fastapi/elasticsearch/tests/features/__init__.py delete mode 100644 stac_fastapi/elasticsearch/tests/features/test_custom_models.py diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index f03ba0e9..00000000 --- a/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM python:3.8-slim as base - -FROM base as builder -# Any python libraries that require system libraries to be installed will likely -# need the following packages in order to build -RUN apt-get update && \ - apt-get install -y build-essential git && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt - -ARG install_dev_dependencies=true - -WORKDIR /app - -COPY . /app - -ENV PATH=$PATH:/install/bin - -RUN mkdir -p /install -RUN pip install --no-cache-dir -e ./stac_fastapi/elasticsearch[dev,server] diff --git a/Dockerfile.deploy b/Dockerfile.deploy new file mode 100644 index 00000000..5a77133b --- /dev/null +++ b/Dockerfile.deploy @@ -0,0 +1,18 @@ +FROM python:3.8-slim + +RUN apt-get update && \ + apt-get -y upgrade && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + +WORKDIR /app + +COPY . /app + +RUN pip install --no-cache-dir ./stac_fastapi/elasticsearch[server] + +EXPOSE 8080 + +CMD ["uvicorn", "stac_fastapi.elasticsearch.app:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 00000000..8386471d --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,18 @@ +FROM python:3.8-slim + + +# update apt pkgs, and install build-essential for ciso8601 +RUN apt-get update && \ + apt-get -y upgrade && \ + apt-get install -y build-essential && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# update certs used by Requests +ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + +WORKDIR /app + +COPY . /app + +RUN pip install --no-cache-dir -e ./stac_fastapi/elasticsearch[dev,server] diff --git a/Makefile b/Makefile index e7bb1f74..51b5202e 100644 --- a/Makefile +++ b/Makefile @@ -13,10 +13,24 @@ run_es = docker-compose \ .PHONY: image image: + docker build -f Dockerfile.deploy -t stac-fastapi-elasticsearch:latest . + +.PHONY: run +run: + docker run -it -p 8080:8080 \ + -e ENVIRONMENT=local \ + -e ES_HOST=docker.for.mac.localhost \ + -e ES_PORT=9200 \ + -e ES_USER=dev \ + -e ES_PASS=stac \ + stac-fastapi-elasticsearch:latest + +.PHONY: image-dev +image-dev: docker-compose build .PHONY: docker-run -docker-run: image +docker-run: image-dev $(run_es) .PHONY: docker-shell diff --git a/README.md b/README.md index a3d131bc..f253448d 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ pre-commit run --all-files` docker-compose build ``` -## Running API on localhost:8083 +## Running API on localhost:8080 ```shell docker-compose up @@ -50,7 +50,7 @@ TBD: how to run this with 8.x with a password enabled and TLS. To create a new Collection: ```shell -curl -X "POST" "http://localhost:8083/collections" \ +curl -X "POST" "http://localhost:8080/collections" \ -H 'Content-Type: application/json; charset=utf-8' \ -d $'{ "id": "my_collection" diff --git a/data_loader/data_loader.py b/data_loader/data_loader.py index 286f49fd..b8d9f93e 100644 --- a/data_loader/data_loader.py +++ b/data_loader/data_loader.py @@ -6,7 +6,7 @@ import requests DATA_DIR = os.path.join(os.path.dirname(__file__), "setup_data/") -STAC_API_BASE_URL = "http://localhost:8083" +STAC_API_BASE_URL = "http://localhost:8080" def load_data(filename): diff --git a/docker-compose.yml b/docker-compose.yml index 99cf4af4..973c693e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,10 +7,10 @@ services: restart: always build: context: . - dockerfile: Dockerfile + dockerfile: Dockerfile.dev environment: - APP_HOST=0.0.0.0 - - APP_PORT=8083 + - APP_PORT=8080 - RELOAD=false - ENVIRONMENT=local - WEB_CONCURRENCY=10 @@ -19,7 +19,7 @@ services: - ES_PORT=9200 - ES_HOST=172.17.0.1 ports: - - "8083:8083" + - "8080:8080" volumes: - ./stac_fastapi:/app/stac_fastapi - ./scripts:/app/scripts diff --git a/postman_collections/stac-fastapi-elasticsearch.postman_collection.json b/postman_collections/stac-fastapi-elasticsearch.postman_collection.json index 2848dc94..a49d5994 100644 --- a/postman_collections/stac-fastapi-elasticsearch.postman_collection.json +++ b/postman_collections/stac-fastapi-elasticsearch.postman_collection.json @@ -11,7 +11,7 @@ "method": "GET", "header": [], "url": { - "raw": "http://localhost:8083/", + "raw": "http://localhost:8080/", "protocol": "http", "host": [ "localhost" @@ -30,7 +30,7 @@ "method": "GET", "header": [], "url": { - "raw": "http://localhost:8083/collections", + "raw": "http://localhost:8080/collections", "protocol": "http", "host": [ "localhost" @@ -49,7 +49,7 @@ "method": "DELETE", "header": [], "url": { - "raw": "http://localhost:8083/collections/test-collection", + "raw": "http://localhost:8080/collections/test-collection", "protocol": "http", "host": [ "localhost" @@ -69,7 +69,7 @@ "method": "DELETE", "header": [], "url": { - "raw": "http://localhost:8083/collections/test-collection/items/test-item", + "raw": "http://localhost:8080/collections/test-collection/items/test-item", "protocol": "http", "host": [ "localhost" @@ -91,7 +91,7 @@ "method": "GET", "header": [], "url": { - "raw": "http://localhost:8083/collections/test-collection", + "raw": "http://localhost:8080/collections/test-collection", "protocol": "http", "host": [ "localhost" @@ -111,7 +111,7 @@ "method": "GET", "header": [], "url": { - "raw": "http://localhost:8083/collections/test-collection/items", + "raw": "http://localhost:8080/collections/test-collection/items", "protocol": "http", "host": [ "localhost" @@ -132,7 +132,7 @@ "method": "GET", "header": [], "url": { - "raw": "http://localhost:8083/collections/test-collection/items/test-item", + "raw": "http://localhost:8080/collections/test-collection/items/test-item", "protocol": "http", "host": [ "localhost" @@ -169,7 +169,7 @@ "raw": "{\n \"id\": \"test-collection\",\n \"stac_extensions\": [\"https://stac-extensions.github.io/eo/v1.0.0/schema.json\"],\n \"type\": \"Collection\",\n \"description\": \"Landat 8 imagery radiometrically calibrated and orthorectified using gound points and Digital Elevation Model (DEM) data to correct relief displacement.\",\n \"stac_version\": \"1.0.0\",\n \"summaries\": {\n \"platform\": [\"landsat-8\"],\n \"instruments\": [\"oli\", \"tirs\"],\n \"gsd\": [30]\n },\n \"extent\": {\n \"spatial\": {\n \"bbox\": [\n [\n -180.0,\n -90.0,\n 180.0,\n 90.0\n ]\n ]\n },\n \"temporal\": {\n \"interval\": [\n [\n \"2013-06-01\",\n null\n ]\n ]\n }\n },\n \"links\": [\n {\n \"href\": \"http://localhost:8081/collections/landsat-8-l1\",\n \"rel\": \"self\",\n \"type\": \"application/json\"\n },\n {\n \"href\": \"http://localhost:8081/\",\n \"rel\": \"parent\",\n \"type\": \"application/json\"\n },\n {\n \"href\": \"http://localhost:8081/collections/landsat-8-l1/items\",\n \"rel\": \"item\",\n \"type\": \"application/geo+json\"\n },\n {\n \"href\": \"http://localhost:8081/\",\n \"rel\": \"root\",\n \"type\": \"application/json\"\n }\n ],\n \"title\": \"Landsat 8 L1\",\n \"keywords\": [\n \"landsat\",\n \"earth observation\",\n \"usgs\"\n ]\n}" }, "url": { - "raw": "http://localhost:8083/collections", + "raw": "http://localhost:8080/collections", "protocol": "http", "host": [ "localhost" @@ -203,7 +203,7 @@ "raw": "{\n \"type\": \"Feature\",\n \"id\": \"test-item\",\n \"stac_version\": \"1.0.0\",\n \"stac_extensions\": [\n \"https://stac-extensions.github.io/eo/v1.0.0/schema.json\",\n \"https://stac-extensions.github.io/projection/v1.0.0/schema.json\"\n ],\n \"geometry\": {\n \"coordinates\": [\n [\n [\n 152.15052873427666,\n -33.82243006904891\n ],\n [\n 150.1000346138806,\n -34.257132625788756\n ],\n [\n 149.5776607193635,\n -32.514709769700254\n ],\n [\n 151.6262528041627,\n -32.08081674221862\n ],\n [\n 152.15052873427666,\n -33.82243006904891\n ]\n ]\n ],\n \"type\": \"Polygon\"\n },\n \"properties\": {\n \"datetime\": \"2018-02-12T12:30:22Z\",\n \"landsat:scene_id\": \"LC82081612020043LGN00\",\n \"landsat:row\": \"161\",\n \"gsd\": 15,\n \"landsat:revision\": \"00\",\n \"view:sun_azimuth\": -148.83296771,\n \"instrument\": \"OLI_TIRS\",\n \"landsat:product_id\": \"LC08_L1GT_208161_20200212_20200212_01_RT\",\n \"eo:cloud_cover\": 0,\n \"landsat:tier\": \"RT\",\n \"landsat:processing_level\": \"L1GT\",\n \"landsat:column\": \"208\",\n \"platform\": \"landsat-8\",\n \"proj:epsg\": 32756,\n \"view:sun_elevation\": -37.30791534,\n \"view:off_nadir\": 0,\n \"height\": 2500,\n \"width\": 2500\n },\n \"bbox\": [\n 149.57574,\n -34.25796,\n 152.15194,\n -32.07915\n ],\n \"collection\": \"test-collection\",\n \"assets\": {},\n \"links\": [\n {\n \"href\": \"http://localhost:8081/collections/landsat-8-l1/items/LC82081612020043\",\n \"rel\": \"self\",\n \"type\": \"application/geo+json\"\n },\n {\n \"href\": \"http://localhost:8081/collections/landsat-8-l1\",\n \"rel\": \"parent\",\n \"type\": \"application/json\"\n },\n {\n \"href\": \"http://localhost:8081/collections/landsat-8-l1\",\n \"rel\": \"collection\",\n \"type\": \"application/json\"\n },\n {\n \"href\": \"http://localhost:8081/\",\n \"rel\": \"root\",\n \"type\": \"application/json\"\n }\n ]\n}" }, "url": { - "raw": "http://localhost:8083/collections/test-collection/items", + "raw": "http://localhost:8080/collections/test-collection/items", "protocol": "http", "host": [ "localhost" @@ -239,7 +239,7 @@ "raw": "{\n \"collections\":[\"test-collection\"],\n \"intersects\":{\"type\": \"Point\", \"coordinates\": [150.04, -33.14]}\n}" }, "url": { - "raw": "http://localhost:8083/search", + "raw": "http://localhost:8080/search", "protocol": "http", "host": [ "localhost" @@ -273,7 +273,7 @@ "raw": "{\n \"collections\":[\"sentinel-s2-l2a\"],\n \"bbox\": [-69.433594,-10.660608,-47.285156,3.513421]\n}" }, "url": { - "raw": "http://localhost:8083/search", + "raw": "http://localhost:8080/search", "protocol": "http", "host": [ "localhost" @@ -298,7 +298,7 @@ } ], "url": { - "raw": "http://localhost:8083/search?query={\"gsd\": {\"gt\": 14}}", + "raw": "http://localhost:8080/search?query={\"gsd\": {\"gt\": 14}}", "protocol": "http", "host": [ "localhost" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index da175af7..923ad975 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -18,6 +18,7 @@ "elasticsearch-dsl==7.4.0", "pystac[validation]", "uvicorn", + "overrides", ] extra_reqs = { @@ -28,7 +29,6 @@ "pre-commit", "requests", "ciso8601", - "overrides", "httpx", ], "docs": ["mkdocs", "mkdocs-material", "pdocs"], diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index e1e639cb..1ee27aa7 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -118,7 +118,7 @@ async def test_app_fields_extension(load_test_data, app_client, txn_client): resp_json = resp.json() assert list(resp_json["features"][0]["properties"]) == ["datetime"] - txn_client.delete_item(item["id"], item["collection"], request=MockRequest) + txn_client.delete_item(item["id"], item["collection"]) async def test_app_query_extension_gt(app_client, ctx): diff --git a/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py b/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py index 9cde8493..db7972a7 100644 --- a/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py +++ b/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py @@ -17,7 +17,7 @@ async def test_create_collection(app_client, ctx, core_client, txn_client): await txn_client.create_collection(in_coll, request=MockRequest) got_coll = await core_client.get_collection(in_coll["id"], request=MockRequest) assert got_coll["id"] == in_coll["id"] - await txn_client.delete_collection(in_coll["id"], request=MockRequest) + await txn_client.delete_collection(in_coll["id"]) async def test_create_collection_already_exists(app_client, ctx, txn_client): @@ -29,7 +29,7 @@ async def test_create_collection_already_exists(app_client, ctx, txn_client): with pytest.raises(ConflictError): await txn_client.create_collection(data, request=MockRequest) - await txn_client.delete_collection(data["id"], request=MockRequest) + await txn_client.delete_collection(data["id"]) async def test_update_collection( @@ -46,7 +46,7 @@ async def test_update_collection( coll = await core_client.get_collection(data["id"], request=MockRequest) assert "new keyword" in coll["keywords"] - await txn_client.delete_collection(data["id"], request=MockRequest) + await txn_client.delete_collection(data["id"]) async def test_delete_collection( @@ -57,7 +57,7 @@ async def test_delete_collection( data = load_test_data("test_collection.json") await txn_client.create_collection(data, request=MockRequest) - await txn_client.delete_collection(data["id"], request=MockRequest) + await txn_client.delete_collection(data["id"]) with pytest.raises(NotFoundError): await core_client.get_collection(data["id"], request=MockRequest) @@ -73,7 +73,7 @@ async def test_get_collection( coll = await core_client.get_collection(data["id"], request=MockRequest) assert coll["id"] == data["id"] - await txn_client.delete_collection(data["id"], request=MockRequest) + await txn_client.delete_collection(data["id"]) async def test_get_item(app_client, ctx, core_client): diff --git a/stac_fastapi/elasticsearch/tests/features/__init__.py b/stac_fastapi/elasticsearch/tests/features/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/stac_fastapi/elasticsearch/tests/features/test_custom_models.py b/stac_fastapi/elasticsearch/tests/features/test_custom_models.py deleted file mode 100644 index 400c14ec..00000000 --- a/stac_fastapi/elasticsearch/tests/features/test_custom_models.py +++ /dev/null @@ -1,75 +0,0 @@ -# from typing import Type -# -# import sqlalchemy as sa -# from starlette.testclient import TestClient -# -# # TODO: move these -# from stac_api.models.database import Item -# from stac_api.models.schemas import Collection -# -# from stac_fastapi.api.app import StacApi -# from stac_fastapi.extensions.core import TransactionExtension -# from stac_fastapi.postgres.core import CoreCrudClient, Session -# from stac_fastapi.postgres.transactions import TransactionsClient -# from stac_fastapi.postgres.config import PostgresSettings -# -# -# from ..conftest import MockStarletteRequest -# -# -# class CustomItem(Item): -# foo = sa.Column(sa.VARCHAR(10)) -# -# -# def create_app(item_model: Type[Item], db_session: Session) -> StacApi: -# """Create application with a custom sqlalchemy item""" -# api = StacApi( -# settings=PostgresSettings(indexed_fields={"datetime", "foo"}), -# extensions=[ -# TransactionExtension( -# client=TransactionsClient(item_table=item_model, session=db_session) -# ) -# ], -# client=CoreCrudClient(item_table=item_model, session=db_session), -# ) -# return api -# -# -# def test_custom_item(load_test_data, postgres_transactions, db_session): -# api = create_app(CustomItem, db_session) -# transactions = TransactionsClient(item_table=CustomItem, session=db_session) -# -# with TestClient(api.app) as test_client: -# # Ingest a collection -# coll = Collection.parse_obj(load_test_data("test_collection.json")) -# transactions.create_collection(coll, request=MockStarletteRequest) -# -# # Modify the table to match our custom item -# # This would typically be done with alembic -# db_session.writer.cached_engine.execute( -# "ALTER TABLE data.items ADD COLUMN foo VARCHAR(10)" -# ) -# -# # Post an item -# test_item = load_test_data("test_item.json") -# test_item["properties"]["foo"] = "hello" -# resp = test_client.post( -# f"/collections/{test_item['collection']}/items", json=test_item -# ) -# assert resp.status_code == 200 -# assert resp.json()["properties"]["foo"] == "hello" -# -# # Search for the item -# body = {"query": {"foo": {"eq": "hello"}}} -# resp = test_client.post("/search", json=body) -# assert resp.status_code == 200 -# resp_json = resp.json() -# assert len(resp_json["features"]) == 1 -# assert resp_json["features"][0]["properties"]["foo"] == "hello" -# -# # Cleanup -# transactions.delete_item(test_item["id"], request=MockStarletteRequest) -# transactions.delete_collection(coll.id, request=MockStarletteRequest) -# db_session.writer.cached_engine.execute( -# "ALTER TABLE data.items DROP COLUMN foo" -# ) From 3a4bb17799d2f525075bdc7950236819f4793518 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Wed, 23 Mar 2022 12:54:21 -0400 Subject: [PATCH 3/3] add EOF newline to dockerfile --- Dockerfile.deploy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.deploy b/Dockerfile.deploy index 5a77133b..29494721 100644 --- a/Dockerfile.deploy +++ b/Dockerfile.deploy @@ -15,4 +15,4 @@ RUN pip install --no-cache-dir ./stac_fastapi/elasticsearch[server] EXPOSE 8080 -CMD ["uvicorn", "stac_fastapi.elasticsearch.app:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file +CMD ["uvicorn", "stac_fastapi.elasticsearch.app:app", "--host", "0.0.0.0", "--port", "8080"]