Skip to content

Commit 527bc83

Browse files
ZaczeroZaczero
committed
Deduplicate common database_logic
1 parent b016945 commit 527bc83

File tree

5 files changed

+242
-422
lines changed

5 files changed

+242
-422
lines changed

‎stac_fastapi/core/stac_fastapi/core/core.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,6 @@
4141

4242
logger = logging.getLogger(__name__)
4343

44-
NumType = Union[float, int]
45-
4644

4745
@attr.s
4846
class CoreClient(AsyncBaseCoreClient):

‎stac_fastapi/core/stac_fastapi/core/database_logic.py

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
import os
2+
from typing import Any, List, Optional, Protocol
3+
4+
from stac_fastapi.types.stac import Item
5+
6+
7+
# stac_pydantic classes extend _GeometryBase, which doesn't have a type field,
8+
# So create our own Protocol for typing
9+
# Union[ Point, MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon, GeometryCollection]
10+
class Geometry(Protocol): # noqa
11+
type: str
12+
coordinates: Any
13+
14+
15+
COLLECTIONS_INDEX = os.getenv("STAC_COLLECTIONS_INDEX", "collections")
16+
ITEMS_INDEX_PREFIX = os.getenv("STAC_ITEMS_INDEX_PREFIX", "items_")
17+
ES_INDEX_NAME_UNSUPPORTED_CHARS = {
18+
"\\",
19+
"/",
20+
"*",
21+
"?",
22+
'"',
23+
"<",
24+
">",
25+
"|",
26+
" ",
27+
",",
28+
"#",
29+
":",
30+
}
31+
32+
ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}*"
33+
34+
DEFAULT_SORT = {
35+
"properties.datetime": {"order": "desc"},
36+
"id": {"order": "desc"},
37+
"collection": {"order": "desc"},
38+
}
39+
40+
ES_ITEMS_SETTINGS = {
41+
"index": {
42+
"sort.field": list(DEFAULT_SORT.keys()),
43+
"sort.order": [v["order"] for v in DEFAULT_SORT.values()],
44+
}
45+
}
46+
47+
ES_MAPPINGS_DYNAMIC_TEMPLATES = [
48+
# Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md
49+
{
50+
"descriptions": {
51+
"match_mapping_type": "string",
52+
"match": "description",
53+
"mapping": {"type": "text"},
54+
}
55+
},
56+
{
57+
"titles": {
58+
"match_mapping_type": "string",
59+
"match": "title",
60+
"mapping": {"type": "text"},
61+
}
62+
},
63+
# Projection Extension https://github.com/stac-extensions/projection
64+
{"proj_epsg": {"match": "proj:epsg", "mapping": {"type": "integer"}}},
65+
{
66+
"proj_projjson": {
67+
"match": "proj:projjson",
68+
"mapping": {"type": "object", "enabled": False},
69+
}
70+
},
71+
{
72+
"proj_centroid": {
73+
"match": "proj:centroid",
74+
"mapping": {"type": "geo_point"},
75+
}
76+
},
77+
{
78+
"proj_geometry": {
79+
"match": "proj:geometry",
80+
"mapping": {"type": "object", "enabled": False},
81+
}
82+
},
83+
{
84+
"no_index_href": {
85+
"match": "href",
86+
"mapping": {"type": "text", "index": False},
87+
}
88+
},
89+
# Default all other strings not otherwise specified to keyword
90+
{"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}},
91+
{"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}},
92+
]
93+
94+
ES_ITEMS_MAPPINGS = {
95+
"numeric_detection": False,
96+
"dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES,
97+
"properties": {
98+
"id": {"type": "keyword"},
99+
"collection": {"type": "keyword"},
100+
"geometry": {"type": "geo_shape"},
101+
"assets": {"type": "object", "enabled": False},
102+
"links": {"type": "object", "enabled": False},
103+
"properties": {
104+
"type": "object",
105+
"properties": {
106+
# Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md
107+
"datetime": {"type": "date"},
108+
"start_datetime": {"type": "date"},
109+
"end_datetime": {"type": "date"},
110+
"created": {"type": "date"},
111+
"updated": {"type": "date"},
112+
# Satellite Extension https://github.com/stac-extensions/sat
113+
"sat:absolute_orbit": {"type": "integer"},
114+
"sat:relative_orbit": {"type": "integer"},
115+
},
116+
},
117+
},
118+
}
119+
120+
ES_COLLECTIONS_MAPPINGS = {
121+
"numeric_detection": False,
122+
"dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES,
123+
"properties": {
124+
"id": {"type": "keyword"},
125+
"extent.spatial.bbox": {"type": "long"},
126+
"extent.temporal.interval": {"type": "date"},
127+
"providers": {"type": "object", "enabled": False},
128+
"links": {"type": "object", "enabled": False},
129+
"item_assets": {"type": "object", "enabled": False},
130+
},
131+
}
132+
133+
134+
def index_by_collection_id(collection_id: str) -> str:
135+
"""
136+
Translate a collection id into an Elasticsearch index name.
137+
138+
Args:
139+
collection_id (str): The collection id to translate into an index name.
140+
141+
Returns:
142+
str: The index name derived from the collection id.
143+
"""
144+
return f"{ITEMS_INDEX_PREFIX}{''.join(c for c in collection_id.lower() if c not in ES_INDEX_NAME_UNSUPPORTED_CHARS)}_{collection_id.encode('utf-8').hex()}"
145+
146+
147+
def index_alias_by_collection_id(collection_id: str) -> str:
148+
"""
149+
Translate a collection id into an Elasticsearch index alias.
150+
151+
Args:
152+
collection_id (str): The collection id to translate into an index alias.
153+
154+
Returns:
155+
str: The index alias derived from the collection id.
156+
"""
157+
return f"{ITEMS_INDEX_PREFIX}{''.join(c for c in collection_id if c not in ES_INDEX_NAME_UNSUPPORTED_CHARS)}"
158+
159+
160+
def indices(collection_ids: Optional[List[str]]) -> str:
161+
"""
162+
Get a comma-separated string of index names for a given list of collection ids.
163+
164+
Args:
165+
collection_ids: A list of collection ids.
166+
167+
Returns:
168+
A string of comma-separated index names. If `collection_ids` is None, returns the default indices.
169+
"""
170+
if collection_ids is None or collection_ids == []:
171+
return ITEM_INDICES
172+
else:
173+
return ",".join([index_alias_by_collection_id(c) for c in collection_ids])
174+
175+
176+
def mk_item_id(item_id: str, collection_id: str):
177+
"""Create the document id for an Item in Elasticsearch.
178+
179+
Args:
180+
item_id (str): The id of the Item.
181+
collection_id (str): The id of the Collection that the Item belongs to.
182+
183+
Returns:
184+
str: The document id for the Item, combining the Item id and the Collection id, separated by a `|` character.
185+
"""
186+
return f"{item_id}|{collection_id}"
187+
188+
189+
def mk_actions(collection_id: str, processed_items: List[Item]):
190+
"""Create Elasticsearch bulk actions for a list of processed items.
191+
192+
Args:
193+
collection_id (str): The identifier for the collection the items belong to.
194+
processed_items (List[Item]): The list of processed items to be bulk indexed.
195+
196+
Returns:
197+
List[Dict[str, Union[str, Dict]]]: The list of bulk actions to be executed,
198+
each action being a dictionary with the following keys:
199+
- `_index`: the index to store the document in.
200+
- `_id`: the document's identifier.
201+
- `_source`: the source of the document.
202+
"""
203+
return [
204+
{
205+
"_index": index_alias_by_collection_id(collection_id),
206+
"_id": mk_item_id(item["id"], item["collection"]),
207+
"_source": item,
208+
}
209+
for item in processed_items
210+
]

‎stac_fastapi/core/stac_fastapi/core/extensions/query.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717

1818
logger = logging.getLogger("uvicorn")
1919
logger.setLevel(logging.INFO)
20-
# Be careful: https://github.com/samuelcolvin/pydantic/issues/1423#issuecomment-642797287
21-
NumType = Union[float, int]
2220

2321

2422
class Operator(str, AutoValueEnum):

0 commit comments

Comments
 (0)