Skip to content

Commit 7d6b741

Browse files
YuriZmytrakovYuri Zmytrakov
YuriZmytrakov
and
Yuri Zmytrakov
authored
USE_DATETIME env var for filter behavior (#452)
**Related Issue(s):** #403 **Description:** This PR introduces a new env var USE_DATETIME to control the datetime filtering behavior in the search. **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --------- Co-authored-by: Yuri Zmytrakov <[email protected]>
1 parent 5e41770 commit 7d6b741

File tree

5 files changed

+274
-88
lines changed

5 files changed

+274
-88
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
99

1010
### Added
1111

12+
- Added `USE_DATETIME` environment variable to configure datetime search behavior in SFEOS. [#452](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/452)
1213
- GET `/collections` collection search sort extension ex. `/collections?sortby=+id`. [#456](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/456)
1314

1415
### Changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ You can customize additional settings in your `.env` file:
247247
| `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional |
248248
| `STAC_INDEX_ASSETS` | Controls if Assets are indexed when added to Elasticsearch/Opensearch. This allows asset fields to be included in search queries. | `false` | Optional |
249249
| `ENV_MAX_LIMIT` | Configures the environment variable in SFEOS to override the default `MAX_LIMIT`, which controls the limit parameter for returned items and STAC collections. | `10,000` | Optional |
250+
| `USE_DATETIME` | Configures the datetime search behavior in SFEOS. When enabled, searches both datetime field and falls back to start_datetime/end_datetime range for items with null datetime. When disabled, searches only by start_datetime/end_datetime range. | True | Optional |
250251

251252
> [!NOTE]
252253
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.

‎stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py‎

Lines changed: 93 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
1919
from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
20-
from stac_fastapi.core.utilities import bbox2polygon, get_max_limit
20+
from stac_fastapi.core.utilities import bbox2polygon, get_bool_env, get_max_limit
2121
from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
2222
from stac_fastapi.elasticsearch.config import (
2323
ElasticsearchSettings as SyncElasticsearchSettings,
@@ -310,26 +310,99 @@ def apply_datetime_filter(
310310
Returns:
311311
The filtered search object.
312312
"""
313+
# USE_DATETIME env var
314+
# True: Search by datetime, if null search by start/end datetime
315+
# False: Always search only by start/end datetime
316+
USE_DATETIME = get_bool_env("USE_DATETIME", default=True)
317+
313318
datetime_search = return_date(datetime)
314319

315320
if not datetime_search:
316321
return search, datetime_search
317322

318-
if "eq" in datetime_search:
319-
# For exact matches, include:
320-
# 1. Items with matching exact datetime
321-
# 2. Items with datetime:null where the time falls within their range
322-
should = [
323-
Q(
324-
"bool",
325-
filter=[
326-
Q("exists", field="properties.datetime"),
327-
Q("term", **{"properties__datetime": datetime_search["eq"]}),
328-
],
329-
),
330-
Q(
323+
if USE_DATETIME:
324+
if "eq" in datetime_search:
325+
# For exact matches, include:
326+
# 1. Items with matching exact datetime
327+
# 2. Items with datetime:null where the time falls within their range
328+
should = [
329+
Q(
330+
"bool",
331+
filter=[
332+
Q("exists", field="properties.datetime"),
333+
Q(
334+
"term",
335+
**{"properties__datetime": datetime_search["eq"]},
336+
),
337+
],
338+
),
339+
Q(
340+
"bool",
341+
must_not=[Q("exists", field="properties.datetime")],
342+
filter=[
343+
Q("exists", field="properties.start_datetime"),
344+
Q("exists", field="properties.end_datetime"),
345+
Q(
346+
"range",
347+
properties__start_datetime={
348+
"lte": datetime_search["eq"]
349+
},
350+
),
351+
Q(
352+
"range",
353+
properties__end_datetime={"gte": datetime_search["eq"]},
354+
),
355+
],
356+
),
357+
]
358+
else:
359+
# For date ranges, include:
360+
# 1. Items with datetime in the range
361+
# 2. Items with datetime:null that overlap the search range
362+
should = [
363+
Q(
364+
"bool",
365+
filter=[
366+
Q("exists", field="properties.datetime"),
367+
Q(
368+
"range",
369+
properties__datetime={
370+
"gte": datetime_search["gte"],
371+
"lte": datetime_search["lte"],
372+
},
373+
),
374+
],
375+
),
376+
Q(
377+
"bool",
378+
must_not=[Q("exists", field="properties.datetime")],
379+
filter=[
380+
Q("exists", field="properties.start_datetime"),
381+
Q("exists", field="properties.end_datetime"),
382+
Q(
383+
"range",
384+
properties__start_datetime={
385+
"lte": datetime_search["lte"]
386+
},
387+
),
388+
Q(
389+
"range",
390+
properties__end_datetime={
391+
"gte": datetime_search["gte"]
392+
},
393+
),
394+
],
395+
),
396+
]
397+
398+
return (
399+
search.query(Q("bool", should=should, minimum_should_match=1)),
400+
datetime_search,
401+
)
402+
else:
403+
if "eq" in datetime_search:
404+
filter_query = Q(
331405
"bool",
332-
must_not=[Q("exists", field="properties.datetime")],
333406
filter=[
334407
Q("exists", field="properties.start_datetime"),
335408
Q("exists", field="properties.end_datetime"),
@@ -342,29 +415,10 @@ def apply_datetime_filter(
342415
properties__end_datetime={"gte": datetime_search["eq"]},
343416
),
344417
],
345-
),
346-
]
347-
else:
348-
# For date ranges, include:
349-
# 1. Items with datetime in the range
350-
# 2. Items with datetime:null that overlap the search range
351-
should = [
352-
Q(
353-
"bool",
354-
filter=[
355-
Q("exists", field="properties.datetime"),
356-
Q(
357-
"range",
358-
properties__datetime={
359-
"gte": datetime_search["gte"],
360-
"lte": datetime_search["lte"],
361-
},
362-
),
363-
],
364-
),
365-
Q(
418+
)
419+
else:
420+
filter_query = Q(
366421
"bool",
367-
must_not=[Q("exists", field="properties.datetime")],
368422
filter=[
369423
Q("exists", field="properties.start_datetime"),
370424
Q("exists", field="properties.end_datetime"),
@@ -377,13 +431,8 @@ def apply_datetime_filter(
377431
properties__end_datetime={"gte": datetime_search["gte"]},
378432
),
379433
],
380-
),
381-
]
382-
383-
return (
384-
search.query(Q("bool", should=should, minimum_should_match=1)),
385-
datetime_search,
386-
)
434+
)
435+
return search.query(filter_query), datetime_search
387436

388437
@staticmethod
389438
def apply_bbox_filter(search: Search, bbox: List):

‎stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py‎

Lines changed: 93 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
1919
from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
20-
from stac_fastapi.core.utilities import bbox2polygon, get_max_limit
20+
from stac_fastapi.core.utilities import bbox2polygon, get_bool_env, get_max_limit
2121
from stac_fastapi.extensions.core.transaction.request import (
2222
PartialCollection,
2323
PartialItem,
@@ -318,21 +318,94 @@ def apply_datetime_filter(
318318
if not datetime_search:
319319
return search, datetime_search
320320

321-
if "eq" in datetime_search:
322-
# For exact matches, include:
323-
# 1. Items with matching exact datetime
324-
# 2. Items with datetime:null where the time falls within their range
325-
should = [
326-
Q(
327-
"bool",
328-
filter=[
329-
Q("exists", field="properties.datetime"),
330-
Q("term", **{"properties__datetime": datetime_search["eq"]}),
331-
],
332-
),
333-
Q(
321+
# USE_DATETIME env var
322+
# True: Search by datetime, if null search by start/end datetime
323+
# False: Always search only by start/end datetime
324+
USE_DATETIME = get_bool_env("USE_DATETIME", default=True)
325+
326+
if USE_DATETIME:
327+
if "eq" in datetime_search:
328+
# For exact matches, include:
329+
# 1. Items with matching exact datetime
330+
# 2. Items with datetime:null where the time falls within their range
331+
should = [
332+
Q(
333+
"bool",
334+
filter=[
335+
Q("exists", field="properties.datetime"),
336+
Q(
337+
"term",
338+
**{"properties__datetime": datetime_search["eq"]},
339+
),
340+
],
341+
),
342+
Q(
343+
"bool",
344+
must_not=[Q("exists", field="properties.datetime")],
345+
filter=[
346+
Q("exists", field="properties.start_datetime"),
347+
Q("exists", field="properties.end_datetime"),
348+
Q(
349+
"range",
350+
properties__start_datetime={
351+
"lte": datetime_search["eq"]
352+
},
353+
),
354+
Q(
355+
"range",
356+
properties__end_datetime={"gte": datetime_search["eq"]},
357+
),
358+
],
359+
),
360+
]
361+
else:
362+
# For date ranges, include:
363+
# 1. Items with datetime in the range
364+
# 2. Items with datetime:null that overlap the search range
365+
should = [
366+
Q(
367+
"bool",
368+
filter=[
369+
Q("exists", field="properties.datetime"),
370+
Q(
371+
"range",
372+
properties__datetime={
373+
"gte": datetime_search["gte"],
374+
"lte": datetime_search["lte"],
375+
},
376+
),
377+
],
378+
),
379+
Q(
380+
"bool",
381+
must_not=[Q("exists", field="properties.datetime")],
382+
filter=[
383+
Q("exists", field="properties.start_datetime"),
384+
Q("exists", field="properties.end_datetime"),
385+
Q(
386+
"range",
387+
properties__start_datetime={
388+
"lte": datetime_search["lte"]
389+
},
390+
),
391+
Q(
392+
"range",
393+
properties__end_datetime={
394+
"gte": datetime_search["gte"]
395+
},
396+
),
397+
],
398+
),
399+
]
400+
401+
return (
402+
search.query(Q("bool", should=should, minimum_should_match=1)),
403+
datetime_search,
404+
)
405+
else:
406+
if "eq" in datetime_search:
407+
filter_query = Q(
334408
"bool",
335-
must_not=[Q("exists", field="properties.datetime")],
336409
filter=[
337410
Q("exists", field="properties.start_datetime"),
338411
Q("exists", field="properties.end_datetime"),
@@ -345,29 +418,10 @@ def apply_datetime_filter(
345418
properties__end_datetime={"gte": datetime_search["eq"]},
346419
),
347420
],
348-
),
349-
]
350-
else:
351-
# For date ranges, include:
352-
# 1. Items with datetime in the range
353-
# 2. Items with datetime:null that overlap the search range
354-
should = [
355-
Q(
356-
"bool",
357-
filter=[
358-
Q("exists", field="properties.datetime"),
359-
Q(
360-
"range",
361-
properties__datetime={
362-
"gte": datetime_search["gte"],
363-
"lte": datetime_search["lte"],
364-
},
365-
),
366-
],
367-
),
368-
Q(
421+
)
422+
else:
423+
filter_query = Q(
369424
"bool",
370-
must_not=[Q("exists", field="properties.datetime")],
371425
filter=[
372426
Q("exists", field="properties.start_datetime"),
373427
Q("exists", field="properties.end_datetime"),
@@ -380,13 +434,8 @@ def apply_datetime_filter(
380434
properties__end_datetime={"gte": datetime_search["gte"]},
381435
),
382436
],
383-
),
384-
]
385-
386-
return (
387-
search.query(Q("bool", should=should, minimum_should_match=1)),
388-
datetime_search,
389-
)
437+
)
438+
return search.query(filter_query), datetime_search
390439

391440
@staticmethod
392441
def apply_bbox_filter(search: Search, bbox: List):

0 commit comments

Comments
 (0)