Merge branch 'main' into CAT-1382

Author: YuriZmytrakov

.pre-commit-config.yaml

@@ -31,7 +31,8 @@ repos:
         ]
         additional_dependencies: [
           "types-attrs",
-          "types-requests"
+          "types-requests",
+          "types-redis"
         ]
   - repo: https://github.com/PyCQA/pydocstyle
     rev: 6.1.1

CHANGELOG.md

@@ -9,8 +9,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- GET `/collections` collection search free text extension ex. `/collections?q=sentinel`. [#470](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/470)
 - Added `USE_DATETIME` environment variable to configure datetime search behavior in SFEOS. [#452](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/452)
 - GET `/collections` collection search sort extension ex. `/collections?sortby=+id`. [#456](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/456)
+- GET `/collections` collection search fields extension ex. `/collections?fields=id,title`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465)
+- Improved error messages for sorting on unsortable fields in collection search, including guidance on how to make fields sortable. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465)
+- Added field alias for `temporal` to enable easier sorting by temporal extent, alongside `extent.temporal.interval`. [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465)
+- Added `ENABLE_COLLECTIONS_SEARCH` environment variable to make collection search extensions optional (defaults to enabled). [#465](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/465)
 
 ### Changed
 

Makefile

@@ -63,22 +63,22 @@ docker-shell-os:
 
 .PHONY: test-elasticsearch
 test-elasticsearch:
-	-$(run_es) /bin/bash -c 'pip install redis==6.4.0 export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest'
+	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest'
 	docker compose down
 
 .PHONY: test-opensearch
 test-opensearch:
-	-$(run_os) /bin/bash -c 'pip install redis==6.4.0 export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest'
+	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest'
 	docker compose down
 
 .PHONY: test-datetime-filtering-es
 test-datetime-filtering-es:
-	-$(run_es) /bin/bash -c 'pip install redis==6.4.0 && export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering'
+	-$(run_es) /bin/bash -c 'export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering'
 	docker compose down
 
 .PHONY: test-datetime-filtering-os
 test-datetime-filtering-os:
-	-$(run_os) /bin/bash -c 'pip install redis==6.4.0 && export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering'
+	-$(run_os) /bin/bash -c 'export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering'
 	docker compose down
 
 .PHONY: test

README.md

@@ -36,11 +36,10 @@ SFEOS (stac-fastapi-elasticsearch-opensearch) is a high-performance, scalable AP
 - **Scale to millions of geospatial assets** with fast search performance through optimized spatial indexing and query capabilities
 - **Support OGC-compliant filtering** including spatial operations (intersects, contains, etc.) and temporal queries
 - **Perform geospatial aggregations** to analyze data distribution across space and time
+- **Enhanced collection search capabilities** with support for sorting and field selection
 
 This implementation builds on the STAC-FastAPI framework, providing a production-ready solution specifically optimized for Elasticsearch and OpenSearch databases. It's ideal for organizations managing large geospatial data catalogs who need efficient discovery and access capabilities through standardized APIs.
 
-
-
 ## Common Deployment Patterns
 
 stac-fastapi-elasticsearch-opensearch can be deployed in several ways depending on your needs:
@@ -72,6 +71,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
   - [Common Deployment Patterns](#common-deployment-patterns)
   - [Technologies](#technologies)
   - [Table of Contents](#table-of-contents)
+  - [Collection Search Extensions](#collection-search-extensions)
   - [Documentation \& Resources](#documentation--resources)
   - [Package Structure](#package-structure)
   - [Examples](#examples)
@@ -113,6 +113,37 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
   - [Gitter Chat](https://app.gitter.im/#/room/#stac-fastapi-elasticsearch_community:gitter.im) - For real-time discussions
   - [GitHub Discussions](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/discussions) - For longer-form questions and answers
 
+## Collection Search Extensions
+
+SFEOS implements extended capabilities for the `/collections` endpoint, allowing for more powerful collection discovery:
+
+- **Sorting**: Sort collections by sortable fields using the `sortby` parameter
+  - Example: `/collections?sortby=+id` (ascending sort by ID)
+  - Example: `/collections?sortby=-id` (descending sort by ID)
+  - Example: `/collections?sortby=-temporal` (descending sort by temporal extent)
+
+- **Field Selection**: Request only specific fields to be returned using the `fields` parameter
+  - Example: `/collections?fields=id,title,description`
+  - This helps reduce payload size when only certain fields are needed
+
+- **Free Text Search**: Search across collection text fields using the `q` parameter
+  - Example: `/collections?q=landsat`
+  - Searches across multiple text fields including title, description, and keywords
+  - Supports partial word matching and relevance-based sorting
+
+These extensions make it easier to build user interfaces that display and navigate through collections efficiently.
+
+> **Configuration**: Collection search extensions can be disabled by setting the `ENABLE_COLLECTIONS_SEARCH` environment variable to `false`. By default, these extensions are enabled.
+
+> **Note**: Sorting is only available on fields that are indexed for sorting in Elasticsearch/OpenSearch. With the default mappings, you can sort on:
+> - `id` (keyword field)
+> - `extent.temporal.interval` (date field)
+> - `temporal` (alias to extent.temporal.interval)
+>
+> Text fields like `title` and `description` are not sortable by default as they use text analysis for better search capabilities. Attempting to sort on these fields will result in a user-friendly error message explaining which fields are sortable and how to make additional fields sortable by updating the mappings.
+>
+> **Important**: Adding keyword fields to make text fields sortable can significantly increase the index size, especially for large text fields. Consider the storage implications when deciding which fields to make sortable.
+
 ## Package Structure
 
 This project is organized into several packages, each with a specific purpose:
@@ -243,6 +274,7 @@ You can customize additional settings in your `.env` file:
 | `ENABLE_DIRECT_RESPONSE`     | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false`                  | Optional                       |
 | `RAISE_ON_BULK_ERROR`        | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
 | `DATABASE_REFRESH`           | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional |
+| `ENABLE_COLLECTIONS_SEARCH`  | Enable collection search extensions (sort, fields).                                 | `true`                   | Optional                                                                                    |
 | `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. If set to `false`, the POST `/collections` route and related transaction endpoints (including bulk transaction operations) will be unavailable in the API. This is useful for deployments where mutating the catalog via the API should be prevented. | `true` | Optional |
 | `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional |
 | `STAC_INDEX_ASSETS` | Controls if Assets are indexed when added to Elasticsearch/Opensearch. This allows asset fields to be included in search queries. | `false` | Optional |
@@ -389,6 +421,10 @@ The system uses a precise naming convention:
 - **Root Path Configuration**: The application root path is the base URL by default.
   - For AWS Lambda with Gateway API: Set `STAC_FASTAPI_ROOT_PATH` to match the Gateway API stage name (e.g., `/v1`)
 
+- **Feature Configuration**: Control which features are enabled:
+  - `ENABLE_COLLECTIONS_SEARCH`: Set to `true` (default) to enable collection search extensions (sort, fields). Set to `false` to disable.
+  - `ENABLE_TRANSACTIONS_EXTENSIONS`: Set to `true` (default) to enable transaction extensions. Set to `false` to disable.
+
 
 ## Collection Pagination
 

docker-compose.redis.yml

@@ -18,3 +18,4 @@ COPY . /app
 RUN pip install --no-cache-dir -e ./stac_fastapi/core
 RUN pip install --no-cache-dir -e ./stac_fastapi/sfeos_helpers
 RUN pip install --no-cache-dir -e ./stac_fastapi/elasticsearch[dev,server]
+RUN pip install --no-cache-dir redis types-redis

dockerfiles/Dockerfile.dev.es

@@ -230,11 +230,18 @@ async def landing_page(self, **kwargs) -> stac_types.LandingPage:
         return landing_page
 
     async def all_collections(
-        self, sortby: Optional[str] = None, **kwargs
+        self,
+        fields: Optional[List[str]] = None,
+        sortby: Optional[str] = None,
+        q: Optional[Union[str, List[str]]] = None,
+        **kwargs,
     ) -> stac_types.Collections:
         """Read all collections from the database.
 
         Args:
+            fields (Optional[List[str]]): Fields to include or exclude from the results.
+            sortby (Optional[str]): Sorting options for the results.
+            q (Optional[List[str]]): Free text search terms.
             **kwargs: Keyword arguments from the request.
 
         Returns:
@@ -245,6 +252,15 @@ async def all_collections(
         limit = int(request.query_params.get("limit", os.getenv("STAC_ITEM_LIMIT", 10)))
         token = request.query_params.get("token")
 
+        # Process fields parameter for filtering collection properties
+        includes, excludes = set(), set()
+        if fields and self.extension_is_enabled("FieldsExtension"):
+            for field in fields:
+                if field[0] == "-":
+                    excludes.add(field[1:])
+                else:
+                    includes.add(field[1:] if field[0] in "+ " else field)
+
         sort = None
         if sortby:
             parsed_sort = []
@@ -267,10 +283,24 @@ async def all_collections(
         except Exception:
             redis = None
 
+        # Convert q to a list if it's a string
+        q_list = None
+        if q is not None:
+            q_list = [q] if isinstance(q, str) else q
+
         collections, next_token = await self.database.get_all_collections(
-            token=token, limit=limit, request=request, sort=sort
+            token=token, limit=limit, request=request, sort=sort, q=q_list
         )
 
+        # Apply field filtering if fields parameter was provided
+        if fields and self.extension_is_enabled("FieldsExtension"):
+            filtered_collections = [
+                filter_fields(collection, includes, excludes)
+                for collection in collections
+            ]
+        else:
+            filtered_collections = collections
+
         links = [
             {"rel": Relations.root.value, "type": MimeTypes.json, "href": base_url},
             {"rel": Relations.parent.value, "type": MimeTypes.json, "href": base_url},
@@ -301,7 +331,7 @@ async def all_collections(
             next_link = PagingLinks(next=next_token, request=request).link_next()
             links.append(next_link)
 
-        return stac_types.Collections(collections=collections, links=links)
+        return stac_types.Collections(collections=filtered_collections, links=links)
 
     async def get_collection(
         self, collection_id: str, **kwargs

mypy.ini

@@ -15,7 +15,7 @@ class RedisSentinelSettings(BaseSettings):
     REDIS_SENTINEL_HOSTS: str = ""
     REDIS_SENTINEL_PORTS: str = "26379"
     REDIS_SENTINEL_MASTER_NAME: str = "master"
-    REDIS_DB: int = 15
+    REDIS_DB: int = 0
 
     REDIS_MAX_CONNECTIONS: int = 10
     REDIS_RETRY_TIMEOUT: bool = True
@@ -25,7 +25,7 @@ class RedisSentinelSettings(BaseSettings):
 
 
 class RedisSettings(BaseSettings):
-    """Configuration for connecting Redis Sentinel."""
+    """Configuration for connecting Redis."""
 
     REDIS_HOST: str = ""
     REDIS_PORT: int = 6379
@@ -42,36 +42,11 @@ class RedisSettings(BaseSettings):
 redis_settings: BaseSettings = RedisSentinelSettings()
 
 
-async def connect_redis(settings: Optional[RedisSettings] = None) -> aioredis.Redis:
-    """Return a Redis connection."""
-    global redis_pool
-    settings = settings or redis_settings
-
-    if not settings.REDIS_HOST or not settings.REDIS_PORT:
-        return None
-
-    if redis_pool is None:
-        pool = aioredis.ConnectionPool(
-            host=settings.REDIS_HOST,
-            port=settings.REDIS_PORT,
-            db=settings.REDIS_DB,
-            max_connections=settings.REDIS_MAX_CONNECTIONS,
-            decode_responses=settings.REDIS_DECODE_RESPONSES,
-            retry_on_timeout=settings.REDIS_RETRY_TIMEOUT,
-            health_check_interval=settings.REDIS_HEALTH_CHECK_INTERVAL,
-        )
-        redis_pool = aioredis.Redis(
-            connection_pool=pool, client_name=settings.REDIS_CLIENT_NAME
-        )
-    return redis_pool
-
-
 async def connect_redis_sentinel(
     settings: Optional[RedisSentinelSettings] = None,
 ) -> Optional[aioredis.Redis]:
-    """Return a Redis Sentinel connection."""
+    """Return Redis Sentinel connection."""
     global redis_pool
-
     settings = settings or redis_settings
 
     if (
@@ -89,7 +64,7 @@ async def connect_redis_sentinel(
     if redis_pool is None:
         try:
             sentinel = Sentinel(
-                [(h, p) for h, p in zip(hosts, ports)],
+                [(host, port) for host, port in zip(hosts, ports)],
                 decode_responses=settings.REDIS_DECODE_RESPONSES,
             )
             master = sentinel.master_for(
@@ -109,16 +84,40 @@ async def connect_redis_sentinel(
     return redis_pool
 
 
+async def connect_redis(settings: Optional[RedisSettings] = None) -> aioredis.Redis:
+    """Return Redis connection."""
+    global redis_pool
+    settings = settings or redis_settings
+
+    if not settings.REDIS_HOST or not settings.REDIS_PORT:
+        return None
+
+    if redis_pool is None:
+        pool = aioredis.ConnectionPool(
+            host=settings.REDIS_HOST,
+            port=settings.REDIS_PORT,
+            db=settings.REDIS_DB,
+            max_connections=settings.REDIS_MAX_CONNECTIONS,
+            decode_responses=settings.REDIS_DECODE_RESPONSES,
+            retry_on_timeout=settings.REDIS_RETRY_TIMEOUT,
+            health_check_interval=settings.REDIS_HEALTH_CHECK_INTERVAL,
+        )
+        redis_pool = aioredis.Redis(
+            connection_pool=pool, client_name=settings.REDIS_CLIENT_NAME
+        )
+    return redis_pool
+
+
 async def save_self_link(
     redis: aioredis.Redis, token: Optional[str], self_href: str
 ) -> None:
-    """Save the self link for the current token with 30 min TTL."""
+    """Add the self link for next page as prev link for the current token."""
     if token:
         await redis.setex(f"nav:self:{token}", 1800, self_href)
 
 
 async def get_prev_link(redis: aioredis.Redis, token: Optional[str]) -> Optional[str]:
-    """Get the previous page link for the current token (if exists)."""
+    """Pull the prev page link for the current token."""
     if not token:
         return None
     return await redis.get(f"nav:self:{token}")