feat: add support for sorting in list v2 endpoint

itslenny · itslenny · commit 24b869c64574 · 2025-08-14T17:22:54.000-04:00
diff --git a/migrations/tenant/0039-add-search-v2-sort-support.sql b/migrations/tenant/0039-add-search-v2-sort-support.sql
@@ -0,0 +1,100 @@
+CREATE OR REPLACE FUNCTION storage.search_v2 (
+    prefix text,
+    bucket_name text,
+    limits int DEFAULT 100,
+    levels int DEFAULT 1,
+    start_after text DEFAULT '',
+    sortcolumn text DEFAULT 'name',
+    sortorder text DEFAULT 'asc'
+) RETURNS TABLE (
+    key text,
+    name text,
+    id uuid,
+    updated_at timestamptz,
+    created_at timestamptz,
+    last_accessed_at timestamptz,
+    metadata jsonb
+)
+SECURITY INVOKER
+AS $func$
+DECLARE
+    sort_col text;
+    sort_ord text;
+    cursor_op text;
+    cursor_expr text;
+    collate_clause text := '';
+BEGIN
+    -- Validate sortorder
+    sort_ord := lower(sortorder);
+    IF sort_ord NOT IN ('asc', 'desc') THEN
+        sort_ord := 'asc';
+    END IF;
+
+    -- Determine cursor comparison operator
+    IF sort_ord = 'asc' THEN
+        cursor_op := '>';
+    ELSE
+        cursor_op := '<';
+    END IF;
+
+    -- Validate sortcolumn
+    sort_col := lower(sortcolumn);
+    IF sort_col IN ('updated_at', 'created_at', 'last_accessed_at') THEN
+        cursor_expr := format('($5 = '''' OR %I %s $5::timestamptz)', sort_col, cursor_op);
+    ELSE
+        sort_col := 'name';
+        collate_clause := ' COLLATE "C"';
+        cursor_expr := format('($5 = '''' OR %I%s %s $5)', sort_col, collate_clause, cursor_op);
+    END IF;
+
+    RETURN QUERY EXECUTE format(
+        $sql$
+        SELECT * FROM (
+            (
+                SELECT
+                    split_part(name, '/', $4) AS key,
+                    name || '/' AS name,
+                    NULL::uuid AS id,
+                    NULL::timestamptz AS updated_at,
+                    NULL::timestamptz AS created_at,
+                    NULL::timestamptz AS last_accessed_at,
+                    NULL::jsonb AS metadata
+                FROM storage.prefixes
+                WHERE name COLLATE "C" LIKE $1 || '%%'
+                    AND bucket_id = $2
+                    AND level = $4
+                    AND %s
+                ORDER BY %I%s %s
+                LIMIT $3
+            )
+            UNION ALL
+            (
+                SELECT
+                    split_part(name, '/', $4) AS key,
+                    name,
+                    id,
+                    updated_at,
+                    created_at,
+                    last_accessed_at,
+                    metadata
+                FROM storage.objects
+                WHERE name COLLATE "C" LIKE $1 || '%%'
+                    AND bucket_id = $2
+                    AND level = $4
+                    AND %s
+                ORDER BY %I%s %s
+                LIMIT $3
+            )
+        ) obj
+        ORDER BY %I %s
+        LIMIT $3
+        $sql$,
+        cursor_expr,         -- prefixes WHERE
+        sort_col, collate_clause, sort_ord,  -- prefixes ORDER BY
+        cursor_expr,         -- objects WHERE
+        sort_col, collate_clause, sort_ord,  -- objects ORDER BY
+        sort_col, collate_clause, sort_ord   -- final ORDER BY
+    )
+    USING prefix, bucket_name, limits, levels, start_after;
+END;
+$func$ LANGUAGE plpgsql STABLE;
diff --git a/src/http/routes/object/listObjectsV2.ts b/src/http/routes/object/listObjectsV2.ts
@@ -23,6 +23,14 @@ const searchRequestBodySchema = {
     limit: { type: 'integer', minimum: 1, examples: [10] },
     cursor: { type: 'string' },
     with_delimiter: { type: 'boolean' },
+    sortBy: {
+      type: 'object',
+      properties: {
+        column: { type: 'string', enum: ['name', 'updated_at', 'created_at', 'last_accessed_at'] },
+        order: { type: 'string', enum: ['asc', 'desc'] },
+      },
+      required: ['column'],
+    },
   },
 } as const
 interface searchRequestInterface extends AuthenticatedRequest {
@@ -57,13 +65,14 @@ export default async function routes(fastify: FastifyInstance) {
       }
 
       const { bucketName } = request.params
-      const { limit, with_delimiter, cursor, prefix } = request.body
+      const { limit, with_delimiter, cursor, prefix, sortBy } = request.body
 
       const results = await request.storage.from(bucketName).listObjectsV2({
         prefix,
         delimiter: with_delimiter ? '/' : undefined,
         maxKeys: limit,
         cursor,
+        sortBy,
       })
 
       return response.status(200).send(results)
diff --git a/src/internal/database/migrations/types.ts b/src/internal/database/migrations/types.ts
@@ -37,4 +37,5 @@ export const DBMigration = {
   'optimise-existing-functions': 36,
   'add-bucket-name-length-trigger': 37,
   'iceberg-catalog-flag-on-buckets': 38,
+  'add-search-v2-sort-support': 39,
 }
diff --git a/src/storage/database/adapter.ts b/src/storage/database/adapter.ts
@@ -105,6 +105,10 @@ export interface Database {
       nextToken?: string
       maxKeys?: number
       startAfter?: string
+      sortBy?: {
+        column?: string
+        order?: string
+      }
     }
   ): Promise<Obj[]>
 
diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts
@@ -269,27 +269,44 @@ export class StorageKnexDB implements Database {
       nextToken?: string
       maxKeys?: number
       startAfter?: string
+      sortBy?: {
+        column?: string
+        order?: string
+      }
     }
   ) {
     return this.runQuery('ListObjectsV2', async (knex) => {
       if (!options?.delimiter) {
         const query = knex
           .table('objects')
           .where('bucket_id', bucketId)
-          .select(['id', 'name', 'metadata', 'updated_at'])
+          .select(['id', 'name', 'metadata', 'updated_at', 'created_at', 'last_accessed_at'])
           .limit(options?.maxKeys || 100)
 
+        // only allow these values for sort columns, "name" is excluded intentionally so it will use the default value (to include collate)
+        const allowedSortColumns = new Set(['updated_at', 'created_at', 'last_accessed_at'])
+        const allowedSortOrders = new Set(['asc', 'desc'])
+        const sortColumn =
+          options?.sortBy?.column && allowedSortColumns.has(options.sortBy.column)
+            ? options.sortBy.column
+            : 'name COLLATE "C"'
+        const sortOrder =
+          options?.sortBy?.order && allowedSortOrders.has(options.sortBy.order)
+            ? options.sortBy.order
+            : 'asc'
+
         // knex typing is wrong, it doesn't accept a knex.raw on orderBy, even though is totally legit
         // eslint-disable-next-line @typescript-eslint/ban-ts-comment
         // @ts-ignore
-        query.orderBy(knex.raw('name COLLATE "C"'))
+        query.orderBy(knex.raw(`${sortColumn}`), sortOrder)
 
         if (options?.prefix) {
           query.where('name', 'like', `${options.prefix}%`)
         }
 
         if (options?.nextToken) {
-          query.andWhere(knex.raw('name COLLATE "C" > ?', [options?.nextToken]))
+          const pageOperator = sortOrder === 'asc' ? '>' : '<'
+          query.andWhere(knex.raw(`${sortColumn} ${pageOperator} ?`, [options.nextToken]))
         }
 
         return query
@@ -302,13 +319,21 @@ export class StorageKnexDB implements Database {
       }
 
       if (useNewSearchVersion2 && options?.delimiter === '/') {
+        let paramPlaceholders = '?,?,?,?,?'
+        const sortParams: string[] = []
+        // this migration adds 2 more parameters to search v2 support sorting
+        if (await tenantHasMigrations(this.tenantId, 'add-search-v2-sort-support')) {
+          paramPlaceholders += ',?,?'
+          sortParams.push(options?.sortBy?.column || 'name', options?.sortBy?.order || 'asc')
+        }
         const levels = !options?.prefix ? 1 : options.prefix.split('/').length
-        const query = await knex.raw('select * from storage.search_v2(?,?,?,?,?)', [
+        const query = await knex.raw(`select * from storage.search_v2(${paramPlaceholders})`, [
           options?.prefix || '',
           bucketId,
           options?.maxKeys || 1000,
           levels,
           options?.startAfter || '',
+          ...sortParams,
         ])
 
         return query.rows
diff --git a/src/storage/object.ts b/src/storage/object.ts
@@ -586,18 +586,25 @@ export class ObjectStorage {
     startAfter?: string
     maxKeys?: number
     encodingType?: 'url'
+    sortBy?: {
+      column?: 'name' | 'created_at' | 'updated_at' | 'last_accessed_at'
+      order?: string
+    }
   }) {
     const limit = Math.min(options?.maxKeys || 1000, 1000)
     const prefix = options?.prefix || ''
     const delimiter = options?.delimiter
 
-    const cursor = options?.cursor ? decodeContinuationToken(options?.cursor) : undefined
+    const cursor = options?.cursor
+      ? decodeContinuationToken(options?.sortBy?.column || 'name', options?.cursor)
+      : undefined
     let searchResult = await this.db.listObjectsV2(this.bucketId, {
       prefix: options?.prefix,
       delimiter: options?.delimiter,
       maxKeys: limit + 1,
       nextToken: cursor,
       startAfter: cursor || options?.startAfter,
+      sortBy: options?.sortBy,
     })
 
     let prevPrefix = ''
@@ -644,9 +651,15 @@ export class ObjectStorage {
       })
     })
 
-    const nextContinuationToken = isTruncated
-      ? encodeContinuationToken(searchResult[searchResult.length - 1].name)
-      : undefined
+    let nextContinuationToken: string | undefined
+    if (isTruncated) {
+      const sortColumn = options?.sortBy?.column || 'name'
+      const tokenValue =
+        sortColumn === 'name'
+          ? searchResult[searchResult.length - 1].name
+          : new Date(searchResult[searchResult.length - 1][sortColumn] || '').toISOString()
+      nextContinuationToken = encodeContinuationToken(sortColumn, tokenValue)
+    }
 
     return {
       hasNext: isTruncated,
@@ -806,16 +819,21 @@ export class ObjectStorage {
   }
 }
 
-function encodeContinuationToken(name: string) {
-  return Buffer.from(`l:${name}`).toString('base64')
+function encodeContinuationToken(sortKey: string, value: string) {
+  return Buffer.from(`${sortKey}:${value}`).toString('base64')
 }
 
-function decodeContinuationToken(token: string) {
+function decodeContinuationToken(sortKey: string, token: string) {
   const decoded = Buffer.from(token, 'base64').toString().split(':')
 
   if (decoded.length === 0) {
     throw new Error('Invalid continuation token')
   }
 
-  return decoded[1]
+  const tokenSort = decoded.shift()
+  if (tokenSort !== sortKey) {
+    throw new Error('Invalid continuation token. Sort column mismatch')
+  }
+
+  return decoded.join(':')
 }

Original file line number	Diff line number	Diff line change
`@@ -37,4 +37,5 @@ export const DBMigration = {`
`37`	`37`	`'optimise-existing-functions': 36,`
`38`	`38`	`'add-bucket-name-length-trigger': 37,`
`39`	`39`	`'iceberg-catalog-flag-on-buckets': 38,`
	`40`	`+ 'add-search-v2-sort-support': 39,`
`40`	`41`	`}`