From f5851e07a779693e9f1f1e10921a51b593f0f14f Mon Sep 17 00:00:00 2001 From: Lenny Date: Mon, 8 Sep 2025 19:27:58 -0400 Subject: [PATCH 1/4] feat: add support for sorting in list v2 endpoint --- .../0039-add-search-v2-sort-support.sql | 127 +++++++ src/http/routes/object/listObjectsV2.ts | 11 +- src/internal/database/migrations/types.ts | 1 + src/storage/database/adapter.ts | 5 + src/storage/database/knex.ts | 46 ++- src/storage/object.ts | 87 ++++- src/test/object-list-v2.test.ts | 348 ++++++++++++++++++ 7 files changed, 604 insertions(+), 21 deletions(-) create mode 100644 migrations/tenant/0039-add-search-v2-sort-support.sql create mode 100644 src/test/object-list-v2.test.ts diff --git a/migrations/tenant/0039-add-search-v2-sort-support.sql b/migrations/tenant/0039-add-search-v2-sort-support.sql new file mode 100644 index 00000000..17b60068 --- /dev/null +++ b/migrations/tenant/0039-add-search-v2-sort-support.sql @@ -0,0 +1,127 @@ +CREATE OR REPLACE FUNCTION storage.search_v2 ( + prefix text, + bucket_name text, + limits int DEFAULT 100, + levels int DEFAULT 1, + start_after text DEFAULT '', + sort_order text DEFAULT 'asc', + sort_column text DEFAULT 'name', + sort_column_after text DEFAULT '' +) RETURNS TABLE ( + key text, + name text, + id uuid, + updated_at timestamptz, + created_at timestamptz, + last_accessed_at timestamptz, + metadata jsonb, + item_type text +) +SECURITY INVOKER +AS $func$ +DECLARE + sort_col text; + sort_ord text; + cursor_op text; + cursor_expr text; + cursor_expr_prefixes text; + sort_expr text; + sort_expr_prefixes text; + collate_clause text := ''; +BEGIN + -- Validate sort_order + sort_ord := lower(sort_order); + IF sort_ord NOT IN ('asc', 'desc') THEN + sort_ord := 'asc'; + END IF; + + -- Determine cursor comparison operator + IF sort_ord = 'asc' THEN + cursor_op := '>'; + ELSE + cursor_op := '<'; + END IF; + + sort_col := lower(sort_column); + -- Validate sort column + IF sort_col IN ('updated_at', 'created_at') THEN + -- For prefixes: use NULL (which becomes epoch) since we ignore the actual created_at column + cursor_expr_prefixes := format( + '($5 = '''' OR ROW(''epoch''::timestamptz, name COLLATE "C") %s ROW(COALESCE(NULLIF($6, '''')::timestamptz, ''epoch''::timestamptz), $5))', + cursor_op + ); + -- For objects: truncate timestamp precision to match cursor + cursor_expr := format( + '($5 = '''' OR ROW(date_trunc(''milliseconds'', %I), name COLLATE "C") %s ROW(COALESCE(NULLIF($6, '''')::timestamptz, ''epoch''::timestamptz), $5))', + sort_col, cursor_op + ); + -- For prefixes: always sort by epoch (NULL) timestamp, then name + sort_expr_prefixes := format( + '''epoch''::timestamptz %s, name COLLATE "C" %s', + sort_ord, sort_ord + ); + -- For objects and outer query: truncate timestamp precision to match cursor + sort_expr := format( + 'COALESCE(date_trunc(''milliseconds'', %I), ''epoch''::timestamptz) %s, name COLLATE "C" %s', + sort_col, sort_ord, sort_ord + ); + ELSE + cursor_expr_prefixes := format('($5 = '''' OR name COLLATE "C" %s $5)', cursor_op); + cursor_expr := format('($5 = '''' OR name COLLATE "C" %s $5)', cursor_op); + sort_expr_prefixes := format('name COLLATE "C" %s', sort_ord); + sort_expr := format('name COLLATE "C" %s', sort_ord); + END IF; + + RETURN QUERY EXECUTE format( + $sql$ + SELECT * FROM ( + ( + SELECT + split_part(name, '/', $4) AS key, + name, + NULL::uuid AS id, + NULL::timestamptz AS updated_at, + NULL::timestamptz AS created_at, + NULL::timestamptz AS last_accessed_at, + NULL::jsonb AS metadata, + 'folder'::text AS item_type + FROM storage.prefixes + WHERE name COLLATE "C" LIKE $1 || '%%' + AND bucket_id = $2 + AND level = $4 + AND %s + ORDER BY %s + LIMIT $3 + ) + UNION ALL + ( + SELECT + split_part(name, '/', $4) AS key, + name, + id, + updated_at, + created_at, + last_accessed_at, + metadata, + 'object'::text AS item_type + FROM storage.objects + WHERE name COLLATE "C" LIKE $1 || '%%' + AND bucket_id = $2 + AND level = $4 + AND %s + ORDER BY %s + LIMIT $3 + ) + ) obj + ORDER BY %s + LIMIT $3 + $sql$, + cursor_expr_prefixes, -- prefixes WHERE + sort_expr_prefixes, -- prefixes ORDER BY + cursor_expr, -- objects WHERE + sort_expr, -- objects ORDER BY + sort_expr -- final ORDER BY + ) + USING prefix, bucket_name, limits, levels, start_after, sort_column_after; +END; +$func$ LANGUAGE plpgsql STABLE; diff --git a/src/http/routes/object/listObjectsV2.ts b/src/http/routes/object/listObjectsV2.ts index a799c668..60daf1da 100644 --- a/src/http/routes/object/listObjectsV2.ts +++ b/src/http/routes/object/listObjectsV2.ts @@ -23,6 +23,14 @@ const searchRequestBodySchema = { limit: { type: 'integer', minimum: 1, examples: [10] }, cursor: { type: 'string' }, with_delimiter: { type: 'boolean' }, + sortBy: { + type: 'object', + properties: { + column: { type: 'string', enum: ['name', 'updated_at', 'created_at'] }, + order: { type: 'string', enum: ['asc', 'desc'] }, + }, + required: ['column'], + }, }, } as const interface searchRequestInterface extends AuthenticatedRequest { @@ -57,13 +65,14 @@ export default async function routes(fastify: FastifyInstance) { } const { bucketName } = request.params - const { limit, with_delimiter, cursor, prefix } = request.body + const { limit, with_delimiter, cursor, prefix, sortBy } = request.body const results = await request.storage.from(bucketName).listObjectsV2({ prefix, delimiter: with_delimiter ? '/' : undefined, maxKeys: limit, cursor, + sortBy, }) return response.status(200).send(results) diff --git a/src/internal/database/migrations/types.ts b/src/internal/database/migrations/types.ts index 5398e896..2bd9a4f5 100644 --- a/src/internal/database/migrations/types.ts +++ b/src/internal/database/migrations/types.ts @@ -37,4 +37,5 @@ export const DBMigration = { 'optimise-existing-functions': 36, 'add-bucket-name-length-trigger': 37, 'iceberg-catalog-flag-on-buckets': 38, + 'add-search-v2-sort-support': 39, } diff --git a/src/storage/database/adapter.ts b/src/storage/database/adapter.ts index c92aae02..f8a5199a 100644 --- a/src/storage/database/adapter.ts +++ b/src/storage/database/adapter.ts @@ -105,6 +105,11 @@ export interface Database { nextToken?: string maxKeys?: number startAfter?: string + sortOrder?: string + sortBy?: { + column?: string + after?: string + } } ): Promise diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index 9d374cdf..d51f7a51 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -269,32 +269,52 @@ export class StorageKnexDB implements Database { nextToken?: string maxKeys?: number startAfter?: string + sortOrder?: string + sortBy?: { + column: string + after?: string + } } ) { return this.runQuery('ListObjectsV2', async (knex) => { + // console.log('!!!!!!!!! LIST OBJECTS V2') if (!options?.delimiter) { + // console.log('!!!!!!!!! LIST OBJECTS V2... NO delimiter') const query = knex .table('objects') .where('bucket_id', bucketId) - .select(['id', 'name', 'metadata', 'updated_at']) + .select(['id', 'name', 'metadata', 'updated_at', 'created_at', 'last_accessed_at']) .limit(options?.maxKeys || 100) + // only allow these values for sort columns, "name" is excluded intentionally so it will use the default value (which includes collate) + const allowedSortColumns = new Set(['updated_at', 'created_at']) + const allowedSortOrders = new Set(['asc', 'desc']) + const sortColumn = + options?.sortBy?.column && allowedSortColumns.has(options.sortBy.column) + ? options.sortBy.column + : 'name COLLATE "C"' + const sortOrder = + options?.sortOrder && allowedSortOrders.has(options.sortOrder) ? options.sortOrder : 'asc' + // knex typing is wrong, it doesn't accept a knex.raw on orderBy, even though is totally legit // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - query.orderBy(knex.raw('name COLLATE "C"')) + query.orderBy(knex.raw(`${sortColumn}`), sortOrder) if (options?.prefix) { query.where('name', 'like', `${options.prefix}%`) } if (options?.nextToken) { - query.andWhere(knex.raw('name COLLATE "C" > ?', [options?.nextToken])) + const pageOperator = sortOrder === 'asc' ? '>' : '<' + query.andWhere(knex.raw(`${sortColumn} ${pageOperator} ?`, [options.nextToken])) } return query } + // console.log('!!!!!!!!! LIST OBJECTS V2... YES delimiter') + let useNewSearchVersion2 = true if (isMultitenant) { @@ -302,14 +322,30 @@ export class StorageKnexDB implements Database { } if (useNewSearchVersion2 && options?.delimiter === '/') { + // console.log('!!!!!!!!! LIST OBJECTS V2... USE NEW SEARCH V2') + let paramPlaceholders = '?,?,?,?,?' + const sortParams: (string | null)[] = [] + // this migration adds 3 more parameters to search v2 support sorting + if (await tenantHasMigrations(this.tenantId, 'add-search-v2-sort-support')) { + // console.log('!!!!!!!!! LIST OBJECTS V2... USE SEARCH WITH SORT!!!!') + paramPlaceholders += ',?,?,?' + sortParams.push( + options?.sortOrder || 'asc', + options?.sortBy?.column || 'name', + options?.sortBy?.after || null + ) + } const levels = !options?.prefix ? 1 : options.prefix.split('/').length - const query = await knex.raw('select * from storage.search_v2(?,?,?,?,?)', [ + const searchParams = [ options?.prefix || '', bucketId, options?.maxKeys || 1000, levels, options?.startAfter || '', - ]) + ...sortParams, + ] + // console.log('...', paramPlaceholders, searchParams) + const query = await knex.raw(`select * from storage.search_v2(${paramPlaceholders})`, searchParams) return query.rows } diff --git a/src/storage/object.ts b/src/storage/object.ts index 6922ecf4..92492e3b 100644 --- a/src/storage/object.ts +++ b/src/storage/object.ts @@ -41,6 +41,12 @@ interface CopyObjectParams { ifUnmodifiedSince?: Date } } +export interface ListObjectsV2Result { + folders: Obj[] + objects: Obj[] + hasNext: boolean + nextCursor?: string +} /** * ObjectStorage @@ -586,18 +592,27 @@ export class ObjectStorage { startAfter?: string maxKeys?: number encodingType?: 'url' - }) { + sortBy?: { + column: 'name' | 'created_at' | 'updated_at' + order?: string + } + }): Promise { const limit = Math.min(options?.maxKeys || 1000, 1000) const prefix = options?.prefix || '' const delimiter = options?.delimiter - const cursor = options?.cursor ? decodeContinuationToken(options?.cursor) : undefined + const cursor = options?.cursor ? decodeContinuationToken(options.cursor) : undefined let searchResult = await this.db.listObjectsV2(this.bucketId, { prefix: options?.prefix, delimiter: options?.delimiter, maxKeys: limit + 1, - nextToken: cursor, - startAfter: cursor || options?.startAfter, + nextToken: cursor?.startAfter, + startAfter: cursor?.startAfter || options?.startAfter, + sortOrder: cursor?.sortOrder || options?.sortBy?.order, + sortBy: { + column: cursor?.sortColumn || options?.sortBy?.column, + after: cursor?.sortColumnAfter, + }, }) let prevPrefix = '' @@ -638,15 +653,31 @@ export class ObjectStorage { const objects: Obj[] = [] searchResult.forEach((obj) => { const target = obj.id === null ? folders : objects + const name = obj.id === null && !obj.name.endsWith('/') ? obj.name + '/' : obj.name target.push({ ...obj, - name: options?.encodingType === 'url' ? encodeURIComponent(obj.name) : obj.name, + name: options?.encodingType === 'url' ? encodeURIComponent(name) : name, }) }) - const nextContinuationToken = isTruncated - ? encodeContinuationToken(searchResult[searchResult.length - 1].name) - : undefined + let nextContinuationToken: string | undefined + if (isTruncated) { + const sortColumn = (cursor?.sortColumn || options?.sortBy?.column) as + | 'name' + | 'created_at' + | 'updated_at' + | undefined + + nextContinuationToken = encodeContinuationToken({ + startAfter: searchResult[searchResult.length - 1].name, + sortOrder: cursor?.sortOrder || options?.sortBy?.order, + sortColumn, + sortColumnAfter: + sortColumn && sortColumn !== 'name' && searchResult[searchResult.length - 1][sortColumn] + ? new Date(searchResult[searchResult.length - 1][sortColumn] || '').toISOString() + : undefined, + }) + } return { hasNext: isTruncated, @@ -806,16 +837,42 @@ export class ObjectStorage { } } -function encodeContinuationToken(name: string) { - return Buffer.from(`l:${name}`).toString('base64') +interface ContinuationToken { + startAfter: string + sortOrder?: string // 'asc' | 'desc' + sortColumn?: string + sortColumnAfter?: string } -function decodeContinuationToken(token: string) { - const decoded = Buffer.from(token, 'base64').toString().split(':') +const CONTINUATION_TOKEN_PART_MAP: Record = { + l: 'startAfter', + o: 'sortOrder', + c: 'sortColumn', + a: 'sortColumnAfter', +} - if (decoded.length === 0) { - throw new Error('Invalid continuation token') +function encodeContinuationToken(tokenInfo: ContinuationToken) { + let result = '' + for (const [k, v] of Object.entries(CONTINUATION_TOKEN_PART_MAP)) { + if (tokenInfo[v]) { + result += `${k}:${tokenInfo[v]}\n` + } } + return Buffer.from(result.slice(0, -1)).toString('base64') +} - return decoded[1] +function decodeContinuationToken(token: string): ContinuationToken { + const decodedParts = Buffer.from(token, 'base64').toString().split('\n') + const result: ContinuationToken = { + startAfter: '', + sortOrder: 'asc', + } + for (const part of decodedParts) { + const partMatch = part.match(/^(\S):(.*)/) + if (!partMatch || partMatch.length !== 3 || !(partMatch[1] in CONTINUATION_TOKEN_PART_MAP)) { + throw new Error('Invalid continuation token') + } + result[CONTINUATION_TOKEN_PART_MAP[partMatch[1]]] = partMatch[2] + } + return result } diff --git a/src/test/object-list-v2.test.ts b/src/test/object-list-v2.test.ts new file mode 100644 index 00000000..9e61eaa3 --- /dev/null +++ b/src/test/object-list-v2.test.ts @@ -0,0 +1,348 @@ +'use strict' + +import app from '../app' +import { getConfig } from '../config' +import { useMockObject, useMockQueue } from './common' +import { Knex } from 'knex' +import { FastifyInstance } from 'fastify' +import { ListObjectsV2Result } from '@storage/object' + +const { serviceKeyAsync } = getConfig() +let appInstance: FastifyInstance +let serviceKey: string = '' + +let tnx: Knex.Transaction | undefined + +useMockObject() +useMockQueue() + +beforeEach(() => { + getConfig({ reload: true }) + appInstance = app() +}) + +afterEach(async () => { + if (tnx) { + await tnx.commit() + } + await appInstance.close() +}) + +const LIST_V2_BUCKET = 'list-v2-sorting-test-bucket' + +// Helper to convert a number into a 3-letter string (aaa ... zzz) +const toName = (n: number): string => { + const a = 97 // 'a' + const first = String.fromCharCode(a + (Math.floor(n / (26 * 26)) % 26)) + const second = String.fromCharCode(a + (Math.floor(n / 26) % 26)) + const third = String.fromCharCode(a + (n % 26)) + return first + second + third +} + +// Statically created sorted list of file paths +// 20 objects (.txt extension) and 20 folders (no extension) - already sorted +const SORTED_OBJECTS: string[] = [] +const SORTED_FOLDERS: string[] = [] +const NESTED_OBJECTS: string[] = [] + +// Generate sorted list of objects/folders +for (let i = 0; i < 30; i++) { + if (i > 5) { + SORTED_OBJECTS.push(toName(i) + '.txt') + } + if (i < 18) { + const folder = toName(i) + '/' + SORTED_FOLDERS.push(folder) + + for (let j = 0; j < 3; j++) { + const objectPath = `${folder}dummy${j}.txt` + NESTED_OBJECTS.push(objectPath) + } + } +} + +// Combine all paths for creation +const ALL_PATHS = [...SORTED_OBJECTS, ...NESTED_OBJECTS].sort() + +const UPDATE_ORDER_OBJECTS: string[] = [] +const CREATION_ORDER_OBJECTS: string[] = [] +const CREATION_ORDER_FOLDERS: string[] = [] +const CREATION_ORDER_ALL: string[] = [] + +beforeAll(async () => { + serviceKey = await serviceKeyAsync + appInstance = app() + + // TODO: remove this, not needed once cleanup is uncommented + // empty if it already exists + await appInstance.inject({ + method: 'POST', + url: `/bucket/${LIST_V2_BUCKET}/empty`, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + + // Create bucket + await appInstance.inject({ + method: 'POST', + url: `/bucket`, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + payload: { + name: LIST_V2_BUCKET, + }, + }) + + function createUpload(name: string, content: string) { + return new File([content], name) + } + + // Shuffle array to create objects in semi-random order (so created_at != name order) + function shuffleArray(array: T[]): T[] { + const shuffled = [...array] + for (let i = shuffled.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)) + ;[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]] + } + return shuffled + } + + const shuffledPaths = shuffleArray(ALL_PATHS) + + // Create all objects in random order + for (const path of shuffledPaths) { + if (path.includes('/')) { + // nested objects + CREATION_ORDER_FOLDERS.push(path.split('/')[0]) + } else { + // root objects + CREATION_ORDER_OBJECTS.push(path) + UPDATE_ORDER_OBJECTS.push(path) + } + CREATION_ORDER_ALL.push(path) + await appInstance.inject({ + method: 'POST', + url: `/object/${LIST_V2_BUCKET}/${path}`, + payload: createUpload(path, 'test content'), + headers: { + authorization: serviceKey, + }, + }) + } + + // update a few objects to make updated_at different than created_at + for (let i = 0; i < 10; i++) { + const firstItem = UPDATE_ORDER_OBJECTS.shift()! + const headers = { + authorization: serviceKey, + 'x-upsert': 'true', + } + await appInstance.inject({ + method: 'POST', + url: `/object/${LIST_V2_BUCKET}/${firstItem}`, + payload: createUpload(firstItem, 'test content'), + headers, + }) + UPDATE_ORDER_OBJECTS.push(firstItem) + } + + await appInstance.close() +}, 300000) + +// TODO... uncomment this to cleanup after tests +// commented for now so I can do manual tests / debugging against test data +// afterAll(async () => { +// appInstance = app() + +// // Empty the bucket +// await appInstance.inject({ +// method: 'POST', +// url: `/bucket/${LIST_V2_BUCKET}/empty`, +// headers: { +// authorization: `Bearer ${serviceKey}`, +// }, +// }) + +// // Delete the bucket +// await appInstance.inject({ +// method: 'DELETE', +// url: `/bucket/${LIST_V2_BUCKET}`, +// headers: { +// authorization: `Bearer ${serviceKey}`, +// }, +// }) + +// await appInstance.close() +// }) + +describe('objects - list v2 sorting tests', () => { + const TEST_CASES = [ + { + desc: 'default sorting (name asc) with delmiter', + options: { + with_delimiter: true, + }, + expected: { objects: SORTED_OBJECTS, folders: SORTED_FOLDERS }, + }, + { + desc: 'name desc with delmiter', + options: { + with_delimiter: true, + sortBy: { + column: 'name', + order: 'desc', + }, + }, + expected: { + objects: SORTED_OBJECTS.slice().reverse(), + folders: SORTED_FOLDERS.slice().reverse(), + }, + }, + + { + desc: 'creation asc with delmiter', + options: { + with_delimiter: true, + sortBy: { + column: 'created_at', + order: 'asc', + }, + }, + expected: { + get objects() { + return CREATION_ORDER_OBJECTS + }, + // folders: CREATION_ORDER_FOLDERS, // folders do not have created at so they're sorted alpha + folders: SORTED_FOLDERS, + }, + }, + { + desc: 'creation desc with delmiter', + options: { + with_delimiter: true, + sortBy: { + column: 'created_at', + order: 'desc', + }, + }, + expected: { + get objects() { + return CREATION_ORDER_OBJECTS.slice().reverse() + }, + // folders: CREATION_ORDER_FOLDERS, // folders do not have created at so they're sorted alpha + folders: SORTED_FOLDERS.slice().reverse(), + }, + }, + + { + desc: 'creation asc with delmiter', + options: { + with_delimiter: true, + sortBy: { + column: 'updated_at', + order: 'asc', + }, + }, + expected: { + get objects() { + return UPDATE_ORDER_OBJECTS + }, + // folders: CREATION_ORDER_FOLDERS, // folders do not have created at so they're sorted alpha + folders: SORTED_FOLDERS, + }, + }, + { + desc: 'creation desc with delmiter', + options: { + with_delimiter: true, + sortBy: { + column: 'updated_at', + order: 'desc', + }, + }, + expected: { + get objects() { + return UPDATE_ORDER_OBJECTS.slice().reverse() + }, + // folders: CREATION_ORDER_FOLDERS, // folders do not have created at so they're sorted alpha + folders: SORTED_FOLDERS.slice().reverse(), + }, + }, + + { + desc: 'default sorting (name asc) without delimiter', + options: { + with_delimiter: false, + }, + expected: { objects: ALL_PATHS, folders: [] }, + }, + { + desc: 'name desc without delimiter', + options: { + with_delimiter: false, + sortBy: { + column: 'name', + order: 'desc', + }, + }, + expected: { objects: ALL_PATHS.slice().reverse(), folders: [] }, + }, + ] + + for (let { desc, options, expected } of TEST_CASES) { + test(desc + ' in correct order with pagination', async () => { + const limit = 5 + let cursor: string | undefined = undefined + let pageCount = 0 + let lastObjectIdx = -1 + let lastFolderIdx = -1 + + // Paginate through all results + while (true) { + const response = await appInstance.inject({ + method: 'POST', + url: '/object/list-v2/' + LIST_V2_BUCKET, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + payload: { + ...options, + limit, + cursor, + }, + }) + + const data = response.json() + expect(response.statusCode).toBe(200) + + // Verify each object is the expected next one in sequence + data.objects.forEach((obj) => { + const expObj = expected.objects[++lastObjectIdx] + expect(obj.name).toBe(expObj) + }) + + // Verify each folder is the expected next one in sequence + data.folders.forEach((folder) => { + const expFolder = expected.folders[++lastFolderIdx] + expect(folder.name).toBe(expFolder) + }) + pageCount++ + + if (!data.hasNext) { + expect(data.nextCursor).toBeUndefined() + break + } + + cursor = data.nextCursor as string + expect(cursor).toBeDefined() + } + + // Verify we processed all expected items + expect(lastObjectIdx).toBe(expected.objects.length - 1) + expect(lastFolderIdx).toBe(expected.folders.length - 1) + expect(pageCount).toBe(Math.ceil((expected.objects.length + expected.folders.length) / limit)) + }) + } +}) From 542373e9ecd5e92768c9147de6ee3f8180bffecd Mon Sep 17 00:00:00 2001 From: Lenny Date: Wed, 10 Sep 2025 18:44:21 -0400 Subject: [PATCH 2/4] add test cases, fix time sorting in flat file listings --- .../0039-add-search-v2-sort-support.sql | 39 +- src/storage/database/knex.ts | 32 +- src/test/object-list-v2.test.ts | 380 ++++++++++++++---- 3 files changed, 329 insertions(+), 122 deletions(-) diff --git a/migrations/tenant/0039-add-search-v2-sort-support.sql b/migrations/tenant/0039-add-search-v2-sort-support.sql index 17b60068..68d02bdc 100644 --- a/migrations/tenant/0039-add-search-v2-sort-support.sql +++ b/migrations/tenant/0039-add-search-v2-sort-support.sql @@ -14,8 +14,7 @@ CREATE OR REPLACE FUNCTION storage.search_v2 ( updated_at timestamptz, created_at timestamptz, last_accessed_at timestamptz, - metadata jsonb, - item_type text + metadata jsonb ) SECURITY INVOKER AS $func$ @@ -24,9 +23,7 @@ DECLARE sort_ord text; cursor_op text; cursor_expr text; - cursor_expr_prefixes text; sort_expr text; - sort_expr_prefixes text; collate_clause text := ''; BEGIN -- Validate sort_order @@ -45,30 +42,16 @@ BEGIN sort_col := lower(sort_column); -- Validate sort column IF sort_col IN ('updated_at', 'created_at') THEN - -- For prefixes: use NULL (which becomes epoch) since we ignore the actual created_at column - cursor_expr_prefixes := format( - '($5 = '''' OR ROW(''epoch''::timestamptz, name COLLATE "C") %s ROW(COALESCE(NULLIF($6, '''')::timestamptz, ''epoch''::timestamptz), $5))', - cursor_op - ); - -- For objects: truncate timestamp precision to match cursor cursor_expr := format( '($5 = '''' OR ROW(date_trunc(''milliseconds'', %I), name COLLATE "C") %s ROW(COALESCE(NULLIF($6, '''')::timestamptz, ''epoch''::timestamptz), $5))', sort_col, cursor_op ); - -- For prefixes: always sort by epoch (NULL) timestamp, then name - sort_expr_prefixes := format( - '''epoch''::timestamptz %s, name COLLATE "C" %s', - sort_ord, sort_ord - ); - -- For objects and outer query: truncate timestamp precision to match cursor sort_expr := format( 'COALESCE(date_trunc(''milliseconds'', %I), ''epoch''::timestamptz) %s, name COLLATE "C" %s', sort_col, sort_ord, sort_ord ); ELSE - cursor_expr_prefixes := format('($5 = '''' OR name COLLATE "C" %s $5)', cursor_op); cursor_expr := format('($5 = '''' OR name COLLATE "C" %s $5)', cursor_op); - sort_expr_prefixes := format('name COLLATE "C" %s', sort_ord); sort_expr := format('name COLLATE "C" %s', sort_ord); END IF; @@ -80,11 +63,10 @@ BEGIN split_part(name, '/', $4) AS key, name, NULL::uuid AS id, - NULL::timestamptz AS updated_at, - NULL::timestamptz AS created_at, + updated_at, + created_at, NULL::timestamptz AS last_accessed_at, - NULL::jsonb AS metadata, - 'folder'::text AS item_type + NULL::jsonb AS metadata FROM storage.prefixes WHERE name COLLATE "C" LIKE $1 || '%%' AND bucket_id = $2 @@ -102,8 +84,7 @@ BEGIN updated_at, created_at, last_accessed_at, - metadata, - 'object'::text AS item_type + metadata FROM storage.objects WHERE name COLLATE "C" LIKE $1 || '%%' AND bucket_id = $2 @@ -116,11 +97,11 @@ BEGIN ORDER BY %s LIMIT $3 $sql$, - cursor_expr_prefixes, -- prefixes WHERE - sort_expr_prefixes, -- prefixes ORDER BY - cursor_expr, -- objects WHERE - sort_expr, -- objects ORDER BY - sort_expr -- final ORDER BY + cursor_expr, -- prefixes WHERE + sort_expr, -- prefixes ORDER BY + cursor_expr, -- objects WHERE + sort_expr, -- objects ORDER BY + sort_expr -- final ORDER BY ) USING prefix, bucket_name, limits, levels, start_after, sort_column_after; END; diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index d51f7a51..993442cf 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -277,29 +277,30 @@ export class StorageKnexDB implements Database { } ) { return this.runQuery('ListObjectsV2', async (knex) => { - // console.log('!!!!!!!!! LIST OBJECTS V2') if (!options?.delimiter) { - // console.log('!!!!!!!!! LIST OBJECTS V2... NO delimiter') const query = knex .table('objects') .where('bucket_id', bucketId) .select(['id', 'name', 'metadata', 'updated_at', 'created_at', 'last_accessed_at']) .limit(options?.maxKeys || 100) - // only allow these values for sort columns, "name" is excluded intentionally so it will use the default value (which includes collate) + // only allow these values for sort columns, "name" is excluded intentionally as it is the default and used as tie breaker when sorting by other columns const allowedSortColumns = new Set(['updated_at', 'created_at']) const allowedSortOrders = new Set(['asc', 'desc']) const sortColumn = options?.sortBy?.column && allowedSortColumns.has(options.sortBy.column) ? options.sortBy.column - : 'name COLLATE "C"' + : undefined const sortOrder = options?.sortOrder && allowedSortOrders.has(options.sortOrder) ? options.sortOrder : 'asc' + if (sortColumn) { + query.orderBy(sortColumn, sortOrder) + } // knex typing is wrong, it doesn't accept a knex.raw on orderBy, even though is totally legit // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - query.orderBy(knex.raw(`${sortColumn}`), sortOrder) + query.orderBy(knex.raw(`name COLLATE "C"`), sortOrder) if (options?.prefix) { query.where('name', 'like', `${options.prefix}%`) @@ -307,14 +308,21 @@ export class StorageKnexDB implements Database { if (options?.nextToken) { const pageOperator = sortOrder === 'asc' ? '>' : '<' - query.andWhere(knex.raw(`${sortColumn} ${pageOperator} ?`, [options.nextToken])) + if (sortColumn && options.sortBy?.after) { + query.andWhere( + knex.raw( + `ROW(date_trunc('milliseconds', ${sortColumn}), name COLLATE "C") ${pageOperator} ROW(COALESCE(NULLIF(?, '')::timestamptz, 'epoch'::timestamptz), ?)`, + [options.sortBy.after, options.nextToken] + ) + ) + } else { + query.andWhere(knex.raw(`name COLLATE "C" ${pageOperator} ?`, [options.nextToken])) + } } return query } - // console.log('!!!!!!!!! LIST OBJECTS V2... YES delimiter') - let useNewSearchVersion2 = true if (isMultitenant) { @@ -322,12 +330,10 @@ export class StorageKnexDB implements Database { } if (useNewSearchVersion2 && options?.delimiter === '/') { - // console.log('!!!!!!!!! LIST OBJECTS V2... USE NEW SEARCH V2') let paramPlaceholders = '?,?,?,?,?' const sortParams: (string | null)[] = [] // this migration adds 3 more parameters to search v2 support sorting if (await tenantHasMigrations(this.tenantId, 'add-search-v2-sort-support')) { - // console.log('!!!!!!!!! LIST OBJECTS V2... USE SEARCH WITH SORT!!!!') paramPlaceholders += ',?,?,?' sortParams.push( options?.sortOrder || 'asc', @@ -344,8 +350,10 @@ export class StorageKnexDB implements Database { options?.startAfter || '', ...sortParams, ] - // console.log('...', paramPlaceholders, searchParams) - const query = await knex.raw(`select * from storage.search_v2(${paramPlaceholders})`, searchParams) + const query = await knex.raw( + `select * from storage.search_v2(${paramPlaceholders})`, + searchParams + ) return query.rows } diff --git a/src/test/object-list-v2.test.ts b/src/test/object-list-v2.test.ts index 9e61eaa3..4c2f0839 100644 --- a/src/test/object-list-v2.test.ts +++ b/src/test/object-list-v2.test.ts @@ -31,7 +31,7 @@ afterEach(async () => { const LIST_V2_BUCKET = 'list-v2-sorting-test-bucket' // Helper to convert a number into a 3-letter string (aaa ... zzz) -const toName = (n: number): string => { +function toName(n: number): string { const a = 97 // 'a' const first = String.fromCharCode(a + (Math.floor(n / (26 * 26)) % 26)) const second = String.fromCharCode(a + (Math.floor(n / 26) % 26)) @@ -39,24 +39,42 @@ const toName = (n: number): string => { return first + second + third } -// Statically created sorted list of file paths -// 20 objects (.txt extension) and 20 folders (no extension) - already sorted +function createUpload(name: string, content: string) { + return new File([content], name) +} + +function shuffleArray(array: T[]): T[] { + const shuffled = [...array] + for (let i = shuffled.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)) + ;[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]] + } + return shuffled +} + const SORTED_OBJECTS: string[] = [] const SORTED_FOLDERS: string[] = [] const NESTED_OBJECTS: string[] = [] +const PREFIX_OBJECTS: Record = + {} +const TEST_PREFIX = 'aal' // Generate sorted list of objects/folders for (let i = 0; i < 30; i++) { + const name = toName(i) if (i > 5) { - SORTED_OBJECTS.push(toName(i) + '.txt') + SORTED_OBJECTS.push(name + '.txt') } if (i < 18) { - const folder = toName(i) + '/' + const folder = name + '/' SORTED_FOLDERS.push(folder) - for (let j = 0; j < 3; j++) { - const objectPath = `${folder}dummy${j}.txt` + const nestedCount = name === TEST_PREFIX ? 9 : 3 + for (let j = 0; j < nestedCount; j++) { + const objectPath = `${folder}dummy-${name}-${j}.txt` NESTED_OBJECTS.push(objectPath) + PREFIX_OBJECTS[folder] ??= { sorted: [], created: [], updated: [] } + PREFIX_OBJECTS[folder].sorted.push(objectPath) } } } @@ -64,25 +82,17 @@ for (let i = 0; i < 30; i++) { // Combine all paths for creation const ALL_PATHS = [...SORTED_OBJECTS, ...NESTED_OBJECTS].sort() -const UPDATE_ORDER_OBJECTS: string[] = [] +// Lists of objects and folders in sorted const CREATION_ORDER_OBJECTS: string[] = [] +const UPDATE_ORDER_OBJECTS: string[] = [] const CREATION_ORDER_FOLDERS: string[] = [] const CREATION_ORDER_ALL: string[] = [] +const UPDATE_ORDER_ALL: string[] = [] beforeAll(async () => { serviceKey = await serviceKeyAsync appInstance = app() - // TODO: remove this, not needed once cleanup is uncommented - // empty if it already exists - await appInstance.inject({ - method: 'POST', - url: `/bucket/${LIST_V2_BUCKET}/empty`, - headers: { - authorization: `Bearer ${serviceKey}`, - }, - }) - // Create bucket await appInstance.inject({ method: 'POST', @@ -95,33 +105,25 @@ beforeAll(async () => { }, }) - function createUpload(name: string, content: string) { - return new File([content], name) - } - - // Shuffle array to create objects in semi-random order (so created_at != name order) - function shuffleArray(array: T[]): T[] { - const shuffled = [...array] - for (let i = shuffled.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)) - ;[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]] - } - return shuffled - } - const shuffledPaths = shuffleArray(ALL_PATHS) // Create all objects in random order for (const path of shuffledPaths) { if (path.includes('/')) { - // nested objects - CREATION_ORDER_FOLDERS.push(path.split('/')[0]) + // root folders in creation order + const rootFolder = path.split('/')[0] + '/' + if (!CREATION_ORDER_FOLDERS.includes(rootFolder)) { + CREATION_ORDER_FOLDERS.push(rootFolder) + } + PREFIX_OBJECTS[rootFolder].created.push(path) + PREFIX_OBJECTS[rootFolder].updated.push(path) } else { - // root objects + // root objects in creation order CREATION_ORDER_OBJECTS.push(path) UPDATE_ORDER_OBJECTS.push(path) } CREATION_ORDER_ALL.push(path) + UPDATE_ORDER_ALL.push(path) await appInstance.inject({ method: 'POST', url: `/object/${LIST_V2_BUCKET}/${path}`, @@ -132,13 +134,14 @@ beforeAll(async () => { }) } + const headers = { + authorization: serviceKey, + 'x-upsert': 'true', + } + // update a few objects to make updated_at different than created_at for (let i = 0; i < 10; i++) { const firstItem = UPDATE_ORDER_OBJECTS.shift()! - const headers = { - authorization: serviceKey, - 'x-upsert': 'true', - } await appInstance.inject({ method: 'POST', url: `/object/${LIST_V2_BUCKET}/${firstItem}`, @@ -146,48 +149,67 @@ beforeAll(async () => { headers, }) UPDATE_ORDER_OBJECTS.push(firstItem) + + // re-arrange item in flat object list to updated order + UPDATE_ORDER_ALL.splice(UPDATE_ORDER_ALL.indexOf(firstItem), 1) + UPDATE_ORDER_ALL.push(firstItem) } + // switch to Object.entries(PREFIX_OBJECTS) to test all prefixes + const prefixRoot = TEST_PREFIX + '/' + const obj = PREFIX_OBJECTS[prefixRoot] + const firstPrefixItem = obj.updated.shift()! + await appInstance.inject({ + method: 'POST', + url: `/object/${LIST_V2_BUCKET}/${firstPrefixItem}`, + payload: createUpload(firstPrefixItem, 'test content'), + headers, + }) + PREFIX_OBJECTS[prefixRoot].updated.push(firstPrefixItem) + + // re-arrange item in flat object list to updated order of nested item + UPDATE_ORDER_ALL.splice(UPDATE_ORDER_ALL.indexOf(firstPrefixItem), 1) + UPDATE_ORDER_ALL.push(firstPrefixItem) + await appInstance.close() }, 300000) -// TODO... uncomment this to cleanup after tests -// commented for now so I can do manual tests / debugging against test data -// afterAll(async () => { -// appInstance = app() - -// // Empty the bucket -// await appInstance.inject({ -// method: 'POST', -// url: `/bucket/${LIST_V2_BUCKET}/empty`, -// headers: { -// authorization: `Bearer ${serviceKey}`, -// }, -// }) - -// // Delete the bucket -// await appInstance.inject({ -// method: 'DELETE', -// url: `/bucket/${LIST_V2_BUCKET}`, -// headers: { -// authorization: `Bearer ${serviceKey}`, -// }, -// }) - -// await appInstance.close() -// }) +afterAll(async () => { + appInstance = app() + + // Empty the bucket + await appInstance.inject({ + method: 'POST', + url: `/bucket/${LIST_V2_BUCKET}/empty`, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + + // Delete the bucket + await appInstance.inject({ + method: 'DELETE', + url: `/bucket/${LIST_V2_BUCKET}`, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + + await appInstance.close() +}) describe('objects - list v2 sorting tests', () => { const TEST_CASES = [ + // WITH DELIMITER { - desc: 'default sorting (name asc) with delmiter', + desc: 'with delimiter - default sorting (name asc)', options: { with_delimiter: true, }, expected: { objects: SORTED_OBJECTS, folders: SORTED_FOLDERS }, }, { - desc: 'name desc with delmiter', + desc: 'with delimiter - name desc', options: { with_delimiter: true, sortBy: { @@ -202,7 +224,7 @@ describe('objects - list v2 sorting tests', () => { }, { - desc: 'creation asc with delmiter', + desc: 'with delimiter - created asc', options: { with_delimiter: true, sortBy: { @@ -214,12 +236,13 @@ describe('objects - list v2 sorting tests', () => { get objects() { return CREATION_ORDER_OBJECTS }, - // folders: CREATION_ORDER_FOLDERS, // folders do not have created at so they're sorted alpha - folders: SORTED_FOLDERS, + get folders() { + return CREATION_ORDER_FOLDERS + }, }, }, { - desc: 'creation desc with delmiter', + desc: 'with delimiter - created desc', options: { with_delimiter: true, sortBy: { @@ -231,13 +254,14 @@ describe('objects - list v2 sorting tests', () => { get objects() { return CREATION_ORDER_OBJECTS.slice().reverse() }, - // folders: CREATION_ORDER_FOLDERS, // folders do not have created at so they're sorted alpha - folders: SORTED_FOLDERS.slice().reverse(), + get folders() { + return CREATION_ORDER_FOLDERS.slice().reverse() + }, }, }, { - desc: 'creation asc with delmiter', + desc: 'with delimiter - updated asc', options: { with_delimiter: true, sortBy: { @@ -249,12 +273,13 @@ describe('objects - list v2 sorting tests', () => { get objects() { return UPDATE_ORDER_OBJECTS }, - // folders: CREATION_ORDER_FOLDERS, // folders do not have created at so they're sorted alpha - folders: SORTED_FOLDERS, + get folders() { + return CREATION_ORDER_FOLDERS + }, }, }, { - desc: 'creation desc with delmiter', + desc: 'with delimiter - updated desc', options: { with_delimiter: true, sortBy: { @@ -266,20 +291,22 @@ describe('objects - list v2 sorting tests', () => { get objects() { return UPDATE_ORDER_OBJECTS.slice().reverse() }, - // folders: CREATION_ORDER_FOLDERS, // folders do not have created at so they're sorted alpha - folders: SORTED_FOLDERS.slice().reverse(), + get folders() { + return CREATION_ORDER_FOLDERS.slice().reverse() + }, }, }, + // WITHOUT DELIMITER { - desc: 'default sorting (name asc) without delimiter', + desc: 'without delimiter - default sorting (name asc)', options: { with_delimiter: false, }, expected: { objects: ALL_PATHS, folders: [] }, }, { - desc: 'name desc without delimiter', + desc: 'without delimiter - name desc without delimiter', options: { with_delimiter: false, sortBy: { @@ -289,9 +316,200 @@ describe('objects - list v2 sorting tests', () => { }, expected: { objects: ALL_PATHS.slice().reverse(), folders: [] }, }, + + { + desc: 'without delimiter - created asc', + options: { + with_delimiter: false, + sortBy: { + column: 'created_at', + order: 'asc', + }, + }, + expected: { + get objects() { + return CREATION_ORDER_ALL + }, + folders: [], + }, + }, + { + desc: 'without delimiter - created desc', + options: { + with_delimiter: false, + sortBy: { + column: 'created_at', + order: 'desc', + }, + }, + expected: { + get objects() { + return CREATION_ORDER_ALL.slice().reverse() + }, + folders: [], + }, + }, + + { + desc: 'without delimiter - updated asc', + options: { + with_delimiter: false, + sortBy: { + column: 'updated_at', + order: 'asc', + }, + }, + expected: { + get objects() { + return UPDATE_ORDER_ALL + }, + folders: [], + }, + }, + { + desc: 'without delimiter - updated desc', + options: { + with_delimiter: false, + sortBy: { + column: 'updated_at', + order: 'desc', + }, + }, + expected: { + get objects() { + return UPDATE_ORDER_ALL.slice().reverse() + }, + folders: [], + }, + }, + + // WITH PREFIX + { + desc: `prefix - with delimiter - default sorting (name asc)`, + options: { + with_delimiter: true, + prefix: TEST_PREFIX + '/', + }, + expected: { objects: PREFIX_OBJECTS[TEST_PREFIX + '/'].sorted, folders: [] }, + }, + { + desc: 'prefix - with delimiter - name desc', + options: { + with_delimiter: true, + prefix: TEST_PREFIX + '/', + sortBy: { + column: 'name', + order: 'desc', + }, + }, + expected: { + objects: PREFIX_OBJECTS[TEST_PREFIX + '/'].sorted.slice().reverse(), + folders: [], + }, + }, + + { + desc: 'prefix - with delimiter - created asc', + options: { + with_delimiter: true, + prefix: TEST_PREFIX + '/', + sortBy: { + column: 'created_at', + order: 'asc', + }, + }, + expected: { + get objects() { + return PREFIX_OBJECTS[TEST_PREFIX + '/'].created + }, + folders: [], + }, + }, + { + desc: 'prefix - with delimiter - created desc', + options: { + with_delimiter: true, + prefix: TEST_PREFIX + '/', + sortBy: { + column: 'created_at', + order: 'desc', + }, + }, + expected: { + get objects() { + return PREFIX_OBJECTS[TEST_PREFIX + '/'].created.slice().reverse() + }, + folders: [], + }, + }, + + { + desc: 'prefix - with delimiter - updated asc', + options: { + with_delimiter: true, + prefix: TEST_PREFIX + '/', + sortBy: { + column: 'updated_at', + order: 'asc', + }, + }, + expected: { + get objects() { + return PREFIX_OBJECTS[TEST_PREFIX + '/'].updated + }, + folders: [], + }, + }, + { + desc: 'prefix - with delimiter - updated desc', + options: { + with_delimiter: true, + prefix: TEST_PREFIX + '/', + sortBy: { + column: 'updated_at', + order: 'desc', + }, + }, + expected: { + get objects() { + return PREFIX_OBJECTS[TEST_PREFIX + '/'].updated.slice().reverse() + }, + folders: [], + }, + }, + + { + desc: 'prefix with slash - without delimiter', + options: { + with_delimiter: false, + prefix: TEST_PREFIX + '/', + }, + expected: { objects: PREFIX_OBJECTS[TEST_PREFIX + '/'].sorted, folders: [] }, + }, + + { + desc: 'prefix without slash - with delimiter', + options: { + with_delimiter: true, + prefix: TEST_PREFIX, + }, + expected: { objects: [TEST_PREFIX + '.txt'], folders: [TEST_PREFIX + '/'] }, + }, + + { + desc: 'prefix without slash - without delimiter', + options: { + with_delimiter: false, + prefix: TEST_PREFIX, + }, + expected: { + objects: [TEST_PREFIX + '.txt', ...PREFIX_OBJECTS[TEST_PREFIX + '/'].sorted], + folders: [], + }, + }, ] - for (let { desc, options, expected } of TEST_CASES) { + for (const { desc, options, expected } of TEST_CASES) { test(desc + ' in correct order with pagination', async () => { const limit = 5 let cursor: string | undefined = undefined From 9a28a1f08ed6b0a773ff7bdf4b5c1fc0fd7b315c Mon Sep 17 00:00:00 2001 From: Lenny Date: Thu, 11 Sep 2025 13:51:51 -0400 Subject: [PATCH 3/4] move order into sortby in database adapter --- src/storage/database/adapter.ts | 2 +- src/storage/database/knex.ts | 10 ++++++---- src/storage/object.ts | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/storage/database/adapter.ts b/src/storage/database/adapter.ts index f8a5199a..e54f0b62 100644 --- a/src/storage/database/adapter.ts +++ b/src/storage/database/adapter.ts @@ -105,8 +105,8 @@ export interface Database { nextToken?: string maxKeys?: number startAfter?: string - sortOrder?: string sortBy?: { + order?: string column?: string after?: string } diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index 993442cf..23844daf 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -269,9 +269,9 @@ export class StorageKnexDB implements Database { nextToken?: string maxKeys?: number startAfter?: string - sortOrder?: string sortBy?: { - column: string + order?: string + column?: string after?: string } } @@ -292,7 +292,9 @@ export class StorageKnexDB implements Database { ? options.sortBy.column : undefined const sortOrder = - options?.sortOrder && allowedSortOrders.has(options.sortOrder) ? options.sortOrder : 'asc' + options?.sortBy?.order && allowedSortOrders.has(options.sortBy.order) + ? options.sortBy.order + : 'asc' if (sortColumn) { query.orderBy(sortColumn, sortOrder) @@ -336,7 +338,7 @@ export class StorageKnexDB implements Database { if (await tenantHasMigrations(this.tenantId, 'add-search-v2-sort-support')) { paramPlaceholders += ',?,?,?' sortParams.push( - options?.sortOrder || 'asc', + options?.sortBy?.order || 'asc', options?.sortBy?.column || 'name', options?.sortBy?.after || null ) diff --git a/src/storage/object.ts b/src/storage/object.ts index 92492e3b..f8698074 100644 --- a/src/storage/object.ts +++ b/src/storage/object.ts @@ -608,8 +608,8 @@ export class ObjectStorage { maxKeys: limit + 1, nextToken: cursor?.startAfter, startAfter: cursor?.startAfter || options?.startAfter, - sortOrder: cursor?.sortOrder || options?.sortBy?.order, sortBy: { + order: cursor?.sortOrder || options?.sortBy?.order, column: cursor?.sortColumn || options?.sortBy?.column, after: cursor?.sortColumnAfter, }, From e0bdadf571436567fcce74d9f5c21a90c33bcc43 Mon Sep 17 00:00:00 2001 From: Lenny Date: Fri, 12 Sep 2025 11:52:27 -0400 Subject: [PATCH 4/4] remove unused variable in query --- migrations/tenant/0039-add-search-v2-sort-support.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/migrations/tenant/0039-add-search-v2-sort-support.sql b/migrations/tenant/0039-add-search-v2-sort-support.sql index 68d02bdc..a325c0d1 100644 --- a/migrations/tenant/0039-add-search-v2-sort-support.sql +++ b/migrations/tenant/0039-add-search-v2-sort-support.sql @@ -24,7 +24,6 @@ DECLARE cursor_op text; cursor_expr text; sort_expr text; - collate_clause text := ''; BEGIN -- Validate sort_order sort_ord := lower(sort_order);