diff --git a/.changeset/two-lamps-wave.md b/.changeset/two-lamps-wave.md new file mode 100644 index 000000000..ef2cd032a --- /dev/null +++ b/.changeset/two-lamps-wave.md @@ -0,0 +1,5 @@ +--- +"@tanstack/db": patch +--- + +Adds an onDeduplicate callback on the DeduplicatedLoadSubset class which is called when a loadSubset call is deduplicated diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index fa8172559..a7c6d7c6a 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -12,8 +12,15 @@ import type { LoadSubsetOptions } from "../types.js" * Tracks what data has been loaded and avoids redundant calls by applying * subset logic to predicates. * + * @param opts - The options for the DeduplicatedLoadSubset + * @param opts.loadSubset - The underlying loadSubset function to wrap + * @param opts.onDeduplicate - An optional callback function that is invoked when a loadSubset call is deduplicated. + * If the call is deduplicated because the requested data is being loaded by an inflight request, + * then this callback is invoked when the inflight request completes successfully and the data is fully loaded. + * This callback is useful if you need to track rows per query, in which case you can't ignore deduplicated calls + * because you need to know which rows were loaded for each query. * @example - * const dedupe = new DeduplicatedLoadSubset(myLoadSubset) + * const dedupe = new DeduplicatedLoadSubset({ loadSubset: myLoadSubset, onDeduplicate: (opts) => console.log(`Call was deduplicated:`, opts) }) * * // First call - fetches data * await dedupe.loadSubset({ where: gt(ref('age'), val(10)) }) @@ -30,6 +37,11 @@ export class DeduplicatedLoadSubset { options: LoadSubsetOptions ) => true | Promise + // An optional callback function that is invoked when a loadSubset call is deduplicated. + private readonly onDeduplicate: + | ((options: LoadSubsetOptions) => void) + | undefined + // Combined where predicate for all unlimited calls (no limit) private unlimitedWhere: BasicExpression | undefined = undefined @@ -52,10 +64,12 @@ export class DeduplicatedLoadSubset { // check if their captured generation matches before updating tracking state private generation = 0 - constructor( + constructor(opts: { loadSubset: (options: LoadSubsetOptions) => true | Promise - ) { - this._loadSubset = loadSubset + onDeduplicate?: (options: LoadSubsetOptions) => void + }) { + this._loadSubset = opts.loadSubset + this.onDeduplicate = opts.onDeduplicate } /** @@ -71,6 +85,7 @@ export class DeduplicatedLoadSubset { loadSubset = (options: LoadSubsetOptions): true | Promise => { // If we've loaded all data, everything is covered if (this.hasLoadedAllData) { + this.onDeduplicate?.(options) return true } @@ -78,6 +93,7 @@ export class DeduplicatedLoadSubset { // If we've loaded all data matching a where clause, we don't need to refetch subsets if (this.unlimitedWhere !== undefined && options.where !== undefined) { if (isWhereSubset(options.where, this.unlimitedWhere)) { + this.onDeduplicate?.(options) return true // Data already loaded via unlimited call } } @@ -89,6 +105,7 @@ export class DeduplicatedLoadSubset { ) if (alreadyLoaded) { + this.onDeduplicate?.(options) return true // Already loaded } } @@ -103,7 +120,10 @@ export class DeduplicatedLoadSubset { // An in-flight call will load data that covers this request // Return the same promise so this caller waits for the data to load // The in-flight promise already handles tracking updates when it completes - return matchingInflight.promise + const prom = matchingInflight.promise + // Call `onDeduplicate` when the inflight request has loaded the data + prom.then(() => this.onDeduplicate?.(options)).catch() // ignore errors + return prom } // Not fully covered by existing data diff --git a/packages/db/tests/predicate-utils.test.ts b/packages/db/tests/query/predicate-utils.test.ts similarity index 99% rename from packages/db/tests/predicate-utils.test.ts rename to packages/db/tests/query/predicate-utils.test.ts index bf973f15c..d2373f1fb 100644 --- a/packages/db/tests/predicate-utils.test.ts +++ b/packages/db/tests/query/predicate-utils.test.ts @@ -6,10 +6,14 @@ import { isWhereSubset, minusWherePredicates, unionWherePredicates, -} from "../src/query/predicate-utils" -import { Func, PropRef, Value } from "../src/query/ir" -import type { BasicExpression, OrderBy, OrderByClause } from "../src/query/ir" -import type { LoadSubsetOptions } from "../src/types" +} from "../../src/query/predicate-utils" +import { Func, PropRef, Value } from "../../src/query/ir" +import type { + BasicExpression, + OrderBy, + OrderByClause, +} from "../../src/query/ir" +import type { LoadSubsetOptions } from "../../src/types" // Helper functions to build expressions more easily function ref(path: string | Array): PropRef { diff --git a/packages/db/tests/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts similarity index 70% rename from packages/db/tests/subset-dedupe.test.ts rename to packages/db/tests/query/subset-dedupe.test.ts index 3417f95a5..59aa8c6a3 100644 --- a/packages/db/tests/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -1,12 +1,12 @@ -import { describe, expect, it } from "vitest" +import { describe, expect, it, vi } from "vitest" import { DeduplicatedLoadSubset, cloneOptions, -} from "../src/query/subset-dedupe" -import { Func, PropRef, Value } from "../src/query/ir" -import { minusWherePredicates } from "../src/query/predicate-utils" -import type { BasicExpression, OrderBy } from "../src/query/ir" -import type { LoadSubsetOptions } from "../src/types" +} from "../../src/query/subset-dedupe" +import { Func, PropRef, Value } from "../../src/query/ir" +import { minusWherePredicates } from "../../src/query/predicate-utils" +import type { BasicExpression, OrderBy } from "../../src/query/ir" +import type { LoadSubsetOptions } from "../../src/types" // Helper functions to build expressions more easily function ref(path: string | Array): PropRef { @@ -53,7 +53,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) await deduplicated.loadSubset({ where: gt(ref(`age`), val(10)) }) expect(callCount).toBe(1) @@ -66,7 +68,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) // First call: age > 10 await deduplicated.loadSubset({ where: gt(ref(`age`), val(10)) }) @@ -87,7 +91,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) // First call: age > 20 await deduplicated.loadSubset({ where: gt(ref(`age`), val(20)) }) @@ -105,7 +111,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) // First call: age > 20 await deduplicated.loadSubset({ where: gt(ref(`age`), val(20)) }) @@ -130,7 +138,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) const orderBy1: OrderBy = [ { @@ -168,7 +178,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) const orderBy1: OrderBy = [ { @@ -205,7 +217,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) const orderBy1: OrderBy = [ { @@ -240,7 +254,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) const orderBy1: OrderBy = [ { @@ -275,7 +291,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) // First call: no where clause (all data) await deduplicated.loadSubset({}) @@ -298,7 +316,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) const orderBy1: OrderBy = [ { @@ -353,7 +373,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) // First call: age > 20 (loads data for age > 20) await deduplicated.loadSubset({ where: gt(ref(`age`), val(20)) }) @@ -377,7 +399,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) // First call: status IN ['B', 'C'] (loads data for B and C) await deduplicated.loadSubset({ @@ -405,7 +429,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) // First call: age > 10 (loads data for age > 10) await deduplicated.loadSubset({ where: gt(ref(`age`), val(10)) }) @@ -428,7 +454,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) // First call: age > 20 AND status = 'active' const firstPredicate = and( @@ -468,7 +496,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) const orderBy1: OrderBy = [ { @@ -509,7 +539,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) // First call: age > 20 await deduplicated.loadSubset({ where: gt(ref(`age`), val(20)) }) @@ -532,7 +564,9 @@ describe(`createDeduplicatedLoadSubset`, () => { return Promise.resolve() } - const deduplicated = new DeduplicatedLoadSubset(mockLoadSubset) + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + }) // First call: age > 20 await deduplicated.loadSubset({ where: gt(ref(`age`), val(20)) }) @@ -559,4 +593,152 @@ describe(`createDeduplicatedLoadSubset`, () => { */ }) }) + + describe(`onDeduplicate callback`, () => { + it(`should call onDeduplicate when all data already loaded`, async () => { + let callCount = 0 + const mockLoadSubset = () => { + callCount++ + return Promise.resolve() + } + + const onDeduplicate = vi.fn() + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + onDeduplicate, + }) + + // Load all data + await deduplicated.loadSubset({}) + expect(callCount).toBe(1) + + // Any subsequent request should be deduplicated + const subsetOptions = { where: gt(ref(`age`), val(10)) } + const result = await deduplicated.loadSubset(subsetOptions) + expect(result).toBe(true) + expect(callCount).toBe(1) + expect(onDeduplicate).toHaveBeenCalledTimes(1) + expect(onDeduplicate).toHaveBeenCalledWith(subsetOptions) + }) + + it(`should call onDeduplicate when unlimited superset already loaded`, async () => { + let callCount = 0 + const mockLoadSubset = () => { + callCount++ + return Promise.resolve() + } + + const onDeduplicate = vi.fn() + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + onDeduplicate: onDeduplicate, + }) + + // First call loads a broader set + await deduplicated.loadSubset({ where: gt(ref(`age`), val(10)) }) + expect(callCount).toBe(1) + + // Second call is a subset of the first; should dedupe and call callback + const subsetOptions = { where: gt(ref(`age`), val(20)) } + const result = await deduplicated.loadSubset(subsetOptions) + expect(result).toBe(true) + expect(callCount).toBe(1) + expect(onDeduplicate).toHaveBeenCalledTimes(1) + expect(onDeduplicate).toHaveBeenCalledWith(subsetOptions) + }) + + it(`should call onDeduplicate for limited subset requests`, async () => { + let callCount = 0 + const mockLoadSubset = () => { + callCount++ + return Promise.resolve() + } + + const onDeduplicate = vi.fn() + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + onDeduplicate, + }) + + const orderBy1: OrderBy = [ + { + expression: ref(`age`), + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `lexical`, + }, + }, + ] + + // First limited call + await deduplicated.loadSubset({ + where: gt(ref(`age`), val(10)), + orderBy: orderBy1, + limit: 10, + }) + expect(callCount).toBe(1) + + // Second limited call is a subset (stricter where and smaller limit) + const subsetOptions = { + where: gt(ref(`age`), val(20)), + orderBy: orderBy1, + limit: 5, + } + const result = await deduplicated.loadSubset(subsetOptions) + expect(result).toBe(true) + expect(callCount).toBe(1) + expect(onDeduplicate).toHaveBeenCalledTimes(1) + expect(onDeduplicate).toHaveBeenCalledWith(subsetOptions) + }) + + it(`should delay onDeduplicate until covering in-flight request completes`, async () => { + let resolveFirst: (() => void) | undefined + let callCount = 0 + const firstPromise = new Promise((resolve) => { + resolveFirst = () => resolve() + }) + + // First call will remain in-flight until we resolve it + let first = true + const mockLoadSubset = (_options: LoadSubsetOptions) => { + callCount++ + if (first) { + first = false + return firstPromise + } + return Promise.resolve() + } + + const onDeduplicate = vi.fn() + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: mockLoadSubset, + onDeduplicate: onDeduplicate, + }) + + // Start a broad in-flight request + const inflightOptions = { where: gt(ref(`age`), val(10)) } + const inflight = deduplicated.loadSubset(inflightOptions) + expect(inflight).toBeInstanceOf(Promise) + expect(callCount).toBe(1) + + // Issue a subset request while first is still in-flight + const subsetOptions = { where: gt(ref(`age`), val(20)) } + const subsetPromise = deduplicated.loadSubset(subsetOptions) + expect(subsetPromise).toBeInstanceOf(Promise) + + // onDeduplicate should NOT have fired yet + expect(onDeduplicate).not.toHaveBeenCalled() + + // Complete the first request + resolveFirst?.() + + // Wait for the subset promise to settle (which chains the first) + await subsetPromise + + // Now the callback should have been called exactly once, with the subset options + expect(onDeduplicate).toHaveBeenCalledTimes(1) + expect(onDeduplicate).toHaveBeenCalledWith(subsetOptions) + }) + }) })