[Assistants] Filter on names (#841)

* [Assistants] Filter on names

* Add `$text` index on `assistant.name` (#844)

* add maxlength

* better experience

* use `$meta: "textScore"`

* Update src/routes/assistants/+page.server.ts

Co-authored-by: Eliott C. <[email protected]>

* null, not undefined

* [Assistants] Filter on names (using searchTokens) (#873)

Filter with `searchTokens`

* input

* rm extra whitespace

* hide UI before migration

* rm ad-hoc migration

---------

Co-authored-by: Eliott C. <[email protected]>
Co-authored-by: Nathan Sarrazin <[email protected]>
Co-authored-by: Victor Mustar <[email protected]>

Author: 4 people

src/lib/server/database.ts

@@ -117,6 +117,7 @@ client.on("open", () => {
 	assistants.createIndex({ userCount: 1 }).catch(console.error);
 	assistants.createIndex({ featured: 1, userCount: -1 }).catch(console.error);
 	assistants.createIndex({ modelId: 1, userCount: -1 }).catch(console.error);
+	assistants.createIndex({ searchTokens: 1 }).catch(console.error);
 	reports.createIndex({ assistantId: 1 }).catch(console.error);
 	reports.createIndex({ createdBy: 1, assistantId: 1 }).catch(console.error);
 });

src/lib/types/Assistant.ts

@@ -14,4 +14,5 @@ export interface Assistant extends Timestamps {
 	preprompt: string;
 	userCount?: number;
 	featured?: boolean;
+	searchTokens: string[];
 }

src/lib/utils/debounce.ts

@@ -0,0 +1,17 @@
+/**
+ * A debounce function that works in both browser and Nodejs.
+ * For pure Nodejs work, prefer the `Debouncer` class.
+ */
+export function debounce<T extends unknown[]>(
+	callback: (...rest: T) => unknown,
+	limit: number
+): (...rest: T) => void {
+	let timer: ReturnType<typeof setTimeout>;
+
+	return function (...rest) {
+		clearTimeout(timer);
+		timer = setTimeout(() => {
+			callback(...rest);
+		}, limit);
+	};
+}

src/lib/utils/searchTokens.ts

@@ -0,0 +1,33 @@
+const PUNCTUATION_REGEX = /\p{P}/gu;
+
+function removeDiacritics(s: string, form: "NFD" | "NFKD" = "NFD"): string {
+	return s.normalize(form).replace(/[\u0300-\u036f]/g, "");
+}
+
+export function generateSearchTokens(value: string): string[] {
+	const fullTitleToken = removeDiacritics(value)
+		.replace(PUNCTUATION_REGEX, "")
+		.replaceAll(/\s+/g, "")
+		.toLowerCase();
+	return [
+		...new Set([
+			...removeDiacritics(value)
+				.split(/\s+/)
+				.map((word) => word.replace(PUNCTUATION_REGEX, "").toLowerCase())
+				.filter((word) => word.length),
+			...(fullTitleToken.length ? [fullTitleToken] : []),
+		]),
+	];
+}
+
+function escapeForRegExp(s: string): string {
+	return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string
+}
+
+export function generateQueryTokens(query: string): RegExp[] {
+	return removeDiacritics(query)
+		.split(/\s+/)
+		.map((word) => word.replace(PUNCTUATION_REGEX, "").toLowerCase())
+		.filter((word) => word.length)
+		.map((token) => new RegExp(`^${escapeForRegExp(token)}`));
+}

src/routes/assistants/+page.server.ts

@@ -3,6 +3,7 @@ import { ENABLE_ASSISTANTS } from "$env/static/private";
 import { collections } from "$lib/server/database.js";
 import type { Assistant } from "$lib/types/Assistant";
 import type { User } from "$lib/types/User";
+import { generateQueryTokens } from "$lib/utils/searchTokens.js";
 import { error, redirect } from "@sveltejs/kit";
 import type { Filter } from "mongodb";
 
@@ -16,6 +17,7 @@ export const load = async ({ url, locals }) => {
 	const modelId = url.searchParams.get("modelId");
 	const pageIndex = parseInt(url.searchParams.get("p") ?? "0");
 	const username = url.searchParams.get("user");
+	const query = url.searchParams.get("q")?.trim() ?? null;
 	const createdByCurrentUser = locals.user?.username && locals.user.username === username;
 
 	let user: Pick<User, "_id"> | null = null;
@@ -34,6 +36,7 @@ export const load = async ({ url, locals }) => {
 		...(modelId && { modelId }),
 		...(!createdByCurrentUser && { userCount: { $gt: 1 } }),
 		...(user ? { createdById: user._id } : { featured: true }),
+		...(query && { searchTokens: { $all: generateQueryTokens(query) } }),
 	};
 	const assistants = await collections.assistants
 		.find(filter)
@@ -49,5 +52,6 @@ export const load = async ({ url, locals }) => {
 		selectedModel: modelId ?? "",
 		numTotalItems,
 		numItemsPerPage: NUM_PER_PAGE,
+		query,
 	};
 };

src/routes/assistants/+page.svelte

@@ -4,6 +4,7 @@
 	import { PUBLIC_APP_ASSETS, PUBLIC_ORIGIN } from "$env/static/public";
 	import { isHuggingChat } from "$lib/utils/isHuggingChat";
 
+	import { tick } from "svelte";
 	import { goto } from "$app/navigation";
 	import { base } from "$app/paths";
 	import { page } from "$app/stores";
@@ -14,16 +15,22 @@
 	import CarbonArrowUpRight from "~icons/carbon/arrow-up-right";
 	import CarbonEarthAmerica from "~icons/carbon/earth-americas-filled";
 	import CarbonUserMultiple from "~icons/carbon/user-multiple";
+	import CarbonSearch from "~icons/carbon/search";
 	import Pagination from "$lib/components/Pagination.svelte";
 	import { formatUserCount } from "$lib/utils/formatUserCount";
 	import { getHref } from "$lib/utils/getHref";
+	import { debounce } from "$lib/utils/debounce";
 	import { useSettingsStore } from "$lib/stores/settings";
 
 	export let data: PageData;
 
 	$: assistantsCreator = $page.url.searchParams.get("user");
 	$: createdByMe = data.user?.username && data.user.username === assistantsCreator;
 
+	const SEARCH_DEBOUNCE_DELAY = 400;
+	let filterInputEl: HTMLInputElement;
+	let searchDisabled = false;
+
 	const onModelChange = (e: Event) => {
 		const newUrl = getHref($page.url, {
 			newKeys: { modelId: (e.target as HTMLSelectElement).value },
@@ -32,6 +39,18 @@
 		goto(newUrl);
 	};
 
+	const filterOnName = debounce(async (e: Event) => {
+		searchDisabled = true;
+		const value = (e.target as HTMLInputElement).value;
+		const newUrl = getHref($page.url, { newKeys: { q: value } });
+		await goto(newUrl);
+		setTimeout(async () => {
+			searchDisabled = false;
+			await tick();
+			filterInputEl.focus();
+		}, 0);
+	}, SEARCH_DEBOUNCE_DELAY);
+
 	const settings = useSettingsStore();
 </script>
 
@@ -99,7 +118,7 @@
 					{assistantsCreator}'s Assistants
 					<a
 						href={getHref($page.url, {
-							existingKeys: { behaviour: "delete", keys: ["user", "modelId", "p"] },
+							existingKeys: { behaviour: "delete", keys: ["user", "modelId", "p", "q"] },
 						})}
 						class="group"
 						><CarbonClose
@@ -119,7 +138,7 @@
 			{:else}
 				<a
 					href={getHref($page.url, {
-						existingKeys: { behaviour: "delete", keys: ["user", "modelId", "p"] },
+						existingKeys: { behaviour: "delete", keys: ["user", "modelId", "p", "q"] },
 					})}
 					class="flex items-center gap-1.5 rounded-full border px-3 py-1 {!assistantsCreator
 						? 'border-gray-300 bg-gray-50  dark:border-gray-600 dark:bg-gray-700 dark:text-white'
@@ -132,16 +151,31 @@
 					<a
 						href={getHref($page.url, {
 							newKeys: { user: data.user.username },
-							existingKeys: { behaviour: "delete", keys: ["modelId", "p"] },
+							existingKeys: { behaviour: "delete", keys: ["modelId", "p", "q"] },
 						})}
-						class="flex items-center gap-1.5 rounded-full border px-3 py-1 {assistantsCreator &&
+						class="flex items-center gap-1.5 truncate rounded-full border px-3 py-1 {assistantsCreator &&
 						createdByMe
 							? 'border-gray-300 bg-gray-50  dark:border-gray-600 dark:bg-gray-700 dark:text-white'
 							: 'border-transparent text-gray-400 hover:text-gray-800 dark:hover:text-gray-300'}"
 						>{data.user.username}
 					</a>
 				{/if}
 			{/if}
+			<div
+				class="relative ml-auto flex hidden h-[30px] w-40 items-center rounded-full border px-2 has-[:focus]:border-gray-400 sm:w-64 dark:border-gray-600"
+			>
+				<CarbonSearch class="pointer-events-none absolute left-2 text-xs text-gray-400" />
+				<input
+					class="h-[30px] w-full bg-transparent pl-5 focus:outline-none"
+					placeholder="Filter by name"
+					value={data.query}
+					on:input={filterOnName}
+					bind:this={filterInputEl}
+					maxlength="150"
+					type="search"
+					disabled={searchDisabled}
+				/>
+			</div>
 		</div>
 
 		<div class="mt-8 grid grid-cols-2 gap-3 sm:gap-5 md:grid-cols-3 lg:grid-cols-4">

src/routes/settings/assistants/[assistantId]/edit/+page.server.ts

@@ -8,6 +8,7 @@ import { z } from "zod";
 import { sha256 } from "$lib/utils/sha256";
 
 import sharp from "sharp";
+import { generateSearchTokens } from "$lib/utils/searchTokens";
 
 const newAsssistantSchema = z.object({
 	name: z.string().min(1),
@@ -130,6 +131,7 @@ export const actions: Actions = {
 					exampleInputs,
 					avatar: deleteAvatar ? undefined : hash ?? assistant.avatar,
 					updatedAt: new Date(),
+					searchTokens: generateSearchTokens(parse.data.name),
 				},
 			}
 		);

src/routes/settings/assistants/new/+page.server.ts

@@ -7,6 +7,7 @@ import { ObjectId } from "mongodb";
 import { z } from "zod";
 import { sha256 } from "$lib/utils/sha256";
 import sharp from "sharp";
+import { generateSearchTokens } from "$lib/utils/searchTokens";
 
 const newAsssistantSchema = z.object({
 	name: z.string().min(1),
@@ -99,6 +100,7 @@ export const actions: Actions = {
 			updatedAt: new Date(),
 			userCount: 1,
 			featured: false,
+			searchTokens: generateSearchTokens(parse.data.name),
 		});
 
 		// add insertedId to user settings