WIP: Manage org dedupe indices

backend/btrixcloud/colls.py

@@ -537,6 +537,7 @@ async def list_collections(
         sort_direction: int = 1,
         name: Optional[str] = None,
         name_prefix: Optional[str] = None,
+        has_dedupe_index: Optional[bool] = None,
         access: Optional[str] = None,
         headers: Optional[dict] = None,
     ):
@@ -554,6 +555,9 @@ async def list_collections(
             regex_pattern = f"^{name_prefix}"
             match_query["name"] = {"$regex": regex_pattern, "$options": "i"}
 
+        if has_dedupe_index is not None:
+            match_query["hasDedupeIndex"] = has_dedupe_index
+
         if public_colls_out:
             match_query["access"] = CollAccessType.PUBLIC
         elif access:
@@ -1121,6 +1125,7 @@ async def list_collection_all(
         sortDirection: int = 1,
         name: Optional[str] = None,
         namePrefix: Optional[str] = None,
+        hasDedupeIndex: Optional[bool] = None,
         access: Optional[str] = None,
     ):
         # pylint: disable=duplicate-code
@@ -1132,6 +1137,7 @@ async def list_collection_all(
             sort_direction=sortDirection,
             name=name,
             name_prefix=namePrefix,
+            has_dedupe_index=hasDedupeIndex,
             access=access,
             headers=dict(request.headers),
         )

frontend/src/components/ui/data-grid/data-grid.ts

@@ -149,9 +149,14 @@ export class DataGrid<
 
     return html`
       <slot name="label">
-        <label id=${this.formControlLabelId} class="form-label text-xs">
-          ${this.formControlLabel}
-        </label>
+        ${this.formControlLabel
+          ? html`<label
+              id=${this.formControlLabelId}
+              class="form-label text-xs"
+            >
+              ${this.formControlLabel}
+            </label>`
+          : nothing}
       </slot>
       <div
         class=${clsx(

frontend/src/pages/org/settings/components/deduplication.ts

@@ -0,0 +1,168 @@
+import { localized, msg } from "@lit/localize";
+import { Task } from "@lit/task";
+import { html, type PropertyValues } from "lit";
+import { customElement, property, state } from "lit/decorators.js";
+import { when } from "lit/directives/when.js";
+import queryString from "query-string";
+
+import { loadingPanel } from "../templates/loading-panel";
+
+import { BtrixElement } from "@/classes/BtrixElement";
+import { parsePage, type PageChangeEvent } from "@/components/ui/pagination";
+import { OrgTab } from "@/routes";
+import { noData } from "@/strings/ui";
+import type { APIPaginatedList, APIPaginationQuery } from "@/types/api";
+import type { Collection } from "@/types/collection";
+import { isNotEqual } from "@/utils/is-not-equal";
+
+const INITIAL_PAGE_SIZE = 10;
+
+@customElement("btrix-org-settings-deduplication")
+@localized()
+export class OrgSettingsDeduplication extends BtrixElement {
+  @property({ type: Boolean })
+  visible?: boolean;
+
+  @state({ hasChanged: isNotEqual })
+  private pagination: Required<APIPaginationQuery> = {
+    page: parsePage(new URLSearchParams(location.search).get("page")),
+    pageSize: INITIAL_PAGE_SIZE,
+  };
+
+  protected willUpdate(changedProperties: PropertyValues): void {
+    // Reset pagination when tab is hidden
+    if (changedProperties.has("visible") && !this.visible) {
+      this.pagination = {
+        ...this.pagination,
+        page: 1,
+      };
+    }
+  }
+
+  private readonly sources = new Task(this, {
+    task: async ([pagination], { signal }) => {
+      return this.getCollections({ ...pagination }, signal);
+    },
+    args: () => [this.pagination] as const,
+  });
+
+  render() {
+    return html` ${this.sources.render({
+      initial: loadingPanel,
+      pending: () =>
+        this.sources.value
+          ? this.renderTable(this.sources.value)
+          : loadingPanel(),
+      complete: this.renderTable,
+    })}`;
+  }
+
+  private readonly renderTable = (
+    collections: APIPaginatedList<Collection>,
+  ) => {
+    return html`<btrix-table
+        class="whitespace-nowrap [--btrix-table-cell-padding-x:var(--sl-spacing-2x-small)]"
+        style="--btrix-table-grid-template-columns: 10ch 40ch repeat(4, 1fr) min-content"
+      >
+        <btrix-table-head class="mb-2">
+          <btrix-table-header-cell>
+            ${msg("Source Type")}
+          </btrix-table-header-cell>
+          <btrix-table-header-cell>${msg("Name")}</btrix-table-header-cell>
+          <btrix-table-header-cell>
+            ${msg("Archived Items")}
+          </btrix-table-header-cell>
+          <btrix-table-header-cell>
+            ${msg("Index Entries")}
+          </btrix-table-header-cell>
+          <btrix-table-header-cell>
+            ${msg("Index Size")}
+          </btrix-table-header-cell>
+          <btrix-table-header-cell>
+            ${msg("Purgeable Entries")}
+          </btrix-table-header-cell>
+          <btrix-table-header-cell>
+            <span class="sr-only">${msg("Actions")}</span>
+          </btrix-table-header-cell>
+        </btrix-table-head>
+        <btrix-table-body
+          class="rounded border [--btrix-table-cell-padding:var(--sl-spacing-2x-small)]"
+        >
+          ${collections.items.map(
+            (item) => html`
+              <btrix-table-row
+                class="border-t first:border-t-0 last:rounded-b hover:bg-neutral-50"
+              >
+                <btrix-table-cell>
+                  <btrix-badge>${msg("Collection")}</btrix-badge>
+                </btrix-table-cell>
+                <btrix-table-cell>${item.name}</btrix-table-cell>
+                <btrix-table-cell
+                  >${this.localize.number(item.crawlCount)}</btrix-table-cell
+                >
+                <btrix-table-cell>${noData}</btrix-table-cell>
+                <btrix-table-cell>${noData}</btrix-table-cell>
+                <btrix-table-cell>${noData}</btrix-table-cell>
+                <btrix-table-cell>
+                  <sl-tooltip content=${msg("Open in New Tab")}>
+                    <sl-icon-button
+                      name="arrow-up-right"
+                      href="${this.navigate
+                        .orgBasePath}/${OrgTab.Collections}/view/${item.id}"
+                      target="_blank"
+                    >
+                    </sl-icon-button>
+                  </sl-tooltip>
+                  <btrix-overflow-dropdown>
+                    <sl-menu>
+                      <sl-menu-item class="menu-item-warning"
+                        >${msg("Clear Index")}</sl-menu-item
+                      >
+                      <sl-menu-item class="menu-item-danger"
+                        >${msg("Delete Index")}</sl-menu-item
+                      >
+                    </sl-menu>
+                  </btrix-overflow-dropdown>
+                </btrix-table-cell>
+              </btrix-table-row>
+            `,
+          )}
+        </btrix-table-body>
+      </btrix-table>
+      ${when(
+        collections.total > collections.pageSize,
+        () => html`
+          <footer class="mt-6 flex justify-center">
+            <btrix-pagination
+              page=${collections.page}
+              totalCount=${collections.total}
+              size=${collections.pageSize}
+              @page-change=${async (e: PageChangeEvent) => {
+                this.pagination = {
+                  ...this.pagination,
+                  page: e.detail.page,
+                };
+              }}
+            ></btrix-pagination>
+          </footer>
+        `,
+      )} `;
+  };
+
+  private async getCollections(
+    params: APIPaginationQuery,
+    signal: AbortSignal,
+  ) {
+    const query = queryString.stringify({
+      pageSize: 2,
+      ...params,
+      hasDedupeIndex: true,
+    });
+    return this.api.fetch<APIPaginatedList<Collection>>(
+      `/orgs/${this.orgId}/collections?${query}`,
+      {
+        signal,
+      },
+    );
+  }
+}

frontend/src/pages/org/settings/settings.ts

@@ -26,10 +26,16 @@ import { tw } from "@/utils/tailwind";
 import "./components/general";
 import "./components/billing";
 import "./components/crawling-defaults";
+import "./components/deduplication";
 
 const styles = unsafeCSS(stylesheet);
 
-type Tab = "information" | "members" | "billing" | "crawling-defaults";
+type Tab =
+  | "information"
+  | "members"
+  | "billing"
+  | "crawling-defaults"
+  | "deduplication";
 type User = {
   email: string;
   role: AccessCode;
@@ -92,6 +98,7 @@ export class OrgSettings extends BtrixElement {
       members: msg("Members"),
       billing: msg("Billing"),
       "crawling-defaults": msg("Crawling Defaults"),
+      deduplication: msg("Deduplication"),
     };
   }
 
@@ -158,6 +165,7 @@ export class OrgSettings extends BtrixElement {
           this.renderTab("billing", "settings/billing"),
         )}
         ${this.renderTab("crawling-defaults", "settings/crawling-defaults")}
+        ${this.renderTab("deduplication", "settings/deduplication")}
 
         <btrix-tab-group-panel name="information">
           ${this.renderPanelHeader({ title: msg("General") })}
@@ -194,7 +202,7 @@ export class OrgSettings extends BtrixElement {
         </btrix-tab-group-panel>
         <btrix-tab-group-panel name="crawling-defaults">
           ${this.renderPanelHeader({
-            title: msg("Crawling Defaults"),
+            title: this.tabLabels["crawling-defaults"],
             actions: html`
               <sl-tooltip
                 content=${msg(
@@ -210,6 +218,10 @@ export class OrgSettings extends BtrixElement {
           })}
           <btrix-org-settings-crawling-defaults></btrix-org-settings-crawling-defaults>
         </btrix-tab-group-panel>
+        <btrix-tab-group-panel name="deduplication">
+          ${this.renderPanelHeader({ title: msg("Deduplication Sources") })}
+          <btrix-org-settings-deduplication></btrix-org-settings-deduplication>
+        </btrix-tab-group-panel>
       </btrix-tab-group>`;
   }
 
@@ -247,6 +259,11 @@ export class OrgSettings extends BtrixElement {
             "crawling-defaults",
             () => html`<sl-icon name="file-code-fill"></sl-icon>`,
           ],
+          [
+            "deduplication",
+            () =>
+              html`<sl-icon name="file-earmark-scan3" library="app"></sl-icon>`,
+          ],
         ])}
         ${this.tabLabels[name]}
       </btrix-tab-group-tab>

frontend/src/pages/org/settings/templates/loading-panel.ts

@@ -0,0 +1,7 @@
+import { html } from "lit";
+
+export const loadingPanel = () => {
+  return html`<div class="flex justify-center rounded border p-5 text-xl">
+    <sl-spinner></sl-spinner>
+  </div>`;
+};

frontend/src/theme.stylesheet.css

@@ -241,6 +241,10 @@
     @apply part-[base]:text-success part-[base]:hover:bg-success-50 part-[base]:hover:text-success-700 part-[base]:focus-visible:bg-success-50;
   }
 
+  .menu-item-warning {
+    @apply part-[base]:text-warning part-[base]:hover:bg-warning-50 part-[base]:hover:text-warning-700 part-[base]:focus-visible:bg-warning-50;
+  }
+
   .menu-item-danger {
     @apply part-[base]:text-danger part-[base]:hover:bg-danger-50 part-[base]:hover:text-danger-700 part-[base]:focus-visible:bg-danger-50;
   }