Reverseindex search

src/assets/ts/app.tsx

@@ -27,8 +27,8 @@ import { RegisterTypes } from './register';
 import KeyEmitter from './keyEmitter';
 import KeyHandler from './keyHandler';
 import KeyMappings from './keyMappings';
-import { ClientStore, DocumentStore } from './datastore';
-import { SynchronousInMemory, InMemory } from '../../shared/data_backend';
+import { ClientStore, DocumentStore, SearchStore } from './datastore';
+import DataBackend, { SynchronousInMemory, InMemory } from '../../shared/data_backend';
 import {
   BackendType, SynchronousLocalStorageBackend,
   LocalStorageBackend, FirebaseBackend, ClientSocketBackend, IndexedDBBackend
@@ -97,9 +97,14 @@ $(document).ready(async () => {
     renderMain(); // fire and forget
   };
 
+  type Stores = {
+    docStore: DocumentStore,
+    searchStore: SearchStore
+  }
+
   const noLocalStorage = (typeof localStorage === 'undefined' || localStorage === null);
   let clientStore: ClientStore;
-  let docStore: DocumentStore;
+  let docStore: Stores;
   let backend_type: BackendType;
   let doc;
 
@@ -120,11 +125,15 @@ $(document).ready(async () => {
 
   const config: Config = vimConfig;
 
-  function getLocalStore(): DocumentStore {
-     return new DocumentStore(new IndexedDBBackend(docname), docname);
+  function getStores(backend: DataBackend, docname = '') {
+    return { docStore: new DocumentStore(backend, docname), searchStore: new SearchStore(backend, docname) };
   }
 
-  async function getFirebaseStore(): Promise<DocumentStore> {
+  function getLocalStore(): Stores {
+     return getStores(new IndexedDBBackend(docname), docname);
+  }
+
+  async function getFirebaseStore(): Promise<Stores> {
     const firebaseId = clientStore.getDocSetting('firebaseId');
     const firebaseApiKey = clientStore.getDocSetting('firebaseApiKey');
     const firebaseUserEmail = clientStore.getDocSetting('firebaseUserEmail');
@@ -137,14 +146,13 @@ $(document).ready(async () => {
       throw new Error('No firebase API key found');
     }
     const fb_backend = new FirebaseBackend(docname, firebaseId, firebaseApiKey);
-    const dStore = new DocumentStore(fb_backend, docname);
     await fb_backend.init(firebaseUserEmail || '', firebaseUserPassword || '');
 
     logger.info(`Successfully initialized firebase connection: ${firebaseId}`);
-    return dStore;
+    return getStores(fb_backend, docname);
   }
 
-  async function getSocketServerStore(): Promise<DocumentStore> {
+  async function getSocketServerStore(): Promise<Stores> {
     let socketServerHost;
     let socketServerDocument;
     let socketServerPassword;
@@ -164,7 +172,7 @@ $(document).ready(async () => {
     const socket_backend = new ClientSocketBackend();
     // NOTE: we don't pass docname to DocumentStore since we want keys
     // to not have prefixes
-    const dStore = new DocumentStore(socket_backend);
+    const dStore = getStores(socket_backend);
     while (true) {
       try {
         await socket_backend.init(
@@ -205,7 +213,7 @@ $(document).ready(async () => {
       backend_type = 'local';
     }
   } else if (backend_type === 'inmemory') {
-    docStore = new DocumentStore(new InMemory());
+    docStore = getStores(new InMemory());
   } else if (backend_type === 'socketserver') {
     try {
       docStore = await getSocketServerStore();
@@ -229,10 +237,10 @@ $(document).ready(async () => {
     backend_type = 'local';
   }
 
-  doc = new Document(docStore, docname);
+  doc = new Document(docStore.docStore, docStore.searchStore, docname);
 
   let to_load: any = null;
-  if ((await docStore.getChildren(Path.rootRow())).length === 0) {
+  if ((await docStore.docStore.getChildren(Path.rootRow())).length === 0) {
     to_load = config.getDefaultData();
   }
 
@@ -330,7 +338,7 @@ $(document).ready(async () => {
   // load plugins
 
   const pluginManager = new PluginsManager(session, config, keyBindings);
-  let enabledPlugins = await docStore.getSetting('enabledPlugins');
+  let enabledPlugins = await docStore.docStore.getSetting('enabledPlugins');
   if (typeof enabledPlugins.slice === 'undefined') { // for backwards compatibility
     enabledPlugins = Object.keys(enabledPlugins);
   }
@@ -466,7 +474,7 @@ $(document).ready(async () => {
   pluginManager.on('status', renderMain); // fire and forget
 
   pluginManager.on('enabledPluginsChange', function(enabled) {
-    docStore.setSetting('enabledPlugins', enabled);
+    docStore.docStore.setSetting('enabledPlugins', enabled);
     renderMain(); // fire and forget
   });
 

src/assets/ts/components/settings.tsx

@@ -9,7 +9,7 @@ import { MODES } from '../modes';
 
 import Path from '../path';
 import Document from '../document';
-import { DocumentStore, ClientStore } from '../datastore';
+import { DocumentStore, ClientStore, SearchStore } from '../datastore';
 import { InMemory } from '../../../shared/data_backend';
 import Session from '../session';
 import Menu from '../menu';
@@ -82,7 +82,8 @@ export default class SettingsComponent extends React.Component<Props, State> {
     this.initial_theme = getCurrentTheme(props.session.clientStore);
 
     (async () => {
-      const preview_document = new Document(new DocumentStore(new InMemory()));
+      const backend = new InMemory();
+      const preview_document = new Document(new DocumentStore(backend), new SearchStore(backend));
       await preview_document.load([
         { text: 'Preview document', children: [
           { text: 'Breadcrumbs', children: [

src/assets/ts/cursor.ts

@@ -53,6 +53,7 @@ export default class Cursor extends EventEmitter {
 
   public async _setPath(path: Path) {
     await this.emitAsync('rowChange', this.path, path);
+    this.session.document.searcher.update(this.path.row);
     this.path = path;
   }
 

src/assets/ts/datastore.ts

@@ -204,6 +204,104 @@ const decodeParents = (parents: number | Array<number>): Array<number> => {
   return parents;
 };
 
+export class SearchStore {
+  private prefix: string;
+  private docname: string;
+  private backend: DataBackend;
+  private cache: {[key: string]: any} = {};
+  private use_cache: boolean = true;
+
+  constructor(backend: DataBackend, docname = '') {
+    this.backend = backend;
+    this.docname = docname;
+    this.prefix = `${docname}save`;
+  }
+
+  private async _get<T>(
+    key: string,
+    default_value: T,
+    decode: (value: any) => T = fn_utils.id
+  ): Promise<T> {
+    if (simulateDelay) { await timeout(simulateDelay * Math.random()); }
+
+    if (this.use_cache) {
+      if (key in this.cache) {
+        return this.cache[key];
+      }
+    }
+    let value: any = await this.backend.get(key);
+    try {
+      // need typeof check because of backwards compatibility plus stupidness like
+      // JSON.parse([106]) === 106
+      if (typeof value === 'string') {
+        value = JSON.parse(value);
+      }
+    } catch (e) { /* do nothing - this should only happen for historical reasons */ }
+    let decodedValue: T;
+    if (value === null) {
+      decodedValue = default_value;
+      logger.debug('tried getting', key, 'defaulted to', decodedValue);
+    } else {
+      decodedValue = decode(value);
+      logger.debug('got from storage', key, decodedValue);
+    }
+    if (this.use_cache) {
+      this.cache[key] = decodedValue;
+    }
+    return decodedValue;
+  }
+
+  private async _set(
+    key: string, value: any, encode: (value: any) => any = fn_utils.id
+  ): Promise<void> {
+    if (simulateDelay) { await timeout(simulateDelay * Math.random()); }
+
+    if (this.use_cache) {
+      this.cache[key] = value;
+    }
+    const encodedValue = encode(value);
+    logger.debug('setting to storage', key, encodedValue);
+    // NOTE: fire and forget
+    this.backend.set(key, JSON.stringify(encodedValue)).catch((err) => {
+      setTimeout(() => { throw err; });
+    });
+  }
+
+  private hash(token: string) {
+    // https://stackoverflow.com/questions/7616461/generate-a-hash-from-string-in-javascript
+    let hash = 0, i, chr;
+    for (i = 0; i < token.length; i++) {
+      chr = token.charCodeAt(i);
+      hash = ((hash << 5) - hash) + chr;
+      hash |= 0; // Convert to 32bit integer
+    }
+    return hash;
+  }
+
+  private _rowsKey_(token: string): string {
+    return `${this.prefix}:rows_${this.hash(token)}`;
+  }
+
+  private _lastRowKey_(): string {
+    return `${this.prefix}:lastRow`;
+  }
+
+  public async setRows(token: string, rows: Set<Row>) {
+    return this._set(this._rowsKey_(token), Array.from(rows));
+  }
+
+  public async setLastRow(last: number) {
+    return this._set(this._lastRowKey_(), last);
+  }
+
+  public async getRows(token: string) {
+    return new Set(await this._get(this._rowsKey_(token), new Array<Row>()));
+  }
+
+  public async getLastRow() {
+    return this._get(this._lastRowKey_(), -1);
+  }
+}
 export class DocumentStore {
   private lastId: number | null;
   private prefix: string;
@@ -368,6 +466,10 @@ export class DocumentStore {
     return await this._get(this._pluginDataKey_(plugin, key), default_value);
   }
 
+  public async getLastIDKey() {
+    return await this._get(this._lastIDKey_(), 0);
+  }
+
   // get next row ID
   // public so test case can override
   public async getId(): Promise<number> {

src/assets/ts/document.ts

@@ -7,11 +7,12 @@ import * as fn_utils from './utils/functional';
 // import logger from './utils/logger';
 import { isWhitespace } from './utils/text';
 import Path from './path';
-import { DocumentStore } from './datastore';
+import { DocumentStore, SearchStore } from './datastore';
 import { InMemory } from '../../shared/data_backend';
 import {
   Row, Col, Char, Line, SerializedLine, SerializedBlock
 } from './types';
+import { Searcher } from './searcher';
 
 type RowInfo = {
   readonly line: Line;
@@ -210,13 +211,17 @@ export default class Document extends EventEmitter {
   public store: DocumentStore;
   public name: string;
   public root: Path;
+  public searcher: Searcher;
 
-  constructor(store: DocumentStore, name = '') {
+  constructor(store: DocumentStore, searchStore: SearchStore, name = '') {
     super();
     this.cache = new DocumentCache();
     this.store = store;
     this.name = name;
     this.root = Path.root();
+    this.searcher = new Searcher(searchStore);
+
+    this.initSearcher();
     return this;
   }
 
@@ -317,6 +322,8 @@ export default class Document extends EventEmitter {
   }
 
   public async setLine(row: Row, line: Line) {
+    const oldLine = await this.getText(row);
+    this.searcher.rowChange(row, oldLine, line.join(''));
     this.cache.setLine(row, line);
     await this.store.setLine(row, line);
   }
@@ -713,22 +720,17 @@ export default class Document extends EventEmitter {
     return path.child(row);
   }
 
-  private async* traverseSubtree(root: Path): AsyncIterableIterator<Path> {
-    const visited_rows: {[row: number]: boolean} = {};
-    let that = this;
-
-    async function* helper(path: Path): AsyncIterableIterator<Path> {
-      if (path.row in visited_rows) {
-        return;
-      }
-      visited_rows[path.row] = true;
-      yield path;
-      const children = await that.getChildren(path);
-      for (let i = 0; i < children.length; i++) {
-        yield* await helper(children[i]);
+  public async initSearcher() {
+    const lastInserted = await this.searcher.searchStore.getLastRow();
+    const lastRow = await this.store.getLastIDKey();
+    for (let i = lastInserted + 1; i <= lastRow; i++) {
+      if (await this.isAttached(i)) {
+        //console.log('inserting row', i, 'out of', lastRow, 'into search store');
+        this.searcher.rowChange(i, '', await this.getText(i));
+        await this.searcher.update(i);
+        await this.searcher.searchStore.setLastRow(i);
       }
     }
-    yield* await helper(root);
   }
 
   public async search(root: Path, query: string, options: SearchOptions = {}) {
@@ -746,25 +748,41 @@ export default class Document extends EventEmitter {
     const query_words =
       query.split(/\s/g).filter(x => x.length).map(canonicalize);
 
-    const paths = this.traverseSubtree(root);
-    for await (let path of paths) {
-      const text = await this.getText(path.row);
-      const line = canonicalize(text);
-      const matches: Array<number> = [];
-      if (_.every(query_words.map((word) => {
-        const index = line.indexOf(word);
-        if (index === -1) { return false; }
-        for (let j = index; j < index + word.length; j++) {
-          matches.push(j);
-        }
-        return true;
-      }))) {
-        results.push({ path, matches });
-      }
-      if (nresults > 0 && results.length === nresults) {
+    const possibleRows = await this.searcher.search(query_words);
+    if (possibleRows === null) {
+      return results;
+    }
+    const possibleRowsArr = Array.from(possibleRows);
+    const chunkedRows = _.chunk(possibleRowsArr, 20);
+    for (let chunk of chunkedRows) {
+      if (nresults > 0 && results.length >= nresults) {
         break;
       }
+      await Promise.all(chunk.map(async (row) => {
+        const text = await this.getText(row);
+        const line = canonicalize(text);
+        const matches: Array<number> = [];
+        const path = await this.canonicalPath(row);
+
+        if (path === null || !path.isDescendant(root)) {  // might not work with cloned rows
+          return;
+        }
+
+        if (_.every(query_words.map((word) => {
+          const index = line.indexOf(word);
+          if (index === -1) { return false; }
+          for (let j = index; j < index + word.length; j++) {
+            matches.push(j);
+          }
+          return true;
+        }))) {
+          if (path && (nresults == 0 || results.length < nresults)) {
+            results.push({ path, matches });
+          }
+        }
+      }));
     }
+    //console.log('Search results', results);
     return results;
   }
 
@@ -891,6 +909,7 @@ export default class Document extends EventEmitter {
 
 export class InMemoryDocument extends Document {
   constructor() {
-    super(new DocumentStore(new InMemory()));
+    const backend = new InMemory();
+    super(new DocumentStore(backend), new SearchStore(backend));
   }
 }