Added support for secondary indices for fast lookups

Summary:
Add support for secondary indices, to allow for cheap lookups, like a set of events for a specific UI element in the events table:

```

#### getAllRecordsByIndex

Usage: `getAllRecordsByIndex({ indexedAttribute: value, indexAttribute2: value2, .... })`

This method allows fast lookups for objects that match specific attributes exactly.
Returns all items matching the specified index query.
Note that the results are unordered, unless
records have not been updated using upsert / update, in that case
insertion order is maintained.
If no index has been specified for this exact keyset in the indexQuery (see options.indices), this method will throw.

Example:
```

```
const ds = createDataSource([eatCookie, drinkCoffee, submitBug], {
  key: 'id',
  indices: [
    ['title']
    ['id', 'title'],
    ['title', 'done'],
  ],
});

// Find first element with title === cookie (or undefined)
const todo = ds.getFirstRecordByIndex({
    title: 'cookie',
})

// Find all elements where title === cookie, and done === false
const todos = ds.getAllRecordsByIndex({
  title: 'cookie',
  done: false,
})
```

Reviewed By: antonk52

Differential Revision: D47396435

fbshipit-source-id: 20c4527be83532863b9b07ab20ebf20a80c3c35d

desktop/flipper-plugin-core/src/data-source/DataSource.tsx

@@ -97,27 +97,44 @@ export type DataSourceOptionKey<K extends PropertyKey> = {
    */
   key?: K;
 };
-export type DataSourceOptions = {
+export type DataSourceOptions<T> = {
   /**
    * The maximum amount of records that this DataSource will store.
    * If the limit is exceeded, the oldest records will automatically be dropped to make place for the new ones
    */
   limit?: number;
+  /**
+   * Secondary indices, that can be used to perform O(1) lookups on specific keys later on.
+   * A combination of keys is allowed.
+   *
+   * For example:
+   * indices: [["title"], ["id", "title"]]
+   *
+   * Enables:
+   * dataSource.getAllRecordsByIndex({
+   *   id: 123,
+   *   title: "Test"
+   * })
+   */
+  indices?: IndexDefinition<T>[];
 };
 
+type IndexDefinition<T> = Array<keyof T>;
+type IndexQuery<T> = Partial<T>;
+
 export function createDataSource<T, Key extends keyof T>(
   initialSet: readonly T[],
-  options: DataSourceOptions & DataSourceOptionKey<Key>,
+  options: DataSourceOptions<T> & DataSourceOptionKey<Key>,
 ): DataSource<T, T[Key] extends string | number ? T[Key] : never>;
 export function createDataSource<T>(
   initialSet?: readonly T[],
-  options?: DataSourceOptions,
+  options?: DataSourceOptions<T>,
 ): DataSource<T, never>;
 export function createDataSource<T, Key extends keyof T>(
   initialSet: readonly T[] = [],
-  options?: DataSourceOptions & DataSourceOptionKey<Key>,
+  options?: DataSourceOptions<T> & DataSourceOptionKey<Key>,
 ): DataSource<T, Key> {
-  const ds = new DataSource<T, Key>(options?.key);
+  const ds = new DataSource<T, Key>(options?.key, options?.indices);
   if (options?.limit !== undefined) {
     ds.limit = options.limit;
   }
@@ -129,6 +146,14 @@ export class DataSource<T extends any, KeyType = never> {
   private nextId = 0;
   private _records: Entry<T>[] = [];
   private _recordsById: Map<KeyType, T> = new Map();
+
+  // secondary indices are used to allow fast lookups for items that match certain columns exactly
+  private _secondaryIndices: Map<
+    string /* normalized string */,
+    IndexDefinition<T> /* sorted keys */
+  >;
+  private _recordsBySecondaryIndex: Map<string, Map<string, T[]>> = new Map();
+
   /**
    * @readonly
    */
@@ -155,8 +180,25 @@ export class DataSource<T extends any, KeyType = never> {
     [viewId: string]: DataSourceView<T, KeyType>;
   };
 
-  constructor(keyAttribute: keyof T | undefined) {
+  constructor(
+    keyAttribute: keyof T | undefined,
+    secondaryIndices: IndexDefinition<T>[] = [],
+  ) {
     this.keyAttribute = keyAttribute;
+    this._secondaryIndices = new Map(
+      secondaryIndices.map((index) => {
+        const sortedKeys = index.slice().sort();
+        const key = sortedKeys.join(':');
+        // immediately reserve a map per index
+        this._recordsBySecondaryIndex.set(key, new Map());
+        return [key, sortedKeys];
+      }),
+    );
+    if (this._secondaryIndices.size !== secondaryIndices.length) {
+      throw new Error(
+        `Duplicate index definition in ${JSON.stringify(secondaryIndices)}`,
+      );
+    }
     this.view = new DataSourceView<T, KeyType>(this, DEFAULT_VIEW_ID);
     this.additionalViews = {};
   }
@@ -246,6 +288,7 @@ export class DataSource<T extends any, KeyType = never> {
       this._recordsById.set(key, value);
       this.storeIndexOfKey(key, this._records.length);
     }
+    this.storeSecondaryIndices(value);
     const visibleMap: {[viewId: string]: boolean} = {[DEFAULT_VIEW_ID]: false};
     const approxIndexMap: {[viewId: string]: number} = {[DEFAULT_VIEW_ID]: -1};
     Object.keys(this.additionalViews).forEach((viewId) => {
@@ -309,6 +352,8 @@ export class DataSource<T extends any, KeyType = never> {
       this._recordsById.set(key, value);
       this.storeIndexOfKey(key, index);
     }
+    this.removeSecondaryIndices(oldValue);
+    this.storeSecondaryIndices(value);
     this.emitDataEvent({
       type: 'update',
       entry,
@@ -343,6 +388,7 @@ export class DataSource<T extends any, KeyType = never> {
         });
       }
     }
+    this.removeSecondaryIndices(entry.value);
     this.emitDataEvent({
       type: 'remove',
       index,
@@ -387,6 +433,7 @@ export class DataSource<T extends any, KeyType = never> {
         this.idToIndex.delete(key);
       });
     }
+    removed.forEach((entry) => this.removeSecondaryIndices(entry.value));
 
     if (
       this.view.isSorted &&
@@ -411,10 +458,13 @@ export class DataSource<T extends any, KeyType = never> {
    * The clear operation removes any records stored, but will keep the current view preferences such as sorting and filtering
    */
   public clear() {
-    this._records = [];
-    this._recordsById = new Map();
+    this._records.splice(0);
+    this._recordsById.clear();
+    for (const m of this._recordsBySecondaryIndex.values()) {
+      m.clear();
+    }
     this.shiftOffset = 0;
-    this.idToIndex = new Map();
+    this.idToIndex.clear();
     this.rebuild();
   }
 
@@ -502,6 +552,91 @@ export class DataSource<T extends any, KeyType = never> {
     });
   }
 
+  private storeSecondaryIndices(value: T) {
+    for (const [indexKey, sortedIndex] of this._secondaryIndices.entries()) {
+      const indexValue = this.getSecondaryIndexValueFromRecord(
+        value,
+        sortedIndex,
+      );
+      // maps are already set up in constructor
+      const m = this._recordsBySecondaryIndex.get(indexKey)!;
+      const a = m.get(indexValue);
+      if (!a) {
+        // not seen this index value yet
+        m.set(indexValue, [value]);
+      } else {
+        a.push(value);
+      }
+    }
+  }
+
+  private removeSecondaryIndices(value: T) {
+    for (const [indexKey, sortedIndex] of this._secondaryIndices.entries()) {
+      const indexValue = this.getSecondaryIndexValueFromRecord(
+        value,
+        sortedIndex,
+      );
+      // maps are already set up in constructor
+      const m = this._recordsBySecondaryIndex.get(indexKey)!;
+      // code belows assumes that we have an entry for this secondary
+      const a = m.get(indexValue)!;
+      a.splice(a.indexOf(value), 1);
+    }
+  }
+
+  /**
+   * Returns all items matching the specified index query.
+   *
+   * Note that the results are unordered, unless
+   * records have not been updated using upsert / update, in that case
+   * insertion order is maintained.
+   *
+   * Example:
+   * `ds.getAllRecordsByIndex({title: 'subit a bug', done: false})`
+   *
+   * If no index has been specified for this exact keyset in the indexQuery (see options.indices), this method will throw
+   *
+   * @param indexQuery
+   * @returns
+   */
+  public getAllRecordsByIndex(indexQuery: IndexQuery<T>): readonly T[] {
+    // normalise indexKey, incl sorting
+    const sortedKeys: (keyof T)[] = Object.keys(indexQuery).sort() as any;
+    const indexKey = sortedKeys.join(':');
+    const recordsByIndex = this._recordsBySecondaryIndex.get(indexKey);
+    if (!recordsByIndex) {
+      throw new Error(
+        `No index has been defined for the keys ${JSON.stringify(
+          Object.keys(indexQuery),
+        )}`,
+      );
+    }
+    const indexValue = JSON.stringify(
+      // query object needs reordering and normalised to produce correct indexValue
+      Object.fromEntries(sortedKeys.map((k) => [k, String(indexQuery[k])])),
+    );
+    return recordsByIndex.get(indexValue) ?? [];
+  }
+
+  /**
+   * Like getAllRecords, but returns the first match only.
+   * @param indexQuery
+   * @returns
+   */
+  public getFirstRecordByIndex(indexQuery: IndexQuery<T>): T | undefined {
+    return this.getAllRecordsByIndex(indexQuery)[0];
+  }
+
+  private getSecondaryIndexValueFromRecord(
+    record: T,
+    // assumes keys is already ordered
+    keys: IndexDefinition<T>,
+  ): any {
+    return JSON.stringify(
+      Object.fromEntries(keys.map((k) => [k, String(record[k])])),
+    );
+  }
+
   /**
    * @private
    */

desktop/flipper-plugin-core/src/data-source/__tests__/datasource-basics.node.tsx

@@ -856,3 +856,185 @@ test('DataSource.view can iterate', () => {
   ds.clear();
   expect([...ds.view]).toEqual([]);
 });
+
+test('secondary keys - doesnt allow duplicate keys', () => {
+  expect(() =>
+    createDataSource([], {
+      indices: [['title'], ['title']],
+    }),
+  ).toThrowErrorMatchingInlineSnapshot(
+    `"Duplicate index definition in [["title"],["title"]]"`,
+  );
+
+  expect(() =>
+    createDataSource([], {
+      // these are the same!
+      indices: [
+        ['id', 'title'],
+        ['title', 'id'],
+      ],
+    }),
+  ).toThrowErrorMatchingInlineSnapshot(
+    `"Duplicate index definition in [["id","title"],["title","id"]]"`,
+  );
+});
+
+test('secondary keys - doesnt allow lookup with nonexisting key', () => {
+  const ds = createDataSource<Todo>([], {indices: [['done']]});
+  expect(() =>
+    expect(ds.getAllRecordsByIndex({title: 'subit a bug', done: false})),
+  ).toThrowErrorMatchingInlineSnapshot(
+    `"No index has been defined for the keys ["title","done"]"`,
+  );
+});
+
+test('secondary keys - lookup by single key', () => {
+  const ds = createDataSource([eatCookie, drinkCoffee, submitBug], {
+    indices: [['id'], ['title'], ['done']],
+  });
+
+  expect(
+    ds.getAllRecordsByIndex({
+      title: 'eat a cookie',
+    }),
+  ).toEqual([eatCookie]);
+
+  const cookie2 = {...eatCookie, done: false};
+  ds.append(cookie2);
+  expect(
+    ds.getAllRecordsByIndex({
+      title: 'eat a cookie',
+    }),
+  ).toEqual([eatCookie, cookie2]);
+
+  expect(
+    ds.getAllRecordsByIndex({
+      done: false,
+    }),
+  ).toEqual([submitBug, cookie2]);
+
+  expect(
+    ds.getFirstRecordByIndex({
+      done: false,
+    }),
+  ).toEqual(submitBug);
+
+  ds.delete(0); // eat Cookie
+  expect(
+    ds.getAllRecordsByIndex({
+      title: 'eat a cookie',
+    }),
+  ).toEqual([cookie2]);
+
+  // replace submit Bug
+  const n = {
+    id: 'bug',
+    title: 'eat a cookie',
+    done: false,
+  };
+  ds.update(1, n);
+
+  expect(
+    ds.getAllRecordsByIndex({
+      title: 'eat a cookie',
+    }),
+  ).toEqual([cookie2, n]);
+
+  expect(
+    ds.getFirstRecordByIndex({
+      title: 'submit a bug',
+    }),
+  ).toBeUndefined();
+
+  // removes drinkCoffe, n
+  ds.shift(2);
+  expect(
+    ds.getAllRecordsByIndex({
+      title: 'eat a cookie',
+    }),
+  ).toEqual([cookie2]);
+});
+
+test('secondary keys - lookup by combined keys', () => {
+  const ds = createDataSource([eatCookie, drinkCoffee, submitBug], {
+    key: 'id',
+    indices: [
+      ['id', 'title'],
+      ['title', 'done'],
+    ],
+  });
+
+  expect(
+    ds.getAllRecordsByIndex({
+      id: 'cookie',
+      title: 'eat a cookie',
+    }),
+  ).toEqual([eatCookie]);
+  expect(
+    ds.getAllRecordsByIndex({
+      // order doesn't matter
+      title: 'eat a cookie',
+      id: 'cookie',
+    }),
+  ).toEqual([eatCookie]);
+
+  // Note: different key order
+  const cookie2 = {id: 'cookie2', done: true, title: 'eat a cookie'};
+  ds.append(cookie2);
+  expect(
+    ds.getAllRecordsByIndex({
+      id: 'cookie2',
+      title: 'eat a cookie',
+    }),
+  ).toEqual([cookie2]);
+
+  expect(
+    ds.getAllRecordsByIndex({
+      done: true,
+      title: 'eat a cookie',
+    }),
+  ).toEqual([eatCookie, cookie2]);
+
+  const upsertedCookie = {
+    id: 'cookie',
+    title: 'eat a cookie',
+    done: false,
+  };
+  ds.upsert(upsertedCookie);
+  expect(
+    ds.getAllRecordsByIndex({
+      done: true,
+      title: 'eat a cookie',
+    }),
+  ).toEqual([cookie2]);
+  expect(
+    ds.getAllRecordsByIndex({
+      done: false,
+      title: 'eat a cookie',
+    }),
+  ).toEqual([upsertedCookie]);
+
+  ds.deleteByKey('cookie'); // eat Cookie
+  expect(
+    ds.getFirstRecordByIndex({
+      title: 'eat a cookie',
+      done: false,
+    }),
+  ).toEqual(undefined);
+
+  ds.clear();
+  expect(
+    ds.getAllRecordsByIndex({
+      done: true,
+      title: 'eat a cookie',
+    }),
+  ).toEqual([]);
+
+  ds.append(cookie2);
+  expect(
+    ds.getAllRecordsByIndex({
+      id: 'cookie2',
+      title: 'eat a cookie',
+    }),
+  ).toEqual([cookie2]);
+});

desktop/flipper-plugin-core/src/state/createDataSource.tsx

@@ -15,7 +15,7 @@ import {
 } from '../data-source/DataSource';
 import {registerStorageAtom} from '../plugin/PluginBase';
 
-type DataSourceOptions = BaseDataSourceOptions & {
+type DataSourceOptions<T> = BaseDataSourceOptions<T> & {
   /**
    * Should this state persist when exporting a plugin?
    * If set, the dataSource will be saved / loaded under the key provided
@@ -25,15 +25,15 @@ type DataSourceOptions = BaseDataSourceOptions & {
 
 export function createDataSource<T, Key extends keyof T>(
   initialSet: readonly T[],
-  options: DataSourceOptions & BaseDataSourceOptionKey<Key>,
+  options: DataSourceOptions<T> & BaseDataSourceOptionKey<Key>,
 ): DataSource<T, T[Key] extends string | number ? T[Key] : never>;
 export function createDataSource<T>(
   initialSet?: readonly T[],
-  options?: DataSourceOptions,
+  options?: DataSourceOptions<T>,
 ): DataSource<T, never>;
 export function createDataSource<T, Key extends keyof T>(
   initialSet: readonly T[] = [],
-  options?: DataSourceOptions & BaseDataSourceOptionKey<Key>,
+  options?: DataSourceOptions<T> & BaseDataSourceOptionKey<Key>,
 ): DataSource<T, T[Key] extends string | number ? T[Key] : never> {
   const ds = options
     ? baseCreateDataSource(initialSet, options)

desktop/flipper-plugin/src/data-source/README.md

@@ -29,6 +29,7 @@ This library builds a map-reduce inspired data processing pipeline that stores d
 * Virtualization (windowing) is built in.
 * Dynamic row wrapping is supported when rendering tables.
 * Any stable JS function can be used for sorting and filtering, without impacting performance significantly.
+* Support for secondary indices to support close to O(1) lookups for known values.
 
 This library is designed with the following constraints in mind:
 

docs/extending/flipper-plugin.mdx

@@ -588,6 +588,7 @@ Valid `options` are:
 * `key` - if a key is set, the given field of the records is assumed to be unique, and its value can be used to perform lookups and upserts.
 * `limit` - the maximum number of records that this DataSource will store. If the limit is exceeded, the oldest records will automatically be dropped to make place for the new ones. Defaults to 100.000 records.
 * `persist` - see the `createState` `persist` option: If set, this data source will automatically be part of Flipper imports / exports; it's recommended to set this option.
+* `indices` - If set, secondary indices will be maintained for this table that allows fast lookups. Indices is an array of keys with 1 or more items. See `getAllRecordsByIndex` for more details.
 
 All records stored in a data source should be treated as being immutable. To update a record, replace it with a new value using the `update` or `upsert` operations.
 
@@ -686,6 +687,41 @@ Usage: `clear()`. Removes all records from this data source.
 
 Usage: `getAdditionalView(viewId: string)`. Gets an additional `DataSourceView` of the `DataSource` by passing in an identifier `viewId`. If there already exists a `DataSourceView` with the `viewId`, we simply return that view instead.
 
+#### getAllRecordsByIndex
+
+Usage: `getAllRecordsByIndex({ indexedAttribute: value, indexAttribute2: value2, .... })`
+
+This method allows fast lookups for objects that match specific attributes exactly. 
+Returns all items matching the specified index query.
+Note that the results are unordered, unless
+records have not been updated using upsert / update, in that case
+insertion order is maintained.
+If no index has been specified for this exact keyset in the indexQuery (see options.indices), this method will throw.
+
+Example:
+
+```javascript
+const ds = createDataSource([eatCookie, drinkCoffee, submitBug], {
+  key: 'id',
+  indices: [
+    ['title']
+    ['id', 'title'],
+    ['title', 'done'],
+  ],
+});
+
+// Find first element with title === cookie (or undefined)
+const todo = ds.getFirstRecordByIndex({
+    title: 'cookie',
+})
+
+// Find all elements where title === cookie, and done === false
+const todos = ds.getAllRecordsByIndex({
+  title: 'cookie',
+  done: false,
+})
+```
+
 #### DataSourceView
 
 A materialized view on a DataSource, which can apply windowing, sorting and filtering and will be kept incrementally up to date with the underlying datasource.