diff --git a/desktop/flipper-plugin-core/src/data-source/DataSource.tsx b/desktop/flipper-plugin-core/src/data-source/DataSource.tsx index bc722c079..9b5bb860c 100644 --- a/desktop/flipper-plugin-core/src/data-source/DataSource.tsx +++ b/desktop/flipper-plugin-core/src/data-source/DataSource.tsx @@ -97,27 +97,44 @@ export type DataSourceOptionKey = { */ key?: K; }; -export type DataSourceOptions = { +export type DataSourceOptions = { /** * The maximum amount of records that this DataSource will store. * If the limit is exceeded, the oldest records will automatically be dropped to make place for the new ones */ limit?: number; + /** + * Secondary indices, that can be used to perform O(1) lookups on specific keys later on. + * A combination of keys is allowed. + * + * For example: + * indices: [["title"], ["id", "title"]] + * + * Enables: + * dataSource.getAllRecordsByIndex({ + * id: 123, + * title: "Test" + * }) + */ + indices?: IndexDefinition[]; }; +type IndexDefinition = Array; +type IndexQuery = Partial; + export function createDataSource( initialSet: readonly T[], - options: DataSourceOptions & DataSourceOptionKey, + options: DataSourceOptions & DataSourceOptionKey, ): DataSource; export function createDataSource( initialSet?: readonly T[], - options?: DataSourceOptions, + options?: DataSourceOptions, ): DataSource; export function createDataSource( initialSet: readonly T[] = [], - options?: DataSourceOptions & DataSourceOptionKey, + options?: DataSourceOptions & DataSourceOptionKey, ): DataSource { - const ds = new DataSource(options?.key); + const ds = new DataSource(options?.key, options?.indices); if (options?.limit !== undefined) { ds.limit = options.limit; } @@ -129,6 +146,14 @@ export class DataSource { private nextId = 0; private _records: Entry[] = []; private _recordsById: Map = new Map(); + + // secondary indices are used to allow fast lookups for items that match certain columns exactly + private _secondaryIndices: Map< + string /* normalized string */, + IndexDefinition /* sorted keys */ + >; + private _recordsBySecondaryIndex: Map> = new Map(); + /** * @readonly */ @@ -155,8 +180,25 @@ export class DataSource { [viewId: string]: DataSourceView; }; - constructor(keyAttribute: keyof T | undefined) { + constructor( + keyAttribute: keyof T | undefined, + secondaryIndices: IndexDefinition[] = [], + ) { this.keyAttribute = keyAttribute; + this._secondaryIndices = new Map( + secondaryIndices.map((index) => { + const sortedKeys = index.slice().sort(); + const key = sortedKeys.join(':'); + // immediately reserve a map per index + this._recordsBySecondaryIndex.set(key, new Map()); + return [key, sortedKeys]; + }), + ); + if (this._secondaryIndices.size !== secondaryIndices.length) { + throw new Error( + `Duplicate index definition in ${JSON.stringify(secondaryIndices)}`, + ); + } this.view = new DataSourceView(this, DEFAULT_VIEW_ID); this.additionalViews = {}; } @@ -246,6 +288,7 @@ export class DataSource { this._recordsById.set(key, value); this.storeIndexOfKey(key, this._records.length); } + this.storeSecondaryIndices(value); const visibleMap: {[viewId: string]: boolean} = {[DEFAULT_VIEW_ID]: false}; const approxIndexMap: {[viewId: string]: number} = {[DEFAULT_VIEW_ID]: -1}; Object.keys(this.additionalViews).forEach((viewId) => { @@ -309,6 +352,8 @@ export class DataSource { this._recordsById.set(key, value); this.storeIndexOfKey(key, index); } + this.removeSecondaryIndices(oldValue); + this.storeSecondaryIndices(value); this.emitDataEvent({ type: 'update', entry, @@ -343,6 +388,7 @@ export class DataSource { }); } } + this.removeSecondaryIndices(entry.value); this.emitDataEvent({ type: 'remove', index, @@ -387,6 +433,7 @@ export class DataSource { this.idToIndex.delete(key); }); } + removed.forEach((entry) => this.removeSecondaryIndices(entry.value)); if ( this.view.isSorted && @@ -411,10 +458,13 @@ export class DataSource { * The clear operation removes any records stored, but will keep the current view preferences such as sorting and filtering */ public clear() { - this._records = []; - this._recordsById = new Map(); + this._records.splice(0); + this._recordsById.clear(); + for (const m of this._recordsBySecondaryIndex.values()) { + m.clear(); + } this.shiftOffset = 0; - this.idToIndex = new Map(); + this.idToIndex.clear(); this.rebuild(); } @@ -502,6 +552,91 @@ export class DataSource { }); } + private storeSecondaryIndices(value: T) { + for (const [indexKey, sortedIndex] of this._secondaryIndices.entries()) { + const indexValue = this.getSecondaryIndexValueFromRecord( + value, + sortedIndex, + ); + // maps are already set up in constructor + const m = this._recordsBySecondaryIndex.get(indexKey)!; + const a = m.get(indexValue); + if (!a) { + // not seen this index value yet + m.set(indexValue, [value]); + } else { + a.push(value); + } + } + } + + private removeSecondaryIndices(value: T) { + for (const [indexKey, sortedIndex] of this._secondaryIndices.entries()) { + const indexValue = this.getSecondaryIndexValueFromRecord( + value, + sortedIndex, + ); + // maps are already set up in constructor + const m = this._recordsBySecondaryIndex.get(indexKey)!; + // code belows assumes that we have an entry for this secondary + const a = m.get(indexValue)!; + a.splice(a.indexOf(value), 1); + } + } + + /** + * Returns all items matching the specified index query. + * + * Note that the results are unordered, unless + * records have not been updated using upsert / update, in that case + * insertion order is maintained. + * + * Example: + * `ds.getAllRecordsByIndex({title: 'subit a bug', done: false})` + * + * If no index has been specified for this exact keyset in the indexQuery (see options.indices), this method will throw + * + * @param indexQuery + * @returns + */ + public getAllRecordsByIndex(indexQuery: IndexQuery): readonly T[] { + // normalise indexKey, incl sorting + const sortedKeys: (keyof T)[] = Object.keys(indexQuery).sort() as any; + const indexKey = sortedKeys.join(':'); + const recordsByIndex = this._recordsBySecondaryIndex.get(indexKey); + if (!recordsByIndex) { + throw new Error( + `No index has been defined for the keys ${JSON.stringify( + Object.keys(indexQuery), + )}`, + ); + } + const indexValue = JSON.stringify( + // query object needs reordering and normalised to produce correct indexValue + Object.fromEntries(sortedKeys.map((k) => [k, String(indexQuery[k])])), + ); + return recordsByIndex.get(indexValue) ?? []; + } + + /** + * Like getAllRecords, but returns the first match only. + * @param indexQuery + * @returns + */ + public getFirstRecordByIndex(indexQuery: IndexQuery): T | undefined { + return this.getAllRecordsByIndex(indexQuery)[0]; + } + + private getSecondaryIndexValueFromRecord( + record: T, + // assumes keys is already ordered + keys: IndexDefinition, + ): any { + return JSON.stringify( + Object.fromEntries(keys.map((k) => [k, String(record[k])])), + ); + } + /** * @private */ diff --git a/desktop/flipper-plugin-core/src/data-source/__tests__/datasource-basics.node.tsx b/desktop/flipper-plugin-core/src/data-source/__tests__/datasource-basics.node.tsx index cb78e07c6..c9bedac46 100644 --- a/desktop/flipper-plugin-core/src/data-source/__tests__/datasource-basics.node.tsx +++ b/desktop/flipper-plugin-core/src/data-source/__tests__/datasource-basics.node.tsx @@ -856,3 +856,185 @@ test('DataSource.view can iterate', () => { ds.clear(); expect([...ds.view]).toEqual([]); }); + +test('secondary keys - doesnt allow duplicate keys', () => { + expect(() => + createDataSource([], { + indices: [['title'], ['title']], + }), + ).toThrowErrorMatchingInlineSnapshot( + `"Duplicate index definition in [["title"],["title"]]"`, + ); + + expect(() => + createDataSource([], { + // these are the same! + indices: [ + ['id', 'title'], + ['title', 'id'], + ], + }), + ).toThrowErrorMatchingInlineSnapshot( + `"Duplicate index definition in [["id","title"],["title","id"]]"`, + ); +}); + +test('secondary keys - doesnt allow lookup with nonexisting key', () => { + const ds = createDataSource([], {indices: [['done']]}); + expect(() => + expect(ds.getAllRecordsByIndex({title: 'subit a bug', done: false})), + ).toThrowErrorMatchingInlineSnapshot( + `"No index has been defined for the keys ["title","done"]"`, + ); +}); + +test('secondary keys - lookup by single key', () => { + const ds = createDataSource([eatCookie, drinkCoffee, submitBug], { + indices: [['id'], ['title'], ['done']], + }); + + expect( + ds.getAllRecordsByIndex({ + title: 'eat a cookie', + }), + ).toEqual([eatCookie]); + + const cookie2 = {...eatCookie, done: false}; + ds.append(cookie2); + expect( + ds.getAllRecordsByIndex({ + title: 'eat a cookie', + }), + ).toEqual([eatCookie, cookie2]); + + expect( + ds.getAllRecordsByIndex({ + done: false, + }), + ).toEqual([submitBug, cookie2]); + + expect( + ds.getFirstRecordByIndex({ + done: false, + }), + ).toEqual(submitBug); + + ds.delete(0); // eat Cookie + expect( + ds.getAllRecordsByIndex({ + title: 'eat a cookie', + }), + ).toEqual([cookie2]); + + // replace submit Bug + const n = { + id: 'bug', + title: 'eat a cookie', + done: false, + }; + ds.update(1, n); + + expect( + ds.getAllRecordsByIndex({ + title: 'eat a cookie', + }), + ).toEqual([cookie2, n]); + + expect( + ds.getFirstRecordByIndex({ + title: 'submit a bug', + }), + ).toBeUndefined(); + + // removes drinkCoffe, n + ds.shift(2); + expect( + ds.getAllRecordsByIndex({ + title: 'eat a cookie', + }), + ).toEqual([cookie2]); +}); + +test('secondary keys - lookup by combined keys', () => { + const ds = createDataSource([eatCookie, drinkCoffee, submitBug], { + key: 'id', + indices: [ + ['id', 'title'], + ['title', 'done'], + ], + }); + + expect( + ds.getAllRecordsByIndex({ + id: 'cookie', + title: 'eat a cookie', + }), + ).toEqual([eatCookie]); + expect( + ds.getAllRecordsByIndex({ + // order doesn't matter + title: 'eat a cookie', + id: 'cookie', + }), + ).toEqual([eatCookie]); + + // Note: different key order + const cookie2 = {id: 'cookie2', done: true, title: 'eat a cookie'}; + ds.append(cookie2); + expect( + ds.getAllRecordsByIndex({ + id: 'cookie2', + title: 'eat a cookie', + }), + ).toEqual([cookie2]); + + expect( + ds.getAllRecordsByIndex({ + done: true, + title: 'eat a cookie', + }), + ).toEqual([eatCookie, cookie2]); + + const upsertedCookie = { + id: 'cookie', + title: 'eat a cookie', + done: false, + }; + ds.upsert(upsertedCookie); + expect( + ds.getAllRecordsByIndex({ + done: true, + title: 'eat a cookie', + }), + ).toEqual([cookie2]); + expect( + ds.getAllRecordsByIndex({ + done: false, + title: 'eat a cookie', + }), + ).toEqual([upsertedCookie]); + + ds.deleteByKey('cookie'); // eat Cookie + expect( + ds.getFirstRecordByIndex({ + title: 'eat a cookie', + done: false, + }), + ).toEqual(undefined); + + ds.clear(); + expect( + ds.getAllRecordsByIndex({ + done: true, + title: 'eat a cookie', + }), + ).toEqual([]); + + ds.append(cookie2); + expect( + ds.getAllRecordsByIndex({ + id: 'cookie2', + title: 'eat a cookie', + }), + ).toEqual([cookie2]); +}); diff --git a/desktop/flipper-plugin-core/src/state/createDataSource.tsx b/desktop/flipper-plugin-core/src/state/createDataSource.tsx index 10f7b2dca..f5d5964df 100644 --- a/desktop/flipper-plugin-core/src/state/createDataSource.tsx +++ b/desktop/flipper-plugin-core/src/state/createDataSource.tsx @@ -15,7 +15,7 @@ import { } from '../data-source/DataSource'; import {registerStorageAtom} from '../plugin/PluginBase'; -type DataSourceOptions = BaseDataSourceOptions & { +type DataSourceOptions = BaseDataSourceOptions & { /** * Should this state persist when exporting a plugin? * If set, the dataSource will be saved / loaded under the key provided @@ -25,15 +25,15 @@ type DataSourceOptions = BaseDataSourceOptions & { export function createDataSource( initialSet: readonly T[], - options: DataSourceOptions & BaseDataSourceOptionKey, + options: DataSourceOptions & BaseDataSourceOptionKey, ): DataSource; export function createDataSource( initialSet?: readonly T[], - options?: DataSourceOptions, + options?: DataSourceOptions, ): DataSource; export function createDataSource( initialSet: readonly T[] = [], - options?: DataSourceOptions & BaseDataSourceOptionKey, + options?: DataSourceOptions & BaseDataSourceOptionKey, ): DataSource { const ds = options ? baseCreateDataSource(initialSet, options) diff --git a/desktop/flipper-plugin/src/data-source/README.md b/desktop/flipper-plugin/src/data-source/README.md index cf39149ac..fde8f1b39 100644 --- a/desktop/flipper-plugin/src/data-source/README.md +++ b/desktop/flipper-plugin/src/data-source/README.md @@ -29,6 +29,7 @@ This library builds a map-reduce inspired data processing pipeline that stores d * Virtualization (windowing) is built in. * Dynamic row wrapping is supported when rendering tables. * Any stable JS function can be used for sorting and filtering, without impacting performance significantly. +* Support for secondary indices to support close to O(1) lookups for known values. This library is designed with the following constraints in mind: diff --git a/docs/extending/flipper-plugin.mdx b/docs/extending/flipper-plugin.mdx index b183f996d..69e651a09 100644 --- a/docs/extending/flipper-plugin.mdx +++ b/docs/extending/flipper-plugin.mdx @@ -588,6 +588,7 @@ Valid `options` are: * `key` - if a key is set, the given field of the records is assumed to be unique, and its value can be used to perform lookups and upserts. * `limit` - the maximum number of records that this DataSource will store. If the limit is exceeded, the oldest records will automatically be dropped to make place for the new ones. Defaults to 100.000 records. * `persist` - see the `createState` `persist` option: If set, this data source will automatically be part of Flipper imports / exports; it's recommended to set this option. +* `indices` - If set, secondary indices will be maintained for this table that allows fast lookups. Indices is an array of keys with 1 or more items. See `getAllRecordsByIndex` for more details. All records stored in a data source should be treated as being immutable. To update a record, replace it with a new value using the `update` or `upsert` operations. @@ -686,6 +687,41 @@ Usage: `clear()`. Removes all records from this data source. Usage: `getAdditionalView(viewId: string)`. Gets an additional `DataSourceView` of the `DataSource` by passing in an identifier `viewId`. If there already exists a `DataSourceView` with the `viewId`, we simply return that view instead. +#### getAllRecordsByIndex + +Usage: `getAllRecordsByIndex({ indexedAttribute: value, indexAttribute2: value2, .... })` + +This method allows fast lookups for objects that match specific attributes exactly. +Returns all items matching the specified index query. +Note that the results are unordered, unless +records have not been updated using upsert / update, in that case +insertion order is maintained. +If no index has been specified for this exact keyset in the indexQuery (see options.indices), this method will throw. + +Example: + +```javascript +const ds = createDataSource([eatCookie, drinkCoffee, submitBug], { + key: 'id', + indices: [ + ['title'] + ['id', 'title'], + ['title', 'done'], + ], +}); + +// Find first element with title === cookie (or undefined) +const todo = ds.getFirstRecordByIndex({ + title: 'cookie', +}) + +// Find all elements where title === cookie, and done === false +const todos = ds.getAllRecordsByIndex({ + title: 'cookie', + done: false, +}) +``` + #### DataSourceView A materialized view on a DataSource, which can apply windowing, sorting and filtering and will be kept incrementally up to date with the underlying datasource.