da5856138d
Reviewed By: LukeDefeo Differential Revision: D51026560 fbshipit-source-id: 8348c6765633d7eecf0d1c80bc5bbd5af8130298
1119 lines
33 KiB
TypeScript
1119 lines
33 KiB
TypeScript
/**
|
|
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
*
|
|
* This source code is licensed under the MIT license found in the
|
|
* LICENSE file in the root directory of this source tree.
|
|
*
|
|
* @format
|
|
*/
|
|
|
|
import sortedIndexBy from 'lodash/sortedIndexBy';
|
|
import sortedLastIndexBy from 'lodash/sortedLastIndexBy';
|
|
import property from 'lodash/property';
|
|
import lodashSort from 'lodash/sortBy';
|
|
import EventEmitter from 'eventemitter3';
|
|
|
|
// If the dataSource becomes to large, after how many records will we start to drop items?
|
|
const dropFactor = 0.1;
|
|
// what is the default maximum amount of records before we start shifting the data set?
|
|
const defaultLimit = 100 * 1000;
|
|
// if a shift on a sorted dataset exceeds this tresholds, we assume it is faster to re-sort the entire set,
|
|
// rather than search and remove the affected individual items
|
|
const shiftRebuildTreshold = 0.05;
|
|
|
|
const DEFAULT_VIEW_ID = '0';
|
|
|
|
type AppendEvent<T> = {
|
|
type: 'append';
|
|
entry: Entry<T>;
|
|
};
|
|
type UpdateEvent<T> = {
|
|
type: 'update';
|
|
entry: Entry<T>;
|
|
oldValue: T;
|
|
oldVisible: {
|
|
[viewId: string]: boolean;
|
|
};
|
|
index: number;
|
|
};
|
|
type RemoveEvent<T> = {
|
|
type: 'remove';
|
|
entry: Entry<T>;
|
|
index: number;
|
|
};
|
|
type ShiftEvent<T> = {
|
|
type: 'shift';
|
|
entries: Entry<T>[];
|
|
amount: number;
|
|
};
|
|
|
|
type DataEvent<T> =
|
|
| AppendEvent<T>
|
|
| UpdateEvent<T>
|
|
| RemoveEvent<T>
|
|
| ShiftEvent<T>;
|
|
|
|
type Entry<T> = {
|
|
value: T;
|
|
id: number; // insertion based
|
|
visible: {
|
|
[viewId: string]: boolean;
|
|
}; // matches current filter?
|
|
approxIndex: {
|
|
[viewId: string]: number;
|
|
}; // we could possible live at this index in the output. No guarantees.
|
|
};
|
|
|
|
type Primitive = number | string | boolean | null | undefined;
|
|
|
|
type OutputChange =
|
|
| {
|
|
type: 'shift';
|
|
index: number;
|
|
location: 'before' | 'in' | 'after'; // relative to current window
|
|
delta: number;
|
|
newCount: number;
|
|
}
|
|
| {
|
|
// an item, inside the current window, was changed
|
|
type: 'update';
|
|
index: number;
|
|
}
|
|
| {
|
|
// something big and awesome happened. Drop earlier updates to the floor and start again
|
|
// like: clear, filter or sorting change, etc
|
|
type: 'reset';
|
|
newCount: number;
|
|
}
|
|
| {
|
|
type: 'windowChange';
|
|
newStart: number;
|
|
newEnd: number;
|
|
};
|
|
|
|
export type DataSourceOptionKey<K extends PropertyKey> = {
|
|
/**
|
|
* If a key is set, the given field of the records is assumed to be unique,
|
|
* and it's value can be used to perform lookups and upserts.
|
|
*/
|
|
key?: K;
|
|
};
|
|
export type DataSourceOptions<T> = {
|
|
/**
|
|
* The maximum amount of records that this DataSource will store.
|
|
* If the limit is exceeded, the oldest records will automatically be dropped to make place for the new ones
|
|
*/
|
|
limit?: number;
|
|
/**
|
|
* Secondary indices, that can be used to perform O(1) lookups on specific keys later on.
|
|
* A combination of keys is allowed.
|
|
*
|
|
* For example:
|
|
* indices: [["title"], ["id", "title"]]
|
|
*
|
|
* Enables:
|
|
* dataSource.getAllRecordsByIndex({
|
|
* id: 123,
|
|
* title: "Test"
|
|
* })
|
|
*/
|
|
indices?: IndexDefinition<T>[];
|
|
};
|
|
|
|
type IndexDefinition<T> = Array<keyof T>;
|
|
type IndexQuery<T> = Partial<T>;
|
|
|
|
export function createDataSource<T, Key extends keyof T>(
|
|
initialSet: readonly T[],
|
|
options: DataSourceOptions<T> & DataSourceOptionKey<Key>,
|
|
): DataSource<T, T[Key] extends string | number ? T[Key] : never>;
|
|
export function createDataSource<T>(
|
|
initialSet?: readonly T[],
|
|
options?: DataSourceOptions<T>,
|
|
): DataSource<T, never>;
|
|
export function createDataSource<T, Key extends keyof T>(
|
|
initialSet: readonly T[] = [],
|
|
options?: DataSourceOptions<T> & DataSourceOptionKey<Key>,
|
|
): DataSource<T, Key> {
|
|
const ds = new DataSource<T, Key>(options?.key, options?.indices);
|
|
if (options?.limit !== undefined) {
|
|
ds.limit = options.limit;
|
|
}
|
|
initialSet.forEach((value) => ds.append(value));
|
|
return ds as any;
|
|
}
|
|
|
|
export class DataSource<T extends any, KeyType = never> {
|
|
private nextId = 0;
|
|
private _records: Entry<T>[] = [];
|
|
private _recordsById: Map<KeyType, T> = new Map();
|
|
|
|
// secondary indices are used to allow fast lookups for items that match certain columns exactly
|
|
private _secondaryIndices: Map<
|
|
string /* normalized string */,
|
|
IndexDefinition<T> /* sorted keys */
|
|
>;
|
|
private _recordsBySecondaryIndex: Map<string, Map<string, T[]>> = new Map();
|
|
|
|
/**
|
|
* @readonly
|
|
*/
|
|
public keyAttribute: keyof T | undefined;
|
|
private idToIndex: Map<KeyType, number> = new Map();
|
|
|
|
// if we shift the window, we increase shiftOffset to correct idToIndex results, rather than remapping all values
|
|
private shiftOffset = 0;
|
|
|
|
/**
|
|
* The maximum amount of records this DataSource can have
|
|
*/
|
|
public limit = defaultLimit;
|
|
|
|
/**
|
|
* The default view on this data source. A view applies
|
|
* sorting, filtering and windowing to get more constrained output.
|
|
*
|
|
* Additional views can created through the fork method.
|
|
*/
|
|
public readonly view: DataSourceView<T, KeyType>;
|
|
|
|
public readonly additionalViews: {
|
|
[viewId: string]: DataSourceView<T, KeyType>;
|
|
};
|
|
|
|
public readonly outputEventEmitter = new EventEmitter();
|
|
|
|
constructor(
|
|
keyAttribute: keyof T | undefined,
|
|
secondaryIndices: IndexDefinition<T>[] = [],
|
|
) {
|
|
this.keyAttribute = keyAttribute;
|
|
this._secondaryIndices = new Map(
|
|
secondaryIndices.map((index) => {
|
|
const sortedKeys = index.slice().sort();
|
|
const key = sortedKeys.join(':');
|
|
// immediately reserve a map per index
|
|
this._recordsBySecondaryIndex.set(key, new Map());
|
|
return [key, sortedKeys];
|
|
}),
|
|
);
|
|
if (this._secondaryIndices.size !== secondaryIndices.length) {
|
|
throw new Error(
|
|
`Duplicate index definition in ${JSON.stringify(secondaryIndices)}`,
|
|
);
|
|
}
|
|
this.view = new DataSourceView<T, KeyType>(this, DEFAULT_VIEW_ID);
|
|
this.additionalViews = {};
|
|
}
|
|
|
|
public get size() {
|
|
return this._records.length;
|
|
}
|
|
|
|
/**
|
|
* Returns a defensive copy of the stored records.
|
|
* This is a O(n) operation! Prefer using .size and .get instead if only a subset is needed.
|
|
*/
|
|
public records(): readonly T[] {
|
|
return this._records.map(unwrap);
|
|
}
|
|
|
|
public get(index: number) {
|
|
return unwrap(this._records[index]);
|
|
}
|
|
|
|
public has(key: KeyType) {
|
|
this.assertKeySet();
|
|
return this._recordsById.has(key);
|
|
}
|
|
|
|
public getById(key: KeyType): T | undefined {
|
|
this.assertKeySet();
|
|
return this._recordsById.get(key);
|
|
}
|
|
|
|
public keys(): IterableIterator<KeyType> {
|
|
this.assertKeySet();
|
|
return this._recordsById.keys();
|
|
}
|
|
|
|
public entries(): IterableIterator<[KeyType, T]> {
|
|
this.assertKeySet();
|
|
return this._recordsById.entries();
|
|
}
|
|
|
|
public [Symbol.iterator](): IterableIterator<T> {
|
|
const self = this;
|
|
let offset = 0;
|
|
return {
|
|
next() {
|
|
offset++;
|
|
if (offset > self.size) {
|
|
return {done: true, value: undefined};
|
|
} else {
|
|
return {
|
|
value: self._records[offset - 1].value,
|
|
};
|
|
}
|
|
},
|
|
[Symbol.iterator]() {
|
|
return this;
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Returns the index of a specific key in the *records* set.
|
|
* Returns -1 if the record wansn't found
|
|
*/
|
|
public getIndexOfKey(key: KeyType): number {
|
|
this.assertKeySet();
|
|
const stored = this.idToIndex.get(key);
|
|
return stored === undefined ? -1 : stored + this.shiftOffset;
|
|
}
|
|
|
|
public append(value: T) {
|
|
if (this._records.length >= this.limit) {
|
|
// we're full! let's free up some space
|
|
this.shift(Math.ceil(this.limit * dropFactor));
|
|
}
|
|
if (this.keyAttribute) {
|
|
const key = this.getKey(value);
|
|
if (this._recordsById.has(key)) {
|
|
const existingValue = this._recordsById.get(key);
|
|
console.warn(
|
|
`Tried to append value with duplicate key: ${key} (key attribute is ${this.keyAttribute.toString()}). Old/new values:`,
|
|
existingValue,
|
|
value,
|
|
);
|
|
throw new Error(`Duplicate key`);
|
|
}
|
|
this._recordsById.set(key, value);
|
|
this.storeIndexOfKey(key, this._records.length);
|
|
}
|
|
this.storeSecondaryIndices(value);
|
|
const visibleMap: {[viewId: string]: boolean} = {[DEFAULT_VIEW_ID]: false};
|
|
const approxIndexMap: {[viewId: string]: number} = {[DEFAULT_VIEW_ID]: -1};
|
|
Object.keys(this.additionalViews).forEach((viewId) => {
|
|
visibleMap[viewId] = false;
|
|
approxIndexMap[viewId] = -1;
|
|
});
|
|
const entry = {
|
|
value,
|
|
id: ++this.nextId,
|
|
visible: visibleMap,
|
|
approxIndex: approxIndexMap,
|
|
};
|
|
this._records.push(entry);
|
|
this.emitDataEvent({
|
|
type: 'append',
|
|
entry,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Updates or adds a record. Returns `true` if the record already existed.
|
|
* Can only be used if a key is used.
|
|
*/
|
|
public upsert(value: T): boolean {
|
|
this.assertKeySet();
|
|
const key = this.getKey(value);
|
|
if (this.idToIndex.has(key)) {
|
|
this.update(this.getIndexOfKey(key), value);
|
|
return true;
|
|
} else {
|
|
this.append(value);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Replaces an item in the base data collection.
|
|
* Note that the index is based on the insertion order, and not based on the current view
|
|
*/
|
|
public update(index: number, value: T) {
|
|
const entry = this._records[index];
|
|
const oldValue = entry.value;
|
|
if (value === oldValue) {
|
|
return;
|
|
}
|
|
const oldVisible = {...entry.visible};
|
|
entry.value = value;
|
|
if (this.keyAttribute) {
|
|
const key = this.getKey(value);
|
|
const currentKey = this.getKey(oldValue);
|
|
if (currentKey !== key) {
|
|
const existingIndex = this.getIndexOfKey(key);
|
|
if (existingIndex !== -1 && existingIndex !== index) {
|
|
throw new Error(
|
|
`Trying to insert duplicate key '${key}', which already exist in the collection`,
|
|
);
|
|
}
|
|
this._recordsById.delete(currentKey);
|
|
this.idToIndex.delete(currentKey);
|
|
}
|
|
this._recordsById.set(key, value);
|
|
this.storeIndexOfKey(key, index);
|
|
}
|
|
this.removeSecondaryIndices(oldValue);
|
|
this.storeSecondaryIndices(value);
|
|
this.emitDataEvent({
|
|
type: 'update',
|
|
entry,
|
|
oldValue,
|
|
oldVisible,
|
|
index,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* @param index
|
|
*
|
|
* Warning: this operation can be O(n) if a key is set
|
|
*/
|
|
public delete(index: number) {
|
|
if (index < 0 || index >= this._records.length) {
|
|
throw new Error('Out of bounds: ' + index);
|
|
}
|
|
const entry = this._records.splice(index, 1)[0];
|
|
if (this.keyAttribute) {
|
|
const key = this.getKey(entry.value);
|
|
this._recordsById.delete(key);
|
|
this.idToIndex.delete(key);
|
|
if (index === 0) {
|
|
// lucky happy case, this is more efficient
|
|
this.shiftOffset -= 1;
|
|
} else {
|
|
// Optimization: this is O(n)! Should be done as an async job
|
|
this.idToIndex.forEach((keyIndex, key) => {
|
|
if (keyIndex + this.shiftOffset > index)
|
|
this.storeIndexOfKey(key, keyIndex - 1);
|
|
});
|
|
}
|
|
}
|
|
this.removeSecondaryIndices(entry.value);
|
|
this.emitDataEvent({
|
|
type: 'remove',
|
|
index,
|
|
entry,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Removes the item with the given key from this dataSource.
|
|
* Returns false if no record with the given key was found
|
|
*
|
|
* Warning: this operation can be O(n) if a key is set
|
|
*/
|
|
public deleteByKey(keyValue: KeyType): boolean {
|
|
this.assertKeySet();
|
|
const index = this.getIndexOfKey(keyValue);
|
|
if (index === -1) {
|
|
return false;
|
|
}
|
|
this.delete(index);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Removes the first N entries.
|
|
* @param amount
|
|
*/
|
|
public shift(amount: number) {
|
|
amount = Math.min(amount, this._records.length);
|
|
if (amount === this._records.length) {
|
|
this.clear();
|
|
return;
|
|
}
|
|
// increase an offset variable with amount, and correct idToIndex reads / writes with that
|
|
this.shiftOffset -= amount;
|
|
// removes the affected records for _records, _recordsById and idToIndex
|
|
const removed = this._records.splice(0, amount);
|
|
if (this.keyAttribute) {
|
|
removed.forEach((entry) => {
|
|
const key = this.getKey(entry.value);
|
|
this._recordsById.delete(key);
|
|
this.idToIndex.delete(key);
|
|
});
|
|
}
|
|
removed.forEach((entry) => this.removeSecondaryIndices(entry.value));
|
|
|
|
if (
|
|
this.view.isSorted &&
|
|
removed.length > 10 &&
|
|
removed.length > shiftRebuildTreshold * this._records.length
|
|
) {
|
|
// removing a large amount of items is expensive when doing it sorted,
|
|
// let's fallback to the async processing of all data instead
|
|
// MWE: there is a risk here that rebuilding is too blocking, as this might happen
|
|
// in background when new data arrives, and not explicitly on a user interaction
|
|
this.rebuild();
|
|
} else {
|
|
this.emitDataEvent({
|
|
type: 'shift',
|
|
entries: removed,
|
|
amount,
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The clear operation removes any records stored, but will keep the current view preferences such as sorting and filtering
|
|
*/
|
|
public clear() {
|
|
this._records.splice(0);
|
|
this._recordsById.clear();
|
|
for (const m of this._recordsBySecondaryIndex.values()) {
|
|
m.clear();
|
|
}
|
|
this.shiftOffset = 0;
|
|
this.idToIndex.clear();
|
|
this.rebuild();
|
|
}
|
|
|
|
/**
|
|
* The rebuild function that would support rebuilding multiple views all at once
|
|
*/
|
|
public rebuild() {
|
|
this.view.rebuild();
|
|
Object.entries(this.additionalViews).forEach(([, dataView]) => {
|
|
dataView.rebuild();
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Returns a fork of this dataSource, that shares the source data with this dataSource,
|
|
* but has it's own FSRW pipeline, to allow multiple views on the same data
|
|
*/
|
|
private fork(viewId: string): DataSourceView<T, KeyType> {
|
|
this._records.forEach((entry) => {
|
|
entry.visible[viewId] = entry.visible[DEFAULT_VIEW_ID];
|
|
entry.approxIndex[viewId] = entry.approxIndex[DEFAULT_VIEW_ID];
|
|
});
|
|
const newView = new DataSourceView<T, KeyType>(this, viewId);
|
|
// Refresh the new view so that it has all the existing records.
|
|
newView.rebuild();
|
|
return newView;
|
|
}
|
|
|
|
/**
|
|
* Returns a new view of the `DataSource` if there doesn't exist a `DataSourceView` with the `viewId` passed in.
|
|
* The view will allow different filters and sortings on the `DataSource` which can be helpful in cases
|
|
* where multiple tables/views are needed.
|
|
* @param viewId id for the `DataSourceView`
|
|
* @returns `DataSourceView` that corresponds to the `viewId`
|
|
*/
|
|
public getAdditionalView(viewId: string): DataSourceView<T, KeyType> {
|
|
if (viewId in this.additionalViews) {
|
|
return this.additionalViews[viewId];
|
|
}
|
|
this.additionalViews[viewId] = this.fork(viewId);
|
|
return this.additionalViews[viewId];
|
|
}
|
|
|
|
public deleteView(viewId: string): void {
|
|
if (viewId in this.additionalViews) {
|
|
delete this.additionalViews[viewId];
|
|
// TODO: Ideally remove the viewId in the visible and approxIndex of DataView outputs
|
|
this._records.forEach((entry) => {
|
|
delete entry.visible[viewId];
|
|
delete entry.approxIndex[viewId];
|
|
});
|
|
}
|
|
}
|
|
|
|
private assertKeySet() {
|
|
if (!this.keyAttribute) {
|
|
throw new Error(
|
|
'No key has been set. Records cannot be looked up by key',
|
|
);
|
|
}
|
|
}
|
|
|
|
private getKey(value: T): KeyType;
|
|
private getKey(value: any): any {
|
|
this.assertKeySet();
|
|
const key = value[this.keyAttribute!];
|
|
if ((typeof key === 'string' || typeof key === 'number') && key !== '') {
|
|
return key;
|
|
}
|
|
throw new Error(`Invalid key value: '${key}'`);
|
|
}
|
|
|
|
private storeIndexOfKey(key: KeyType, index: number) {
|
|
// de-normalize the index, so that on later look ups its corrected again
|
|
this.idToIndex.set(key, index - this.shiftOffset);
|
|
}
|
|
|
|
private emitDataEvent(event: DataEvent<T>) {
|
|
// Optimization: potentially we could schedule this to happen async,
|
|
// using a queue,
|
|
// or only if there is an active view (although that could leak memory)
|
|
this.view.processEvent(event);
|
|
Object.entries(this.additionalViews).forEach(([, dataView]) => {
|
|
dataView.processEvent(event);
|
|
});
|
|
this.outputEventEmitter.emit(event.type, event);
|
|
}
|
|
|
|
private storeSecondaryIndices(value: T) {
|
|
for (const [indexKey, sortedIndex] of this._secondaryIndices.entries()) {
|
|
const indexValue = this.getSecondaryIndexValueFromRecord(
|
|
value,
|
|
sortedIndex,
|
|
);
|
|
// maps are already set up in constructor
|
|
const m = this._recordsBySecondaryIndex.get(indexKey)!;
|
|
const a = m.get(indexValue);
|
|
if (!a) {
|
|
// not seen this index value yet
|
|
m.set(indexValue, [value]);
|
|
} else {
|
|
a.push(value);
|
|
}
|
|
}
|
|
}
|
|
|
|
private removeSecondaryIndices(value: T) {
|
|
for (const [indexKey, sortedIndex] of this._secondaryIndices.entries()) {
|
|
const indexValue = this.getSecondaryIndexValueFromRecord(
|
|
value,
|
|
sortedIndex,
|
|
);
|
|
// maps are already set up in constructor
|
|
const m = this._recordsBySecondaryIndex.get(indexKey)!;
|
|
// code belows assumes that we have an entry for this secondary
|
|
const a = m.get(indexValue)!;
|
|
a.splice(a.indexOf(value), 1);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Returns all items matching the specified index query.
|
|
*
|
|
* Note that the results are unordered, unless
|
|
* records have not been updated using upsert / update, in that case
|
|
* insertion order is maintained.
|
|
*
|
|
* Example:
|
|
* `ds.getAllRecordsByIndex({title: 'subit a bug', done: false})`
|
|
*
|
|
* If no index has been specified for this exact keyset in the indexQuery (see options.indices), this method will throw
|
|
*
|
|
* @param indexQuery
|
|
* @returns
|
|
*/
|
|
public getAllRecordsByIndex(indexQuery: IndexQuery<T>): readonly T[] {
|
|
// normalise indexKey, incl sorting
|
|
const sortedKeys: (keyof T)[] = Object.keys(indexQuery).sort() as any;
|
|
const indexKey = sortedKeys.join(':');
|
|
const recordsByIndex = this._recordsBySecondaryIndex.get(indexKey);
|
|
if (!recordsByIndex) {
|
|
throw new Error(
|
|
`No index has been defined for the keys ${JSON.stringify(
|
|
Object.keys(indexQuery),
|
|
)}`,
|
|
);
|
|
}
|
|
const indexValue = JSON.stringify(
|
|
// query object needs reordering and normalised to produce correct indexValue
|
|
Object.fromEntries(sortedKeys.map((k) => [k, String(indexQuery[k])])),
|
|
);
|
|
return recordsByIndex.get(indexValue) ?? [];
|
|
}
|
|
|
|
/**
|
|
* Like getAllRecords, but returns the first match only.
|
|
* @param indexQuery
|
|
* @returns
|
|
*/
|
|
public getFirstRecordByIndex(indexQuery: IndexQuery<T>): T | undefined {
|
|
return this.getAllRecordsByIndex(indexQuery)[0];
|
|
}
|
|
|
|
private getSecondaryIndexValueFromRecord(
|
|
record: T,
|
|
// assumes keys is already ordered
|
|
keys: IndexDefinition<T>,
|
|
): any {
|
|
return JSON.stringify(
|
|
Object.fromEntries(keys.map((k) => [k, String(record[k])])),
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @private
|
|
*/
|
|
serialize(): readonly T[] {
|
|
return this.records();
|
|
}
|
|
|
|
/**
|
|
* @private
|
|
*/
|
|
deserialize(value: any[]) {
|
|
this.clear();
|
|
value.forEach((record) => {
|
|
this.append(record);
|
|
});
|
|
}
|
|
}
|
|
|
|
function unwrap<T>(entry: Entry<T>): T {
|
|
return entry?.value;
|
|
}
|
|
|
|
export class DataSourceView<T, KeyType> {
|
|
public readonly datasource: DataSource<T, KeyType>;
|
|
private sortBy: undefined | ((a: T) => Primitive) = undefined;
|
|
private reverse: boolean = false;
|
|
private filter?: (value: T) => boolean = undefined;
|
|
private filterExceptions?: Set<KeyType> = undefined;
|
|
|
|
/**
|
|
* @readonly
|
|
*/
|
|
public windowStart = 0;
|
|
/**
|
|
* @readonly
|
|
*/
|
|
public windowEnd = 0;
|
|
private viewId;
|
|
|
|
private outputChangeListeners = new Set<(change: OutputChange) => void>();
|
|
|
|
/**
|
|
* This is the base view data, that is filtered and sorted, but not reversed or windowed
|
|
*/
|
|
private _output: Entry<T>[] = [];
|
|
|
|
constructor(datasource: DataSource<T, KeyType>, viewId: string) {
|
|
this.datasource = datasource;
|
|
this.viewId = viewId;
|
|
}
|
|
|
|
public get size() {
|
|
return this._output.length;
|
|
}
|
|
|
|
public get isSorted() {
|
|
return !!this.sortBy;
|
|
}
|
|
|
|
public get isFiltered() {
|
|
return !!this.filter;
|
|
}
|
|
|
|
public get isReversed() {
|
|
return this.reverse;
|
|
}
|
|
|
|
/**
|
|
* Returns a defensive copy of the current output.
|
|
* Sort, filter, reverse and are applied.
|
|
* Start and end behave like slice, and default to the currently active window.
|
|
*/
|
|
public output(start = this.windowStart, end = this.windowEnd): readonly T[] {
|
|
if (this.reverse) {
|
|
return this._output
|
|
.slice(this._output.length - end, this._output.length - start)
|
|
.reverse()
|
|
.map((e) => e.value);
|
|
} else {
|
|
return this._output.slice(start, end).map((e) => e.value);
|
|
}
|
|
}
|
|
|
|
getViewIndex(entry: T): number {
|
|
return this._output.findIndex((x) => x.value === entry);
|
|
}
|
|
|
|
public setWindow(start: number, end: number) {
|
|
this.windowStart = start;
|
|
this.windowEnd = end;
|
|
this.notifyAllListeners({
|
|
type: 'windowChange',
|
|
newStart: start,
|
|
newEnd: end,
|
|
});
|
|
}
|
|
|
|
public addListener(listener: (change: OutputChange) => void) {
|
|
this.outputChangeListeners.add(listener);
|
|
return () => {
|
|
this.outputChangeListeners.delete(listener);
|
|
};
|
|
}
|
|
|
|
public setSortBy(sortBy: undefined | keyof T | ((a: T) => Primitive)) {
|
|
if (this.sortBy === sortBy) {
|
|
return;
|
|
}
|
|
if (
|
|
typeof sortBy === 'string' &&
|
|
(!this.sortBy || (this.sortBy as any).sortByKey !== sortBy)
|
|
) {
|
|
sortBy = property(sortBy);
|
|
Object.assign(sortBy, {
|
|
sortByKey: sortBy,
|
|
});
|
|
}
|
|
this.sortBy = sortBy as any;
|
|
this.rebuild();
|
|
}
|
|
|
|
public setFilter(filter: undefined | ((value: T) => boolean)) {
|
|
if (this.filter !== filter) {
|
|
this.filter = filter;
|
|
// Filter exceptions are relevant for one filter only
|
|
this.filterExceptions = undefined;
|
|
this.rebuild();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Granular control over filters to add one-off exceptions to them.
|
|
* They allow us to add singular items to table views.
|
|
* Extremely useful for Bloks Debugger where we have to jump between multiple types of rows that could be filtered out
|
|
*/
|
|
public setFilterExpections(ids: KeyType[] | undefined) {
|
|
this.filterExceptions = ids ? new Set(ids) : undefined;
|
|
this.rebuild();
|
|
}
|
|
|
|
public toggleReversed() {
|
|
this.setReversed(!this.reverse);
|
|
}
|
|
|
|
public setReversed(reverse: boolean) {
|
|
if (this.reverse !== reverse) {
|
|
this.reverse = reverse;
|
|
this.notifyReset(this._output.length);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The reset operation resets any view preferences such as sorting and filtering, but keeps the current set of records.
|
|
*/
|
|
reset() {
|
|
this.sortBy = undefined;
|
|
this.reverse = false;
|
|
this.filter = undefined;
|
|
this.filterExceptions = undefined;
|
|
this.windowStart = 0;
|
|
this.windowEnd = 0;
|
|
this.rebuild();
|
|
}
|
|
|
|
private normalizeIndex(viewIndex: number): number {
|
|
return this.reverse ? this._output.length - 1 - viewIndex : viewIndex;
|
|
}
|
|
|
|
public get(viewIndex: number): T {
|
|
return this._output[this.normalizeIndex(viewIndex)]?.value;
|
|
}
|
|
|
|
public getEntry(viewIndex: number): Entry<T> {
|
|
return this._output[this.normalizeIndex(viewIndex)];
|
|
}
|
|
|
|
public getViewIndexOfEntry(entry: Entry<T>) {
|
|
// Note: this function leverages the fact that entry is an internal structure that is mutable,
|
|
// so any changes in the entry being moved around etc will be reflected in the original `entry` object,
|
|
// and we just want to verify that this entry is indeed still the same element, visible, and still present in
|
|
// the output data set.
|
|
if (
|
|
entry.visible[this.viewId] &&
|
|
entry.id === this._output[entry.approxIndex[this.viewId]]?.id
|
|
) {
|
|
return this.normalizeIndex(entry.approxIndex[this.viewId]);
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
public [Symbol.iterator](): IterableIterator<T> {
|
|
const self = this;
|
|
let offset = this.windowStart;
|
|
return {
|
|
next() {
|
|
offset++;
|
|
if (offset > self.windowEnd || offset > self.size) {
|
|
return {done: true, value: undefined};
|
|
} else {
|
|
return {
|
|
value: self.get(offset - 1),
|
|
};
|
|
}
|
|
},
|
|
[Symbol.iterator]() {
|
|
return this;
|
|
},
|
|
};
|
|
}
|
|
|
|
private notifyAllListeners(change: OutputChange) {
|
|
this.outputChangeListeners.forEach((listener) => listener(change));
|
|
}
|
|
|
|
private notifyItemUpdated(viewIndex: number) {
|
|
viewIndex = this.normalizeIndex(viewIndex);
|
|
if (
|
|
!this.outputChangeListeners.size ||
|
|
viewIndex < this.windowStart ||
|
|
viewIndex >= this.windowEnd
|
|
) {
|
|
return;
|
|
}
|
|
this.notifyAllListeners({
|
|
type: 'update',
|
|
index: viewIndex,
|
|
});
|
|
}
|
|
|
|
private notifyItemShift(index: number, delta: number) {
|
|
if (!this.outputChangeListeners.size) {
|
|
return;
|
|
}
|
|
let viewIndex = this.normalizeIndex(index);
|
|
if (this.reverse && delta < 0) {
|
|
viewIndex -= delta; // we need to correct for normalize already using the new length after applying this change
|
|
}
|
|
// Idea: we could add an option to automatically shift the window for before events.
|
|
this.notifyAllListeners({
|
|
type: 'shift',
|
|
delta,
|
|
index: viewIndex,
|
|
newCount: this._output.length,
|
|
location:
|
|
viewIndex < this.windowStart
|
|
? 'before'
|
|
: viewIndex >= this.windowEnd
|
|
? 'after'
|
|
: 'in',
|
|
});
|
|
}
|
|
|
|
private notifyReset(count: number) {
|
|
this.notifyAllListeners({
|
|
type: 'reset',
|
|
newCount: count,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* @private
|
|
*/
|
|
processEvent(event: DataEvent<T>) {
|
|
const {_output: output, sortBy, filter} = this;
|
|
switch (event.type) {
|
|
case 'append': {
|
|
const {entry} = event;
|
|
entry.visible[this.viewId] = filter ? filter(entry.value) : true;
|
|
this.applyFilterExceptions(entry);
|
|
if (!entry.visible[this.viewId]) {
|
|
// not in filter? skip this entry
|
|
return;
|
|
}
|
|
if (!sortBy) {
|
|
// no sorting? insert at the end, or beginning
|
|
entry.approxIndex[this.viewId] = output.length;
|
|
output.push(entry);
|
|
this.notifyItemShift(entry.approxIndex[this.viewId], 1);
|
|
} else {
|
|
this.insertSorted(entry);
|
|
}
|
|
break;
|
|
}
|
|
case 'update': {
|
|
const {entry} = event;
|
|
entry.visible[this.viewId] = filter ? filter(entry.value) : true;
|
|
this.applyFilterExceptions(entry);
|
|
// short circuit; no view active so update straight away
|
|
if (!filter && !sortBy) {
|
|
output[event.index].approxIndex[this.viewId] = event.index;
|
|
this.notifyItemUpdated(event.index);
|
|
} else if (!event.oldVisible[this.viewId]) {
|
|
if (!entry.visible[this.viewId]) {
|
|
// Done!
|
|
} else {
|
|
// insertion, not visible before
|
|
this.insertSorted(entry);
|
|
}
|
|
} else {
|
|
// Entry was visible previously
|
|
const existingIndex = this.getSortedIndex(entry, event.oldValue);
|
|
if (!entry.visible[this.viewId]) {
|
|
// Remove from output
|
|
output.splice(existingIndex, 1);
|
|
this.notifyItemShift(existingIndex, -1);
|
|
} else {
|
|
// Entry was and still is visible
|
|
if (
|
|
!this.sortBy ||
|
|
this.sortBy(event.oldValue) === this.sortBy(entry.value)
|
|
) {
|
|
// Still at same position, so done!
|
|
this.notifyItemUpdated(existingIndex);
|
|
} else {
|
|
// item needs to be moved cause of sorting
|
|
// possible optimization: if we discover that old and new index would be the same,
|
|
// despite different sort values, we could still emit only an update instead of two shifts
|
|
output.splice(existingIndex, 1);
|
|
this.notifyItemShift(existingIndex, -1);
|
|
// find new sort index
|
|
this.insertSorted(entry);
|
|
}
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
case 'remove': {
|
|
this.processRemoveEvent(event.index, event.entry);
|
|
break;
|
|
}
|
|
case 'shift': {
|
|
// no sorting? then all items are removed from the start so optimize for that
|
|
if (!sortBy) {
|
|
let amount = 0;
|
|
if (!filter) {
|
|
amount = event.amount;
|
|
} else {
|
|
// if there is a filter, count the visibles and shift those
|
|
for (let i = 0; i < event.entries.length; i++)
|
|
if (event.entries[i].visible[this.viewId]) amount++;
|
|
}
|
|
output.splice(0, amount);
|
|
this.notifyItemShift(0, -amount);
|
|
} else {
|
|
// we have sorting, so we need to remove item by item
|
|
// we do this backward, so that approxIndex is more likely to be correct
|
|
for (let i = event.entries.length - 1; i >= 0; i--) {
|
|
this.processRemoveEvent(i, event.entries[i]);
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
default:
|
|
throw new Error('unknown event type');
|
|
}
|
|
}
|
|
|
|
private processRemoveEvent(index: number, entry: Entry<T>) {
|
|
const {_output: output, sortBy, filter} = this;
|
|
|
|
// filter active, and not visible? short circuilt
|
|
if (!entry.visible[this.viewId]) {
|
|
return;
|
|
}
|
|
// no sorting, no filter?
|
|
if (!sortBy && !filter) {
|
|
output.splice(index, 1);
|
|
this.notifyItemShift(index, -1);
|
|
} else {
|
|
// sorting or filter is active, find the actual location
|
|
const existingIndex = this.getSortedIndex(entry, entry.value);
|
|
output.splice(existingIndex, 1);
|
|
this.notifyItemShift(existingIndex, -1);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Rebuilds the entire view. Typically there should be no need to call this manually
|
|
* @private
|
|
*/
|
|
rebuild() {
|
|
// Pending on the size, should we batch this in smaller non-blocking steps,
|
|
// which we update in a double-buffering mechanism, report progress, and swap out when done?
|
|
//
|
|
// MWE: 9-3-2020 postponed for now, one massive sort seems fine. It might shortly block,
|
|
// but that happens only (exception: limit caused shifts) on user interaction at very roughly 1ms per 1000 records.
|
|
// See also comment below
|
|
const {sortBy, filter, sortHelper} = this;
|
|
// copy base array or run filter (with side effecty update of visible)
|
|
// @ts-ignore prevent making _record public
|
|
const records: Entry<T>[] = this.datasource._records;
|
|
let output = filter
|
|
? records.filter((entry) => {
|
|
entry.visible[this.viewId] = filter(entry.value);
|
|
this.applyFilterExceptions(entry);
|
|
return entry.visible[this.viewId];
|
|
})
|
|
: records.slice();
|
|
if (sortBy) {
|
|
// Pending on the size, should we batch this in smaller steps?
|
|
// The following sorthing method can be taskified, however,
|
|
// the implementation is 20x slower than a native sort. So for now we stick to a
|
|
// blocking sort, until we have some more numbers that this is hanging for anyone
|
|
// const filtered = output;
|
|
// output = [];
|
|
// filtered.forEach((entry) => {
|
|
// const insertionIndex = sortedLastIndexBy(output, entry, sortHelper);
|
|
// output.splice(insertionIndex, 0, entry);
|
|
// });
|
|
output = lodashSort(output, sortHelper); // uses array.sort under the hood
|
|
}
|
|
|
|
// write approx indexes for faster lookup of entries in visible output
|
|
for (let i = 0; i < output.length; i++) {
|
|
output[i].approxIndex[this.viewId] = i;
|
|
}
|
|
this._output = output;
|
|
this.notifyReset(output.length);
|
|
}
|
|
|
|
private sortHelper = (a: Entry<T>) =>
|
|
this.sortBy ? this.sortBy(a.value) : a.id;
|
|
|
|
private getSortedIndex(entry: Entry<T>, oldValue: T) {
|
|
const {_output: output} = this;
|
|
if (output[entry.approxIndex[this.viewId]] === entry) {
|
|
// yay!
|
|
return entry.approxIndex[this.viewId];
|
|
}
|
|
let index = sortedIndexBy(
|
|
output,
|
|
{
|
|
value: oldValue,
|
|
id: -1,
|
|
visible: entry.visible,
|
|
approxIndex: entry.approxIndex,
|
|
},
|
|
this.sortHelper,
|
|
);
|
|
index--;
|
|
// the item we are looking for is not necessarily the first one at the insertion index
|
|
while (output[index] !== entry) {
|
|
index++;
|
|
if (index >= output.length) {
|
|
throw new Error('illegal state: sortedIndex not found'); // sanity check to avoid browser freeze if people mess up with internals
|
|
}
|
|
}
|
|
|
|
return index;
|
|
}
|
|
|
|
private insertSorted(entry: Entry<T>) {
|
|
// apply sorting
|
|
const insertionIndex = sortedLastIndexBy(
|
|
this._output,
|
|
entry,
|
|
this.sortHelper,
|
|
);
|
|
entry.approxIndex[this.viewId] = insertionIndex;
|
|
this._output.splice(insertionIndex, 0, entry);
|
|
this.notifyItemShift(insertionIndex, 1);
|
|
}
|
|
|
|
private applyFilterExceptions(entry: Entry<T>) {
|
|
if (
|
|
this.datasource.keyAttribute &&
|
|
this.filter &&
|
|
this.filterExceptions &&
|
|
!entry.visible[this.viewId]
|
|
) {
|
|
const keyValue = entry.value[this.datasource.keyAttribute] as KeyType;
|
|
entry.visible[this.viewId] = this.filterExceptions.has(keyValue);
|
|
}
|
|
}
|
|
}
|