Reorganise for easier extraction

Summary:
To make the DataSource abstraction reusable for other teams and an upcoming talk, this diff moves all DataSource storage & virtualization logic in one folder.

Will set up a build process and demo project in later diffs.

Reviewed By: nikoant

Differential Revision: D28056700

fbshipit-source-id: 7cfe5b40bbbe387da711f765a604a45029d451c7

desktop/flipper-plugin/src/__tests__/test-utils.node.tsx

@@ -14,7 +14,7 @@ import {PluginClient} from '../plugin/Plugin';
 import {DevicePluginClient} from '../plugin/DevicePlugin';
 import mockConsole from 'jest-mock-console';
 import {sleep} from '../utils/sleep';
-import {createDataSource} from '../state/DataSource';
+import {createDataSource} from '../state/createDataSource';
 
 test('it can start a plugin and lifecycle events', () => {
   const {instance, ...p} = TestUtils.startPlugin(testPlugin);

desktop/flipper-plugin/src/data-source/DataSource.tsx

@@ -13,7 +13,6 @@ import {
   property,
   sortBy as lodashSort,
 } from 'lodash';
-import {Persistable, registerStorageAtom} from '../plugin/PluginBase';
 
 // If the dataSource becomes to large, after how many records will we start to drop items?
 const dropFactor = 0.1;
@@ -23,7 +22,7 @@ const defaultLimit = 100 * 1000;
 // rather than search and remove the affected individual items
 const shiftRebuildTreshold = 0.05;
 
-type ExtractKeyType<T, KEY extends keyof T> = T[KEY] extends string
+export type ExtractKeyType<T, KEY extends keyof T> = T[KEY] extends string
   ? string
   : T[KEY] extends number
   ? number
@@ -90,7 +89,7 @@ export class DataSource<
   T = any,
   KEY extends keyof T = any,
   KEY_TYPE extends string | number | never = ExtractKeyType<T, KEY>
-> implements Persistable {
+> {
   private nextId = 0;
   private _records: Entry<T>[] = [];
   private _recordsById: Map<KEY_TYPE, T> = new Map();
@@ -425,44 +424,6 @@ export class DataSource<
   }
 }
 
-type CreateDataSourceOptions<T, K extends keyof T> = {
-  /**
-   * If a key is set, the given field of the records is assumed to be unique,
-   * and it's value can be used to perform lookups and upserts.
-   */
-  key?: K;
-  /**
-   * The maximum amount of records that this DataSource will store.
-   * If the limit is exceeded, the oldest records will automatically be dropped to make place for the new ones
-   */
-  limit?: number;
-  /**
-   * Should this state persist when exporting a plugin?
-   * If set, the dataSource will be saved / loaded under the key provided
-   */
-  persist?: string;
-};
-
-export function createDataSource<T, KEY extends keyof T = any>(
-  initialSet: T[],
-  options: CreateDataSourceOptions<T, KEY>,
-): DataSource<T, KEY, ExtractKeyType<T, KEY>>;
-export function createDataSource<T>(
-  initialSet?: T[],
-): DataSource<T, never, never>;
-export function createDataSource<T, KEY extends keyof T>(
-  initialSet: T[] = [],
-  options?: CreateDataSourceOptions<T, KEY>,
-): DataSource<T, any, any> {
-  const ds = new DataSource<T, KEY>(options?.key);
-  if (options?.limit !== undefined) {
-    ds.limit = options.limit;
-  }
-  registerStorageAtom(options?.persist, ds);
-  initialSet.forEach((value) => ds.append(value));
-  return ds;
-}
-
 function unwrap<T>(entry: Entry<T>): T {
   return entry?.value;
 }

desktop/flipper-plugin/src/data-source/DataSourceRendererStatic.tsx

@@ -7,12 +7,10 @@
  * @format
  */
 
+import {DataSource} from './DataSource';
 import React, {memo, useCallback, useEffect, useState} from 'react';
-import {DataSource} from '../../state/DataSource';
-import {useVirtual} from 'react-virtual';
-import {RedrawContext} from './DataSourceRenderer';
 
-export type DataSourceVirtualizer = ReturnType<typeof useVirtual>;
+import {RedrawContext} from './DataSourceRendererVirtual';
 
 type DataSourceProps<T extends object, C> = {
   /**
@@ -41,9 +39,9 @@ type DataSourceProps<T extends object, C> = {
  * This component is UI agnostic, and just takes care of rendering all items in the DataSource.
  * This component does not apply virtualization, so don't use it for large datasets!
  */
-export const StaticDataSourceRenderer: <T extends object, C>(
+export const DataSourceRendererStatic: <T extends object, C>(
   props: DataSourceProps<T, C>,
-) => React.ReactElement = memo(function StaticDataSourceRenderer({
+) => React.ReactElement = memo(function DataSourceRendererStatic({
   dataSource,
   useFixedRowHeight,
   context,
@@ -98,7 +96,7 @@ export const StaticDataSourceRenderer: <T extends object, C>(
 
   return (
     <RedrawContext.Provider value={redraw}>
-      <div onKeyDown={onKeyDown}>
+      <div onKeyDown={onKeyDown} tabIndex={0}>
         {records.length === 0
           ? emptyRenderer?.(dataSource)
           : records.map((item, index) => (

desktop/flipper-plugin/src/data-source/DataSourceRendererVirtual.tsx

@@ -18,11 +18,9 @@ import React, {
   useContext,
   createContext,
 } from 'react';
-import {DataSource} from '../../state/DataSource';
+import {DataSource} from './DataSource';
 import {useVirtual} from 'react-virtual';
-import styled from '@emotion/styled';
 import observeRect from '@reach/observe-rect';
-import {useInUnitTest} from '../../utils/useInUnitTest()';
 
 // how fast we update if updates are low-prio (e.g. out of window and not super significant)
 const LOW_PRIO_UPDATE = 1000; //ms
@@ -75,9 +73,9 @@ type DataSourceProps<T extends object, C> = {
  * This component is UI agnostic, and just takes care of virtualizing the provided dataSource, and render it as efficiently a possibible,
  * de priorizing off screen updates etc.
  */
-export const DataSourceRenderer: <T extends object, C>(
+export const DataSourceRendererVirtual: <T extends object, C>(
   props: DataSourceProps<T, C>,
-) => React.ReactElement = memo(function DataSourceRenderer({
+) => React.ReactElement = memo(function DataSourceRendererVirtual({
   dataSource,
   defaultRowHeight,
   useFixedRowHeight,
@@ -286,12 +284,15 @@ export const DataSourceRenderer: <T extends object, C>(
    */
   return (
     <RedrawContext.Provider value={redraw}>
-      <TableContainer ref={parentRef} onScroll={onScroll}>
+      <div ref={parentRef} onScroll={onScroll} style={tableContainerStyle}>
         {virtualizer.virtualItems.length === 0
           ? emptyRenderer?.(dataSource)
           : null}
-        <TableWindow
+        <div
-          height={virtualizer.totalSize}
+          style={{
+            ...tableWindowStyle,
+            height: virtualizer.totalSize,
+          }}
           onKeyDown={onKeyDown}
           tabIndex={0}>
           {virtualizer.virtualItems.map((virtualRow) => {
@@ -314,27 +315,30 @@ export const DataSourceRenderer: <T extends object, C>(
               </div>
             );
           })}
-        </TableWindow>
+        </div>
-      </TableContainer>
+      </div>
     </RedrawContext.Provider>
   );
 }) as any;
 
-const TableContainer = styled.div({
+const tableContainerStyle = {
   overflowY: 'auto',
   overflowX: 'hidden',
   display: 'flex',
   flex: 1,
-});
+} as const;
 
-const TableWindow = styled.div<{height: number}>(({height}) => ({
+const tableWindowStyle = {
-  height,
   position: 'relative',
   width: '100%',
-}));
+} as const;
 
 export const RedrawContext = createContext<undefined | (() => void)>(undefined);
 
 export function useTableRedraw() {
   return useContext(RedrawContext);
 }
+
+function useInUnitTest(): boolean {
+  return process.env.NODE_ENV === 'test';
+}

desktop/flipper-plugin/src/data-source/README.md

@@ -0,0 +1,76 @@
+# DataSource
+
+_Library to power streamig data visualisations_
+
+## Contstraints & Benefits
+
+This library builds a map-reduce inspired data processing pipeline that stores data, and can incrementally update existing visualizations when new data arrives or existing data is updated. It achieves this by emitting events that describe how a visualisation should be _changed_ over time, rather than computing & providing a fresh immutable dataset when the stored data is updated. Some benefits:
+
+* Appending or updating records is roughly `O(update_size)` instead of `O(dataset_size)`, while still respecting filtering, sorting and windowing
+* Virtualization (windowing) is built in.
+* Dynamic row wrapping is supported when rendering tables.
+* Any stable JS function can be used for sorting and filtering, without impacting performance significantly.
+
+This library is designed with the following constraints in mind:
+
+* The full dataset is kept in memory (automatically removing old items is supported to prevent unlimited growth)
+* New data or data updates arrive 'streaming' over time. For rendering a large fixed dataset that doesn't evolve over time this abstraction offers no benefits.
+
+![CPU load snapshot](img/logs.png)
+
+After applying this abstraction (right two sections), in Flipper we saw a 10-20 fold framerate increase while displaying 100K log items, to which new records where added at a rate of ~50/sec. The above image is taken while tailing the logs (so it continuesly scrolls with the arrival of new data), while a search filter is also active. In the first two sections, scripting ate away most of the CPU, result in just a couple of frames per second. After applying these changes CPU time is primarily spend on cranking out more frames resulting in a smooth rather than stuttering scroll (see the lightning talk below for a demo). On top of that the base situation uses a fixed row-height, while the new situation supports text wrapping.
+
+## Links
+
+* [DataSource API documentation](https://fbflipper.com/docs/extending/flipper-plugin#createdatasource)
+* [DataSource project plan](https://fb.quip.com/noJDArpLF7Fe)
+* Introduction talk (TODO)
+* [Lightning talk using DataSource in Logs view](https://fb.workplace.com/groups/427492358561913/permalink/432720091372473/)
+
+## More detailed explanation
+
+![FSRW pipeline](img/FSRW.png)
+
+### DataSource
+
+Many visualization and underlying storage abstractions are optimised for large but fixed datasets.
+This abstractions is optimised for visualization that need to present datasets that are continuesly updated / expanded.
+
+The significant difference to many other solutions is that DataSource doesn't produce an immutable dataset that is swapped out every time the data is changed.
+Instead, it keeps internally a mutable dataset (the records stored themselves are still immutable but can be replaced) to which new entries are added.
+However, instead of propagating the dataset to the rendering layer, events are emitted instead.
+
+### DataSourceView
+
+Conceptually, `DataSourceView` is a materialized view of a `DataSource`.
+For visualizations, typically the following transformations need to be applied: filter/search, sorting and windowing.
+
+Where many libraries applies these transformations as part of the _rendering_, DataSourceView applies these operations directly when updates to the dataset are received.
+As a result the transformations need to be applied only to the newly arriving data.
+For example, if a new record arrives for a sorted dataset, we will apply a binary inseration sort for the new entry, avoiding the need for a full re-sort of the dataset during Rendering.
+
+Once the dataset is updated, the DataSource will emit _events_ to the `DataSourceRenderer`, rather than providing it a new dataset.
+The events will describe how the current view should be updated to reflect the data changes.
+
+### DataSourceRendererVirtual
+
+`DataSourceRendererVirtual` is one of the possible visualizations of a DataSourceView.
+It takes care of subscribing to the events emitted by the `DataSourceView`, and applies them when they are relevant (e.g. within the visible window).
+Beyond that, it manages virtualizations (using the `react-virtual` library), so that for example scroll interactions are used to move the window of the`DataSourceView`.
+
+Typically this component is used as underlying abstraction for a Table representation. 
+
+### DataSourceRendererStatic
+
+A simplified (and not very efficient) render for DataSource that doens't use virtualization. Use this as basic for a natural growing representaiton.
+
+### DataSourceChartRenderer
+
+## Future work
+
+* [ ] **Give this thing a proper name**
+* [ ] **Leverage React concurrent mode**: Currently there is custom scheduler logic to handle high- and low- (outside window) priority updates. In principle this could probably be achieved through React concurrent mode as well, but ANT.design (which is used in Flipper) doesn't support it yet.
+* [ ] **Support multiple DataSourceView's per DataSource**: Currently there is a one view per source limitation because we didn't need more yet.
+* [ ] **Break up operations that process the full data set in smaller tasks**: There are several operations that process the full data set, for example changing the sort / filter criteria. Currently this is done synchronously (and we debounce changing the filter), in the future we will split up the filtering in smaller taks to make it efficient. But we don't have a way to efficiently break down sorting into smaller tasks as using insertion sorting is 20x slower than the native sorting mechanism if the full data set needs to be processed.
+* [ ] **Publish as open source / standalone package**
+* [ ] **Reduce build size**. Currently half lodash is baked in, but basically we only need it's binary sort function :).

desktop/flipper-plugin/src/data-source/__tests__/datasource-basics.node.tsx

@@ -7,7 +7,8 @@
  * @format
  */
 
-import {createDataSource, DataSource} from '../DataSource';
+import {createDataSource} from '../../state/createDataSource';
+import {DataSource} from '../DataSource';
 
 type Todo = {
   id: string;

desktop/flipper-plugin/src/data-source/__tests__/datasource-perf.node.tsx

@@ -7,7 +7,8 @@
  * @format
  */
 
-import {createDataSource, DataSource} from '../DataSource';
+import {createDataSource} from '../../state/createDataSource';
+import {DataSource} from '../DataSource';
 
 type Todo = {
   id: string;

desktop/flipper-plugin/src/data-source/index.tsx

@@ -0,0 +1,15 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+export {DataSource} from './DataSource';
+export {
+  DataSourceRendererVirtual,
+  DataSourceVirtualizer,
+} from './DataSourceRendererVirtual';
+export {DataSourceRendererStatic} from './DataSourceRendererStatic';

desktop/flipper-plugin/src/index.ts

@@ -83,7 +83,8 @@ export {
 } from './utils/Logger';
 export {Idler} from './utils/Idler';
 
-export {createDataSource, DataSource} from './state/DataSource';
+export {DataSource} from './data-source/DataSource';
+export {createDataSource} from './state/createDataSource';
 
 export {DataTable, DataTableColumn} from './ui/data-table/DataTable';
 export {DataTableManager} from './ui/data-table/DataTableManager';

desktop/flipper-plugin/src/state/createDataSource.tsx

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import {DataSource, ExtractKeyType} from '../data-source/DataSource';
+import {registerStorageAtom} from '../plugin/PluginBase';
+
+type CreateDataSourceOptions<T, K extends keyof T> = {
+  /**
+   * If a key is set, the given field of the records is assumed to be unique,
+   * and it's value can be used to perform lookups and upserts.
+   */
+  key?: K;
+  /**
+   * The maximum amount of records that this DataSource will store.
+   * If the limit is exceeded, the oldest records will automatically be dropped to make place for the new ones
+   */
+  limit?: number;
+  /**
+   * Should this state persist when exporting a plugin?
+   * If set, the dataSource will be saved / loaded under the key provided
+   */
+  persist?: string;
+};
+
+export function createDataSource<T, KEY extends keyof T = any>(
+  initialSet: T[],
+  options: CreateDataSourceOptions<T, KEY>,
+): DataSource<T, KEY, ExtractKeyType<T, KEY>>;
+export function createDataSource<T>(
+  initialSet?: T[],
+): DataSource<T, never, never>;
+export function createDataSource<T, KEY extends keyof T>(
+  initialSet: T[] = [],
+  options?: CreateDataSourceOptions<T, KEY>,
+): DataSource<T, any, any> {
+  const ds = new DataSource<T, KEY>(options?.key);
+  if (options?.limit !== undefined) {
+    ds.limit = options.limit;
+  }
+  registerStorageAtom(options?.persist, ds);
+  initialSet.forEach((value) => ds.append(value));
+  return ds;
+}

desktop/flipper-plugin/src/ui/DataFormatter.tsx

@@ -18,7 +18,7 @@ import React, {createElement, Fragment, isValidElement, useState} from 'react';
 import {tryGetFlipperLibImplementation} from '../plugin/FlipperLib';
 import {safeStringify} from '../utils/safeStringify';
 import {urlRegex} from '../utils/urlRegex';
-import {useTableRedraw} from './data-table/DataSourceRenderer';
+import {useTableRedraw} from '../data-source/DataSourceRendererVirtual';
 import {theme} from './theme';
 
 /**

desktop/flipper-plugin/src/ui/data-inspector/DataInspectorNode.tsx

@@ -26,7 +26,7 @@ import {useHighlighter, HighlightManager} from '../Highlight';
 import {Dropdown, Menu, Tooltip} from 'antd';
 import {tryGetFlipperLibImplementation} from '../../plugin/FlipperLib';
 import {safeStringify} from '../../utils/safeStringify';
-import {useInUnitTest} from '../../utils/useInUnitTest()';
+import {useInUnitTest} from '../../utils/useInUnitTest';
 
 export {DataValueExtractor} from './DataPreview';
 

desktop/flipper-plugin/src/ui/data-table/DataTable.tsx

@@ -20,11 +20,15 @@ import React, {
   useReducer,
 } from 'react';
 import {TableRow, DEFAULT_ROW_HEIGHT} from './TableRow';
-import {DataSource} from '../../state/DataSource';
 import {Layout} from '../Layout';
 import {TableHead} from './TableHead';
 import {Percentage} from '../../utils/widthUtils';
-import {DataSourceRenderer, DataSourceVirtualizer} from './DataSourceRenderer';
+import {
+  DataSourceRendererVirtual,
+  DataSourceRendererStatic,
+  DataSource,
+  DataSourceVirtualizer,
+} from '../../data-source';
 import {
   computeDataTableFilter,
   createDataTableManager,
@@ -46,8 +50,7 @@ import {useAssertStableRef} from '../../utils/useAssertStableRef';
 import {Formatter} from '../DataFormatter';
 import {usePluginInstance} from '../../plugin/PluginContext';
 import {debounce} from 'lodash';
-import {StaticDataSourceRenderer} from './StaticDataSourceRenderer';
+import {useInUnitTest} from '../../utils/useInUnitTest';
-import {useInUnitTest} from '../../utils/useInUnitTest()';
 
 interface DataTableBaseProps<T = any> {
   columns: DataTableColumn<T>[];
@@ -458,7 +461,7 @@ export function DataTable<T extends object>(
   const mainSection = props.scrollable ? (
     <Layout.Top>
       {header}
-      <DataSourceRenderer<T, TableRowRenderContext<T>>
+      <DataSourceRendererVirtual<T, TableRowRenderContext<T>>
         dataSource={dataSource}
         autoScroll={tableState.autoScroll && !dragging.current}
         useFixedRowHeight={!tableState.usesWrapping}
@@ -475,7 +478,7 @@ export function DataTable<T extends object>(
   ) : (
     <Layout.Container>
       {header}
-      <StaticDataSourceRenderer<T, TableRowRenderContext<T>>
+      <DataSourceRendererStatic<T, TableRowRenderContext<T>>
         dataSource={dataSource}
         useFixedRowHeight={!tableState.usesWrapping}
         defaultRowHeight={DEFAULT_ROW_HEIGHT}

desktop/flipper-plugin/src/ui/data-table/DataTableManager.tsx

@@ -10,8 +10,8 @@
 import type {DataTableColumn} from './DataTable';
 import {Percentage} from '../../utils/widthUtils';
 import {MutableRefObject, Reducer} from 'react';
-import {DataSource} from '../../state/DataSource';
+import {DataSource} from '../../data-source/DataSource';
-import {DataSourceVirtualizer} from './DataSourceRenderer';
+import {DataSourceVirtualizer} from '../../data-source/DataSourceRendererVirtual';
 import produce, {castDraft, immerable, original} from 'immer';
 
 export type OnColumnResize = (id: string, size: number | Percentage) => void;

desktop/flipper-plugin/src/ui/data-table/TableContextMenu.tsx

@@ -18,8 +18,8 @@ import {
 import React from 'react';
 import {tryGetFlipperLibImplementation} from '../../plugin/FlipperLib';
 import {DataTableColumn} from './DataTable';
-import {DataSource} from '../../state/DataSource';
 import {toFirstUpper} from '../../utils/toFirstUpper';
+import {DataSource} from '../../data-source/DataSource';
 
 const {Item, SubMenu} = Menu;
 

desktop/flipper-plugin/src/ui/data-table/__tests__/DataTable.node.tsx

@@ -10,7 +10,7 @@
 import React, {createRef} from 'react';
 import {DataTable, DataTableColumn} from '../DataTable';
 import {render, act} from '@testing-library/react';
-import {createDataSource} from '../../../state/DataSource';
+import {createDataSource} from '../../../state/createDataSource';
 import {computeDataTableFilter, DataTableManager} from '../DataTableManager';
 import {Button} from 'antd';
 

desktop/flipper-plugin/src/utils/createTablePlugin.tsx

@@ -8,13 +8,14 @@
  */
 
 import {notification, Typography} from 'antd';
+import {DataSource} from '../data-source/DataSource';
 import React from 'react';
 import {PluginClient} from '../plugin/Plugin';
 import {usePlugin} from '../plugin/PluginContext';
 import {createState} from '../state/atom';
-import {createDataSource, DataSource} from '../state/DataSource';
 import {DataTableColumn} from '../ui/data-table/DataTable';
 import {MasterDetail} from '../ui/MasterDetail';
+import {createDataSource} from '../state/createDataSource';
 
 type PluginResult<Raw, Row> = {
   plugin(