Process queues before exporting plugins

Summary: This diff makes sure that pending queues for plugins that are selected are processed before making a flipper export. Reviewed By: jknoxville Differential Revision: D19194211 fbshipit-source-id: e076375889450407e7f94384051719f3bbc415ee
2020-01-02 07:12:06 -08:00
parent 0494a84d98
commit b8e752412e
8 changed files with 222 additions and 131 deletions
--- a/headless/index.tsx
+++ b/headless/index.tsx
@@ -177,9 +177,7 @@ async function exitActions(
          );
          outputAndExit(payload);
        } else {
-          const {serializedString, errorArray} = await exportStore(
-            store.getState(),
-          );
+          const {serializedString, errorArray} = await exportStore(store);
          errorArray.forEach(console.error);
          outputAndExit(serializedString);
        }
@@ -256,7 +254,7 @@ async function startFlipper(userArguments: UserArguments) {
              errorAndExit(e);
            });
        } else {
-          exportStore(store.getState())
+          exportStore(store)
            .then(({serializedString}) => {
              outputAndExit(serializedString);
            })
--- a/src/chrome/ExportDataPluginSheet.tsx
+++ b/src/chrome/ExportDataPluginSheet.tsx
@@ -13,6 +13,7 @@ import {ShareType} from '../reducers/application';
 import {State as PluginState} from '../reducers/plugins';
 import {State as PluginStatesState} from '../reducers/pluginStates';
 import {State as Store} from '../reducers';
+import {State as PluginMessageQueueState} from '../reducers/pluginMessageQueue';
 import {ActiveSheet} from '../reducers/application';
 import {selectedPlugins as actionForSelectedPlugins} from '../reducers/plugins';
 import {getActivePersistentPlugins} from '../utils/pluginUtils';
@@ -35,6 +36,7 @@ type StateFromProps = {
  share: ShareType | null;
  plugins: PluginState;
  pluginStates: PluginStatesState;
+  pluginMessageQueue: PluginMessageQueueState;
  selectedClient: Client | undefined;
 };

@@ -57,13 +59,20 @@ const Container = styled(FlexColumn)({

 class ExportDataPluginSheet extends Component<Props> {
  render() {
-    const {plugins, pluginStates, onHide, selectedClient} = this.props;
+    const {
+      plugins,
+      pluginStates,
+      pluginMessageQueue,
+      onHide,
+      selectedClient,
+    } = this.props;
    const onHideWithUnsettingShare = () => {
      this.props.unsetShare();
      onHide();
    };
    const pluginsToExport = getActivePersistentPlugins(
      pluginStates,
+      pluginMessageQueue,
      plugins,
      selectedClient,
    );
@@ -121,6 +130,7 @@ export default connect<StateFromProps, DispatchFromProps, OwnProps, Store>(
    application: {share},
    plugins,
    pluginStates,
+    pluginMessageQueue,
    connections: {selectedApp, clients},
  }) => {
    const selectedClient = clients.find(o => {
@@ -130,6 +140,7 @@ export default connect<StateFromProps, DispatchFromProps, OwnProps, Store>(
      share,
      plugins,
      pluginStates,
+      pluginMessageQueue,
      selectedClient,
    };
  },
--- a/src/store.tsx
+++ b/src/store.tsx
@@ -10,6 +10,7 @@
 import {createStore} from 'redux';
 import reducers, {Actions, State as StoreState} from './reducers/index';
 import {stateSanitizer} from './utils/reduxDevToolsConfig';
+import isProduction from './utils/isProduction';

 export const store = createStore<StoreState, Actions, any, any>(
  reducers,
@@ -20,3 +21,9 @@ export const store = createStore<StoreState, Actions, any, any>(
      })
    : undefined,
 );
+
+if (!isProduction()) {
+  // For debugging purposes only
+  // @ts-ignore
+  window.flipperStore = store;
+}
--- a/src/utils/tests/pluginUtils.node.js
+++ b/src/utils/tests/pluginUtils.node.js
@@ -14,8 +14,10 @@ import {
 import type {State as PluginsState} from '../../reducers/plugins.tsx';
 import type {State as PluginStatesState} from '../../reducers/pluginStates.tsx';
 import type {PluginDefinition} from '../../dispatcher/plugins.tsx';
+import type {State as PluginMessageQueueState} from '../../reducers/pluginStates.tsx';
 import {FlipperBasePlugin} from 'flipper';
 import type {ReduxState} from '../../reducers/index.tsx';
+
 class MockFlipperPluginWithDefaultPersistedState extends FlipperBasePlugin<
  *,
  *,
@@ -157,11 +159,12 @@ test('getActivePersistentPlugins, where the non persistent plugins getting exclu
    'serial#app#ClientPlugin1': {msg: 'ClientPlugin1'},
    'serial#app#ClientPlugin2': {msg: 'ClientPlugin2'},
  };
-  const list = getActivePersistentPlugins(plugins, state);
+  const queues: PluginMessageQueueState = {};
+  const list = getActivePersistentPlugins(plugins, queues, state);
  expect(list).toEqual(['ClientPlugin1', 'DevicePlugin2']);
 });

-test('getActivePersistentPlugins, where the plugins not in pluginState gets excluded', () => {
+test('getActivePersistentPlugins, where the plugins not in pluginState or queue gets excluded', () => {
  const state: PluginsState = {
    devicePlugins: new Map([
      ['DevicePlugin1', MockFlipperPluginWithDefaultPersistedState],
@@ -170,6 +173,7 @@ test('getActivePersistentPlugins, where the plugins not in pluginState gets excl
    clientPlugins: new Map([
      ['ClientPlugin1', MockFlipperPluginWithDefaultPersistedState],
      ['ClientPlugin2', MockFlipperPluginWithDefaultPersistedState],
+      ['ClientPlugin3', MockFlipperPluginWithDefaultPersistedState],
    ]),
    gatekeepedPlugins: [],
    disabledPlugins: [],
@@ -180,6 +184,11 @@ test('getActivePersistentPlugins, where the plugins not in pluginState gets excl
    'serial#app#DevicePlugin1': {msg: 'DevicePlugin1'},
    'serial#app#ClientPlugin2': {msg: 'ClientPlugin2'},
  };
-  const list = getActivePersistentPlugins(plugins, state);
-  expect(list).toEqual(['ClientPlugin2', 'DevicePlugin1']);
+  const queues: PluginMessageQueueState = {
+    'serial#app#ClientPlugin3': [
+      {method: 'msg', params: {msg: 'ClientPlugin3'}},
+    ],
+  };
+  const list = getActivePersistentPlugins(plugins, queues, state);
+  expect(list).toEqual(['ClientPlugin2', 'ClientPlugin3', 'DevicePlugin1']);
 });
--- a/src/utils/exportData.tsx
+++ b/src/utils/exportData.tsx
@@ -11,7 +11,7 @@ import os from 'os';
 import path from 'path';
 import electron from 'electron';
 import {getInstance as getLogger} from '../fb-stubs/Logger';
-import {Store, State as ReduxState} from '../reducers';
+import {Store, State as ReduxState, MiddlewareAPI} from '../reducers';
 import {DeviceExport} from '../devices/BaseDevice';
 import {State as PluginStatesState} from '../reducers/pluginStates';
 import {PluginNotification} from '../reducers/notifications';
@@ -41,6 +41,7 @@ import {
 import {setSelectPluginsToExportActiveSheet} from '../reducers/application';
 import {deconstructClientId, deconstructPluginKey} from '../utils/clientUtils';
 import {performance} from 'perf_hooks';
+import {processMessageQueue} from './messageQueue';

 export const IMPORT_FLIPPER_TRACE_EVENT = 'import-flipper-trace';
 export const EXPORT_FLIPPER_TRACE_EVENT = 'export-flipper-trace';
@@ -82,6 +83,13 @@ type SerializePluginStatesOptions = {
  pluginStates: PluginStatesState;
 };

+type PluginsToProcess = {
+  pluginKey: string;
+  plugin: string;
+  pluginClass: typeof FlipperPlugin | typeof FlipperDevicePlugin;
+  client: Client;
+}[];
+
 type AddSaltToDeviceSerialOptions = {
  salt: string;
  device: BaseDevice;
@@ -381,38 +389,18 @@ export const processStore = async (
 };

 export async function fetchMetadata(
-  clients: Client[],
+  pluginsToProcess: PluginsToProcess,
  pluginStates: PluginStatesState,
-  pluginsMap: Map<string, typeof FlipperDevicePlugin | typeof FlipperPlugin>,
  state: ReduxState,
  statusUpdate?: (msg: string) => void,
  idler?: Idler,
 ): Promise<{pluginStates: PluginStatesState; errorArray: Array<Error>}> {
  const newPluginState = {...pluginStates};
  const errorArray: Array<Error> = [];
-  const selectedDevice = state.connections.selectedDevice;
-  for (const client of clients) {
-    if (
-      !selectedDevice ||
-      selectedDevice.isArchived ||
-      !client.id.includes(selectedDevice.serial)
-    ) {
-      continue;
-    }
-    const selectedPlugins = state.plugins.selectedPlugins;
-    const selectedFilteredPlugins =
-      selectedPlugins.length > 0
-        ? client.plugins.filter(plugin => selectedPlugins.includes(plugin))
-        : client.plugins;
-    for (const plugin of selectedFilteredPlugins) {
-      const pluginClass:
-        | typeof FlipperDevicePlugin
-        | typeof FlipperPlugin
-        | undefined
-        | null = plugin ? pluginsMap.get(plugin) : null;
+
+  for (const {plugin, pluginClass, client, pluginKey} of pluginsToProcess) {
    const exportState = pluginClass ? pluginClass.exportPersistedState : null;
    if (exportState) {
-        const key = pluginKey(client.id, plugin);
      const fetchMetaDataMarker = `${EXPORT_FLIPPER_TRACE_EVENT}:fetch-meta-data-per-plugin`;
      performance.mark(fetchMetaDataMarker);
      try {
@@ -422,18 +410,17 @@ export async function fetchMetadata(
          240000, // Fetching MobileConfig data takes ~ 3 mins, thus keeping timeout at 4 mins.
          exportState(
            callClient(client, plugin),
-              newPluginState[key],
+            newPluginState[pluginKey],
            state,
            idler,
            statusUpdate,
          ),
-
          `Timed out while collecting data for ${plugin}`,
        );
        getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, {
          plugin,
        });
-          newPluginState[key] = data;
+        newPluginState[pluginKey] = data;
      } catch (e) {
        errorArray.push(e);
        getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, {
@@ -444,22 +431,52 @@ export async function fetchMetadata(
      }
    }
  }
-  }
+
  return {pluginStates: newPluginState, errorArray};
 }

-export async function getStoreExport(
-  state: ReduxState,
+async function processQueues(
+  store: MiddlewareAPI,
+  pluginsToProcess: PluginsToProcess,
  statusUpdate?: (msg: string) => void,
  idler?: Idler,
-): Promise<{exportData: ExportType | null; errorArray: Array<Error>}> {
+) {
+  for (const {plugin, pluginKey, pluginClass} of pluginsToProcess) {
+    if (pluginClass.persistedStateReducer) {
+      const processQueueMarker = `${EXPORT_FLIPPER_TRACE_EVENT}:process-queue-per-plugin`;
+      performance.mark(processQueueMarker);
+
+      await processMessageQueue(
+        pluginClass,
+        pluginKey,
+        store,
+        ({current, total}) => {
+          statusUpdate?.(
+            `Processing event ${current} / ${total} (${Math.round(
+              (current / total) * 100,
+            )}%) for plugin ${plugin}`,
+          );
+        },
+        idler,
+      );
+
+      getLogger().trackTimeSince(processQueueMarker, processQueueMarker, {
+        plugin,
+      });
+    }
+  }
+}
+
+function getSelection(
+  store: MiddlewareAPI,
+): {client: Client; device: BaseDevice} {
+  const state = store.getState();
  const {clients} = state.connections;
  const client = clients.find(
    client => client.id === state.connections.selectedApp,
  );
-  const {pluginStates} = state;
-  const {plugins} = state;
  const {selectedDevice} = state.connections;
+  // TODO: T59434642 remove these restrictions
  if (!selectedDevice) {
    throw new Error('Please select a device before exporting data.');
  }
@@ -468,28 +485,68 @@ export async function getStoreExport(
  if (!client) {
    throw new Error('Please select a client before exporting data.');
  }
+  return {client, device: selectedDevice};
+}

-  const pluginsMap: Map<
-    string,
-    typeof FlipperDevicePlugin | typeof FlipperPlugin
-  > = new Map([]);
-  plugins.clientPlugins.forEach((val, key) => {
-    pluginsMap.set(key, val);
-  });
-  plugins.devicePlugins.forEach((val, key) => {
-    pluginsMap.set(key, val);
+export function determinePluginsToProcess(
+  store: MiddlewareAPI,
+): PluginsToProcess {
+  const state = store.getState();
+  const {plugins} = state;
+  const {clients} = state.connections;
+  const {client, device} = getSelection(store);
+
+  const pluginsToProcess: PluginsToProcess = [];
+  const selectedPlugins = state.plugins.selectedPlugins;
+  const selectedFilteredPlugins =
+    selectedPlugins.length > 0
+      ? client.plugins.filter(plugin => selectedPlugins.includes(plugin))
+      : client.plugins;
+  for (const client of clients) {
+    if (!device || device.isArchived || !client.id.includes(device.serial)) {
+      continue;
+    }
+    for (const plugin of selectedFilteredPlugins) {
+      const pluginClass =
+        plugins.clientPlugins.get(plugin) || plugins.devicePlugins.get(plugin);
+      if (pluginClass) {
+        const key = pluginKey(client.id, plugin);
+        pluginsToProcess.push({
+          pluginKey: key,
+          client,
+          plugin,
+          pluginClass,
        });
+      }
+    }
+  }
+  return pluginsToProcess;
+}
+
+export async function getStoreExport(
+  store: MiddlewareAPI,
+  statusUpdate?: (msg: string) => void,
+  idler?: Idler,
+): Promise<{exportData: ExportType | null; errorArray: Array<Error>}> {
+  const pluginsToProcess = determinePluginsToProcess(store);
+
+  statusUpdate?.('Preparing to process data queues for plugins...');
+  await processQueues(store, pluginsToProcess, statusUpdate, idler);
+
  statusUpdate && statusUpdate('Preparing to fetch metadata from client...');
  const fetchMetaDataMarker = `${EXPORT_FLIPPER_TRACE_EVENT}:fetch-meta-data`;
  performance.mark(fetchMetaDataMarker);
+
+  const {device, client} = getSelection(store);
+  const state = store.getState();
  const metadata = await fetchMetadata(
-    [client],
-    pluginStates,
-    pluginsMap,
+    pluginsToProcess,
+    state.pluginStates,
    state,
    statusUpdate,
    idler,
  );
+
  getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, {
    plugins: state.plugins.selectedPlugins,
  });
@@ -501,7 +558,7 @@ export async function getStoreExport(
  const exportData = await processStore(
    {
      activeNotifications,
-      device: selectedDevice,
+      device,
      pluginStates: newPluginState,
      clients: [client.toJSON()],
      devicePlugins,
@@ -515,18 +572,17 @@ export async function getStoreExport(
  return {exportData, errorArray};
 }

-export function exportStore(
-  state: ReduxState,
+export async function exportStore(
+  store: MiddlewareAPI,
  idler?: Idler,
  statusUpdate?: (msg: string) => void,
 ): Promise<{serializedString: string; errorArray: Array<Error>}> {
  getLogger().track('usage', EXPORT_FLIPPER_TRACE_EVENT);
-  return new Promise(async (resolve, reject) => {
  performance.mark(EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT);
-    try {
  statusUpdate && statusUpdate('Preparing to export Flipper data...');
+  const state = store.getState();
  const {exportData, errorArray} = await getStoreExport(
-        state,
+    store,
    statusUpdate,
    idler,
  );
@@ -542,7 +598,7 @@ export function exportStore(
    statusUpdate && statusUpdate('Serializing Flipper data...');
    const serializedString = JSON.stringify(exportData);
    if (serializedString.length <= 0) {
-          reject(new Error('Serialize function returned empty string'));
+      throw new Error('Serialize function returned empty string');
    }
    getLogger().trackTimeSince(
      EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT,
@@ -551,15 +607,11 @@ export function exportStore(
        plugins: state.plugins.selectedPlugins,
      },
    );
-        resolve({serializedString, errorArray});
+    return {serializedString, errorArray};
  } else {
    console.error('Make sure a device is connected');
-        reject(new Error('No device is selected'));
+    throw new Error('No device is selected');
  }
-    } catch (e) {
-      reject(e);
-    }
-  });
 }

 export const exportStoreToFile = (
@@ -568,7 +620,7 @@ export const exportStoreToFile = (
  idler?: Idler,
  statusUpdate?: (msg: string) => void,
 ): Promise<{errorArray: Array<Error>}> => {
-  return exportStore(store.getState(), idler, statusUpdate).then(
+  return exportStore(store, idler, statusUpdate).then(
    ({serializedString, errorArray}) => {
      return promisify(fs.writeFile)(exportFilePath, serializedString).then(
        () => {
--- a/src/utils/exportMetrics.tsx
+++ b/src/utils/exportMetrics.tsx
@@ -12,7 +12,11 @@ import {serialize} from './serialization';
 import {State as PluginStatesState} from '../reducers/pluginStates';
 import {Store} from '../reducers';
 import fs from 'fs';
-import {ExportType, fetchMetadata} from './exportData';
+import {
+  ExportType,
+  fetchMetadata,
+  determinePluginsToProcess,
+} from './exportData';
 import {deserializeObject} from './serialization';
 import {deconstructPluginKey} from './clientUtils';
 import {pluginsClassMap} from './pluginUtils';
@@ -67,10 +71,10 @@ export async function exportMetricsWithoutTrace(
    string,
    typeof FlipperDevicePlugin | typeof FlipperPlugin
  > = pluginsClassMap(store.getState().plugins);
+  const pluginsToProcess = determinePluginsToProcess(store);
  const metadata = await fetchMetadata(
-    store.getState().connections.clients,
+    pluginsToProcess,
    pluginStates,
-    pluginsMap,
    store.getState(),
  );
  const newPluginStates = metadata.pluginStates;
--- a/src/utils/messageQueue.tsx
+++ b/src/utils/messageQueue.tsx
@@ -8,7 +8,7 @@
 */

 import {PersistedStateReducer} from '../plugin';
-import {Store, State} from '../reducers/index';
+import {State, MiddlewareAPI} from '../reducers/index';
 import {setPluginState} from '../reducers/pluginStates';
 import {flipperRecorderAddEvent} from './pluginStateRecorder';
 import {
@@ -92,7 +92,7 @@ function processMessage(
 }

 export function processMessageImmediately(
-  store: Store,
+  store: MiddlewareAPI,
  pluginKey: string,
  plugin: {
    defaultPersistedState: any;
@@ -122,7 +122,7 @@ export function processMessageImmediately(
 }

 export function processMessageLater(
-  store: Store,
+  store: MiddlewareAPI,
  pluginKey: string,
  plugin: {
    defaultPersistedState: any;
@@ -168,7 +168,7 @@ export async function processMessageQueue(
    persistedStateReducer: PersistedStateReducer | null;
  },
  pluginKey: string,
-  store: Store,
+  store: MiddlewareAPI,
  progressCallback?: (progress: {current: number; total: number}) => void,
  idler: BaseIdler = new Idler(),
 ) {
@@ -226,6 +226,9 @@ export async function processMessageQueue(
  } while (getPendingMessages(store, pluginKey).length);
 }

-function getPendingMessages(store: Store, pluginKey: string): Message[] {
+function getPendingMessages(
+  store: MiddlewareAPI,
+  pluginKey: string,
+): Message[] {
  return store.getState().pluginMessageQueue[pluginKey] || [];
 }
--- a/src/utils/pluginUtils.tsx
+++ b/src/utils/pluginUtils.tsx
@@ -11,6 +11,7 @@ import {FlipperDevicePlugin, FlipperPlugin, FlipperBasePlugin} from '../plugin';
 import BaseDevice from '../devices/BaseDevice';
 import {State as PluginStatesState} from '../reducers/pluginStates';
 import {State as PluginsState} from '../reducers/plugins';
+import {State as PluginMessageQueueState} from '../reducers/pluginMessageQueue';
 import {PluginDefinition} from '../dispatcher/plugins';
 import {deconstructPluginKey} from './clientUtils';

@@ -73,6 +74,7 @@ export function getPersistedState<PersistedState>(
 */
 export function getActivePersistentPlugins(
  pluginsState: PluginStatesState,
+  pluginsMessageQueue: PluginMessageQueueState,
  plugins: PluginsState,
  selectedClient?: Client,
 ): Array<string> {
@@ -82,7 +84,12 @@ export function getActivePersistentPlugins(
  > = pluginsClassMap(plugins);
  return getPersistentPlugins(plugins).filter(plugin => {
    const pluginClass = pluginsMap.get(plugin);
-    const keys = Object.keys(pluginsState)
+    const keys = [
+      ...new Set([
+        ...Object.keys(pluginsState),
+        ...Object.keys(pluginsMessageQueue),
+      ]),
+    ]
      .filter(k => !selectedClient || k.includes(selectedClient.id))
      .map(key => deconstructPluginKey(key).pluginName);
    let result = plugin == 'DeviceLogs';