Process queues before exporting plugins

Summary: This diff makes sure that pending queues for plugins that are selected are processed before making a flipper export.

Reviewed By: jknoxville

Differential Revision: D19194211

fbshipit-source-id: e076375889450407e7f94384051719f3bbc415ee

headless/index.tsx

@@ -177,9 +177,7 @@ async function exitActions(
           );
           outputAndExit(payload);
         } else {
-          const {serializedString, errorArray} = await exportStore(
+          const {serializedString, errorArray} = await exportStore(store);
-            store.getState(),
-          );
           errorArray.forEach(console.error);
           outputAndExit(serializedString);
         }
@@ -256,7 +254,7 @@ async function startFlipper(userArguments: UserArguments) {
               errorAndExit(e);
             });
         } else {
-          exportStore(store.getState())
+          exportStore(store)
             .then(({serializedString}) => {
               outputAndExit(serializedString);
             })

src/chrome/ExportDataPluginSheet.tsx

@@ -13,6 +13,7 @@ import {ShareType} from '../reducers/application';
 import {State as PluginState} from '../reducers/plugins';
 import {State as PluginStatesState} from '../reducers/pluginStates';
 import {State as Store} from '../reducers';
+import {State as PluginMessageQueueState} from '../reducers/pluginMessageQueue';
 import {ActiveSheet} from '../reducers/application';
 import {selectedPlugins as actionForSelectedPlugins} from '../reducers/plugins';
 import {getActivePersistentPlugins} from '../utils/pluginUtils';
@@ -35,6 +36,7 @@ type StateFromProps = {
   share: ShareType | null;
   plugins: PluginState;
   pluginStates: PluginStatesState;
+  pluginMessageQueue: PluginMessageQueueState;
   selectedClient: Client | undefined;
 };
 
@@ -57,13 +59,20 @@ const Container = styled(FlexColumn)({
 
 class ExportDataPluginSheet extends Component<Props> {
   render() {
-    const {plugins, pluginStates, onHide, selectedClient} = this.props;
+    const {
+      plugins,
+      pluginStates,
+      pluginMessageQueue,
+      onHide,
+      selectedClient,
+    } = this.props;
     const onHideWithUnsettingShare = () => {
       this.props.unsetShare();
       onHide();
     };
     const pluginsToExport = getActivePersistentPlugins(
       pluginStates,
+      pluginMessageQueue,
       plugins,
       selectedClient,
     );
@@ -121,6 +130,7 @@ export default connect<StateFromProps, DispatchFromProps, OwnProps, Store>(
     application: {share},
     plugins,
     pluginStates,
+    pluginMessageQueue,
     connections: {selectedApp, clients},
   }) => {
     const selectedClient = clients.find(o => {
@@ -130,6 +140,7 @@ export default connect<StateFromProps, DispatchFromProps, OwnProps, Store>(
       share,
       plugins,
       pluginStates,
+      pluginMessageQueue,
       selectedClient,
     };
   },

src/store.tsx

@@ -10,6 +10,7 @@
 import {createStore} from 'redux';
 import reducers, {Actions, State as StoreState} from './reducers/index';
 import {stateSanitizer} from './utils/reduxDevToolsConfig';
+import isProduction from './utils/isProduction';
 
 export const store = createStore<StoreState, Actions, any, any>(
   reducers,
@@ -20,3 +21,9 @@ export const store = createStore<StoreState, Actions, any, any>(
       })
     : undefined,
 );
+
+if (!isProduction()) {
+  // For debugging purposes only
+  // @ts-ignore
+  window.flipperStore = store;
+}

src/utils/__tests__/pluginUtils.node.js

@@ -14,8 +14,10 @@ import {
 import type {State as PluginsState} from '../../reducers/plugins.tsx';
 import type {State as PluginStatesState} from '../../reducers/pluginStates.tsx';
 import type {PluginDefinition} from '../../dispatcher/plugins.tsx';
+import type {State as PluginMessageQueueState} from '../../reducers/pluginStates.tsx';
 import {FlipperBasePlugin} from 'flipper';
 import type {ReduxState} from '../../reducers/index.tsx';
+
 class MockFlipperPluginWithDefaultPersistedState extends FlipperBasePlugin<
   *,
   *,
@@ -157,11 +159,12 @@ test('getActivePersistentPlugins, where the non persistent plugins getting exclu
     'serial#app#ClientPlugin1': {msg: 'ClientPlugin1'},
     'serial#app#ClientPlugin2': {msg: 'ClientPlugin2'},
   };
-  const list = getActivePersistentPlugins(plugins, state);
+  const queues: PluginMessageQueueState = {};
+  const list = getActivePersistentPlugins(plugins, queues, state);
   expect(list).toEqual(['ClientPlugin1', 'DevicePlugin2']);
 });
 
-test('getActivePersistentPlugins, where the plugins not in pluginState gets excluded', () => {
+test('getActivePersistentPlugins, where the plugins not in pluginState or queue gets excluded', () => {
   const state: PluginsState = {
     devicePlugins: new Map([
       ['DevicePlugin1', MockFlipperPluginWithDefaultPersistedState],
@@ -170,6 +173,7 @@ test('getActivePersistentPlugins, where the plugins not in pluginState gets excl
     clientPlugins: new Map([
       ['ClientPlugin1', MockFlipperPluginWithDefaultPersistedState],
       ['ClientPlugin2', MockFlipperPluginWithDefaultPersistedState],
+      ['ClientPlugin3', MockFlipperPluginWithDefaultPersistedState],
     ]),
     gatekeepedPlugins: [],
     disabledPlugins: [],
@@ -180,6 +184,11 @@ test('getActivePersistentPlugins, where the plugins not in pluginState gets excl
     'serial#app#DevicePlugin1': {msg: 'DevicePlugin1'},
     'serial#app#ClientPlugin2': {msg: 'ClientPlugin2'},
   };
-  const list = getActivePersistentPlugins(plugins, state);
+  const queues: PluginMessageQueueState = {
-  expect(list).toEqual(['ClientPlugin2', 'DevicePlugin1']);
+    'serial#app#ClientPlugin3': [
+      {method: 'msg', params: {msg: 'ClientPlugin3'}},
+    ],
+  };
+  const list = getActivePersistentPlugins(plugins, queues, state);
+  expect(list).toEqual(['ClientPlugin2', 'ClientPlugin3', 'DevicePlugin1']);
 });

src/utils/exportData.tsx

@@ -11,7 +11,7 @@ import os from 'os';
 import path from 'path';
 import electron from 'electron';
 import {getInstance as getLogger} from '../fb-stubs/Logger';
-import {Store, State as ReduxState} from '../reducers';
+import {Store, State as ReduxState, MiddlewareAPI} from '../reducers';
 import {DeviceExport} from '../devices/BaseDevice';
 import {State as PluginStatesState} from '../reducers/pluginStates';
 import {PluginNotification} from '../reducers/notifications';
@@ -41,6 +41,7 @@ import {
 import {setSelectPluginsToExportActiveSheet} from '../reducers/application';
 import {deconstructClientId, deconstructPluginKey} from '../utils/clientUtils';
 import {performance} from 'perf_hooks';
+import {processMessageQueue} from './messageQueue';
 
 export const IMPORT_FLIPPER_TRACE_EVENT = 'import-flipper-trace';
 export const EXPORT_FLIPPER_TRACE_EVENT = 'export-flipper-trace';
@@ -82,6 +83,13 @@ type SerializePluginStatesOptions = {
   pluginStates: PluginStatesState;
 };
 
+type PluginsToProcess = {
+  pluginKey: string;
+  plugin: string;
+  pluginClass: typeof FlipperPlugin | typeof FlipperDevicePlugin;
+  client: Client;
+}[];
+
 type AddSaltToDeviceSerialOptions = {
   salt: string;
   device: BaseDevice;
@@ -381,85 +389,94 @@ export const processStore = async (
 };
 
 export async function fetchMetadata(
-  clients: Client[],
+  pluginsToProcess: PluginsToProcess,
   pluginStates: PluginStatesState,
-  pluginsMap: Map<string, typeof FlipperDevicePlugin | typeof FlipperPlugin>,
   state: ReduxState,
   statusUpdate?: (msg: string) => void,
   idler?: Idler,
 ): Promise<{pluginStates: PluginStatesState; errorArray: Array<Error>}> {
   const newPluginState = {...pluginStates};
   const errorArray: Array<Error> = [];
-  const selectedDevice = state.connections.selectedDevice;
-  for (const client of clients) {
-    if (
-      !selectedDevice ||
-      selectedDevice.isArchived ||
-      !client.id.includes(selectedDevice.serial)
-    ) {
-      continue;
-    }
-    const selectedPlugins = state.plugins.selectedPlugins;
-    const selectedFilteredPlugins =
-      selectedPlugins.length > 0
-        ? client.plugins.filter(plugin => selectedPlugins.includes(plugin))
-        : client.plugins;
-    for (const plugin of selectedFilteredPlugins) {
-      const pluginClass:
-        | typeof FlipperDevicePlugin
-        | typeof FlipperPlugin
-        | undefined
-        | null = plugin ? pluginsMap.get(plugin) : null;
-      const exportState = pluginClass ? pluginClass.exportPersistedState : null;
-      if (exportState) {
-        const key = pluginKey(client.id, plugin);
-        const fetchMetaDataMarker = `${EXPORT_FLIPPER_TRACE_EVENT}:fetch-meta-data-per-plugin`;
-        performance.mark(fetchMetaDataMarker);
-        try {
-          statusUpdate &&
-            statusUpdate(`Fetching metadata for plugin ${plugin}...`);
-          const data = await promiseTimeout(
-            240000, // Fetching MobileConfig data takes ~ 3 mins, thus keeping timeout at 4 mins.
-            exportState(
-              callClient(client, plugin),
-              newPluginState[key],
-              state,
-              idler,
-              statusUpdate,
-            ),
 
-            `Timed out while collecting data for ${plugin}`,
+  for (const {plugin, pluginClass, client, pluginKey} of pluginsToProcess) {
-          );
+    const exportState = pluginClass ? pluginClass.exportPersistedState : null;
-          getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, {
+    if (exportState) {
-            plugin,
+      const fetchMetaDataMarker = `${EXPORT_FLIPPER_TRACE_EVENT}:fetch-meta-data-per-plugin`;
-          });
+      performance.mark(fetchMetaDataMarker);
-          newPluginState[key] = data;
+      try {
-        } catch (e) {
+        statusUpdate &&
-          errorArray.push(e);
+          statusUpdate(`Fetching metadata for plugin ${plugin}...`);
-          getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, {
+        const data = await promiseTimeout(
-            plugin,
+          240000, // Fetching MobileConfig data takes ~ 3 mins, thus keeping timeout at 4 mins.
-            error: e,
+          exportState(
-          });
+            callClient(client, plugin),
-          continue;
+            newPluginState[pluginKey],
-        }
+            state,
+            idler,
+            statusUpdate,
+          ),
+          `Timed out while collecting data for ${plugin}`,
+        );
+        getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, {
+          plugin,
+        });
+        newPluginState[pluginKey] = data;
+      } catch (e) {
+        errorArray.push(e);
+        getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, {
+          plugin,
+          error: e,
+        });
+        continue;
       }
     }
   }
+
   return {pluginStates: newPluginState, errorArray};
 }
 
-export async function getStoreExport(
+async function processQueues(
-  state: ReduxState,
+  store: MiddlewareAPI,
+  pluginsToProcess: PluginsToProcess,
   statusUpdate?: (msg: string) => void,
   idler?: Idler,
-): Promise<{exportData: ExportType | null; errorArray: Array<Error>}> {
+) {
+  for (const {plugin, pluginKey, pluginClass} of pluginsToProcess) {
+    if (pluginClass.persistedStateReducer) {
+      const processQueueMarker = `${EXPORT_FLIPPER_TRACE_EVENT}:process-queue-per-plugin`;
+      performance.mark(processQueueMarker);
+
+      await processMessageQueue(
+        pluginClass,
+        pluginKey,
+        store,
+        ({current, total}) => {
+          statusUpdate?.(
+            `Processing event ${current} / ${total} (${Math.round(
+              (current / total) * 100,
+            )}%) for plugin ${plugin}`,
+          );
+        },
+        idler,
+      );
+
+      getLogger().trackTimeSince(processQueueMarker, processQueueMarker, {
+        plugin,
+      });
+    }
+  }
+}
+
+function getSelection(
+  store: MiddlewareAPI,
+): {client: Client; device: BaseDevice} {
+  const state = store.getState();
   const {clients} = state.connections;
   const client = clients.find(
     client => client.id === state.connections.selectedApp,
   );
-  const {pluginStates} = state;
-  const {plugins} = state;
   const {selectedDevice} = state.connections;
+  // TODO: T59434642 remove these restrictions
   if (!selectedDevice) {
     throw new Error('Please select a device before exporting data.');
   }
@@ -468,28 +485,68 @@ export async function getStoreExport(
   if (!client) {
     throw new Error('Please select a client before exporting data.');
   }
+  return {client, device: selectedDevice};
+}
+
+export function determinePluginsToProcess(
+  store: MiddlewareAPI,
+): PluginsToProcess {
+  const state = store.getState();
+  const {plugins} = state;
+  const {clients} = state.connections;
+  const {client, device} = getSelection(store);
+
+  const pluginsToProcess: PluginsToProcess = [];
+  const selectedPlugins = state.plugins.selectedPlugins;
+  const selectedFilteredPlugins =
+    selectedPlugins.length > 0
+      ? client.plugins.filter(plugin => selectedPlugins.includes(plugin))
+      : client.plugins;
+  for (const client of clients) {
+    if (!device || device.isArchived || !client.id.includes(device.serial)) {
+      continue;
+    }
+    for (const plugin of selectedFilteredPlugins) {
+      const pluginClass =
+        plugins.clientPlugins.get(plugin) || plugins.devicePlugins.get(plugin);
+      if (pluginClass) {
+        const key = pluginKey(client.id, plugin);
+        pluginsToProcess.push({
+          pluginKey: key,
+          client,
+          plugin,
+          pluginClass,
+        });
+      }
+    }
+  }
+  return pluginsToProcess;
+}
+
+export async function getStoreExport(
+  store: MiddlewareAPI,
+  statusUpdate?: (msg: string) => void,
+  idler?: Idler,
+): Promise<{exportData: ExportType | null; errorArray: Array<Error>}> {
+  const pluginsToProcess = determinePluginsToProcess(store);
+
+  statusUpdate?.('Preparing to process data queues for plugins...');
+  await processQueues(store, pluginsToProcess, statusUpdate, idler);
 
-  const pluginsMap: Map<
-    string,
-    typeof FlipperDevicePlugin | typeof FlipperPlugin
-  > = new Map([]);
-  plugins.clientPlugins.forEach((val, key) => {
-    pluginsMap.set(key, val);
-  });
-  plugins.devicePlugins.forEach((val, key) => {
-    pluginsMap.set(key, val);
-  });
   statusUpdate && statusUpdate('Preparing to fetch metadata from client...');
   const fetchMetaDataMarker = `${EXPORT_FLIPPER_TRACE_EVENT}:fetch-meta-data`;
   performance.mark(fetchMetaDataMarker);
+
+  const {device, client} = getSelection(store);
+  const state = store.getState();
   const metadata = await fetchMetadata(
-    [client],
+    pluginsToProcess,
-    pluginStates,
+    state.pluginStates,
-    pluginsMap,
     state,
     statusUpdate,
     idler,
   );
+
   getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, {
     plugins: state.plugins.selectedPlugins,
   });
@@ -501,7 +558,7 @@ export async function getStoreExport(
   const exportData = await processStore(
     {
       activeNotifications,
-      device: selectedDevice,
+      device,
       pluginStates: newPluginState,
       clients: [client.toJSON()],
       devicePlugins,
@@ -515,51 +572,46 @@ export async function getStoreExport(
   return {exportData, errorArray};
 }
 
-export function exportStore(
+export async function exportStore(
-  state: ReduxState,
+  store: MiddlewareAPI,
   idler?: Idler,
   statusUpdate?: (msg: string) => void,
 ): Promise<{serializedString: string; errorArray: Array<Error>}> {
   getLogger().track('usage', EXPORT_FLIPPER_TRACE_EVENT);
-  return new Promise(async (resolve, reject) => {
+  performance.mark(EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT);
-    performance.mark(EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT);
+  statusUpdate && statusUpdate('Preparing to export Flipper data...');
-    try {
+  const state = store.getState();
-      statusUpdate && statusUpdate('Preparing to export Flipper data...');
+  const {exportData, errorArray} = await getStoreExport(
-      const {exportData, errorArray} = await getStoreExport(
+    store,
-        state,
+    statusUpdate,
-        statusUpdate,
+    idler,
-        idler,
+  );
-      );
+  if (exportData != null) {
-      if (exportData != null) {
+    exportData.supportRequestDetails = {
-        exportData.supportRequestDetails = {
+      ...state.supportForm?.supportFormV2,
-          ...state.supportForm?.supportFormV2,
+      appName:
-          appName:
+        state.connections.selectedApp == null
-            state.connections.selectedApp == null
+          ? ''
-              ? ''
+          : deconstructClientId(state.connections.selectedApp).app,
-              : deconstructClientId(state.connections.selectedApp).app,
+    };
-        };
 
-        statusUpdate && statusUpdate('Serializing Flipper data...');
+    statusUpdate && statusUpdate('Serializing Flipper data...');
-        const serializedString = JSON.stringify(exportData);
+    const serializedString = JSON.stringify(exportData);
-        if (serializedString.length <= 0) {
+    if (serializedString.length <= 0) {
-          reject(new Error('Serialize function returned empty string'));
+      throw new Error('Serialize function returned empty string');
-        }
-        getLogger().trackTimeSince(
-          EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT,
-          EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT,
-          {
-            plugins: state.plugins.selectedPlugins,
-          },
-        );
-        resolve({serializedString, errorArray});
-      } else {
-        console.error('Make sure a device is connected');
-        reject(new Error('No device is selected'));
-      }
-    } catch (e) {
-      reject(e);
     }
-  });
+    getLogger().trackTimeSince(
+      EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT,
+      EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT,
+      {
+        plugins: state.plugins.selectedPlugins,
+      },
+    );
+    return {serializedString, errorArray};
+  } else {
+    console.error('Make sure a device is connected');
+    throw new Error('No device is selected');
+  }
 }
 
 export const exportStoreToFile = (
@@ -568,7 +620,7 @@ export const exportStoreToFile = (
   idler?: Idler,
   statusUpdate?: (msg: string) => void,
 ): Promise<{errorArray: Array<Error>}> => {
-  return exportStore(store.getState(), idler, statusUpdate).then(
+  return exportStore(store, idler, statusUpdate).then(
     ({serializedString, errorArray}) => {
       return promisify(fs.writeFile)(exportFilePath, serializedString).then(
         () => {

src/utils/exportMetrics.tsx

@@ -12,7 +12,11 @@ import {serialize} from './serialization';
 import {State as PluginStatesState} from '../reducers/pluginStates';
 import {Store} from '../reducers';
 import fs from 'fs';
-import {ExportType, fetchMetadata} from './exportData';
+import {
+  ExportType,
+  fetchMetadata,
+  determinePluginsToProcess,
+} from './exportData';
 import {deserializeObject} from './serialization';
 import {deconstructPluginKey} from './clientUtils';
 import {pluginsClassMap} from './pluginUtils';
@@ -67,10 +71,10 @@ export async function exportMetricsWithoutTrace(
     string,
     typeof FlipperDevicePlugin | typeof FlipperPlugin
   > = pluginsClassMap(store.getState().plugins);
+  const pluginsToProcess = determinePluginsToProcess(store);
   const metadata = await fetchMetadata(
-    store.getState().connections.clients,
+    pluginsToProcess,
     pluginStates,
-    pluginsMap,
     store.getState(),
   );
   const newPluginStates = metadata.pluginStates;

src/utils/messageQueue.tsx

@@ -8,7 +8,7 @@
  */
 
 import {PersistedStateReducer} from '../plugin';
-import {Store, State} from '../reducers/index';
+import {State, MiddlewareAPI} from '../reducers/index';
 import {setPluginState} from '../reducers/pluginStates';
 import {flipperRecorderAddEvent} from './pluginStateRecorder';
 import {
@@ -92,7 +92,7 @@ function processMessage(
 }
 
 export function processMessageImmediately(
-  store: Store,
+  store: MiddlewareAPI,
   pluginKey: string,
   plugin: {
     defaultPersistedState: any;
@@ -122,7 +122,7 @@ export function processMessageImmediately(
 }
 
 export function processMessageLater(
-  store: Store,
+  store: MiddlewareAPI,
   pluginKey: string,
   plugin: {
     defaultPersistedState: any;
@@ -168,7 +168,7 @@ export async function processMessageQueue(
     persistedStateReducer: PersistedStateReducer | null;
   },
   pluginKey: string,
-  store: Store,
+  store: MiddlewareAPI,
   progressCallback?: (progress: {current: number; total: number}) => void,
   idler: BaseIdler = new Idler(),
 ) {
@@ -226,6 +226,9 @@ export async function processMessageQueue(
   } while (getPendingMessages(store, pluginKey).length);
 }
 
-function getPendingMessages(store: Store, pluginKey: string): Message[] {
+function getPendingMessages(
+  store: MiddlewareAPI,
+  pluginKey: string,
+): Message[] {
   return store.getState().pluginMessageQueue[pluginKey] || [];
 }

src/utils/pluginUtils.tsx

@@ -11,6 +11,7 @@ import {FlipperDevicePlugin, FlipperPlugin, FlipperBasePlugin} from '../plugin';
 import BaseDevice from '../devices/BaseDevice';
 import {State as PluginStatesState} from '../reducers/pluginStates';
 import {State as PluginsState} from '../reducers/plugins';
+import {State as PluginMessageQueueState} from '../reducers/pluginMessageQueue';
 import {PluginDefinition} from '../dispatcher/plugins';
 import {deconstructPluginKey} from './clientUtils';
 
@@ -73,6 +74,7 @@ export function getPersistedState<PersistedState>(
  */
 export function getActivePersistentPlugins(
   pluginsState: PluginStatesState,
+  pluginsMessageQueue: PluginMessageQueueState,
   plugins: PluginsState,
   selectedClient?: Client,
 ): Array<string> {
@@ -82,7 +84,12 @@ export function getActivePersistentPlugins(
   > = pluginsClassMap(plugins);
   return getPersistentPlugins(plugins).filter(plugin => {
     const pluginClass = pluginsMap.get(plugin);
-    const keys = Object.keys(pluginsState)
+    const keys = [
+      ...new Set([
+        ...Object.keys(pluginsState),
+        ...Object.keys(pluginsMessageQueue),
+      ]),
+    ]
       .filter(k => !selectedClient || k.includes(selectedClient.id))
       .map(key => deconstructPluginKey(key).pluginName);
     let result = plugin == 'DeviceLogs';