diff --git a/headless/index.tsx b/headless/index.tsx index d1b312373..b952c7ace 100644 --- a/headless/index.tsx +++ b/headless/index.tsx @@ -177,9 +177,7 @@ async function exitActions( ); outputAndExit(payload); } else { - const {serializedString, errorArray} = await exportStore( - store.getState(), - ); + const {serializedString, errorArray} = await exportStore(store); errorArray.forEach(console.error); outputAndExit(serializedString); } @@ -256,7 +254,7 @@ async function startFlipper(userArguments: UserArguments) { errorAndExit(e); }); } else { - exportStore(store.getState()) + exportStore(store) .then(({serializedString}) => { outputAndExit(serializedString); }) diff --git a/src/chrome/ExportDataPluginSheet.tsx b/src/chrome/ExportDataPluginSheet.tsx index bc118f995..388cea854 100644 --- a/src/chrome/ExportDataPluginSheet.tsx +++ b/src/chrome/ExportDataPluginSheet.tsx @@ -13,6 +13,7 @@ import {ShareType} from '../reducers/application'; import {State as PluginState} from '../reducers/plugins'; import {State as PluginStatesState} from '../reducers/pluginStates'; import {State as Store} from '../reducers'; +import {State as PluginMessageQueueState} from '../reducers/pluginMessageQueue'; import {ActiveSheet} from '../reducers/application'; import {selectedPlugins as actionForSelectedPlugins} from '../reducers/plugins'; import {getActivePersistentPlugins} from '../utils/pluginUtils'; @@ -35,6 +36,7 @@ type StateFromProps = { share: ShareType | null; plugins: PluginState; pluginStates: PluginStatesState; + pluginMessageQueue: PluginMessageQueueState; selectedClient: Client | undefined; }; @@ -57,13 +59,20 @@ const Container = styled(FlexColumn)({ class ExportDataPluginSheet extends Component { render() { - const {plugins, pluginStates, onHide, selectedClient} = this.props; + const { + plugins, + pluginStates, + pluginMessageQueue, + onHide, + selectedClient, + } = this.props; const onHideWithUnsettingShare = () => { this.props.unsetShare(); onHide(); }; const pluginsToExport = getActivePersistentPlugins( pluginStates, + pluginMessageQueue, plugins, selectedClient, ); @@ -121,6 +130,7 @@ export default connect( application: {share}, plugins, pluginStates, + pluginMessageQueue, connections: {selectedApp, clients}, }) => { const selectedClient = clients.find(o => { @@ -130,6 +140,7 @@ export default connect( share, plugins, pluginStates, + pluginMessageQueue, selectedClient, }; }, diff --git a/src/store.tsx b/src/store.tsx index a7beb4b38..e167d40af 100644 --- a/src/store.tsx +++ b/src/store.tsx @@ -10,6 +10,7 @@ import {createStore} from 'redux'; import reducers, {Actions, State as StoreState} from './reducers/index'; import {stateSanitizer} from './utils/reduxDevToolsConfig'; +import isProduction from './utils/isProduction'; export const store = createStore( reducers, @@ -20,3 +21,9 @@ export const store = createStore( }) : undefined, ); + +if (!isProduction()) { + // For debugging purposes only + // @ts-ignore + window.flipperStore = store; +} diff --git a/src/utils/__tests__/pluginUtils.node.js b/src/utils/__tests__/pluginUtils.node.js index ddfc7b8e7..a330a6af6 100644 --- a/src/utils/__tests__/pluginUtils.node.js +++ b/src/utils/__tests__/pluginUtils.node.js @@ -14,8 +14,10 @@ import { import type {State as PluginsState} from '../../reducers/plugins.tsx'; import type {State as PluginStatesState} from '../../reducers/pluginStates.tsx'; import type {PluginDefinition} from '../../dispatcher/plugins.tsx'; +import type {State as PluginMessageQueueState} from '../../reducers/pluginStates.tsx'; import {FlipperBasePlugin} from 'flipper'; import type {ReduxState} from '../../reducers/index.tsx'; + class MockFlipperPluginWithDefaultPersistedState extends FlipperBasePlugin< *, *, @@ -157,11 +159,12 @@ test('getActivePersistentPlugins, where the non persistent plugins getting exclu 'serial#app#ClientPlugin1': {msg: 'ClientPlugin1'}, 'serial#app#ClientPlugin2': {msg: 'ClientPlugin2'}, }; - const list = getActivePersistentPlugins(plugins, state); + const queues: PluginMessageQueueState = {}; + const list = getActivePersistentPlugins(plugins, queues, state); expect(list).toEqual(['ClientPlugin1', 'DevicePlugin2']); }); -test('getActivePersistentPlugins, where the plugins not in pluginState gets excluded', () => { +test('getActivePersistentPlugins, where the plugins not in pluginState or queue gets excluded', () => { const state: PluginsState = { devicePlugins: new Map([ ['DevicePlugin1', MockFlipperPluginWithDefaultPersistedState], @@ -170,6 +173,7 @@ test('getActivePersistentPlugins, where the plugins not in pluginState gets excl clientPlugins: new Map([ ['ClientPlugin1', MockFlipperPluginWithDefaultPersistedState], ['ClientPlugin2', MockFlipperPluginWithDefaultPersistedState], + ['ClientPlugin3', MockFlipperPluginWithDefaultPersistedState], ]), gatekeepedPlugins: [], disabledPlugins: [], @@ -180,6 +184,11 @@ test('getActivePersistentPlugins, where the plugins not in pluginState gets excl 'serial#app#DevicePlugin1': {msg: 'DevicePlugin1'}, 'serial#app#ClientPlugin2': {msg: 'ClientPlugin2'}, }; - const list = getActivePersistentPlugins(plugins, state); - expect(list).toEqual(['ClientPlugin2', 'DevicePlugin1']); + const queues: PluginMessageQueueState = { + 'serial#app#ClientPlugin3': [ + {method: 'msg', params: {msg: 'ClientPlugin3'}}, + ], + }; + const list = getActivePersistentPlugins(plugins, queues, state); + expect(list).toEqual(['ClientPlugin2', 'ClientPlugin3', 'DevicePlugin1']); }); diff --git a/src/utils/exportData.tsx b/src/utils/exportData.tsx index 55351b9eb..2b4ec52c4 100644 --- a/src/utils/exportData.tsx +++ b/src/utils/exportData.tsx @@ -11,7 +11,7 @@ import os from 'os'; import path from 'path'; import electron from 'electron'; import {getInstance as getLogger} from '../fb-stubs/Logger'; -import {Store, State as ReduxState} from '../reducers'; +import {Store, State as ReduxState, MiddlewareAPI} from '../reducers'; import {DeviceExport} from '../devices/BaseDevice'; import {State as PluginStatesState} from '../reducers/pluginStates'; import {PluginNotification} from '../reducers/notifications'; @@ -41,6 +41,7 @@ import { import {setSelectPluginsToExportActiveSheet} from '../reducers/application'; import {deconstructClientId, deconstructPluginKey} from '../utils/clientUtils'; import {performance} from 'perf_hooks'; +import {processMessageQueue} from './messageQueue'; export const IMPORT_FLIPPER_TRACE_EVENT = 'import-flipper-trace'; export const EXPORT_FLIPPER_TRACE_EVENT = 'export-flipper-trace'; @@ -82,6 +83,13 @@ type SerializePluginStatesOptions = { pluginStates: PluginStatesState; }; +type PluginsToProcess = { + pluginKey: string; + plugin: string; + pluginClass: typeof FlipperPlugin | typeof FlipperDevicePlugin; + client: Client; +}[]; + type AddSaltToDeviceSerialOptions = { salt: string; device: BaseDevice; @@ -381,85 +389,94 @@ export const processStore = async ( }; export async function fetchMetadata( - clients: Client[], + pluginsToProcess: PluginsToProcess, pluginStates: PluginStatesState, - pluginsMap: Map, state: ReduxState, statusUpdate?: (msg: string) => void, idler?: Idler, ): Promise<{pluginStates: PluginStatesState; errorArray: Array}> { const newPluginState = {...pluginStates}; const errorArray: Array = []; - const selectedDevice = state.connections.selectedDevice; - for (const client of clients) { - if ( - !selectedDevice || - selectedDevice.isArchived || - !client.id.includes(selectedDevice.serial) - ) { - continue; - } - const selectedPlugins = state.plugins.selectedPlugins; - const selectedFilteredPlugins = - selectedPlugins.length > 0 - ? client.plugins.filter(plugin => selectedPlugins.includes(plugin)) - : client.plugins; - for (const plugin of selectedFilteredPlugins) { - const pluginClass: - | typeof FlipperDevicePlugin - | typeof FlipperPlugin - | undefined - | null = plugin ? pluginsMap.get(plugin) : null; - const exportState = pluginClass ? pluginClass.exportPersistedState : null; - if (exportState) { - const key = pluginKey(client.id, plugin); - const fetchMetaDataMarker = `${EXPORT_FLIPPER_TRACE_EVENT}:fetch-meta-data-per-plugin`; - performance.mark(fetchMetaDataMarker); - try { - statusUpdate && - statusUpdate(`Fetching metadata for plugin ${plugin}...`); - const data = await promiseTimeout( - 240000, // Fetching MobileConfig data takes ~ 3 mins, thus keeping timeout at 4 mins. - exportState( - callClient(client, plugin), - newPluginState[key], - state, - idler, - statusUpdate, - ), - `Timed out while collecting data for ${plugin}`, - ); - getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, { - plugin, - }); - newPluginState[key] = data; - } catch (e) { - errorArray.push(e); - getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, { - plugin, - error: e, - }); - continue; - } + for (const {plugin, pluginClass, client, pluginKey} of pluginsToProcess) { + const exportState = pluginClass ? pluginClass.exportPersistedState : null; + if (exportState) { + const fetchMetaDataMarker = `${EXPORT_FLIPPER_TRACE_EVENT}:fetch-meta-data-per-plugin`; + performance.mark(fetchMetaDataMarker); + try { + statusUpdate && + statusUpdate(`Fetching metadata for plugin ${plugin}...`); + const data = await promiseTimeout( + 240000, // Fetching MobileConfig data takes ~ 3 mins, thus keeping timeout at 4 mins. + exportState( + callClient(client, plugin), + newPluginState[pluginKey], + state, + idler, + statusUpdate, + ), + `Timed out while collecting data for ${plugin}`, + ); + getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, { + plugin, + }); + newPluginState[pluginKey] = data; + } catch (e) { + errorArray.push(e); + getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, { + plugin, + error: e, + }); + continue; } } } + return {pluginStates: newPluginState, errorArray}; } -export async function getStoreExport( - state: ReduxState, +async function processQueues( + store: MiddlewareAPI, + pluginsToProcess: PluginsToProcess, statusUpdate?: (msg: string) => void, idler?: Idler, -): Promise<{exportData: ExportType | null; errorArray: Array}> { +) { + for (const {plugin, pluginKey, pluginClass} of pluginsToProcess) { + if (pluginClass.persistedStateReducer) { + const processQueueMarker = `${EXPORT_FLIPPER_TRACE_EVENT}:process-queue-per-plugin`; + performance.mark(processQueueMarker); + + await processMessageQueue( + pluginClass, + pluginKey, + store, + ({current, total}) => { + statusUpdate?.( + `Processing event ${current} / ${total} (${Math.round( + (current / total) * 100, + )}%) for plugin ${plugin}`, + ); + }, + idler, + ); + + getLogger().trackTimeSince(processQueueMarker, processQueueMarker, { + plugin, + }); + } + } +} + +function getSelection( + store: MiddlewareAPI, +): {client: Client; device: BaseDevice} { + const state = store.getState(); const {clients} = state.connections; const client = clients.find( client => client.id === state.connections.selectedApp, ); - const {pluginStates} = state; - const {plugins} = state; const {selectedDevice} = state.connections; + // TODO: T59434642 remove these restrictions if (!selectedDevice) { throw new Error('Please select a device before exporting data.'); } @@ -468,28 +485,68 @@ export async function getStoreExport( if (!client) { throw new Error('Please select a client before exporting data.'); } + return {client, device: selectedDevice}; +} + +export function determinePluginsToProcess( + store: MiddlewareAPI, +): PluginsToProcess { + const state = store.getState(); + const {plugins} = state; + const {clients} = state.connections; + const {client, device} = getSelection(store); + + const pluginsToProcess: PluginsToProcess = []; + const selectedPlugins = state.plugins.selectedPlugins; + const selectedFilteredPlugins = + selectedPlugins.length > 0 + ? client.plugins.filter(plugin => selectedPlugins.includes(plugin)) + : client.plugins; + for (const client of clients) { + if (!device || device.isArchived || !client.id.includes(device.serial)) { + continue; + } + for (const plugin of selectedFilteredPlugins) { + const pluginClass = + plugins.clientPlugins.get(plugin) || plugins.devicePlugins.get(plugin); + if (pluginClass) { + const key = pluginKey(client.id, plugin); + pluginsToProcess.push({ + pluginKey: key, + client, + plugin, + pluginClass, + }); + } + } + } + return pluginsToProcess; +} + +export async function getStoreExport( + store: MiddlewareAPI, + statusUpdate?: (msg: string) => void, + idler?: Idler, +): Promise<{exportData: ExportType | null; errorArray: Array}> { + const pluginsToProcess = determinePluginsToProcess(store); + + statusUpdate?.('Preparing to process data queues for plugins...'); + await processQueues(store, pluginsToProcess, statusUpdate, idler); - const pluginsMap: Map< - string, - typeof FlipperDevicePlugin | typeof FlipperPlugin - > = new Map([]); - plugins.clientPlugins.forEach((val, key) => { - pluginsMap.set(key, val); - }); - plugins.devicePlugins.forEach((val, key) => { - pluginsMap.set(key, val); - }); statusUpdate && statusUpdate('Preparing to fetch metadata from client...'); const fetchMetaDataMarker = `${EXPORT_FLIPPER_TRACE_EVENT}:fetch-meta-data`; performance.mark(fetchMetaDataMarker); + + const {device, client} = getSelection(store); + const state = store.getState(); const metadata = await fetchMetadata( - [client], - pluginStates, - pluginsMap, + pluginsToProcess, + state.pluginStates, state, statusUpdate, idler, ); + getLogger().trackTimeSince(fetchMetaDataMarker, fetchMetaDataMarker, { plugins: state.plugins.selectedPlugins, }); @@ -501,7 +558,7 @@ export async function getStoreExport( const exportData = await processStore( { activeNotifications, - device: selectedDevice, + device, pluginStates: newPluginState, clients: [client.toJSON()], devicePlugins, @@ -515,51 +572,46 @@ export async function getStoreExport( return {exportData, errorArray}; } -export function exportStore( - state: ReduxState, +export async function exportStore( + store: MiddlewareAPI, idler?: Idler, statusUpdate?: (msg: string) => void, ): Promise<{serializedString: string; errorArray: Array}> { getLogger().track('usage', EXPORT_FLIPPER_TRACE_EVENT); - return new Promise(async (resolve, reject) => { - performance.mark(EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT); - try { - statusUpdate && statusUpdate('Preparing to export Flipper data...'); - const {exportData, errorArray} = await getStoreExport( - state, - statusUpdate, - idler, - ); - if (exportData != null) { - exportData.supportRequestDetails = { - ...state.supportForm?.supportFormV2, - appName: - state.connections.selectedApp == null - ? '' - : deconstructClientId(state.connections.selectedApp).app, - }; + performance.mark(EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT); + statusUpdate && statusUpdate('Preparing to export Flipper data...'); + const state = store.getState(); + const {exportData, errorArray} = await getStoreExport( + store, + statusUpdate, + idler, + ); + if (exportData != null) { + exportData.supportRequestDetails = { + ...state.supportForm?.supportFormV2, + appName: + state.connections.selectedApp == null + ? '' + : deconstructClientId(state.connections.selectedApp).app, + }; - statusUpdate && statusUpdate('Serializing Flipper data...'); - const serializedString = JSON.stringify(exportData); - if (serializedString.length <= 0) { - reject(new Error('Serialize function returned empty string')); - } - getLogger().trackTimeSince( - EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT, - EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT, - { - plugins: state.plugins.selectedPlugins, - }, - ); - resolve({serializedString, errorArray}); - } else { - console.error('Make sure a device is connected'); - reject(new Error('No device is selected')); - } - } catch (e) { - reject(e); + statusUpdate && statusUpdate('Serializing Flipper data...'); + const serializedString = JSON.stringify(exportData); + if (serializedString.length <= 0) { + throw new Error('Serialize function returned empty string'); } - }); + getLogger().trackTimeSince( + EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT, + EXPORT_FLIPPER_TRACE_TIME_SERIALIZATION_EVENT, + { + plugins: state.plugins.selectedPlugins, + }, + ); + return {serializedString, errorArray}; + } else { + console.error('Make sure a device is connected'); + throw new Error('No device is selected'); + } } export const exportStoreToFile = ( @@ -568,7 +620,7 @@ export const exportStoreToFile = ( idler?: Idler, statusUpdate?: (msg: string) => void, ): Promise<{errorArray: Array}> => { - return exportStore(store.getState(), idler, statusUpdate).then( + return exportStore(store, idler, statusUpdate).then( ({serializedString, errorArray}) => { return promisify(fs.writeFile)(exportFilePath, serializedString).then( () => { diff --git a/src/utils/exportMetrics.tsx b/src/utils/exportMetrics.tsx index c498db141..30b520189 100644 --- a/src/utils/exportMetrics.tsx +++ b/src/utils/exportMetrics.tsx @@ -12,7 +12,11 @@ import {serialize} from './serialization'; import {State as PluginStatesState} from '../reducers/pluginStates'; import {Store} from '../reducers'; import fs from 'fs'; -import {ExportType, fetchMetadata} from './exportData'; +import { + ExportType, + fetchMetadata, + determinePluginsToProcess, +} from './exportData'; import {deserializeObject} from './serialization'; import {deconstructPluginKey} from './clientUtils'; import {pluginsClassMap} from './pluginUtils'; @@ -67,10 +71,10 @@ export async function exportMetricsWithoutTrace( string, typeof FlipperDevicePlugin | typeof FlipperPlugin > = pluginsClassMap(store.getState().plugins); + const pluginsToProcess = determinePluginsToProcess(store); const metadata = await fetchMetadata( - store.getState().connections.clients, + pluginsToProcess, pluginStates, - pluginsMap, store.getState(), ); const newPluginStates = metadata.pluginStates; diff --git a/src/utils/messageQueue.tsx b/src/utils/messageQueue.tsx index da3908148..d238fa6e4 100644 --- a/src/utils/messageQueue.tsx +++ b/src/utils/messageQueue.tsx @@ -8,7 +8,7 @@ */ import {PersistedStateReducer} from '../plugin'; -import {Store, State} from '../reducers/index'; +import {State, MiddlewareAPI} from '../reducers/index'; import {setPluginState} from '../reducers/pluginStates'; import {flipperRecorderAddEvent} from './pluginStateRecorder'; import { @@ -92,7 +92,7 @@ function processMessage( } export function processMessageImmediately( - store: Store, + store: MiddlewareAPI, pluginKey: string, plugin: { defaultPersistedState: any; @@ -122,7 +122,7 @@ export function processMessageImmediately( } export function processMessageLater( - store: Store, + store: MiddlewareAPI, pluginKey: string, plugin: { defaultPersistedState: any; @@ -168,7 +168,7 @@ export async function processMessageQueue( persistedStateReducer: PersistedStateReducer | null; }, pluginKey: string, - store: Store, + store: MiddlewareAPI, progressCallback?: (progress: {current: number; total: number}) => void, idler: BaseIdler = new Idler(), ) { @@ -226,6 +226,9 @@ export async function processMessageQueue( } while (getPendingMessages(store, pluginKey).length); } -function getPendingMessages(store: Store, pluginKey: string): Message[] { +function getPendingMessages( + store: MiddlewareAPI, + pluginKey: string, +): Message[] { return store.getState().pluginMessageQueue[pluginKey] || []; } diff --git a/src/utils/pluginUtils.tsx b/src/utils/pluginUtils.tsx index 7021aa1b8..70076c02f 100644 --- a/src/utils/pluginUtils.tsx +++ b/src/utils/pluginUtils.tsx @@ -11,6 +11,7 @@ import {FlipperDevicePlugin, FlipperPlugin, FlipperBasePlugin} from '../plugin'; import BaseDevice from '../devices/BaseDevice'; import {State as PluginStatesState} from '../reducers/pluginStates'; import {State as PluginsState} from '../reducers/plugins'; +import {State as PluginMessageQueueState} from '../reducers/pluginMessageQueue'; import {PluginDefinition} from '../dispatcher/plugins'; import {deconstructPluginKey} from './clientUtils'; @@ -73,6 +74,7 @@ export function getPersistedState( */ export function getActivePersistentPlugins( pluginsState: PluginStatesState, + pluginsMessageQueue: PluginMessageQueueState, plugins: PluginsState, selectedClient?: Client, ): Array { @@ -82,7 +84,12 @@ export function getActivePersistentPlugins( > = pluginsClassMap(plugins); return getPersistentPlugins(plugins).filter(plugin => { const pluginClass = pluginsMap.get(plugin); - const keys = Object.keys(pluginsState) + const keys = [ + ...new Set([ + ...Object.keys(pluginsState), + ...Object.keys(pluginsMessageQueue), + ]), + ] .filter(k => !selectedClient || k.includes(selectedClient.id)) .map(key => deconstructPluginKey(key).pluginName); let result = plugin == 'DeviceLogs';