From 20db85adf4f5fb881db4a6f4a21c089ac2c26497 Mon Sep 17 00:00:00 2001 From: John Knox Date: Fri, 21 Feb 2020 07:17:32 -0800 Subject: [PATCH] Add archived device visualizer Summary: Adds a visual indicator of layout nodes when using an archived device. The "device" window isn't movable for some reason. I've tried the movable and draggable attributes but with no luck. It would obviously be good to fix that but I think its probably shippable as is, and I don't have any more ideas about how to do it. Reviewed By: passy Differential Revision: D19885719 fbshipit-source-id: 186ba38c85afee18ce111e30187bdccd9b919025 --- src/devices/AndroidDevice.tsx | 1 + src/devices/ArchivedDevice.tsx | 8 +- src/devices/MetroDevice.tsx | 1 + src/fb-stubs/user.tsx | 6 + src/index.tsx | 1 + src/plugins/layout/Inspector.tsx | 6 +- src/plugins/layout/ProxyArchiveClient.tsx | 12 +- src/plugins/layout/index.tsx | 128 +++++++++++++++++- .../elements-inspector/Visualizer.tsx | 113 ++++++++++++++++ .../elements-inspector/elements.tsx | 4 +- src/ui/index.tsx | 1 + src/utils/__tests__/exportData.electron.tsx | 24 +++- src/utils/exportData.tsx | 5 +- static/index.js | 1 + 14 files changed, 302 insertions(+), 9 deletions(-) create mode 100644 src/ui/components/elements-inspector/Visualizer.tsx diff --git a/src/devices/AndroidDevice.tsx b/src/devices/AndroidDevice.tsx index 8980aa10f..2a773f702 100644 --- a/src/devices/AndroidDevice.tsx +++ b/src/devices/AndroidDevice.tsx @@ -89,6 +89,7 @@ export default class AndroidDevice extends BaseDevice { this.title, this.os, [...this.logEntries], + null, ); } diff --git a/src/devices/ArchivedDevice.tsx b/src/devices/ArchivedDevice.tsx index d8ef642a3..4542751ff 100644 --- a/src/devices/ArchivedDevice.tsx +++ b/src/devices/ArchivedDevice.tsx @@ -28,6 +28,7 @@ export default class ArchivedDevice extends BaseDevice { title: string, os: OS, logEntries: Array, + screenshotHandle: string | null, source: string = '', supportRequestDetails?: SupportFormRequestDetailsState, ) { @@ -35,10 +36,11 @@ export default class ArchivedDevice extends BaseDevice { this.logs = logEntries; this.source = source; this.supportRequestDetails = supportRequestDetails; + this.archivedScreenshotHandle = screenshotHandle; } logs: Array; - + archivedScreenshotHandle: string | null; isArchived = true; displayTitle(): string { @@ -59,4 +61,8 @@ export default class ArchivedDevice extends BaseDevice { spawnShell(): DeviceShell | undefined | null { return null; } + + getArchivedScreenshotHandle(): string | null { + return this.archivedScreenshotHandle; + } } diff --git a/src/devices/MetroDevice.tsx b/src/devices/MetroDevice.tsx index e1a38969e..b95b1bd3c 100644 --- a/src/devices/MetroDevice.tsx +++ b/src/devices/MetroDevice.tsx @@ -206,6 +206,7 @@ export default class MetroDevice extends BaseDevice { this.title, this.os, [...this.logEntries], + null, ); } } diff --git a/src/fb-stubs/user.tsx b/src/fb-stubs/user.tsx index 24fb30658..fdcbc1bd6 100644 --- a/src/fb-stubs/user.tsx +++ b/src/fb-stubs/user.tsx @@ -62,3 +62,9 @@ export async function uploadFlipperMedia( ): Promise { throw new Error('Feature not implemented'); } +export async function getFlipperMediaCDN( + _uploadID: string, + _kind: 'Image' | 'Video', +): Promise { + throw new Error('Feature not implemented'); +} diff --git a/src/index.tsx b/src/index.tsx index 015d49470..0e86583ee 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -170,3 +170,4 @@ export {InspectorSidebar} from './ui/components/elements-inspector/sidebar'; export {Console} from './ui/components/console'; export {default as Sheet} from './ui/components/Sheet'; export {KeyboardActions} from './MenuBar'; +export {getFlipperMediaCDN} from './fb-stubs/user'; diff --git a/src/plugins/layout/Inspector.tsx b/src/plugins/layout/Inspector.tsx index bf25ed783..8d6271264 100644 --- a/src/plugins/layout/Inspector.tsx +++ b/src/plugins/layout/Inspector.tsx @@ -296,12 +296,12 @@ export default class Inspector extends Component { this.props.onSelect(selectedKey); }); - onElementHovered = debounce((key: ElementID | null | undefined) => + onElementHovered = debounce((key: ElementID | null | undefined) => { this.props.client.call(this.call().SET_HIGHLIGHTED, { id: key, isAlignmentMode: this.props.inAlignmentMode, - }), - ); + }); + }); onElementExpanded = ( id: ElementID, diff --git a/src/plugins/layout/ProxyArchiveClient.tsx b/src/plugins/layout/ProxyArchiveClient.tsx index 0648ea64d..77e5d7e4d 100644 --- a/src/plugins/layout/ProxyArchiveClient.tsx +++ b/src/plugins/layout/ProxyArchiveClient.tsx @@ -84,10 +84,15 @@ export function searchNodes( } class ProxyArchiveClient { - constructor(persistedState: PersistedState) { + constructor( + persistedState: PersistedState, + onElementHighlighted?: (id: string) => void, + ) { this.persistedState = cloneDeep(persistedState); + this.onElementHighlighted = onElementHighlighted; } persistedState: PersistedState; + onElementHighlighted: ((id: string) => void) | undefined; subscribe(_method: string, _callback: (params: any) => void): void { return; } @@ -175,6 +180,11 @@ class ProxyArchiveClient { case 'isConsoleEnabled': { return Promise.resolve(false); } + case 'setHighlighted': { + const id = paramaters?.id; + this.onElementHighlighted && this.onElementHighlighted(id); + return Promise.resolve(); + } default: { return Promise.resolve(); } diff --git a/src/plugins/layout/index.tsx b/src/plugins/layout/index.tsx index f3067011f..64e9e3e1b 100644 --- a/src/plugins/layout/index.tsx +++ b/src/plugins/layout/index.tsx @@ -27,6 +27,7 @@ import { SupportRequestFormV2, constants, ReduxState, + ArchivedDevice, } from 'flipper'; import Inspector from './Inspector'; import ToolbarIcon from './ToolbarIcon'; @@ -34,6 +35,8 @@ import InspectorSidebar from './InspectorSidebar'; import Search from './Search'; import ProxyArchiveClient from './ProxyArchiveClient'; import React from 'react'; +import {VisualizerPortal} from 'flipper'; +import {getFlipperMediaCDN} from 'flipper'; type State = { init: boolean; @@ -42,7 +45,11 @@ type State = { inAlignmentMode: boolean; selectedElement: ElementID | null | undefined; selectedAXElement: ElementID | null | undefined; + highlightedElement: ElementID | null; searchResults: ElementSearchResultSet | null; + visualizerWindow: Window | null; + visualizerScreenshot: string | null; + screenDimensions: {width: number; height: number} | null; }; export type ElementMap = {[key: string]: Element}; @@ -125,6 +132,10 @@ export default class Layout extends FlipperPlugin { return JSON.parse(serializedString); }; + teardown() { + this.state.visualizerWindow?.close(); + } + static defaultPersistedState = { rootElement: null, rootAXElement: null, @@ -140,6 +151,10 @@ export default class Layout extends FlipperPlugin { selectedElement: null, selectedAXElement: null, searchResults: null, + visualizerWindow: null, + highlightedElement: null, + visualizerScreenshot: null, + screenDimensions: null, }; init() { @@ -160,6 +175,22 @@ export default class Layout extends FlipperPlugin { } }); + if (this.props.isArchivedDevice) { + this.getDevice() + .then(d => { + const handle = (d as ArchivedDevice).getArchivedScreenshotHandle(); + if (!handle) { + throw new Error('No screenshot attached.'); + } + return handle; + }) + .then(handle => getFlipperMediaCDN(handle, 'Image')) + .then(url => this.setState({visualizerScreenshot: url})) + .catch(_ => { + // Not all exports have screenshots. This is ok. + }); + } + this.setState({ init: true, selectedElement: this.props.deepLinkPayload @@ -180,7 +211,9 @@ export default class Layout extends FlipperPlugin { getClient(): PluginClient { return this.props.isArchivedDevice - ? new ProxyArchiveClient(this.props.persistedState) + ? new ProxyArchiveClient(this.props.persistedState, (id: string) => { + this.setState({highlightedElement: id}); + }) : this.client; } onToggleAlignmentMode = () => { @@ -193,6 +226,34 @@ export default class Layout extends FlipperPlugin { this.setState({inAlignmentMode: !this.state.inAlignmentMode}); }; + onToggleVisualizer = () => { + if (this.state.visualizerWindow) { + this.state.visualizerWindow.close(); + } else { + const screenDimensions = this.state.screenDimensions; + if (!screenDimensions) { + return; + } + const visualizerWindow = window.open( + '', + 'visualizer', + `width=${screenDimensions.width},height=${screenDimensions.height}`, + ); + if (!visualizerWindow) { + return; + } + visualizerWindow.onunload = () => { + this.setState({visualizerWindow: null}); + }; + visualizerWindow.onresize = () => { + this.setState({visualizerWindow: visualizerWindow}); + }; + visualizerWindow.onload = () => { + this.setState({visualizerWindow: visualizerWindow}); + }; + } + }; + onDataValueChanged = (path: Array, value: any) => { const id = this.state.inAXMode ? this.state.selectedAXElement @@ -205,6 +266,47 @@ export default class Layout extends FlipperPlugin { }); }; showFlipperADBar: boolean = false; + + getScreenDimensions(): {width: number; height: number} | null { + if (this.state.screenDimensions) { + return this.state.screenDimensions; + } + + requestIdleCallback(() => { + // Walk the layout tree from root node down until a node with width and height is found. + // Assume these are the dimensions of the screen. + let elementId = this.props.persistedState.rootElement; + while (elementId != null) { + const element = this.props.persistedState.elements[elementId]; + if (!element) { + return null; + } + if (element.data.View?.width) { + break; + } + elementId = element.children[0]; + } + if (elementId == null) { + return null; + } + const element = this.props.persistedState.elements[elementId]; + if ( + element == null || + typeof element.data.View?.width != 'object' || + typeof element.data.View?.height != 'object' + ) { + return null; + } + const screenDimensions = { + width: element.data.View?.width.value, + height: element.data.View?.height.value, + }; + this.setState({screenDimensions}); + }); + + return null; + } + render() { const inspectorProps = { client: this.getClient(), @@ -248,6 +350,8 @@ export default class Layout extends FlipperPlugin { const showAnalyzeYogaPerformanceButton = GK.get('flipper_yogaperformance'); + const screenDimensions = this.getScreenDimensions(); + return ( {this.state.init && ( @@ -277,6 +381,15 @@ export default class Layout extends FlipperPlugin { active={this.state.inAlignmentMode} /> )} + {this.props.isArchivedDevice && + this.state.visualizerScreenshot && ( + + )} { ) : null} + {this.state.visualizerWindow && + screenDimensions && + (this.state.visualizerScreenshot ? ( + + ) : ( + 'Loading...' + ))} )} diff --git a/src/ui/components/elements-inspector/Visualizer.tsx b/src/ui/components/elements-inspector/Visualizer.tsx new file mode 100644 index 000000000..af8d0ccc6 --- /dev/null +++ b/src/ui/components/elements-inspector/Visualizer.tsx @@ -0,0 +1,113 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +import React from 'react'; +import ReactDOM from 'react-dom'; +import {Element, styled} from '../../../ui'; + +export function VisualizerPortal(props: { + container: HTMLElement; + highlightedElement: string | null; + elements: {[key: string]: Element}; + screenshotURL: string; + screenDimensions: {width: number; height: number}; +}) { + const element: Element | null | '' = + props.highlightedElement && props.elements[props.highlightedElement]; + + const position = + element && + typeof element.data.View?.positionOnScreenX == 'number' && + typeof element.data.View?.positionOnScreenY == 'number' && + typeof element.data.View.width === 'object' && + element.data.View.width.value != null && + typeof element.data.View.height === 'object' && + element.data.View.height.value != null + ? { + x: element.data.View.positionOnScreenX, + y: element.data.View.positionOnScreenY, + width: element.data.View.width.value, + height: element.data.View.height.value, + } + : null; + + return ReactDOM.createPortal( + , + props.container, + ); +} + +const VisualizerContainer = styled.div({ + position: 'relative', + top: 0, + left: 0, + width: '100%', + height: '100%', + userSelect: 'none', +}); + +const DeviceImage = styled.img(({width, height}) => ({ + width, + height, + userSelect: 'none', +})); + +/** + * Component that displays a static picture of a device + * and renders "highlighted" rectangles over arbitrary points on it. + * Used for emulating the layout plugin when a device isn't connected. + */ +function Visualizer(props: { + screenDimensions: {width: number; height: number}; + element: {x: number; y: number; width: number; height: number} | null; + imageURL: string; +}) { + const containerRef: React.Ref = React.createRef(); + const imageRef: React.Ref = React.createRef(); + let w: number = 0; + let h: number = 0; + const [scale, updateScale] = React.useState(1); + + React.useLayoutEffect(() => { + w = containerRef.current?.offsetWidth || 0; + h = containerRef.current?.offsetHeight || 0; + const xScale = props.screenDimensions.width / w; + const yScale = props.screenDimensions.height / h; + updateScale(Math.max(xScale, yScale)); + imageRef.current?.setAttribute('draggable', 'false'); + }); + return ( + + + {props.element && ( +
+ )} + /> + + ); +} diff --git a/src/ui/components/elements-inspector/elements.tsx b/src/ui/components/elements-inspector/elements.tsx index 56a83ba6e..cad5c942a 100644 --- a/src/ui/components/elements-inspector/elements.tsx +++ b/src/ui/components/elements-inspector/elements.tsx @@ -635,7 +635,9 @@ export class Elements extends PureComponent { key={row.key} even={isEven} onElementExpanded={onElementExpanded} - onElementHovered={onElementHovered} + onElementHovered={(key: string | null | undefined) => { + onElementHovered && onElementHovered(key); + }} onElementSelected={onElementSelected} selected={selected === row.key} focused={focused === row.key} diff --git a/src/ui/index.tsx b/src/ui/index.tsx index d33c80f5f..7529fb731 100644 --- a/src/ui/index.tsx +++ b/src/ui/index.tsx @@ -155,6 +155,7 @@ export {Elements} from './components/elements-inspector/elements'; export {ContextMenuExtension} from './components/elements-inspector/elements'; export {default as ElementsInspector} from './components/elements-inspector/ElementsInspector'; export {InspectorSidebar} from './components/elements-inspector/sidebar'; +export {VisualizerPortal} from './components/elements-inspector/Visualizer'; export {Console} from './components/console'; diff --git a/src/utils/__tests__/exportData.electron.tsx b/src/utils/__tests__/exportData.electron.tsx index 11bbcc89c..0eac86439 100644 --- a/src/utils/__tests__/exportData.electron.tsx +++ b/src/utils/__tests__/exportData.electron.tsx @@ -77,6 +77,7 @@ test('test generateClientIndentifierWithSalt helper function', () => { 'TestiPhone', 'iOS', [], + null, ); const identifier = generateClientIdentifier(device, 'app'); const saltIdentifier = generateClientIdentifierWithSalt(identifier, 'salt'); @@ -91,6 +92,7 @@ test('test generateClientFromClientWithSalt helper function', () => { 'TestiPhone', 'iOS', [], + null, ); const client = generateClientFromDevice(device, 'app'); const saltedClient = generateClientFromClientWithSalt(client, 'salt'); @@ -121,6 +123,7 @@ test('test generateClientFromDevice helper function', () => { 'TestiPhone', 'iOS', [], + null, ); const client = generateClientFromDevice(device, 'app'); expect(client).toEqual({ @@ -141,6 +144,7 @@ test('test generateClientIdentifier helper function', () => { 'TestiPhone', 'iOS', [], + null, ); const identifier = generateClientIdentifier(device, 'app'); expect(identifier).toEqual('app#iOS#archivedEmulator#serial'); @@ -173,7 +177,14 @@ test('test processStore function for empty state', () => { test('test processStore function for an iOS device connected', async () => { const json = await processStore({ activeNotifications: [], - device: new ArchivedDevice('serial', 'emulator', 'TestiPhone', 'iOS', []), + device: new ArchivedDevice( + 'serial', + 'emulator', + 'TestiPhone', + 'iOS', + [], + null, + ), pluginStates: {}, clients: [], devicePlugins: new Map(), @@ -209,6 +220,7 @@ test('test processStore function for an iOS device connected with client plugin 'TestiPhone', 'iOS', [], + null, ); const clientIdentifier = generateClientIdentifier(device, 'testapp'); const json = await processStore({ @@ -246,6 +258,7 @@ test('test processStore function to have only the client for the selected device 'TestiPhone', 'iOS', [], + null, ); const unselectedDevice = new ArchivedDevice( 'identifier', @@ -253,6 +266,7 @@ test('test processStore function to have only the client for the selected device 'TestiPhone', 'iOS', [], + null, ); const unselectedDeviceClientIdentifier = generateClientIdentifier( @@ -313,6 +327,7 @@ test('test processStore function to have multiple clients for the selected devic 'TestiPhone', 'iOS', [], + null, ); const clientIdentifierApp1 = generateClientIdentifier( @@ -379,6 +394,7 @@ test('test processStore function for device plugin state and no clients', async 'TestiPhone', 'iOS', [], + null, ); const json = await processStore({ activeNotifications: [], @@ -416,6 +432,7 @@ test('test processStore function for unselected device plugin state and no clien 'TestiPhone', 'iOS', [], + null, ); const json = await processStore({ activeNotifications: [], @@ -449,6 +466,7 @@ test('test processStore function for notifications for selected device', async ( 'TestiPhone', 'iOS', [], + null, ); const client = generateClientFromDevice(selectedDevice, 'testapp1'); const notification = generateNotifications( @@ -498,6 +516,7 @@ test('test processStore function for notifications for unselected device', async 'TestiPhone', 'iOS', [], + null, ); const unselectedDevice = new ArchivedDevice( 'identifier', @@ -505,6 +524,7 @@ test('test processStore function for notifications for unselected device', async 'TestiPhone', 'iOS', [], + null, ); const client = generateClientFromDevice(selectedDevice, 'testapp1'); @@ -552,6 +572,7 @@ test('test processStore function for selected plugins', async () => { 'TestiPhone', 'iOS', [], + null, ); const client = generateClientFromDevice(selectedDevice, 'app'); @@ -602,6 +623,7 @@ test('test processStore function for no selected plugins', async () => { 'TestiPhone', 'iOS', [], + null, ); const client = generateClientFromDevice(selectedDevice, 'app'); const pluginstates = { diff --git a/src/utils/exportData.tsx b/src/utils/exportData.tsx index 7331ea9ea..5a7b9ffd9 100644 --- a/src/utils/exportData.tsx +++ b/src/utils/exportData.tsx @@ -274,6 +274,7 @@ const addSaltToDeviceSerial = async ( device.title, device.os, selectedPlugins.includes('DeviceLogs') ? device.getLogs() : [], + deviceScreenshot, ); statusUpdate && statusUpdate('Adding salt to the selected device id in the client data...'); @@ -674,11 +675,12 @@ export const exportStoreToFile = ( export function importDataToStore(source: string, data: string, store: Store) { getLogger().track('usage', IMPORT_FLIPPER_TRACE_EVENT); const json: ExportType = JSON.parse(data); - const {device, clients, supportRequestDetails} = json; + const {device, clients, supportRequestDetails, deviceScreenshot} = json; if (device == null) { return; } const {serial, deviceType, title, os, logs} = device; + const archivedDevice = new ArchivedDevice( serial, deviceType, @@ -689,6 +691,7 @@ export function importDataToStore(source: string, data: string, store: Store) { return {...l, date: new Date(l.date)}; }) : [], + deviceScreenshot, source, supportRequestDetails, ); diff --git a/static/index.js b/static/index.js index 21db19cdb..6733f9174 100644 --- a/static/index.js +++ b/static/index.js @@ -268,6 +268,7 @@ function tryCreateWindow() { experimentalFeatures: true, nodeIntegration: true, webviewTag: true, + nativeWindowOpen: true, }, }); win.once('ready-to-show', () => win.show());