Add archived device visualizer

Summary:
Adds a visual indicator of layout nodes when using an archived device.

The "device" window isn't movable for some reason. I've tried the movable and draggable attributes but with no luck. It would obviously be good to fix that but I think its probably shippable as is, and I don't have any more ideas about how to do it.

Reviewed By: passy

Differential Revision: D19885719

fbshipit-source-id: 186ba38c85afee18ce111e30187bdccd9b919025

src/devices/AndroidDevice.tsx

@@ -89,6 +89,7 @@ export default class AndroidDevice extends BaseDevice {
       this.title,
       this.os,
       [...this.logEntries],
+      null,
     );
   }
 

src/devices/ArchivedDevice.tsx

@@ -28,6 +28,7 @@ export default class ArchivedDevice extends BaseDevice {
     title: string,
     os: OS,
     logEntries: Array<DeviceLogEntry>,
+    screenshotHandle: string | null,
     source: string = '',
     supportRequestDetails?: SupportFormRequestDetailsState,
   ) {
@@ -35,10 +36,11 @@ export default class ArchivedDevice extends BaseDevice {
     this.logs = logEntries;
     this.source = source;
     this.supportRequestDetails = supportRequestDetails;
+    this.archivedScreenshotHandle = screenshotHandle;
   }
 
   logs: Array<DeviceLogEntry>;
-
+  archivedScreenshotHandle: string | null;
   isArchived = true;
 
   displayTitle(): string {
@@ -59,4 +61,8 @@ export default class ArchivedDevice extends BaseDevice {
   spawnShell(): DeviceShell | undefined | null {
     return null;
   }
+
+  getArchivedScreenshotHandle(): string | null {
+    return this.archivedScreenshotHandle;
+  }
 }

src/devices/MetroDevice.tsx

@@ -206,6 +206,7 @@ export default class MetroDevice extends BaseDevice {
       this.title,
       this.os,
       [...this.logEntries],
+      null,
     );
   }
 }

src/fb-stubs/user.tsx

@@ -62,3 +62,9 @@ export async function uploadFlipperMedia(
 ): Promise<string> {
   throw new Error('Feature not implemented');
 }
+export async function getFlipperMediaCDN(
+  _uploadID: string,
+  _kind: 'Image' | 'Video',
+): Promise<string> {
+  throw new Error('Feature not implemented');
+}

src/index.tsx

@@ -170,3 +170,4 @@ export {InspectorSidebar} from './ui/components/elements-inspector/sidebar';
 export {Console} from './ui/components/console';
 export {default as Sheet} from './ui/components/Sheet';
 export {KeyboardActions} from './MenuBar';
+export {getFlipperMediaCDN} from './fb-stubs/user';

src/plugins/layout/Inspector.tsx

@@ -296,12 +296,12 @@ export default class Inspector extends Component<Props> {
     this.props.onSelect(selectedKey);
   });
 
-  onElementHovered = debounce((key: ElementID | null | undefined) =>
+  onElementHovered = debounce((key: ElementID | null | undefined) => {
     this.props.client.call(this.call().SET_HIGHLIGHTED, {
       id: key,
       isAlignmentMode: this.props.inAlignmentMode,
-    }),
-  );
+    });
+  });
 
   onElementExpanded = (
     id: ElementID,

src/plugins/layout/ProxyArchiveClient.tsx

@@ -84,10 +84,15 @@ export function searchNodes(
 }
 
 class ProxyArchiveClient {
-  constructor(persistedState: PersistedState) {
+  constructor(
+    persistedState: PersistedState,
+    onElementHighlighted?: (id: string) => void,
+  ) {
     this.persistedState = cloneDeep(persistedState);
+    this.onElementHighlighted = onElementHighlighted;
   }
   persistedState: PersistedState;
+  onElementHighlighted: ((id: string) => void) | undefined;
   subscribe(_method: string, _callback: (params: any) => void): void {
     return;
   }
@@ -175,6 +180,11 @@ class ProxyArchiveClient {
       case 'isConsoleEnabled': {
         return Promise.resolve(false);
       }
+      case 'setHighlighted': {
+        const id = paramaters?.id;
+        this.onElementHighlighted && this.onElementHighlighted(id);
+        return Promise.resolve();
+      }
       default: {
         return Promise.resolve();
       }

src/plugins/layout/index.tsx

@@ -27,6 +27,7 @@ import {
   SupportRequestFormV2,
   constants,
   ReduxState,
+  ArchivedDevice,
 } from 'flipper';
 import Inspector from './Inspector';
 import ToolbarIcon from './ToolbarIcon';
@@ -34,6 +35,8 @@ import InspectorSidebar from './InspectorSidebar';
 import Search from './Search';
 import ProxyArchiveClient from './ProxyArchiveClient';
 import React from 'react';
+import {VisualizerPortal} from 'flipper';
+import {getFlipperMediaCDN} from 'flipper';
 
 type State = {
   init: boolean;
@@ -42,7 +45,11 @@ type State = {
   inAlignmentMode: boolean;
   selectedElement: ElementID | null | undefined;
   selectedAXElement: ElementID | null | undefined;
+  highlightedElement: ElementID | null;
   searchResults: ElementSearchResultSet | null;
+  visualizerWindow: Window | null;
+  visualizerScreenshot: string | null;
+  screenDimensions: {width: number; height: number} | null;
 };
 
 export type ElementMap = {[key: string]: Element};
@@ -125,6 +132,10 @@ export default class Layout extends FlipperPlugin<State, any, PersistedState> {
     return JSON.parse(serializedString);
   };
 
+  teardown() {
+    this.state.visualizerWindow?.close();
+  }
+
   static defaultPersistedState = {
     rootElement: null,
     rootAXElement: null,
@@ -140,6 +151,10 @@ export default class Layout extends FlipperPlugin<State, any, PersistedState> {
     selectedElement: null,
     selectedAXElement: null,
     searchResults: null,
+    visualizerWindow: null,
+    highlightedElement: null,
+    visualizerScreenshot: null,
+    screenDimensions: null,
   };
 
   init() {
@@ -160,6 +175,22 @@ export default class Layout extends FlipperPlugin<State, any, PersistedState> {
       }
     });
 
+    if (this.props.isArchivedDevice) {
+      this.getDevice()
+        .then(d => {
+          const handle = (d as ArchivedDevice).getArchivedScreenshotHandle();
+          if (!handle) {
+            throw new Error('No screenshot attached.');
+          }
+          return handle;
+        })
+        .then(handle => getFlipperMediaCDN(handle, 'Image'))
+        .then(url => this.setState({visualizerScreenshot: url}))
+        .catch(_ => {
+          // Not all exports have screenshots. This is ok.
+        });
+    }
+
     this.setState({
       init: true,
       selectedElement: this.props.deepLinkPayload
@@ -180,7 +211,9 @@ export default class Layout extends FlipperPlugin<State, any, PersistedState> {
 
   getClient(): PluginClient {
     return this.props.isArchivedDevice
-      ? new ProxyArchiveClient(this.props.persistedState)
+      ? new ProxyArchiveClient(this.props.persistedState, (id: string) => {
+          this.setState({highlightedElement: id});
+        })
       : this.client;
   }
   onToggleAlignmentMode = () => {
@@ -193,6 +226,34 @@ export default class Layout extends FlipperPlugin<State, any, PersistedState> {
     this.setState({inAlignmentMode: !this.state.inAlignmentMode});
   };
 
+  onToggleVisualizer = () => {
+    if (this.state.visualizerWindow) {
+      this.state.visualizerWindow.close();
+    } else {
+      const screenDimensions = this.state.screenDimensions;
+      if (!screenDimensions) {
+        return;
+      }
+      const visualizerWindow = window.open(
+        '',
+        'visualizer',
+        `width=${screenDimensions.width},height=${screenDimensions.height}`,
+      );
+      if (!visualizerWindow) {
+        return;
+      }
+      visualizerWindow.onunload = () => {
+        this.setState({visualizerWindow: null});
+      };
+      visualizerWindow.onresize = () => {
+        this.setState({visualizerWindow: visualizerWindow});
+      };
+      visualizerWindow.onload = () => {
+        this.setState({visualizerWindow: visualizerWindow});
+      };
+    }
+  };
+
   onDataValueChanged = (path: Array<string>, value: any) => {
     const id = this.state.inAXMode
       ? this.state.selectedAXElement
@@ -205,6 +266,47 @@ export default class Layout extends FlipperPlugin<State, any, PersistedState> {
     });
   };
   showFlipperADBar: boolean = false;
+
+  getScreenDimensions(): {width: number; height: number} | null {
+    if (this.state.screenDimensions) {
+      return this.state.screenDimensions;
+    }
+
+    requestIdleCallback(() => {
+      // Walk the layout tree from root node down until a node with width and height is found.
+      // Assume these are the dimensions of the screen.
+      let elementId = this.props.persistedState.rootElement;
+      while (elementId != null) {
+        const element = this.props.persistedState.elements[elementId];
+        if (!element) {
+          return null;
+        }
+        if (element.data.View?.width) {
+          break;
+        }
+        elementId = element.children[0];
+      }
+      if (elementId == null) {
+        return null;
+      }
+      const element = this.props.persistedState.elements[elementId];
+      if (
+        element == null ||
+        typeof element.data.View?.width != 'object' ||
+        typeof element.data.View?.height != 'object'
+      ) {
+        return null;
+      }
+      const screenDimensions = {
+        width: element.data.View?.width.value,
+        height: element.data.View?.height.value,
+      };
+      this.setState({screenDimensions});
+    });
+
+    return null;
+  }
+
   render() {
     const inspectorProps = {
       client: this.getClient(),
@@ -248,6 +350,8 @@ export default class Layout extends FlipperPlugin<State, any, PersistedState> {
 
     const showAnalyzeYogaPerformanceButton = GK.get('flipper_yogaperformance');
 
+    const screenDimensions = this.getScreenDimensions();
+
     return (
       <FlexColumn grow={true}>
         {this.state.init && (
@@ -277,6 +381,15 @@ export default class Layout extends FlipperPlugin<State, any, PersistedState> {
                   active={this.state.inAlignmentMode}
                 />
               )}
+              {this.props.isArchivedDevice &&
+                this.state.visualizerScreenshot && (
+                  <ToolbarIcon
+                    onClick={this.onToggleVisualizer}
+                    title="Toggle visual recreation of layout"
+                    icon="mobile"
+                    active={!!this.state.visualizerWindow}
+                  />
+                )}
 
               <Search
                 client={this.getClient()}
@@ -317,6 +430,19 @@ export default class Layout extends FlipperPlugin<State, any, PersistedState> {
                 </Button>
               ) : null}
             </DetailSidebar>
+            {this.state.visualizerWindow &&
+              screenDimensions &&
+              (this.state.visualizerScreenshot ? (
+                <VisualizerPortal
+                  container={this.state.visualizerWindow.document.body}
+                  elements={this.props.persistedState.elements}
+                  highlightedElement={this.state.highlightedElement}
+                  screenshotURL={this.state.visualizerScreenshot}
+                  screenDimensions={screenDimensions}
+                />
+              ) : (
+                'Loading...'
+              ))}
           </>
         )}
       </FlexColumn>

src/ui/components/elements-inspector/Visualizer.tsx

@@ -0,0 +1,113 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import React from 'react';
+import ReactDOM from 'react-dom';
+import {Element, styled} from '../../../ui';
+
+export function VisualizerPortal(props: {
+  container: HTMLElement;
+  highlightedElement: string | null;
+  elements: {[key: string]: Element};
+  screenshotURL: string;
+  screenDimensions: {width: number; height: number};
+}) {
+  const element: Element | null | '' =
+    props.highlightedElement && props.elements[props.highlightedElement];
+
+  const position =
+    element &&
+    typeof element.data.View?.positionOnScreenX == 'number' &&
+    typeof element.data.View?.positionOnScreenY == 'number' &&
+    typeof element.data.View.width === 'object' &&
+    element.data.View.width.value != null &&
+    typeof element.data.View.height === 'object' &&
+    element.data.View.height.value != null
+      ? {
+          x: element.data.View.positionOnScreenX,
+          y: element.data.View.positionOnScreenY,
+          width: element.data.View.width.value,
+          height: element.data.View.height.value,
+        }
+      : null;
+
+  return ReactDOM.createPortal(
+    <Visualizer
+      screenDimensions={props.screenDimensions}
+      element={position}
+      imageURL={props.screenshotURL}
+    />,
+    props.container,
+  );
+}
+
+const VisualizerContainer = styled.div({
+  position: 'relative',
+  top: 0,
+  left: 0,
+  width: '100%',
+  height: '100%',
+  userSelect: 'none',
+});
+
+const DeviceImage = styled.img(({width, height}) => ({
+  width,
+  height,
+  userSelect: 'none',
+}));
+
+/**
+ * Component that displays a static picture of a device
+ * and renders "highlighted" rectangles over arbitrary points on it.
+ * Used for emulating the layout plugin when a device isn't connected.
+ */
+function Visualizer(props: {
+  screenDimensions: {width: number; height: number};
+  element: {x: number; y: number; width: number; height: number} | null;
+  imageURL: string;
+}) {
+  const containerRef: React.Ref<HTMLDivElement> = React.createRef();
+  const imageRef: React.Ref<HTMLImageElement> = React.createRef();
+  let w: number = 0;
+  let h: number = 0;
+  const [scale, updateScale] = React.useState(1);
+
+  React.useLayoutEffect(() => {
+    w = containerRef.current?.offsetWidth || 0;
+    h = containerRef.current?.offsetHeight || 0;
+    const xScale = props.screenDimensions.width / w;
+    const yScale = props.screenDimensions.height / h;
+    updateScale(Math.max(xScale, yScale));
+    imageRef.current?.setAttribute('draggable', 'false');
+  });
+  return (
+    <VisualizerContainer ref={containerRef}>
+      <DeviceImage
+        ref={imageRef}
+        src={props.imageURL}
+        width={props.screenDimensions.width / scale}
+        height={props.screenDimensions.height / scale}
+      />
+      {props.element && (
+        <div
+          style={{
+            position: 'absolute',
+            left: props.element.x / scale,
+            top: props.element.y / scale,
+            width: props.element.width / scale,
+            height: props.element.height / scale,
+            backgroundColor: '#637dff',
+            opacity: 0.7,
+            userSelect: 'none',
+          }}></div>
+      )}
+      />
+    </VisualizerContainer>
+  );
+}

src/ui/components/elements-inspector/elements.tsx

@@ -635,7 +635,9 @@ export class Elements extends PureComponent<ElementsProps, ElementsState> {
         key={row.key}
         even={isEven}
         onElementExpanded={onElementExpanded}
-        onElementHovered={onElementHovered}
+        onElementHovered={(key: string | null | undefined) => {
+          onElementHovered && onElementHovered(key);
+        }}
         onElementSelected={onElementSelected}
         selected={selected === row.key}
         focused={focused === row.key}

src/ui/index.tsx

@@ -155,6 +155,7 @@ export {Elements} from './components/elements-inspector/elements';
 export {ContextMenuExtension} from './components/elements-inspector/elements';
 export {default as ElementsInspector} from './components/elements-inspector/ElementsInspector';
 export {InspectorSidebar} from './components/elements-inspector/sidebar';
+export {VisualizerPortal} from './components/elements-inspector/Visualizer';
 
 export {Console} from './components/console';
 

src/utils/__tests__/exportData.electron.tsx

@@ -77,6 +77,7 @@ test('test generateClientIndentifierWithSalt helper function', () => {
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
   const identifier = generateClientIdentifier(device, 'app');
   const saltIdentifier = generateClientIdentifierWithSalt(identifier, 'salt');
@@ -91,6 +92,7 @@ test('test generateClientFromClientWithSalt helper function', () => {
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
   const client = generateClientFromDevice(device, 'app');
   const saltedClient = generateClientFromClientWithSalt(client, 'salt');
@@ -121,6 +123,7 @@ test('test generateClientFromDevice helper function', () => {
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
   const client = generateClientFromDevice(device, 'app');
   expect(client).toEqual({
@@ -141,6 +144,7 @@ test('test generateClientIdentifier helper function', () => {
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
   const identifier = generateClientIdentifier(device, 'app');
   expect(identifier).toEqual('app#iOS#archivedEmulator#serial');
@@ -173,7 +177,14 @@ test('test processStore function for empty state', () => {
 test('test processStore function for an iOS device connected', async () => {
   const json = await processStore({
     activeNotifications: [],
-    device: new ArchivedDevice('serial', 'emulator', 'TestiPhone', 'iOS', []),
+    device: new ArchivedDevice(
+      'serial',
+      'emulator',
+      'TestiPhone',
+      'iOS',
+      [],
+      null,
+    ),
     pluginStates: {},
     clients: [],
     devicePlugins: new Map(),
@@ -209,6 +220,7 @@ test('test processStore function for an iOS device connected with client plugin
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
   const clientIdentifier = generateClientIdentifier(device, 'testapp');
   const json = await processStore({
@@ -246,6 +258,7 @@ test('test processStore function to have only the client for the selected device
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
   const unselectedDevice = new ArchivedDevice(
     'identifier',
@@ -253,6 +266,7 @@ test('test processStore function to have only the client for the selected device
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
 
   const unselectedDeviceClientIdentifier = generateClientIdentifier(
@@ -313,6 +327,7 @@ test('test processStore function to have multiple clients for the selected devic
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
 
   const clientIdentifierApp1 = generateClientIdentifier(
@@ -379,6 +394,7 @@ test('test processStore function for device plugin state and no clients', async
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
   const json = await processStore({
     activeNotifications: [],
@@ -416,6 +432,7 @@ test('test processStore function for unselected device plugin state and no clien
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
   const json = await processStore({
     activeNotifications: [],
@@ -449,6 +466,7 @@ test('test processStore function for notifications for selected device', async (
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
   const client = generateClientFromDevice(selectedDevice, 'testapp1');
   const notification = generateNotifications(
@@ -498,6 +516,7 @@ test('test processStore function for notifications for unselected device', async
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
   const unselectedDevice = new ArchivedDevice(
     'identifier',
@@ -505,6 +524,7 @@ test('test processStore function for notifications for unselected device', async
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
 
   const client = generateClientFromDevice(selectedDevice, 'testapp1');
@@ -552,6 +572,7 @@ test('test processStore function for selected plugins', async () => {
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
 
   const client = generateClientFromDevice(selectedDevice, 'app');
@@ -602,6 +623,7 @@ test('test processStore function for no selected plugins', async () => {
     'TestiPhone',
     'iOS',
     [],
+    null,
   );
   const client = generateClientFromDevice(selectedDevice, 'app');
   const pluginstates = {

src/utils/exportData.tsx

@@ -274,6 +274,7 @@ const addSaltToDeviceSerial = async (
     device.title,
     device.os,
     selectedPlugins.includes('DeviceLogs') ? device.getLogs() : [],
+    deviceScreenshot,
   );
   statusUpdate &&
     statusUpdate('Adding salt to the selected device id in the client data...');
@@ -674,11 +675,12 @@ export const exportStoreToFile = (
 export function importDataToStore(source: string, data: string, store: Store) {
   getLogger().track('usage', IMPORT_FLIPPER_TRACE_EVENT);
   const json: ExportType = JSON.parse(data);
-  const {device, clients, supportRequestDetails} = json;
+  const {device, clients, supportRequestDetails, deviceScreenshot} = json;
   if (device == null) {
     return;
   }
   const {serial, deviceType, title, os, logs} = device;
+
   const archivedDevice = new ArchivedDevice(
     serial,
     deviceType,
@@ -689,6 +691,7 @@ export function importDataToStore(source: string, data: string, store: Store) {
           return {...l, date: new Date(l.date)};
         })
       : [],
+    deviceScreenshot,
     source,
     supportRequestDetails,
   );

static/index.js

@@ -268,6 +268,7 @@ function tryCreateWindow() {
         experimentalFeatures: true,
         nodeIntegration: true,
         webviewTag: true,
+        nativeWindowOpen: true,
       },
     });
     win.once('ready-to-show', () => win.show());