UI preview of selected element

Summary:
This is a prototype for view preview within Flipper for iOS (Android next).

If enabled, a preview of the selected element is rendered in the attribute inspector.

Changelog: Add view preview/snapshot for the Layout plugin on iOS.

Reviewed By: antonk52

Differential Revision: D34990372

fbshipit-source-id: 1984514fbf59041ad236008a8db10569c5fc5f94

desktop/plugins/public/layout/InspectorSidebar.tsx

@@ -18,7 +18,7 @@ import {
   Logger,
 } from 'flipper';
 import {Panel} from 'flipper-plugin';
-import {PureComponent} from 'react';
+import {PureComponent, useState} from 'react';
 import React from 'react';
 import {useMemo, useEffect} from 'react';
 import {kebabCase} from 'lodash';
@@ -85,11 +85,19 @@ type Props = {
   client: PluginClient;
   realClient: Client;
   logger: Logger;
+  inSnapshotMode: boolean;
+};
+
+type ElementSnapshot = {
+  element: Element | null;
+  snapshot: String | null;
 };
 
 const Sidebar: React.FC<Props> = (props: Props) => {
   const {element} = props;
 
+  const [elementSnapshot, setElementSnapshot] = useState<ElementSnapshot>();
+
   const [sectionDefs, sectionKeys] = useMemo(() => {
     const sectionKeys = [];
     const sectionDefs = [];
@@ -135,7 +143,36 @@ const Sidebar: React.FC<Props> = (props: Props) => {
     return [sectionDefs, sectionKeys];
   }, [element]);
 
-  const sections: Array<React.ReactNode> = (
+  useEffect(() => {
+    if (
+      props.inSnapshotMode &&
+      (!elementSnapshot || elementSnapshot.element != element)
+    ) {
+      props.client
+        .call('getSnapshot', {
+          id: props.element?.id,
+          name: props.element?.name,
+        })
+        .then((response) => {
+          setElementSnapshot({element: element, snapshot: response.snapshot});
+        })
+        .catch((e) => {
+          console.log(
+            'ElementsInspector unable to obtain screenshot for the selected item',
+            e,
+          );
+        });
+    }
+  }, [
+    element,
+    elementSnapshot,
+    props.client,
+    props.element?.id,
+    props.element?.name,
+    props.inSnapshotMode,
+  ]);
+
+  const sidebarExtensions =
     (SidebarExtensions &&
       element?.data &&
       Object.entries(SidebarExtensions).map(([ext, Comp]) => (
@@ -147,18 +184,32 @@ const Sidebar: React.FC<Props> = (props: Props) => {
           selectedNode={element}
         />
       ))) ||
-    []
+    [];
-  ).concat(
+
-    sectionDefs.map((def) => (
+  const sidebarInspector = sectionDefs.map((def) => (
-      <InspectorSidebarSection
+    <InspectorSidebarSection
-        tooltips={props.tooltips}
+      tooltips={props.tooltips}
-        key={def.key}
+      key={def.key}
-        id={def.id}
+      id={def.id}
-        data={def.data}
+      data={def.data}
-        onValueChanged={props.onValueChanged}
+      onValueChanged={props.onValueChanged}
-      />
+    />
-    )),
+  ));
-  );
+
+  const sidebarPreview =
+    props.inSnapshotMode && elementSnapshot?.snapshot ? (
+      <Panel key="preview" title="Preview" pad>
+        <img
+          style={{
+            display: 'block',
+            marginLeft: 'auto',
+            marginRight: 'auto',
+            width: '100%',
+          }}
+          src={'data:image/png;base64,' + elementSnapshot?.snapshot}
+        />
+      </Panel>
+    ) : null;
 
   useEffect(() => {
     sectionKeys.map((key) =>
@@ -169,7 +220,13 @@ const Sidebar: React.FC<Props> = (props: Props) => {
   if (!element || !element.data) {
     return <NoData grow>No data</NoData>;
   }
-  return <>{sections}</>;
+  return (
+    <>
+      {sidebarExtensions}
+      {sidebarInspector}
+      {sidebarPreview}
+    </>
+  );
 };
 
 export default Sidebar;

desktop/plugins/public/layout/index.tsx

@@ -40,6 +40,7 @@ import {getFlipperLib} from 'flipper-plugin';
 type State = {
   init: boolean;
   inTargetMode: boolean;
+  inSnapshotMode: boolean;
   inAXMode: boolean;
   inAlignmentMode: boolean;
   selectedElement: ElementID | null | undefined;
@@ -201,6 +202,7 @@ export default class LayoutPlugin extends FlipperPlugin<
   state: State = {
     init: false,
     inTargetMode: false,
+    inSnapshotMode: false,
     inAXMode: false,
     inAlignmentMode: false,
     selectedElement: null,
@@ -317,6 +319,10 @@ export default class LayoutPlugin extends FlipperPlugin<
     }
   };
 
+  onToggleSnapshotMode = () => {
+    this.setState((prevState) => ({inSnapshotMode: !prevState.inSnapshotMode}));
+  };
+
   onToggleAXMode = () => {
     this.setState({inAXMode: !this.state.inAXMode});
   };
@@ -476,14 +482,26 @@ export default class LayoutPlugin extends FlipperPlugin<
           <Toolbar>
             {!this.props.isArchivedDevice && (
               <ToolbarIcon
+                key="targetMode"
                 onClick={this.onToggleTargetMode}
                 title="Toggle target mode"
                 icon="target"
                 active={this.state.inTargetMode}
               />
             )}
+            {!this.props.isArchivedDevice &&
+              this.realClient.query.os === 'iOS' && (
+                <ToolbarIcon
+                  key="snapshotMode"
+                  onClick={this.onToggleSnapshotMode}
+                  title="Toggle to see view snapshots on the attribute inspector"
+                  icon="eye"
+                  active={this.state.inSnapshotMode}
+                />
+              )}
             {this.realClient.query.os === 'Android' && (
               <ToolbarIcon
+                key="axMode"
                 onClick={this.onToggleAXMode}
                 title="Toggle to see the accessibility hierarchy"
                 icon="accessibility"
@@ -492,6 +510,7 @@ export default class LayoutPlugin extends FlipperPlugin<
             )}
             {!this.props.isArchivedDevice && (
               <ToolbarIcon
+                key="alignmentMode"
                 onClick={this.onToggleAlignmentMode}
                 title="Toggle AlignmentMode to show alignment lines"
                 icon="borders"
@@ -500,6 +519,7 @@ export default class LayoutPlugin extends FlipperPlugin<
             )}
             {this.props.isArchivedDevice && this.state.visualizerScreenshot && (
               <ToolbarIcon
+                key="visualizer"
                 onClick={this.onToggleVisualizer}
                 title="Toggle visual recreation of layout"
                 icon="mobile"
@@ -534,6 +554,7 @@ export default class LayoutPlugin extends FlipperPlugin<
             client={this.getClient()}
             realClient={this.realClient}
             element={element}
+            inSnapshotMode={this.state.inSnapshotMode}
             onValueChanged={this.onDataValueChanged}
             logger={this.props.logger}
           />

iOS/Plugins/FlipperKitLayoutPlugin/FlipperKitLayoutPlugin/FlipperKitLayoutPlugin.mm

@@ -155,6 +155,16 @@ NSObject* flattenLayoutEditorMessage(NSObject* field);
                   },
                   responder);
             }];
+
+  [connection receive:@"getSnapshot"
+            withBlock:^(NSDictionary* params, id<FlipperResponder> responder) {
+              FlipperPerformBlockOnMainThread(
+                  ^{
+                    [weakSelf onCallGetSnapshot:params[@"id"]
+                                  withResponder:responder];
+                  },
+                  responder);
+            }];
 }
 
 - (void)didDisconnect {
@@ -320,6 +330,46 @@ NSObject* flattenLayoutEditorMessage(NSObject* field) {
   return;
 }
 
+- (void)onCallGetSnapshot:(NSString*)objectId
+            withResponder:(id<FlipperResponder>)responder {
+  if (objectId == nil || [objectId isKindOfClass:[NSNull class]]) {
+    [responder error:@{@"error" : @"unable to get snapshot for object"}];
+    return;
+  }
+
+  id object = [_trackedObjects objectForKey:objectId];
+  if (object == nil) {
+    [responder error:@{@"error" : @"unable to get snapshot for object"}];
+    return;
+  }
+
+  id lastHighlightedObject = nil;
+  id lastHighlightedDescriptor = nil;
+  if (_lastHighlightedNode != nil) {
+    lastHighlightedObject = [_trackedObjects objectForKey:_lastHighlightedNode];
+    if (lastHighlightedObject != nil) {
+      lastHighlightedDescriptor = [self->_descriptorMapper
+          descriptorForClass:[lastHighlightedObject class]];
+      [lastHighlightedDescriptor setHighlighted:NO
+                                        forNode:lastHighlightedObject];
+    }
+  }
+
+  SKNodeDescriptor* descriptor =
+      [self->_descriptorMapper descriptorForClass:[object class]];
+  UIImage* snapshot = [descriptor getSnapshot:YES forNode:object];
+  NSData* snapshotData = UIImagePNGRepresentation(snapshot);
+  NSString* snapshotBase64 = [snapshotData
+      base64EncodedStringWithOptions:NSDataBase64Encoding64CharacterLineLength];
+
+  if (lastHighlightedDescriptor != nil) {
+    [lastHighlightedDescriptor setHighlighted:YES
+                                      forNode:lastHighlightedObject];
+  }
+
+  [responder success:@{@"snapshot" : snapshotBase64, @"id" : objectId}];
+}
+
 - (void)onCallSetHighlighted:(NSString*)objectId
                withResponder:(id<FlipperResponder>)responder {
   if (_lastHighlightedNode != nil) {

iOS/Plugins/FlipperKitPluginUtils/FlipperKitLayoutHelpers/FlipperKitLayoutHelpers/SKNodeDescriptor.h

@@ -105,6 +105,12 @@ typedef void (^SKNodeUpdateData)(id value);
  */
 - (void)setHighlighted:(BOOL)highlighted forNode:(T)node;
 
+/**
+ Used to grab a snapshot of the specified node which is currently selected in
+ the Flipper application.
+ */
+- (UIImage*)getSnapshot:(BOOL)includeChildren forNode:(T)node;
+
 /**
  Perform hit testing on the given node. Either continue the search in
  one of the children of the node, or finish the hit testing on this

iOS/Plugins/FlipperKitPluginUtils/FlipperKitLayoutHelpers/FlipperKitLayoutHelpers/SKNodeDescriptor.mm

@@ -68,6 +68,10 @@
   return @[];
 }
 
+- (UIImage*)getSnapshot:(BOOL)includeChildren forNode:(id)node {
+  return nil;
+}
+
 - (void)setHighlighted:(BOOL)highlighted forNode:(id)node {
 }
 

iOS/Plugins/FlipperKitPluginUtils/FlipperKitLayoutIOSDescriptors/FlipperKitLayoutIOSDescriptors/SKApplicationDescriptor.m

@@ -32,6 +32,11 @@
   [windowDescriptor setHighlighted:highlighted forNode:[node keyWindow]];
 }
 
+- (UIImage*)getSnapshot:(BOOL)includeChildren forNode:(UIApplication*)node {
+  SKNodeDescriptor* descriptor = [self descriptorForClass:[UIView class]];
+  return [descriptor getSnapshot:includeChildren forNode:[node keyWindow]];
+}
+
 - (void)hitTest:(SKTouch*)touch forNode:(UIApplication*)node {
   bool finish = true;
   for (NSInteger index = [self childCountForNode:node] - 1; index >= 0;

iOS/Plugins/FlipperKitPluginUtils/FlipperKitLayoutIOSDescriptors/FlipperKitLayoutIOSDescriptors/SKButtonDescriptor.mm

@@ -87,6 +87,11 @@ return mutations;
   [viewDescriptor setHighlighted:highlighted forNode:node];
 }
 
+- (UIImage*)getSnapshot:(BOOL)includeChildren forNode:(UIButton*)node {
+  SKNodeDescriptor* descriptor = [self descriptorForClass:[UIView class]];
+  return [descriptor getSnapshot:includeChildren forNode:node];
+}
+
 - (void)hitTest:(SKTouch*)touch forNode:(UIButton*)node {
   [touch finish];
 }

iOS/Plugins/FlipperKitPluginUtils/FlipperKitLayoutIOSDescriptors/FlipperKitLayoutIOSDescriptors/SKScrollViewDescriptor.m

@@ -83,6 +83,11 @@
   }
 }
 
+- (UIImage*)getSnapshot:(BOOL)includeChildren forNode:(UIScrollView*)node {
+  SKNodeDescriptor* descriptor = [self descriptorForClass:[UIView class]];
+  return [descriptor getSnapshot:includeChildren forNode:node];
+}
+
 @end
 
 #endif

iOS/Plugins/FlipperKitPluginUtils/FlipperKitLayoutIOSDescriptors/FlipperKitLayoutIOSDescriptors/SKViewControllerDescriptor.m

@@ -36,6 +36,11 @@
   [descriptor setHighlighted:highlighted forNode:node.view];
 }
 
+- (UIImage*)getSnapshot:(BOOL)includeChildren forNode:(UIViewController*)node {
+  SKNodeDescriptor* descriptor = [self descriptorForClass:[UIView class]];
+  return [descriptor getSnapshot:includeChildren forNode:node.view];
+}
+
 - (void)hitTest:(SKTouch*)touch forNode:(UIViewController*)node {
   [touch continueWithChildIndex:0 withOffset:(CGPoint){0, 0}];
 }

iOS/Plugins/FlipperKitPluginUtils/FlipperKitLayoutIOSDescriptors/FlipperKitLayoutIOSDescriptors/SKViewDescriptor.mm

@@ -497,6 +497,21 @@ return dataMutations;
   }
 }
 
+- (UIImage*)getSnapshot:(BOOL)includeChildren forNode:(UIView*)node {
+  if ([[UIScreen mainScreen] respondsToSelector:@selector(scale)]) {
+    UIGraphicsBeginImageContextWithOptions(
+        node.bounds.size, node.isOpaque, 0.0);
+  } else {
+    UIGraphicsBeginImageContext(node.bounds.size);
+  }
+
+  [node.layer renderInContext:UIGraphicsGetCurrentContext()];
+
+  UIImage* img = UIGraphicsGetImageFromCurrentImageContext();
+  UIGraphicsEndImageContext();
+  return img;
+}
+
 - (void)hitTest:(SKTouch*)touch forNode:(UIView*)node {
   bool finish = true;
   for (NSInteger index = [self childCountForNode:node] - 1; index >= 0;