NodeInfo tree working (besides litho nodes)

Summary: The second tree has access to all AX NodeInfo properties (they are not in the sidebar yet). Infrastructure set up to customize displayed information bassed on what is most useful. Descriptors for views updated to include AX functionality and non-view descriptors AX functions defaulted to null/empty. Non-view nodes (like Fragments, Window, Appication) no longer included in AX tree. Corresponding nodes will be highlighted (although not expanded) on click in either tree.

Differential Revision: D8795800

fbshipit-source-id: cf2333f69bfecca3ff84aae62681c684dfa14bf3

android/plugins/inspector/InspectorSonarPlugin.java

@@ -126,6 +126,8 @@ public class InspectorSonarPlugin implements SonarPlugin {
     connection.receive("setHighlighted", mSetHighlighted);
     connection.receive("setSearchActive", mSetSearchActive);
     connection.receive("getSearchResults", mGetSearchResults);
+    connection.receive("getAXRoot", mGetAXRoot);
+    connection.receive("getAXNodes", mGetAXNodes);
 
     if (mExtensionCommands != null) {
       for (ExtensionCommand extensionCommand : mExtensionCommands) {
@@ -156,6 +158,28 @@ public class InspectorSonarPlugin implements SonarPlugin {
         }
       };
 
+  final SonarReceiver mGetAXRoot =
+      new MainThreadSonarReceiver(mConnection) {
+        @Override
+        public void onReceiveOnMainThread(SonarObject params, SonarResponder responder)
+            throws Exception {
+          List<View> viewRoots = mApplication.getViewRoots();
+          // for now only works if one view root
+          if (viewRoots.size() != 1) {
+            responder.error(
+                new SonarObject.Builder().put("message", "Too many view roots.").build());
+            return;
+          }
+          SonarObject response = getAXNode(trackObject(viewRoots.get(0)));
+          if (response == null) {
+            responder.error(
+                new SonarObject.Builder().put("message", "AX root node returned null.").build());
+            return;
+          }
+          responder.success(response);
+        }
+      };
+
   final SonarReceiver mGetNodes =
       new MainThreadSonarReceiver(mConnection) {
         @Override
@@ -183,6 +207,33 @@ public class InspectorSonarPlugin implements SonarPlugin {
         }
       };
 
+  final SonarReceiver mGetAXNodes =
+      new MainThreadSonarReceiver(mConnection) {
+        @Override
+        public void onReceiveOnMainThread(final SonarObject params, final SonarResponder responder)
+            throws Exception {
+          final SonarArray ids = params.getArray("ids");
+          final SonarArray.Builder result = new SonarArray.Builder();
+
+          for (int i = 0, count = ids.length(); i < count; i++) {
+            final String id = ids.getString(i);
+            final SonarObject node = getAXNode(id);
+            if (node != null) {
+              result.put(node);
+            } else {
+              responder.error(
+                  new SonarObject.Builder()
+                      .put("message", "No node with given id")
+                      .put("id", id)
+                      .build());
+              return;
+            }
+          }
+
+          responder.success(new SonarObject.Builder().put("elements", result).build());
+        }
+      };
+
   final SonarReceiver mSetData =
       new MainThreadSonarReceiver(mConnection) {
         @Override
@@ -435,6 +486,62 @@ public class InspectorSonarPlugin implements SonarPlugin {
         .build();
   }
 
+  private @Nullable SonarObject getAXNode(String id) throws Exception {
+
+    final Object obj = mObjectTracker.get(id);
+    if (obj == null) {
+      return null;
+    }
+
+    final NodeDescriptor<Object> descriptor = descriptorForObject(obj);
+    if (descriptor == null) {
+      return null;
+    }
+
+    final SonarArray.Builder children = new SonarArray.Builder();
+    new ErrorReportingRunnable(mConnection) {
+      @Override
+      protected void runOrThrow() throws Exception {
+        for (int i = 0, count = descriptor.getChildCount(obj); i < count; i++) {
+          final Object child = assertNotNull(descriptor.getAXChildAt(obj, i));
+          children.put(trackObject(child));
+        }
+      }
+    }.run();
+
+    final SonarObject.Builder data = new SonarObject.Builder();
+    new ErrorReportingRunnable(mConnection) {
+      @Override
+      protected void runOrThrow() throws Exception {
+        for (Named<SonarObject> props : descriptor.getAXData(obj)) {
+          data.put(props.getName(), props.getValue());
+        }
+      }
+    }.run();
+
+    final SonarArray.Builder attributes = new SonarArray.Builder();
+    new ErrorReportingRunnable(mConnection) {
+      @Override
+      protected void runOrThrow() throws Exception {
+        for (Named<String> attribute : descriptor.getAXAttributes(obj)) {
+          attributes.put(
+              new SonarObject.Builder()
+                  .put("name", attribute.getName())
+                  .put("value", attribute.getValue())
+                  .build());
+        }
+      }
+    }.run();
+
+    return new SonarObject.Builder()
+        .put("id", descriptor.getId(obj))
+        .put("name", descriptor.getAXName(obj))
+        .put("data", data)
+        .put("children", children)
+        .put("attributes", attributes)
+        .build();
+  }
+
   private String trackObject(Object obj) throws Exception {
     final NodeDescriptor<Object> descriptor = descriptorForObject(obj);
     final String id = descriptor.getId(obj);

android/plugins/inspector/NodeDescriptor.java

@@ -12,7 +12,9 @@ import com.facebook.sonar.core.SonarArray;
 import com.facebook.sonar.core.SonarConnection;
 import com.facebook.sonar.core.SonarDynamic;
 import com.facebook.sonar.core.SonarObject;
+import java.util.Collections;
 import java.util.List;
+import javax.annotation.Nullable;
 
 /**
  * A NodeDescriptor is an object which known how to expose an Object of type T to the ew Inspector.
@@ -90,18 +92,33 @@ public abstract class NodeDescriptor<T> {
    */
   public abstract String getName(T node) throws Exception;
 
+  /** Gets name for AX tree. */
+  public String getAXName(T node) throws Exception {
+    return "";
+  }
+
   /** @return The number of children this node exposes in the inspector. */
   public abstract int getChildCount(T node) throws Exception;
 
   /** @return The child at index. */
   public abstract Object getChildAt(T node, int index) throws Exception;
 
+  /** Gets child at index for AX tree. Ignores non-view children. */
+  public @Nullable Object getAXChildAt(T node, int index) throws Exception {
+    return null;
+  }
+
   /**
    * Get the data to show for this node in the sidebar of the inspector. The object will be showen
    * in order and with a header matching the given name.
    */
   public abstract List<Named<SonarObject>> getData(T node) throws Exception;
 
+  /** Gets data for AX tree */
+  public List<Named<SonarObject>> getAXData(T node) throws Exception {
+    return Collections.EMPTY_LIST;
+  }
+
   /**
    * Set a value on the provided node at the given path. The path will match a key path in the data
    * provided by {@link this#getData(Object)} and the value will be of the same type as the value
@@ -115,6 +132,11 @@ public abstract class NodeDescriptor<T> {
    */
   public abstract List<Named<String>> getAttributes(T node) throws Exception;
 
+  /** Gets attributes for AX tree */
+  public List<Named<String>> getAXAttributes(T node) throws Exception {
+    return Collections.EMPTY_LIST;
+  }
+
   /**
    * Highlight this node. Use {@link HighlightedOverlay} if possible. This is used to highlight a
    * node which is selected in the inspector. The plugin automatically takes care of de-selecting

android/plugins/inspector/descriptors/TextViewDescriptor.java

@@ -44,6 +44,12 @@ public class TextViewDescriptor extends NodeDescriptor<TextView> {
     return descriptor.getName(node);
   }
 
+  @Override
+  public String getAXName(TextView node) throws Exception {
+    final NodeDescriptor descriptor = descriptorForClass(View.class);
+    return descriptor.getAXName(node);
+  }
+
   @Override
   public int getChildCount(TextView node) {
     return 0;
@@ -54,6 +60,11 @@ public class TextViewDescriptor extends NodeDescriptor<TextView> {
     return null;
   }
 
+  @Override
+  public @Nullable Object getAXChildAt(TextView node, int index) {
+    return null;
+  }
+
   @Override
   public List<Named<SonarObject>> getData(TextView node) throws Exception {
     final List<Named<SonarObject>> props = new ArrayList<>();
@@ -76,6 +87,14 @@ public class TextViewDescriptor extends NodeDescriptor<TextView> {
     return props;
   }
 
+  @Override
+  public List<Named<SonarObject>> getAXData(TextView node) throws Exception {
+    final List<Named<SonarObject>> props = new ArrayList<>();
+    final NodeDescriptor descriptor = descriptorForClass(View.class);
+    props.addAll(descriptor.getAXData(node));
+    return props;
+  }
+
   @Override
   public void setValue(TextView node, String[] path, SonarDynamic value) throws Exception {
     switch (path[0]) {
@@ -106,6 +125,12 @@ public class TextViewDescriptor extends NodeDescriptor<TextView> {
     return descriptor.getAttributes(node);
   }
 
+  @Override
+  public List<Named<String>> getAXAttributes(TextView node) throws Exception {
+    final NodeDescriptor descriptor = descriptorForClass(View.class);
+    return descriptor.getAXAttributes(node);
+  }
+
   @Override
   public void setHighlighted(TextView node, boolean selected) throws Exception {
     final NodeDescriptor descriptor = descriptorForClass(View.class);

android/plugins/inspector/descriptors/ViewDescriptor.java

@@ -18,6 +18,7 @@ import android.graphics.drawable.Drawable;
 import android.os.Build;
 import android.support.v4.view.MarginLayoutParamsCompat;
 import android.support.v4.view.ViewCompat;
+import android.support.v4.view.accessibility.AccessibilityNodeInfoCompat;
 import android.util.SparseArray;
 import android.view.Gravity;
 import android.view.View;
@@ -37,10 +38,12 @@ import com.facebook.sonar.plugins.inspector.NodeDescriptor;
 import com.facebook.sonar.plugins.inspector.Touch;
 import com.facebook.sonar.plugins.inspector.descriptors.utils.AccessibilityUtil;
 import com.facebook.sonar.plugins.inspector.descriptors.utils.EnumMapping;
+import com.facebook.sonar.plugins.inspector.descriptors.utils.ViewAccessibilityHelper;
 import com.facebook.stetho.common.android.ResourcesUtil;
 import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import javax.annotation.Nullable;
 
@@ -78,6 +81,17 @@ public class ViewDescriptor extends NodeDescriptor<View> {
     return node.getClass().getSimpleName();
   }
 
+  @Override
+  public String getAXName(View node) {
+    AccessibilityNodeInfoCompat nodeInfo = ViewAccessibilityHelper.createNodeInfoFromView(node);
+    if (nodeInfo == null) {
+      return "NULL NODEINFO";
+    }
+    String name = nodeInfo.getClassName().toString();
+    nodeInfo.recycle();
+    return name;
+  }
+
   @Override
   public int getChildCount(View node) {
     return 0;
@@ -88,6 +102,11 @@ public class ViewDescriptor extends NodeDescriptor<View> {
     return null;
   }
 
+  @Override
+  public @Nullable Object getAXChildAt(View node, int index) {
+    return null;
+  }
+
   @Override
   public List<Named<SonarObject>> getData(View node) {
     final SonarObject.Builder viewProps =
@@ -176,6 +195,11 @@ public class ViewDescriptor extends NodeDescriptor<View> {
         new Named<>("Accessibility", getAccessibilityData(node)));
   }
 
+  @Override
+  public List<Named<SonarObject>> getAXData(View node) {
+    return Arrays.asList(new Named<>("AX Properties", getAccessibilityData(node)));
+  }
+
   private static SonarObject getAccessibilityData(View view) {
     final SonarObject.Builder accessibilityProps = new SonarObject.Builder();
 
@@ -408,6 +432,10 @@ public class ViewDescriptor extends NodeDescriptor<View> {
     return attributes;
   }
 
+  public List<Named<String>> getAXAttributes(View node) throws Exception {
+    return Collections.EMPTY_LIST;
+  }
+
   @Nullable
   private static String getResourceId(View node) {
     final int id = node.getId();

android/plugins/inspector/descriptors/ViewGroupDescriptor.java

@@ -99,6 +99,12 @@ public class ViewGroupDescriptor extends NodeDescriptor<ViewGroup> {
     return descriptor.getName(node);
   }
 
+  @Override
+  public String getAXName(ViewGroup node) throws Exception {
+    NodeDescriptor descriptor = descriptorForClass(View.class);
+    return descriptor.getAXName(node);
+  }
+
   @Override
   public int getChildCount(ViewGroup node) {
     int childCount = 0;
@@ -130,6 +136,21 @@ public class ViewGroupDescriptor extends NodeDescriptor<ViewGroup> {
     return null;
   }
 
+  @Override
+  public @Nullable Object getAXChildAt(ViewGroup node, int index) {
+    for (int i = 0, count = node.getChildCount(); i < count; i++) {
+      final View child = node.getChildAt(i);
+      if (child instanceof HiddenNode) {
+        continue;
+      }
+
+      if (i >= index) {
+        return child;
+      }
+    }
+    return null;
+  }
+
   @Override
   public List<Named<SonarObject>> getData(ViewGroup node) throws Exception {
     final List<Named<SonarObject>> props = new ArrayList<>();
@@ -160,6 +181,14 @@ public class ViewGroupDescriptor extends NodeDescriptor<ViewGroup> {
     return props;
   }
 
+  @Override
+  public List<Named<SonarObject>> getAXData(ViewGroup node) throws Exception {
+    final List<Named<SonarObject>> props = new ArrayList<>();
+    final NodeDescriptor descriptor = descriptorForClass(View.class);
+    props.addAll(descriptor.getAXData(node));
+    return props;
+  }
+
   @Override
   public void setValue(ViewGroup node, String[] path, SonarDynamic value) throws Exception {
     switch (path[0]) {
@@ -200,6 +229,12 @@ public class ViewGroupDescriptor extends NodeDescriptor<ViewGroup> {
     return descriptor.getAttributes(node);
   }
 
+  @Override
+  public List<Named<String>> getAXAttributes(ViewGroup node) throws Exception {
+    final NodeDescriptor descriptor = descriptorForClass(View.class);
+    return descriptor.getAXAttributes(node);
+  }
+
   @Override
   public void setHighlighted(ViewGroup node, boolean selected) throws Exception {
     final NodeDescriptor descriptor = descriptorForClass(View.class);

android/plugins/inspector/litho-sonar/DebugComponentDescriptor.java

@@ -12,6 +12,7 @@ import android.graphics.Rect;
 import android.graphics.drawable.ColorDrawable;
 import android.graphics.drawable.Drawable;
 import android.support.v4.util.Pair;
+import android.support.v4.view.accessibility.AccessibilityNodeInfoCompat;
 import android.view.View;
 import com.facebook.litho.Component;
 import com.facebook.litho.ComponentContext;
@@ -31,6 +32,7 @@ import com.facebook.sonar.plugins.inspector.NodeDescriptor;
 import com.facebook.sonar.plugins.inspector.Touch;
 import com.facebook.sonar.plugins.inspector.descriptors.ObjectDescriptor;
 import com.facebook.sonar.plugins.inspector.descriptors.utils.AccessibilityUtil;
+import com.facebook.sonar.plugins.inspector.descriptors.utils.ViewAccessibilityHelper;
 import com.facebook.yoga.YogaAlign;
 import com.facebook.yoga.YogaDirection;
 import com.facebook.yoga.YogaEdge;
@@ -41,6 +43,7 @@ import com.facebook.yoga.YogaValue;
 import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -122,6 +125,15 @@ public class DebugComponentDescriptor extends NodeDescriptor<DebugComponent> {
     return node.getComponent().getSimpleName();
   }
 
+  @Override
+  public String getAXName(DebugComponent node) {
+    View v = node.getComponentHost();
+    AccessibilityNodeInfoCompat nodeInfo = ViewAccessibilityHelper.createNodeInfoFromView(v);
+    String name = nodeInfo.getClassName().toString();
+    nodeInfo.recycle();
+    return name;
+  }
+
   @Override
   public int getChildCount(DebugComponent node) {
     if (node.getMountedView() != null || node.getMountedDrawable() != null) {
@@ -145,6 +157,17 @@ public class DebugComponentDescriptor extends NodeDescriptor<DebugComponent> {
     }
   }
 
+  @Override
+  public @Nullable Object getAXChildAt(DebugComponent node, int index) {
+    final View mountedView = node.getMountedView();
+
+    if (mountedView != null) {
+      return mountedView;
+    } else {
+      return node.getChildComponents().get(index);
+    }
+  }
+
   @Override
   public List<Named<SonarObject>> getData(DebugComponent node) throws Exception {
     NodeDescriptor componentDescriptor = descriptorForClass(node.getComponent().getClass());
@@ -177,6 +200,16 @@ public class DebugComponentDescriptor extends NodeDescriptor<DebugComponent> {
     return data;
   }
 
+  @Override
+  public List<Named<SonarObject>> getAXData(DebugComponent node) {
+    final List<Named<SonarObject>> data = new ArrayList<>();
+    final SonarObject accessibilityData = getAccessibilityData(node);
+    if (accessibilityData != null) {
+      data.add(new Named<>("Accessibility", accessibilityData));
+    }
+    return data;
+  }
+
   @Nullable
   private static SonarObject getLayoutData(DebugComponent node) {
     final DebugLayoutNode layout = node.getLayoutNode();
@@ -457,6 +490,11 @@ public class DebugComponentDescriptor extends NodeDescriptor<DebugComponent> {
     return attributes;
   }
 
+  @Override
+  public List<Named<String>> getAXAttributes(DebugComponent node) {
+    return Collections.EMPTY_LIST;
+  }
+
   @Override
   public void setHighlighted(DebugComponent node, boolean selected) {
     final LithoView lithoView = node.getLithoView();

android/plugins/inspector/litho-sonar/LithoViewDescriptor.java

@@ -13,6 +13,7 @@ import com.facebook.sonar.plugins.inspector.NodeDescriptor;
 import com.facebook.sonar.plugins.inspector.Touch;
 import java.util.ArrayList;
 import java.util.List;
+import javax.annotation.Nullable;
 
 public class LithoViewDescriptor extends NodeDescriptor<LithoView> {
 
@@ -39,6 +40,12 @@ public class LithoViewDescriptor extends NodeDescriptor<LithoView> {
     return descriptor.getName(node);
   }
 
+  @Override
+  public String getAXName(LithoView node) throws Exception {
+    final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class);
+    return descriptor.getAXName(node);
+  }
+
   @Override
   public int getChildCount(LithoView node) {
     return DebugComponent.getRootInstance(node) == null ? 0 : 1;
@@ -49,6 +56,11 @@ public class LithoViewDescriptor extends NodeDescriptor<LithoView> {
     return DebugComponent.getRootInstance(node);
   }
 
+  @Override
+  public @Nullable Object getAXChildAt(LithoView node, int index) {
+    return DebugComponent.getRootInstance(node);
+  }
+
   @Override
   public List<Named<SonarObject>> getData(LithoView node) throws Exception {
     final List<Named<SonarObject>> props = new ArrayList<>();
@@ -74,6 +86,14 @@ public class LithoViewDescriptor extends NodeDescriptor<LithoView> {
     return props;
   }
 
+  @Override
+  public List<Named<SonarObject>> getAXData(LithoView node) throws Exception {
+    final List<Named<SonarObject>> props = new ArrayList<>();
+    final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class);
+    props.addAll(descriptor.getAXData(node));
+    return props;
+  }
+
   @Override
   public void setValue(LithoView node, String[] path, SonarDynamic value) throws Exception {
     final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class);
@@ -86,6 +106,12 @@ public class LithoViewDescriptor extends NodeDescriptor<LithoView> {
     return descriptor.getAttributes(node);
   }
 
+  @Override
+  public List<Named<String>> getAXAttributes(LithoView node) throws Exception {
+    final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class);
+    return descriptor.getAXAttributes(node);
+  }
+
   @Override
   public void setHighlighted(LithoView node, boolean selected) throws Exception {
     final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class);

src/plugins/layout/index.js

@@ -166,11 +166,6 @@ export default class Layout extends SonarPlugin<InspectorState> {
     SelectElement(state: InspectorState, {key}: SelectElementArgs) {
       return {
         selected: key,
-      };
-    },
-
-    SelectAXElement(state: InspectorState, {key}: SelectElementArgs) {
-      return {
         AXselected: key,
       };
     },
@@ -402,7 +397,7 @@ export default class Layout extends SonarPlugin<InspectorState> {
     });
 
     if (AXToggleButtonEnabled) {
-      this.client.call('getRoot').then((element: Element) => {
+      this.client.call('getAXRoot').then((element: Element) => {
         this.dispatchAction({elements: [element], type: 'UpdateAXElements'});
         this.dispatchAction({root: element.id, type: 'SetAXRoot'});
         this.performInitialExpand(element, true).then(() => {
@@ -511,7 +506,7 @@ export default class Layout extends SonarPlugin<InspectorState> {
     if (ids.length > 0) {
       performance.mark('LayoutInspectorGetNodes');
       return this.client
-        .call('getNodes', {ids})
+        .call(ax ? 'getAXNodes' : 'getNodes', {ids})
         .then(({elements}: GetNodesResult) => {
           this.props.logger.trackTimeSince('LayoutInspectorGetNodes');
           return Promise.resolve(elements);
@@ -629,7 +624,7 @@ export default class Layout extends SonarPlugin<InspectorState> {
   });
 
   onAXElementSelected = debounce((key: ElementID) => {
-    this.dispatchAction({key, type: 'SelectAXElement'});
+    this.dispatchAction({key, type: 'SelectElement'});
     this.client.send('setHighlighted', {id: key});
     this.getNodes([key], true, true).then((elements: Array<Element>) => {
       this.dispatchAction({elements, type: 'UpdateAXElements'});