From 1c5ecce6678ce981906f230be1cfbf52dd2718c8 Mon Sep 17 00:00:00 2001 From: Sara Valderrama Date: Thu, 12 Jul 2018 09:23:29 -0700 Subject: [PATCH] NodeInfo tree working (besides litho nodes) Summary: The second tree has access to all AX NodeInfo properties (they are not in the sidebar yet). Infrastructure set up to customize displayed information bassed on what is most useful. Descriptors for views updated to include AX functionality and non-view descriptors AX functions defaulted to null/empty. Non-view nodes (like Fragments, Window, Appication) no longer included in AX tree. Corresponding nodes will be highlighted (although not expanded) on click in either tree. Differential Revision: D8795800 fbshipit-source-id: cf2333f69bfecca3ff84aae62681c684dfa14bf3 --- .../inspector/InspectorSonarPlugin.java | 107 ++++++++++++++++++ android/plugins/inspector/NodeDescriptor.java | 22 ++++ .../descriptors/TextViewDescriptor.java | 25 ++++ .../inspector/descriptors/ViewDescriptor.java | 28 +++++ .../descriptors/ViewGroupDescriptor.java | 35 ++++++ .../litho-sonar/DebugComponentDescriptor.java | 38 +++++++ .../litho-sonar/LithoViewDescriptor.java | 26 +++++ src/plugins/layout/index.js | 11 +- 8 files changed, 284 insertions(+), 8 deletions(-) diff --git a/android/plugins/inspector/InspectorSonarPlugin.java b/android/plugins/inspector/InspectorSonarPlugin.java index 3f14fe9c3..1cc4d567e 100644 --- a/android/plugins/inspector/InspectorSonarPlugin.java +++ b/android/plugins/inspector/InspectorSonarPlugin.java @@ -126,6 +126,8 @@ public class InspectorSonarPlugin implements SonarPlugin { connection.receive("setHighlighted", mSetHighlighted); connection.receive("setSearchActive", mSetSearchActive); connection.receive("getSearchResults", mGetSearchResults); + connection.receive("getAXRoot", mGetAXRoot); + connection.receive("getAXNodes", mGetAXNodes); if (mExtensionCommands != null) { for (ExtensionCommand extensionCommand : mExtensionCommands) { @@ -156,6 +158,28 @@ public class InspectorSonarPlugin implements SonarPlugin { } }; + final SonarReceiver mGetAXRoot = + new MainThreadSonarReceiver(mConnection) { + @Override + public void onReceiveOnMainThread(SonarObject params, SonarResponder responder) + throws Exception { + List viewRoots = mApplication.getViewRoots(); + // for now only works if one view root + if (viewRoots.size() != 1) { + responder.error( + new SonarObject.Builder().put("message", "Too many view roots.").build()); + return; + } + SonarObject response = getAXNode(trackObject(viewRoots.get(0))); + if (response == null) { + responder.error( + new SonarObject.Builder().put("message", "AX root node returned null.").build()); + return; + } + responder.success(response); + } + }; + final SonarReceiver mGetNodes = new MainThreadSonarReceiver(mConnection) { @Override @@ -183,6 +207,33 @@ public class InspectorSonarPlugin implements SonarPlugin { } }; + final SonarReceiver mGetAXNodes = + new MainThreadSonarReceiver(mConnection) { + @Override + public void onReceiveOnMainThread(final SonarObject params, final SonarResponder responder) + throws Exception { + final SonarArray ids = params.getArray("ids"); + final SonarArray.Builder result = new SonarArray.Builder(); + + for (int i = 0, count = ids.length(); i < count; i++) { + final String id = ids.getString(i); + final SonarObject node = getAXNode(id); + if (node != null) { + result.put(node); + } else { + responder.error( + new SonarObject.Builder() + .put("message", "No node with given id") + .put("id", id) + .build()); + return; + } + } + + responder.success(new SonarObject.Builder().put("elements", result).build()); + } + }; + final SonarReceiver mSetData = new MainThreadSonarReceiver(mConnection) { @Override @@ -435,6 +486,62 @@ public class InspectorSonarPlugin implements SonarPlugin { .build(); } + private @Nullable SonarObject getAXNode(String id) throws Exception { + + final Object obj = mObjectTracker.get(id); + if (obj == null) { + return null; + } + + final NodeDescriptor descriptor = descriptorForObject(obj); + if (descriptor == null) { + return null; + } + + final SonarArray.Builder children = new SonarArray.Builder(); + new ErrorReportingRunnable(mConnection) { + @Override + protected void runOrThrow() throws Exception { + for (int i = 0, count = descriptor.getChildCount(obj); i < count; i++) { + final Object child = assertNotNull(descriptor.getAXChildAt(obj, i)); + children.put(trackObject(child)); + } + } + }.run(); + + final SonarObject.Builder data = new SonarObject.Builder(); + new ErrorReportingRunnable(mConnection) { + @Override + protected void runOrThrow() throws Exception { + for (Named props : descriptor.getAXData(obj)) { + data.put(props.getName(), props.getValue()); + } + } + }.run(); + + final SonarArray.Builder attributes = new SonarArray.Builder(); + new ErrorReportingRunnable(mConnection) { + @Override + protected void runOrThrow() throws Exception { + for (Named attribute : descriptor.getAXAttributes(obj)) { + attributes.put( + new SonarObject.Builder() + .put("name", attribute.getName()) + .put("value", attribute.getValue()) + .build()); + } + } + }.run(); + + return new SonarObject.Builder() + .put("id", descriptor.getId(obj)) + .put("name", descriptor.getAXName(obj)) + .put("data", data) + .put("children", children) + .put("attributes", attributes) + .build(); + } + private String trackObject(Object obj) throws Exception { final NodeDescriptor descriptor = descriptorForObject(obj); final String id = descriptor.getId(obj); diff --git a/android/plugins/inspector/NodeDescriptor.java b/android/plugins/inspector/NodeDescriptor.java index 914a8f3fb..bb6adac89 100644 --- a/android/plugins/inspector/NodeDescriptor.java +++ b/android/plugins/inspector/NodeDescriptor.java @@ -12,7 +12,9 @@ import com.facebook.sonar.core.SonarArray; import com.facebook.sonar.core.SonarConnection; import com.facebook.sonar.core.SonarDynamic; import com.facebook.sonar.core.SonarObject; +import java.util.Collections; import java.util.List; +import javax.annotation.Nullable; /** * A NodeDescriptor is an object which known how to expose an Object of type T to the ew Inspector. @@ -90,18 +92,33 @@ public abstract class NodeDescriptor { */ public abstract String getName(T node) throws Exception; + /** Gets name for AX tree. */ + public String getAXName(T node) throws Exception { + return ""; + } + /** @return The number of children this node exposes in the inspector. */ public abstract int getChildCount(T node) throws Exception; /** @return The child at index. */ public abstract Object getChildAt(T node, int index) throws Exception; + /** Gets child at index for AX tree. Ignores non-view children. */ + public @Nullable Object getAXChildAt(T node, int index) throws Exception { + return null; + } + /** * Get the data to show for this node in the sidebar of the inspector. The object will be showen * in order and with a header matching the given name. */ public abstract List> getData(T node) throws Exception; + /** Gets data for AX tree */ + public List> getAXData(T node) throws Exception { + return Collections.EMPTY_LIST; + } + /** * Set a value on the provided node at the given path. The path will match a key path in the data * provided by {@link this#getData(Object)} and the value will be of the same type as the value @@ -115,6 +132,11 @@ public abstract class NodeDescriptor { */ public abstract List> getAttributes(T node) throws Exception; + /** Gets attributes for AX tree */ + public List> getAXAttributes(T node) throws Exception { + return Collections.EMPTY_LIST; + } + /** * Highlight this node. Use {@link HighlightedOverlay} if possible. This is used to highlight a * node which is selected in the inspector. The plugin automatically takes care of de-selecting diff --git a/android/plugins/inspector/descriptors/TextViewDescriptor.java b/android/plugins/inspector/descriptors/TextViewDescriptor.java index 4426259b5..238c48dad 100644 --- a/android/plugins/inspector/descriptors/TextViewDescriptor.java +++ b/android/plugins/inspector/descriptors/TextViewDescriptor.java @@ -44,6 +44,12 @@ public class TextViewDescriptor extends NodeDescriptor { return descriptor.getName(node); } + @Override + public String getAXName(TextView node) throws Exception { + final NodeDescriptor descriptor = descriptorForClass(View.class); + return descriptor.getAXName(node); + } + @Override public int getChildCount(TextView node) { return 0; @@ -54,6 +60,11 @@ public class TextViewDescriptor extends NodeDescriptor { return null; } + @Override + public @Nullable Object getAXChildAt(TextView node, int index) { + return null; + } + @Override public List> getData(TextView node) throws Exception { final List> props = new ArrayList<>(); @@ -76,6 +87,14 @@ public class TextViewDescriptor extends NodeDescriptor { return props; } + @Override + public List> getAXData(TextView node) throws Exception { + final List> props = new ArrayList<>(); + final NodeDescriptor descriptor = descriptorForClass(View.class); + props.addAll(descriptor.getAXData(node)); + return props; + } + @Override public void setValue(TextView node, String[] path, SonarDynamic value) throws Exception { switch (path[0]) { @@ -106,6 +125,12 @@ public class TextViewDescriptor extends NodeDescriptor { return descriptor.getAttributes(node); } + @Override + public List> getAXAttributes(TextView node) throws Exception { + final NodeDescriptor descriptor = descriptorForClass(View.class); + return descriptor.getAXAttributes(node); + } + @Override public void setHighlighted(TextView node, boolean selected) throws Exception { final NodeDescriptor descriptor = descriptorForClass(View.class); diff --git a/android/plugins/inspector/descriptors/ViewDescriptor.java b/android/plugins/inspector/descriptors/ViewDescriptor.java index b6883ace2..80760b7ee 100644 --- a/android/plugins/inspector/descriptors/ViewDescriptor.java +++ b/android/plugins/inspector/descriptors/ViewDescriptor.java @@ -18,6 +18,7 @@ import android.graphics.drawable.Drawable; import android.os.Build; import android.support.v4.view.MarginLayoutParamsCompat; import android.support.v4.view.ViewCompat; +import android.support.v4.view.accessibility.AccessibilityNodeInfoCompat; import android.util.SparseArray; import android.view.Gravity; import android.view.View; @@ -37,10 +38,12 @@ import com.facebook.sonar.plugins.inspector.NodeDescriptor; import com.facebook.sonar.plugins.inspector.Touch; import com.facebook.sonar.plugins.inspector.descriptors.utils.AccessibilityUtil; import com.facebook.sonar.plugins.inspector.descriptors.utils.EnumMapping; +import com.facebook.sonar.plugins.inspector.descriptors.utils.ViewAccessibilityHelper; import com.facebook.stetho.common.android.ResourcesUtil; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import javax.annotation.Nullable; @@ -78,6 +81,17 @@ public class ViewDescriptor extends NodeDescriptor { return node.getClass().getSimpleName(); } + @Override + public String getAXName(View node) { + AccessibilityNodeInfoCompat nodeInfo = ViewAccessibilityHelper.createNodeInfoFromView(node); + if (nodeInfo == null) { + return "NULL NODEINFO"; + } + String name = nodeInfo.getClassName().toString(); + nodeInfo.recycle(); + return name; + } + @Override public int getChildCount(View node) { return 0; @@ -88,6 +102,11 @@ public class ViewDescriptor extends NodeDescriptor { return null; } + @Override + public @Nullable Object getAXChildAt(View node, int index) { + return null; + } + @Override public List> getData(View node) { final SonarObject.Builder viewProps = @@ -176,6 +195,11 @@ public class ViewDescriptor extends NodeDescriptor { new Named<>("Accessibility", getAccessibilityData(node))); } + @Override + public List> getAXData(View node) { + return Arrays.asList(new Named<>("AX Properties", getAccessibilityData(node))); + } + private static SonarObject getAccessibilityData(View view) { final SonarObject.Builder accessibilityProps = new SonarObject.Builder(); @@ -408,6 +432,10 @@ public class ViewDescriptor extends NodeDescriptor { return attributes; } + public List> getAXAttributes(View node) throws Exception { + return Collections.EMPTY_LIST; + } + @Nullable private static String getResourceId(View node) { final int id = node.getId(); diff --git a/android/plugins/inspector/descriptors/ViewGroupDescriptor.java b/android/plugins/inspector/descriptors/ViewGroupDescriptor.java index 14a096397..271bc9da5 100644 --- a/android/plugins/inspector/descriptors/ViewGroupDescriptor.java +++ b/android/plugins/inspector/descriptors/ViewGroupDescriptor.java @@ -99,6 +99,12 @@ public class ViewGroupDescriptor extends NodeDescriptor { return descriptor.getName(node); } + @Override + public String getAXName(ViewGroup node) throws Exception { + NodeDescriptor descriptor = descriptorForClass(View.class); + return descriptor.getAXName(node); + } + @Override public int getChildCount(ViewGroup node) { int childCount = 0; @@ -130,6 +136,21 @@ public class ViewGroupDescriptor extends NodeDescriptor { return null; } + @Override + public @Nullable Object getAXChildAt(ViewGroup node, int index) { + for (int i = 0, count = node.getChildCount(); i < count; i++) { + final View child = node.getChildAt(i); + if (child instanceof HiddenNode) { + continue; + } + + if (i >= index) { + return child; + } + } + return null; + } + @Override public List> getData(ViewGroup node) throws Exception { final List> props = new ArrayList<>(); @@ -160,6 +181,14 @@ public class ViewGroupDescriptor extends NodeDescriptor { return props; } + @Override + public List> getAXData(ViewGroup node) throws Exception { + final List> props = new ArrayList<>(); + final NodeDescriptor descriptor = descriptorForClass(View.class); + props.addAll(descriptor.getAXData(node)); + return props; + } + @Override public void setValue(ViewGroup node, String[] path, SonarDynamic value) throws Exception { switch (path[0]) { @@ -200,6 +229,12 @@ public class ViewGroupDescriptor extends NodeDescriptor { return descriptor.getAttributes(node); } + @Override + public List> getAXAttributes(ViewGroup node) throws Exception { + final NodeDescriptor descriptor = descriptorForClass(View.class); + return descriptor.getAXAttributes(node); + } + @Override public void setHighlighted(ViewGroup node, boolean selected) throws Exception { final NodeDescriptor descriptor = descriptorForClass(View.class); diff --git a/android/plugins/inspector/litho-sonar/DebugComponentDescriptor.java b/android/plugins/inspector/litho-sonar/DebugComponentDescriptor.java index c29c96a86..f60e422aa 100644 --- a/android/plugins/inspector/litho-sonar/DebugComponentDescriptor.java +++ b/android/plugins/inspector/litho-sonar/DebugComponentDescriptor.java @@ -12,6 +12,7 @@ import android.graphics.Rect; import android.graphics.drawable.ColorDrawable; import android.graphics.drawable.Drawable; import android.support.v4.util.Pair; +import android.support.v4.view.accessibility.AccessibilityNodeInfoCompat; import android.view.View; import com.facebook.litho.Component; import com.facebook.litho.ComponentContext; @@ -31,6 +32,7 @@ import com.facebook.sonar.plugins.inspector.NodeDescriptor; import com.facebook.sonar.plugins.inspector.Touch; import com.facebook.sonar.plugins.inspector.descriptors.ObjectDescriptor; import com.facebook.sonar.plugins.inspector.descriptors.utils.AccessibilityUtil; +import com.facebook.sonar.plugins.inspector.descriptors.utils.ViewAccessibilityHelper; import com.facebook.yoga.YogaAlign; import com.facebook.yoga.YogaDirection; import com.facebook.yoga.YogaEdge; @@ -41,6 +43,7 @@ import com.facebook.yoga.YogaValue; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -122,6 +125,15 @@ public class DebugComponentDescriptor extends NodeDescriptor { return node.getComponent().getSimpleName(); } + @Override + public String getAXName(DebugComponent node) { + View v = node.getComponentHost(); + AccessibilityNodeInfoCompat nodeInfo = ViewAccessibilityHelper.createNodeInfoFromView(v); + String name = nodeInfo.getClassName().toString(); + nodeInfo.recycle(); + return name; + } + @Override public int getChildCount(DebugComponent node) { if (node.getMountedView() != null || node.getMountedDrawable() != null) { @@ -145,6 +157,17 @@ public class DebugComponentDescriptor extends NodeDescriptor { } } + @Override + public @Nullable Object getAXChildAt(DebugComponent node, int index) { + final View mountedView = node.getMountedView(); + + if (mountedView != null) { + return mountedView; + } else { + return node.getChildComponents().get(index); + } + } + @Override public List> getData(DebugComponent node) throws Exception { NodeDescriptor componentDescriptor = descriptorForClass(node.getComponent().getClass()); @@ -177,6 +200,16 @@ public class DebugComponentDescriptor extends NodeDescriptor { return data; } + @Override + public List> getAXData(DebugComponent node) { + final List> data = new ArrayList<>(); + final SonarObject accessibilityData = getAccessibilityData(node); + if (accessibilityData != null) { + data.add(new Named<>("Accessibility", accessibilityData)); + } + return data; + } + @Nullable private static SonarObject getLayoutData(DebugComponent node) { final DebugLayoutNode layout = node.getLayoutNode(); @@ -457,6 +490,11 @@ public class DebugComponentDescriptor extends NodeDescriptor { return attributes; } + @Override + public List> getAXAttributes(DebugComponent node) { + return Collections.EMPTY_LIST; + } + @Override public void setHighlighted(DebugComponent node, boolean selected) { final LithoView lithoView = node.getLithoView(); diff --git a/android/plugins/inspector/litho-sonar/LithoViewDescriptor.java b/android/plugins/inspector/litho-sonar/LithoViewDescriptor.java index 4cb96cf3b..c561130c2 100644 --- a/android/plugins/inspector/litho-sonar/LithoViewDescriptor.java +++ b/android/plugins/inspector/litho-sonar/LithoViewDescriptor.java @@ -13,6 +13,7 @@ import com.facebook.sonar.plugins.inspector.NodeDescriptor; import com.facebook.sonar.plugins.inspector.Touch; import java.util.ArrayList; import java.util.List; +import javax.annotation.Nullable; public class LithoViewDescriptor extends NodeDescriptor { @@ -39,6 +40,12 @@ public class LithoViewDescriptor extends NodeDescriptor { return descriptor.getName(node); } + @Override + public String getAXName(LithoView node) throws Exception { + final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class); + return descriptor.getAXName(node); + } + @Override public int getChildCount(LithoView node) { return DebugComponent.getRootInstance(node) == null ? 0 : 1; @@ -49,6 +56,11 @@ public class LithoViewDescriptor extends NodeDescriptor { return DebugComponent.getRootInstance(node); } + @Override + public @Nullable Object getAXChildAt(LithoView node, int index) { + return DebugComponent.getRootInstance(node); + } + @Override public List> getData(LithoView node) throws Exception { final List> props = new ArrayList<>(); @@ -74,6 +86,14 @@ public class LithoViewDescriptor extends NodeDescriptor { return props; } + @Override + public List> getAXData(LithoView node) throws Exception { + final List> props = new ArrayList<>(); + final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class); + props.addAll(descriptor.getAXData(node)); + return props; + } + @Override public void setValue(LithoView node, String[] path, SonarDynamic value) throws Exception { final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class); @@ -86,6 +106,12 @@ public class LithoViewDescriptor extends NodeDescriptor { return descriptor.getAttributes(node); } + @Override + public List> getAXAttributes(LithoView node) throws Exception { + final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class); + return descriptor.getAXAttributes(node); + } + @Override public void setHighlighted(LithoView node, boolean selected) throws Exception { final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class); diff --git a/src/plugins/layout/index.js b/src/plugins/layout/index.js index 45587335a..21ce8e7b0 100644 --- a/src/plugins/layout/index.js +++ b/src/plugins/layout/index.js @@ -166,11 +166,6 @@ export default class Layout extends SonarPlugin { SelectElement(state: InspectorState, {key}: SelectElementArgs) { return { selected: key, - }; - }, - - SelectAXElement(state: InspectorState, {key}: SelectElementArgs) { - return { AXselected: key, }; }, @@ -402,7 +397,7 @@ export default class Layout extends SonarPlugin { }); if (AXToggleButtonEnabled) { - this.client.call('getRoot').then((element: Element) => { + this.client.call('getAXRoot').then((element: Element) => { this.dispatchAction({elements: [element], type: 'UpdateAXElements'}); this.dispatchAction({root: element.id, type: 'SetAXRoot'}); this.performInitialExpand(element, true).then(() => { @@ -511,7 +506,7 @@ export default class Layout extends SonarPlugin { if (ids.length > 0) { performance.mark('LayoutInspectorGetNodes'); return this.client - .call('getNodes', {ids}) + .call(ax ? 'getAXNodes' : 'getNodes', {ids}) .then(({elements}: GetNodesResult) => { this.props.logger.trackTimeSince('LayoutInspectorGetNodes'); return Promise.resolve(elements); @@ -629,7 +624,7 @@ export default class Layout extends SonarPlugin { }); onAXElementSelected = debounce((key: ElementID) => { - this.dispatchAction({key, type: 'SelectAXElement'}); + this.dispatchAction({key, type: 'SelectElement'}); this.client.send('setHighlighted', {id: key}); this.getNodes([key], true, true).then((elements: Array) => { this.dispatchAction({elements, type: 'UpdateAXElements'});