Use views not components for Litho nodes in AX tree + more sidebar props

Summary: Switches the tree to use the view hierarchy for Litho nodes rather than Litho component hierarchy since Accessibility services interact with the views rendered. Includes a few more properties in the accessibility sidebar and updates to the segmented sidebar based on derived/non-derived properties for all views. Also adds functions to the AccessibilityUtil to be able to work on the accessibility sidebar while still leaving the non-accessibility sidebar unchanged. Eventually the accessibility panel will be removed from 'normal' mode and the original functions will no longer be necessary.

Reviewed By: blavalla

Differential Revision: D8881739

fbshipit-source-id: 9ce37e8f18025538cba2c86c0895ee38d13d024b

android/plugins/inspector/descriptors/ViewDescriptor.java

@@ -49,6 +49,9 @@ import javax.annotation.Nullable;
 
 public class ViewDescriptor extends NodeDescriptor<View> {
 
+  private static final String axViewPropsTitle = "Accessibility View";
+  private static final String axNodeInfoPropsTitle = "NodeInfo & TalkBack";
+
   private static Field sKeyedTagsField;
   private static Field sListenerInfoField;
   private static Field sOnClickListenerField;
@@ -193,8 +196,8 @@ public class ViewDescriptor extends NodeDescriptor<View> {
   @Override
   public List<Named<SonarObject>> getAXData(View node) {
     return Arrays.asList(
-        new Named<>("Derived Props", AccessibilityUtil.getDerivedAXData(node)),
+        new Named<>(axNodeInfoPropsTitle, AccessibilityUtil.getDerivedAXData(node)),
-        new Named<>("AX Props", AccessibilityUtil.getViewAXData(node)));
+        new Named<>(axViewPropsTitle, AccessibilityUtil.getViewAXData(node)));
   }
 
   private static SonarObject getAccessibilityData(View view) {
@@ -223,9 +226,10 @@ public class ViewDescriptor extends NodeDescriptor<View> {
   @Override
   public void setValue(View node, String[] path, SonarDynamic value) {
     if (path[0].equals("Accessibility")
-        || path[0].equals("AX Props")
+        || path[0].equals(axViewPropsTitle)
-        || path[0].equals("Derived Props")) {
+        || path[0].equals(axNodeInfoPropsTitle)) {
       setAccessibilityValue(node, path, value);
+      return;
     }
 
     if (!path[0].equals("View")) {
@@ -404,6 +408,15 @@ public class ViewDescriptor extends NodeDescriptor<View> {
       case "content-description":
         node.setContentDescription(value.asString());
         break;
+      case "label-for":
+        node.setLabelFor(value.asInt());
+        break;
+      case "traversal-after":
+        node.setAccessibilityTraversalAfter(value.asInt());
+        break;
+      case "traversal-before":
+        node.setAccessibilityTraversalBefore(value.asInt());
+        break;
     }
     invalidate(node);
   }

android/plugins/inspector/descriptors/utils/AccessibilityUtil.java

@@ -336,6 +336,90 @@ public final class AccessibilityUtil {
     return nodeInfoProps.build();
   }
 
+  /**
+   * Creates a {@link SonarObject} of useful properties of AccessibilityNodeInfo, to be shown in the
+   * Sonar Layout Inspector extension. All properties are immutable since they are all derived from
+   * various {@link View} properties. This is a more complete list than
+   * getAccessibilityNodeInfoProperties returns.
+   *
+   * @param view The {@link View} to derive the AccessibilityNodeInfo properties from.
+   * @return {@link SonarObject} containing the properties.
+   */
+  @Nullable
+  public static SonarObject getAXNodeInfoProperties(View view) {
+    final AccessibilityNodeInfoCompat nodeInfo =
+        ViewAccessibilityHelper.createNodeInfoFromView(view);
+    if (nodeInfo == null) {
+      return null;
+    }
+
+    final SonarObject.Builder nodeInfoProps = new SonarObject.Builder();
+    final Rect bounds = new Rect();
+
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+      final SonarArray.Builder actionsArrayBuilder = new SonarArray.Builder();
+      for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action :
+          nodeInfo.getActionList()) {
+        final String actionLabel = (String) action.getLabel();
+        if (actionLabel != null) {
+          actionsArrayBuilder.put(actionLabel);
+        } else {
+          actionsArrayBuilder.put(
+              AccessibilityUtil.sAccessibilityActionMapping.get(action.getId(), false));
+        }
+      }
+      nodeInfoProps.put("actions", actionsArrayBuilder.build());
+    }
+
+    nodeInfoProps
+        .put("checkable", nodeInfo.isCheckable())
+        .put("checked", nodeInfo.isChecked())
+        .put("clickable", nodeInfo.isClickable())
+        .put("content-description", nodeInfo.getContentDescription())
+        .put("content-invalid", nodeInfo.isContentInvalid())
+        .put("context-clickable", nodeInfo.isContextClickable())
+        .put("dismissable", nodeInfo.isDismissable())
+        .put("drawing-order", nodeInfo.getDrawingOrder())
+        .put("editable", nodeInfo.isEditable())
+        .put("enabled", nodeInfo.isEnabled())
+        .put("focusable", nodeInfo.isFocusable())
+        .put("focused", nodeInfo.isAccessibilityFocused())
+        .put("important-for-accessibility", nodeInfo.isImportantForAccessibility())
+        .put("long-clickable", nodeInfo.isLongClickable())
+        .put("multiline", nodeInfo.isMultiLine())
+        .put("password", nodeInfo.isPassword())
+        .put("scrollable", nodeInfo.isScrollable())
+        .put("selected", nodeInfo.isSelected())
+        .put("text", nodeInfo.getText())
+        .put("visible-to-user", nodeInfo.isVisibleToUser());
+
+    nodeInfo.getBoundsInParent(bounds);
+    nodeInfoProps.put(
+        "parent-bounds",
+        new SonarObject.Builder()
+            .put("width", bounds.width())
+            .put("height", bounds.height())
+            .put("top", bounds.top)
+            .put("left", bounds.left)
+            .put("bottom", bounds.bottom)
+            .put("right", bounds.right));
+
+    nodeInfo.getBoundsInScreen(bounds);
+    nodeInfoProps.put(
+        "screen-bounds",
+        new SonarObject.Builder()
+            .put("width", bounds.width())
+            .put("height", bounds.height())
+            .put("top", bounds.top)
+            .put("left", bounds.left)
+            .put("bottom", bounds.bottom)
+            .put("right", bounds.right));
+
+    nodeInfo.recycle();
+
+    return nodeInfoProps.build();
+  }
+
   /**
    * Modifies a {@link SonarObject.Builder} to add Talkback-specific Accessibiltiy properties to be
    * shown in the Sonar Layout Inspector.
@@ -392,8 +476,10 @@ public final class AccessibilityUtil {
           .put("talkback-description", description == null ? "" : description);
     }
 
-    SonarObject axProps = getAccessibilityNodeInfoProperties(view);
+    SonarObject axNodeInfo = getAXNodeInfoProperties(view);
-    props.put("node-info", axProps == null ? "null" : axProps);
+    if (axNodeInfo != null) {
+      props.put("node-info", axNodeInfo);
+    }
 
     return props.build();
   }

android/plugins/inspector/litho-sonar/DebugComponentDescriptor.java

@@ -30,6 +30,7 @@ import com.facebook.sonar.plugins.inspector.Named;
 import com.facebook.sonar.plugins.inspector.NodeDescriptor;
 import com.facebook.sonar.plugins.inspector.Touch;
 import com.facebook.sonar.plugins.inspector.descriptors.ObjectDescriptor;
+import com.facebook.sonar.plugins.inspector.descriptors.utils.AccessibilityRoleUtil;
 import com.facebook.sonar.plugins.inspector.descriptors.utils.AccessibilityUtil;
 import com.facebook.yoga.YogaAlign;
 import com.facebook.yoga.YogaDirection;
@@ -41,7 +42,6 @@ import com.facebook.yoga.YogaValue;
 import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -49,6 +49,9 @@ import javax.annotation.Nullable;
 
 public class DebugComponentDescriptor extends NodeDescriptor<DebugComponent> {
 
+  private static final String axViewPropsTitle = "DebugLayoutNode";
+  private static final String axNodeInfoPropsTitle = "NodeInfo & TalkBack";
+
   private Map<String, List<Pair<String[], SonarDynamic>>> mOverrides = new HashMap<>();
   private DebugComponent.Overrider mOverrider =
       new DebugComponent.Overrider() {
@@ -97,7 +100,9 @@ public class DebugComponentDescriptor extends NodeDescriptor<DebugComponent> {
                     override.second);
               } catch (Exception ignored) {
               }
-            } else if (override.first[0].equals("Accessibility")) {
+            } else if (override.first[0].equals("Accessibility")
+                || override.first[0].equals(axViewPropsTitle)
+                || override.first[0].equals(axNodeInfoPropsTitle)) {
               applyAccessibilityOverride(node, override.first[1], override.second);
             }
           }
@@ -123,15 +128,6 @@ public class DebugComponentDescriptor extends NodeDescriptor<DebugComponent> {
     return node.getComponent().getSimpleName();
   }
 
-  @Override
-  public String getAXName(DebugComponent node) throws Exception {
-    NodeDescriptor componentDescriptor = descriptorForClass(node.getComponent().getClass());
-    if (componentDescriptor.getClass() != ObjectDescriptor.class) {
-      return componentDescriptor.getAXName(node.getComponent());
-    }
-    return node.getComponent().getSimpleName();
-  }
-
   @Override
   public int getChildCount(DebugComponent node) {
     if (node.getMountedView() != null || node.getMountedDrawable() != null) {
@@ -141,15 +137,6 @@ public class DebugComponentDescriptor extends NodeDescriptor<DebugComponent> {
     }
   }
 
-  @Override
-  public int getAXChildCount(DebugComponent node) {
-    if (node.getMountedView() != null) {
-      return 1;
-    } else {
-      return node.getChildComponents().size();
-    }
-  }
-
   @Override
   public Object getChildAt(DebugComponent node, int index) {
     final View mountedView = node.getMountedView();
@@ -164,17 +151,6 @@ public class DebugComponentDescriptor extends NodeDescriptor<DebugComponent> {
     }
   }
 
-  @Override
-  public Object getAXChildAt(DebugComponent node, int index) {
-    final View mountedView = node.getMountedView();
-
-    if (mountedView != null) {
-      return mountedView;
-    } else {
-      return node.getChildComponents().get(index);
-    }
-  }
-
   @Override
   public List<Named<SonarObject>> getData(DebugComponent node) throws Exception {
     NodeDescriptor componentDescriptor = descriptorForClass(node.getComponent().getClass());
@@ -213,12 +189,82 @@ public class DebugComponentDescriptor extends NodeDescriptor<DebugComponent> {
     if (componentDescriptor.getClass() != ObjectDescriptor.class) {
       return componentDescriptor.getAXData(node.getComponent());
     }
-    final List<Named<SonarObject>> data = new ArrayList<>();
+    final List<Named<SonarObject>> sections = new ArrayList<>();
-    final SonarObject accessibilityData = getAccessibilityData(node);
+
-    if (accessibilityData != null) {
+    final SonarObject derivedData = getDerivedAXData(node);
-      data.add(new Named<>("Accessibility", accessibilityData));
+    if (derivedData != null) {
+      sections.add(new Named<>(axNodeInfoPropsTitle, derivedData));
     }
-    return data;
+
+    final SonarObject viewData = getViewAXData(node);
+    if (viewData != null) {
+      sections.add(new Named<>(axViewPropsTitle, viewData));
+    }
+
+    return sections;
+  }
+
+  @Nullable
+  private static SonarObject getViewAXData(DebugComponent node) {
+    final DebugLayoutNode layout = node.getLayoutNode();
+    if (layout == null) {
+      return null;
+    }
+
+    final SonarObject.Builder props = new SonarObject.Builder();
+
+    // This needs to be an empty string to be mutable. See t20470623.
+    final CharSequence contentDescription =
+        layout.getContentDescription() != null ? layout.getContentDescription() : "";
+    props.put("content-description", InspectorValue.mutable(contentDescription));
+    props.put("focusable", InspectorValue.mutable(layout.getFocusable()));
+    props.put(
+        "important-for-accessibility",
+        AccessibilityUtil.sImportantForAccessibilityMapping.get(
+            layout.getImportantForAccessibility()));
+
+    return props.build();
+  }
+
+  @Nullable
+  private static SonarObject getDerivedAXData(DebugComponent node) {
+    final DebugLayoutNode layout = node.getLayoutNode();
+    if (layout == null) {
+      return null;
+    }
+
+    final View hostView = node.getComponentHost();
+    final SonarObject.Builder props = new SonarObject.Builder();
+
+    // No host view exists, so this component is inherently not accessible.  Add the reason why this
+    // is the case and then return.
+    if (hostView == node.getLithoView() || hostView == null) {
+      final int importantForAccessibility = layout.getImportantForAccessibility();
+      final boolean isAccessibilityEnabled =
+          AccessibilityUtil.isAccessibilityEnabled(node.getContext());
+      String ignoredReason;
+
+      if (!isAccessibilityEnabled) {
+        ignoredReason = "No accessibility service is running.";
+      } else if (importantForAccessibility == IMPORTANT_FOR_ACCESSIBILITY_NO) {
+        ignoredReason = "Component has importantForAccessibility set to NO.";
+      } else if (importantForAccessibility == IMPORTANT_FOR_ACCESSIBILITY_NO_HIDE_DESCENDANTS) {
+        ignoredReason = "Component has importantForAccessibility set to NO_HIDE_DESCENDANTS.";
+      } else {
+        ignoredReason = "Component does not have content, or accessibility handlers.";
+      }
+
+      props.put("talkback-ignored", true);
+      props.put("talkback-ignored-reasons", ignoredReason);
+
+      return props.build();
+    }
+
+    // host view exists so add node info and TalkBack properties
+    props.put("node-info", AccessibilityUtil.getAXNodeInfoProperties(hostView));
+    AccessibilityUtil.addTalkbackProperties(props, hostView);
+
+    return props.build();
   }
 
   @Nullable
@@ -503,7 +549,13 @@ public class DebugComponentDescriptor extends NodeDescriptor<DebugComponent> {
 
   @Override
   public List<Named<String>> getAXAttributes(DebugComponent node) {
-    return Collections.EMPTY_LIST;
+    final View hostView = node.getComponentHost();
+    List<Named<String>> attributes = new ArrayList<>();
+    String role = AccessibilityRoleUtil.getRole(hostView).toString();
+    if (!role.equals("NONE")) {
+      attributes.add(new Named<>("role", role));
+    }
+    return attributes;
   }
 
   @Override

android/plugins/inspector/litho-sonar/LithoViewDescriptor.java

@@ -51,6 +51,11 @@ public class LithoViewDescriptor extends NodeDescriptor<LithoView> {
     return DebugComponent.getRootInstance(node) == null ? 0 : 1;
   }
 
+  @Override
+  public int getAXChildCount(LithoView node) {
+    return node.getChildCount();
+  }
+
   @Override
   public Object getChildAt(LithoView node, int index) {
     return DebugComponent.getRootInstance(node);
@@ -58,7 +63,7 @@ public class LithoViewDescriptor extends NodeDescriptor<LithoView> {
 
   @Override
   public @Nullable Object getAXChildAt(LithoView node, int index) {
-    return DebugComponent.getRootInstance(node);
+    return node.getChildAt(index);
   }
 
   @Override