Highlight the current talkback-focused element in the accessibility tree

Summary: Highlights the element corresponding to the view talkback is focused on in green in the ax tree (and updates live as talkback moves). Reviewed By: blavalla Differential Revision: D9021542 fbshipit-source-id: c3bf6f5625aacb0cd054032b33a50541b88b2eaf
2018-07-27 16:16:54 -07:00
parent 6939292209
commit 33e6538477
10 changed files with 154 additions and 18 deletions
--- a/android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java
+++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java
@@ -10,6 +10,8 @@ package com.facebook.sonar.plugins.inspector;

 import android.app.Application;
 import android.content.Context;
+import android.support.v4.view.ViewCompat;
+import android.view.accessibility.AccessibilityEvent;
 import android.view.MotionEvent;
 import android.view.View;
 import android.view.ViewGroup;
@@ -170,20 +172,41 @@ public class InspectorSonarPlugin implements SonarPlugin {
        @Override
        public void onReceiveOnMainThread(SonarObject params, SonarResponder responder)
            throws Exception {
-          List<View> viewRoots = mApplication.getViewRoots();
-          // for now only works if one view root
-          if (viewRoots.size() != 1) {
-            responder.error(
-                new SonarObject.Builder().put("message", "Too many view roots.").build());
-            return;
+          final List<View> viewRoots = mApplication.getViewRoots();
+
+          ViewGroup root = null;
+          for (int i = viewRoots.size() - 1; i >= 0; i--) {
+            if (viewRoots.get(i) instanceof ViewGroup) {
+              root = (ViewGroup) viewRoots.get(i);
+              break;
+            }
          }
-          SonarObject response = getAXNode(trackObject(viewRoots.get(0)));
-          if (response == null) {
-            responder.error(
-                new SonarObject.Builder().put("message", "AX root node returned null.").build());
-            return;
+
+          if (root != null) {
+
+            // unlikely, but check to make sure accessibility functionality doesn't change
+            if (!ViewCompat.hasAccessibilityDelegate(root)) {
+              
+              // add delegate to root to catch accessibility events so we can update focus in sonar
+              root.setAccessibilityDelegate(new View.AccessibilityDelegate() {
+
+                @Override
+                public boolean onRequestSendAccessibilityEvent(ViewGroup host, View child, AccessibilityEvent event) {
+                  int eventType = event.getEventType();
+                  if (eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED || eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED) {
+                    mConnection.send("axFocusEvent",
+                            new SonarObject.Builder()
+                                    .put("isFocus", eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED)
+                                    .build());
+                  }
+
+                  return super.onRequestSendAccessibilityEvent(host, child, event);
+                }
+              });
+            }
+
+            responder.success(getAXNode(trackObject(root)));
          }
-          responder.success(response);
        }
      };

@@ -550,6 +573,7 @@ public class InspectorSonarPlugin implements SonarPlugin {
        .put("data", data)
        .put("children", children)
        .put("attributes", attributes)
+        .put("extraInfo", descriptor.getExtraInfo(obj))
        .build();
  }

--- a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/TextViewDescriptor.java
+++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/TextViewDescriptor.java
@@ -131,6 +131,12 @@ public class TextViewDescriptor extends NodeDescriptor<TextView> {
    return descriptor.getAXAttributes(node);
  }

+  @Override
+  public SonarObject getExtraInfo(TextView node) {
+    final NodeDescriptor descriptor = descriptorForClass(View.class);
+    return descriptor.getExtraInfo(node);
+  }
+
  @Override
  public void setHighlighted(TextView node, boolean selected) throws Exception {
    final NodeDescriptor descriptor = descriptorForClass(View.class);
--- a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewDescriptor.java
+++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewDescriptor.java
@@ -444,6 +444,7 @@ public class ViewDescriptor extends NodeDescriptor<View> {
    return attributes;
  }

+  @Override
  public List<Named<String>> getAXAttributes(View node) throws Exception {
    List<Named<String>> attributes = new ArrayList<>();
    String role = AccessibilityRoleUtil.getRole(node).toString();
@@ -453,6 +454,11 @@ public class ViewDescriptor extends NodeDescriptor<View> {
    return attributes;
  }

+  @Override
+  public SonarObject getExtraInfo(View node) {
+    return new SonarObject.Builder().put("focused", AccessibilityUtil.isAXFocused(node)).build();
+  }
+
  @Nullable
  private static String getResourceId(View node) {
    final int id = node.getId();
--- a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewGroupDescriptor.java
+++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewGroupDescriptor.java
@@ -235,6 +235,12 @@ public class ViewGroupDescriptor extends NodeDescriptor<ViewGroup> {
    return descriptor.getAXAttributes(node);
  }

+  @Override
+  public SonarObject getExtraInfo(ViewGroup node) {
+    final NodeDescriptor descriptor = descriptorForClass(View.class);
+    return descriptor.getExtraInfo(node);
+  }
+
  @Override
  public void setHighlighted(ViewGroup node, boolean selected) throws Exception {
    final NodeDescriptor descriptor = descriptorForClass(View.class);
--- a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java
+++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java
@@ -420,6 +420,18 @@ public final class AccessibilityUtil {
    return nodeInfoProps.build();
  }

+  public static boolean isAXFocused(View view) {
+    final AccessibilityNodeInfoCompat nodeInfo =
+            ViewAccessibilityHelper.createNodeInfoFromView(view);
+    if (nodeInfo == null) {
+      return false;
+    } else {
+      boolean focused = nodeInfo.isAccessibilityFocused();
+      nodeInfo.recycle();
+      return focused;
+    }
+  }
+
  /**
   * Modifies a {@link SonarObject.Builder} to add Talkback-specific Accessibiltiy properties to be
   * shown in the Sonar Layout Inspector.
--- a/android/src/main/java/com/facebook/sonar/plugins/litho/LithoViewDescriptor.java
+++ b/android/src/main/java/com/facebook/sonar/plugins/litho/LithoViewDescriptor.java
@@ -117,6 +117,12 @@ public class LithoViewDescriptor extends NodeDescriptor<LithoView> {
    return descriptor.getAXAttributes(node);
  }

+  @Override
+  public SonarObject getExtraInfo(LithoView node) {
+    final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class);
+    return descriptor.getExtraInfo(node);
+  }
+
  @Override
  public void setHighlighted(LithoView node, boolean selected) throws Exception {
    final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class);
--- a/src/fb-stubs/AXLayoutExtender.js
+++ b/src/fb-stubs/AXLayoutExtender.js
@@ -14,6 +14,7 @@ export class AXElementsInspector extends Component<{
  onElementHovered: ?(key: ?ElementID) => void,
  onValueChanged: ?(path: Array<string>, val: any) => void,
  selected: ?ElementID,
+  focused: ?ElementID,
  searchResults?: ?ElementSearchResultSet,
  root: ?ElementID,
  elements: {[key: ElementID]: Element},
--- a/src/plugins/layout/index.js
+++ b/src/plugins/layout/index.js
@@ -38,6 +38,7 @@ export type InspectorState = {|
  AXinitialised: boolean,
  selected: ?ElementID,
  AXselected: ?ElementID,
+  AXfocused: ?ElementID,
  root: ?ElementID,
  AXroot: ?ElementID,
  elements: {[key: ElementID]: Element},
@@ -66,6 +67,15 @@ type UpdateElementsArgs = {|
  elements: Array<$Shape<Element>>,
 |};

+type UpdateAXElementsArgs = {|
+  elements: Array<$Shape<Element>>,
+  forFocusEvent: boolean,
+|};
+
+type AXFocusEventResult = {|
+  isFocus: boolean,
+|};
+
 type SetRootArgs = {|
  root: ElementID,
 |};
@@ -159,6 +169,7 @@ export default class Layout extends SonarPlugin<InspectorState> {
    AXroot: null,
    selected: null,
    AXselected: null,
+    AXfocused: null,
    searchResults: null,
    outstandingSearchQuery: null,
    AXtoNonAXMapping: {},
@@ -278,17 +289,30 @@ export default class Layout extends SonarPlugin<InspectorState> {
      return {elements: updatedElements, AXtoNonAXMapping: updatedMapping};
    },

-    UpdateAXElements(state: InspectorState, {elements}: UpdateElementsArgs) {
+    UpdateAXElements(
+      state: InspectorState,
+      {elements, forFocusEvent}: UpdateAXElementsArgs,
+    ) {
      const updatedElements = state.AXelements;

+      // if focusEvent, previously focused element can be reset
+      let updatedFocus = forFocusEvent ? null : state.AXfocused;
+
      for (const element of elements) {
+        if (element.extraInfo.focused) {
+          updatedFocus = element.id;
+        }
        const current = updatedElements[element.id] || {};
        updatedElements[element.id] = {
          ...current,
          ...element,
        };
      }
-      return {AXelements: updatedElements};
+
+      return {
+        AXelements: updatedElements,
+        AXfocused: updatedFocus,
+      };
    },

    SetRoot(state: InspectorState, {root}: SetRootArgs) {
@@ -442,6 +466,34 @@ export default class Layout extends SonarPlugin<InspectorState> {
      });
    }

+    this.client.subscribe('axFocusEvent', (focusEvent: AXFocusEventResult) => {
+      if (AXToggleButtonEnabled) {
+        // if focusing, need to update all elements in the tree because
+        // we don't know which one now has focus
+        const keys = focusEvent.isFocus
+          ? Object.keys(this.state.AXelements)
+          : [];
+
+        // if unfocusing and currently focused element exists, update only the
+        // focused element (and only if it is loaded in tree)
+        if (
+          !focusEvent.isFocus &&
+          this.state.AXfocused &&
+          this.state.AXelements[this.state.AXfocused]
+        ) {
+          keys.push(this.state.AXfocused);
+        }
+
+        this.getNodes(keys, true, true).then((elements: Array<Element>) => {
+          this.dispatchAction({
+            elements,
+            forFocusEvent: true,
+            type: 'UpdateAXElements',
+          });
+        });
+      }
+    });
+
    this.client.subscribe(
      'invalidate',
      ({nodes}: {nodes: Array<{id: ElementID}>}) => {
@@ -713,6 +765,7 @@ export default class Layout extends SonarPlugin<InspectorState> {
      AXinitialised,
      selected,
      AXselected,
+      AXfocused,
      root,
      AXroot,
      elements,
@@ -792,6 +845,7 @@ export default class Layout extends SonarPlugin<InspectorState> {
              onElementExpanded={this.onElementExpanded}
              onValueChanged={this.onDataValueChanged}
              selected={AXselected}
+              focused={AXfocused}
              searchResults={null}
              root={AXroot}
              elements={AXelements}
--- a/src/ui/components/elements-inspector/ElementsInspector.js
+++ b/src/ui/components/elements-inspector/ElementsInspector.js
@@ -37,6 +37,7 @@ export type ElementAttribute = {|
 export type ElementExtraInfo = {|
  nonAXWithAXChild?: boolean,
  linkedAXNode?: string,
+  focused?: boolean,
 |};

 export type Element = {|
--- a/src/ui/components/elements-inspector/elements.js
+++ b/src/ui/components/elements-inspector/elements.js
@@ -31,6 +31,8 @@ const ElementsRowContainer = ContextMenu.extends(
    backgroundColor: props => {
      if (props.selected) {
        return colors.macOSTitleBarIconSelected;
+      } else if (props.focused) {
+        return colors.lime;
      } else if (props.even) {
        return colors.light02;
      } else {
@@ -47,12 +49,20 @@ const ElementsRowContainer = ContextMenu.extends(
    position: 'relative',

    '& *': {
-      color: props => (props.selected ? `${colors.white} !important` : ''),
+      color: props =>
+        props.selected || props.focused ? `${colors.white} !important` : '',
    },

    '&:hover': {
-      backgroundColor: props =>
-        props.selected ? colors.macOSTitleBarIconSelected : '#EBF1FB',
+      backgroundColor: props => {
+        if (props.selected) {
+          return colors.macOSTitleBarIconSelected;
+        } else if (props.focused) {
+          return colors.lime;
+        } else {
+          return '#EBF1FB';
+        }
+      },
    },
  },
  {
@@ -122,7 +132,9 @@ class PartialHighlight extends PureComponent<{
  static HighlightedText = styled.text({
    backgroundColor: '#ffff33',
    color: props =>
-      props.selected ? `${colors.grapeDark3} !important` : 'auto',
+      props.selected || props.focused
+        ? `${colors.grapeDark3} !important`
+        : 'auto',
  });

  render() {
@@ -162,6 +174,7 @@ class ElementsRowAttribute extends PureComponent<{
  value: string,
  matchingSearchQuery: ?string,
  selected: boolean,
+  focused: boolean,
 }> {
  render() {
    const {name, value, matchingSearchQuery, selected} = this.props;
@@ -195,6 +208,7 @@ type ElementsRowProps = {
  id: ElementID,
  level: number,
  selected: boolean,
+  focused: boolean,
  matchingSearchQuery: ?string,
  element: Element,
  even: boolean,
@@ -268,6 +282,7 @@ class ElementsRow extends PureComponent<ElementsRowProps, ElementsRowState> {
      id,
      level,
      selected,
+      focused,
      style,
      even,
      matchingSearchQuery,
@@ -295,6 +310,7 @@ class ElementsRow extends PureComponent<ElementsRowProps, ElementsRowState> {
            value={attr.value}
            matchingSearchQuery={matchingSearchQuery}
            selected={selected}
+            focused={focused}
          />
        ))
      : [];
@@ -327,6 +343,7 @@ class ElementsRow extends PureComponent<ElementsRowProps, ElementsRowState> {
        key={id}
        level={level}
        selected={selected}
+        focused={focused}
        matchingSearchQuery={matchingSearchQuery}
        even={even}
        onClick={this.onClick}
@@ -368,6 +385,7 @@ const ElementsBox = FlexColumn.extends({
 type ElementsProps = {|
  root: ?ElementID,
  selected: ?ElementID,
+  focused?: ?ElementID,
  searchResults: ?ElementSearchResultSet,
  elements: {[key: ElementID]: Element},
  onElementSelected: (key: ElementID) => void,
@@ -532,6 +550,7 @@ export class Elements extends PureComponent<ElementsProps, ElementsState> {
      onElementHovered,
      onElementSelected,
      selected,
+      focused,
      searchResults,
    } = this.props;
    const {flatElements} = this.state;
@@ -557,6 +576,7 @@ export class Elements extends PureComponent<ElementsProps, ElementsState> {
        onElementHovered={onElementHovered}
        onElementSelected={onElementSelected}
        selected={selected === row.key}
+        focused={focused === row.key}
        matchingSearchQuery={
          searchResults && searchResults.matches.has(row.key)
            ? searchResults.query