Highlight the current talkback-focused element in the accessibility tree

Summary: Highlights the element corresponding to the view talkback is focused on in green in the ax tree (and updates live as talkback moves).

Reviewed By: blavalla

Differential Revision: D9021542

fbshipit-source-id: c3bf6f5625aacb0cd054032b33a50541b88b2eaf

android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java

@@ -10,6 +10,8 @@ package com.facebook.sonar.plugins.inspector;
 
 import android.app.Application;
 import android.content.Context;
+import android.support.v4.view.ViewCompat;
+import android.view.accessibility.AccessibilityEvent;
 import android.view.MotionEvent;
 import android.view.View;
 import android.view.ViewGroup;
@@ -170,20 +172,41 @@ public class InspectorSonarPlugin implements SonarPlugin {
         @Override
         public void onReceiveOnMainThread(SonarObject params, SonarResponder responder)
             throws Exception {
-          List<View> viewRoots = mApplication.getViewRoots();
+          final List<View> viewRoots = mApplication.getViewRoots();
-          // for now only works if one view root
+
-          if (viewRoots.size() != 1) {
+          ViewGroup root = null;
-            responder.error(
+          for (int i = viewRoots.size() - 1; i >= 0; i--) {
-                new SonarObject.Builder().put("message", "Too many view roots.").build());
+            if (viewRoots.get(i) instanceof ViewGroup) {
-            return;
+              root = (ViewGroup) viewRoots.get(i);
+              break;
+            }
           }
-          SonarObject response = getAXNode(trackObject(viewRoots.get(0)));
+
-          if (response == null) {
+          if (root != null) {
-            responder.error(
+
-                new SonarObject.Builder().put("message", "AX root node returned null.").build());
+            // unlikely, but check to make sure accessibility functionality doesn't change
-            return;
+            if (!ViewCompat.hasAccessibilityDelegate(root)) {
+              
+              // add delegate to root to catch accessibility events so we can update focus in sonar
+              root.setAccessibilityDelegate(new View.AccessibilityDelegate() {
+
+                @Override
+                public boolean onRequestSendAccessibilityEvent(ViewGroup host, View child, AccessibilityEvent event) {
+                  int eventType = event.getEventType();
+                  if (eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED || eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED) {
+                    mConnection.send("axFocusEvent",
+                            new SonarObject.Builder()
+                                    .put("isFocus", eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED)
+                                    .build());
+                  }
+
+                  return super.onRequestSendAccessibilityEvent(host, child, event);
+                }
+              });
+            }
+
+            responder.success(getAXNode(trackObject(root)));
           }
-          responder.success(response);
         }
       };
 
@@ -550,6 +573,7 @@ public class InspectorSonarPlugin implements SonarPlugin {
         .put("data", data)
         .put("children", children)
         .put("attributes", attributes)
+        .put("extraInfo", descriptor.getExtraInfo(obj))
         .build();
   }
 

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/TextViewDescriptor.java

@@ -131,6 +131,12 @@ public class TextViewDescriptor extends NodeDescriptor<TextView> {
     return descriptor.getAXAttributes(node);
   }
 
+  @Override
+  public SonarObject getExtraInfo(TextView node) {
+    final NodeDescriptor descriptor = descriptorForClass(View.class);
+    return descriptor.getExtraInfo(node);
+  }
+
   @Override
   public void setHighlighted(TextView node, boolean selected) throws Exception {
     final NodeDescriptor descriptor = descriptorForClass(View.class);

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewDescriptor.java

@@ -444,6 +444,7 @@ public class ViewDescriptor extends NodeDescriptor<View> {
     return attributes;
   }
 
+  @Override
   public List<Named<String>> getAXAttributes(View node) throws Exception {
     List<Named<String>> attributes = new ArrayList<>();
     String role = AccessibilityRoleUtil.getRole(node).toString();
@@ -453,6 +454,11 @@ public class ViewDescriptor extends NodeDescriptor<View> {
     return attributes;
   }
 
+  @Override
+  public SonarObject getExtraInfo(View node) {
+    return new SonarObject.Builder().put("focused", AccessibilityUtil.isAXFocused(node)).build();
+  }
+
   @Nullable
   private static String getResourceId(View node) {
     final int id = node.getId();

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewGroupDescriptor.java

@@ -235,6 +235,12 @@ public class ViewGroupDescriptor extends NodeDescriptor<ViewGroup> {
     return descriptor.getAXAttributes(node);
   }
 
+  @Override
+  public SonarObject getExtraInfo(ViewGroup node) {
+    final NodeDescriptor descriptor = descriptorForClass(View.class);
+    return descriptor.getExtraInfo(node);
+  }
+
   @Override
   public void setHighlighted(ViewGroup node, boolean selected) throws Exception {
     final NodeDescriptor descriptor = descriptorForClass(View.class);

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java

@@ -420,6 +420,18 @@ public final class AccessibilityUtil {
     return nodeInfoProps.build();
   }
 
+  public static boolean isAXFocused(View view) {
+    final AccessibilityNodeInfoCompat nodeInfo =
+            ViewAccessibilityHelper.createNodeInfoFromView(view);
+    if (nodeInfo == null) {
+      return false;
+    } else {
+      boolean focused = nodeInfo.isAccessibilityFocused();
+      nodeInfo.recycle();
+      return focused;
+    }
+  }
+
   /**
    * Modifies a {@link SonarObject.Builder} to add Talkback-specific Accessibiltiy properties to be
    * shown in the Sonar Layout Inspector.

android/src/main/java/com/facebook/sonar/plugins/litho/LithoViewDescriptor.java

@@ -117,6 +117,12 @@ public class LithoViewDescriptor extends NodeDescriptor<LithoView> {
     return descriptor.getAXAttributes(node);
   }
 
+  @Override
+  public SonarObject getExtraInfo(LithoView node) {
+    final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class);
+    return descriptor.getExtraInfo(node);
+  }
+
   @Override
   public void setHighlighted(LithoView node, boolean selected) throws Exception {
     final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class);

src/fb-stubs/AXLayoutExtender.js

@@ -14,6 +14,7 @@ export class AXElementsInspector extends Component<{
   onElementHovered: ?(key: ?ElementID) => void,
   onValueChanged: ?(path: Array<string>, val: any) => void,
   selected: ?ElementID,
+  focused: ?ElementID,
   searchResults?: ?ElementSearchResultSet,
   root: ?ElementID,
   elements: {[key: ElementID]: Element},

src/plugins/layout/index.js

@@ -38,6 +38,7 @@ export type InspectorState = {|
   AXinitialised: boolean,
   selected: ?ElementID,
   AXselected: ?ElementID,
+  AXfocused: ?ElementID,
   root: ?ElementID,
   AXroot: ?ElementID,
   elements: {[key: ElementID]: Element},
@@ -66,6 +67,15 @@ type UpdateElementsArgs = {|
   elements: Array<$Shape<Element>>,
 |};
 
+type UpdateAXElementsArgs = {|
+  elements: Array<$Shape<Element>>,
+  forFocusEvent: boolean,
+|};
+
+type AXFocusEventResult = {|
+  isFocus: boolean,
+|};
+
 type SetRootArgs = {|
   root: ElementID,
 |};
@@ -159,6 +169,7 @@ export default class Layout extends SonarPlugin<InspectorState> {
     AXroot: null,
     selected: null,
     AXselected: null,
+    AXfocused: null,
     searchResults: null,
     outstandingSearchQuery: null,
     AXtoNonAXMapping: {},
@@ -278,17 +289,30 @@ export default class Layout extends SonarPlugin<InspectorState> {
       return {elements: updatedElements, AXtoNonAXMapping: updatedMapping};
     },
 
-    UpdateAXElements(state: InspectorState, {elements}: UpdateElementsArgs) {
+    UpdateAXElements(
+      state: InspectorState,
+      {elements, forFocusEvent}: UpdateAXElementsArgs,
+    ) {
       const updatedElements = state.AXelements;
 
+      // if focusEvent, previously focused element can be reset
+      let updatedFocus = forFocusEvent ? null : state.AXfocused;
+
       for (const element of elements) {
+        if (element.extraInfo.focused) {
+          updatedFocus = element.id;
+        }
         const current = updatedElements[element.id] || {};
         updatedElements[element.id] = {
           ...current,
           ...element,
         };
       }
-      return {AXelements: updatedElements};
+
+      return {
+        AXelements: updatedElements,
+        AXfocused: updatedFocus,
+      };
     },
 
     SetRoot(state: InspectorState, {root}: SetRootArgs) {
@@ -442,6 +466,34 @@ export default class Layout extends SonarPlugin<InspectorState> {
       });
     }
 
+    this.client.subscribe('axFocusEvent', (focusEvent: AXFocusEventResult) => {
+      if (AXToggleButtonEnabled) {
+        // if focusing, need to update all elements in the tree because
+        // we don't know which one now has focus
+        const keys = focusEvent.isFocus
+          ? Object.keys(this.state.AXelements)
+          : [];
+
+        // if unfocusing and currently focused element exists, update only the
+        // focused element (and only if it is loaded in tree)
+        if (
+          !focusEvent.isFocus &&
+          this.state.AXfocused &&
+          this.state.AXelements[this.state.AXfocused]
+        ) {
+          keys.push(this.state.AXfocused);
+        }
+
+        this.getNodes(keys, true, true).then((elements: Array<Element>) => {
+          this.dispatchAction({
+            elements,
+            forFocusEvent: true,
+            type: 'UpdateAXElements',
+          });
+        });
+      }
+    });
+
     this.client.subscribe(
       'invalidate',
       ({nodes}: {nodes: Array<{id: ElementID}>}) => {
@@ -713,6 +765,7 @@ export default class Layout extends SonarPlugin<InspectorState> {
       AXinitialised,
       selected,
       AXselected,
+      AXfocused,
       root,
       AXroot,
       elements,
@@ -792,6 +845,7 @@ export default class Layout extends SonarPlugin<InspectorState> {
               onElementExpanded={this.onElementExpanded}
               onValueChanged={this.onDataValueChanged}
               selected={AXselected}
+              focused={AXfocused}
               searchResults={null}
               root={AXroot}
               elements={AXelements}

src/ui/components/elements-inspector/ElementsInspector.js

@@ -37,6 +37,7 @@ export type ElementAttribute = {|
 export type ElementExtraInfo = {|
   nonAXWithAXChild?: boolean,
   linkedAXNode?: string,
+  focused?: boolean,
 |};
 
 export type Element = {|

src/ui/components/elements-inspector/elements.js

@@ -31,6 +31,8 @@ const ElementsRowContainer = ContextMenu.extends(
     backgroundColor: props => {
       if (props.selected) {
         return colors.macOSTitleBarIconSelected;
+      } else if (props.focused) {
+        return colors.lime;
       } else if (props.even) {
         return colors.light02;
       } else {
@@ -47,12 +49,20 @@ const ElementsRowContainer = ContextMenu.extends(
     position: 'relative',
 
     '& *': {
-      color: props => (props.selected ? `${colors.white} !important` : ''),
+      color: props =>
+        props.selected || props.focused ? `${colors.white} !important` : '',
     },
 
     '&:hover': {
-      backgroundColor: props =>
+      backgroundColor: props => {
-        props.selected ? colors.macOSTitleBarIconSelected : '#EBF1FB',
+        if (props.selected) {
+          return colors.macOSTitleBarIconSelected;
+        } else if (props.focused) {
+          return colors.lime;
+        } else {
+          return '#EBF1FB';
+        }
+      },
     },
   },
   {
@@ -122,7 +132,9 @@ class PartialHighlight extends PureComponent<{
   static HighlightedText = styled.text({
     backgroundColor: '#ffff33',
     color: props =>
-      props.selected ? `${colors.grapeDark3} !important` : 'auto',
+      props.selected || props.focused
+        ? `${colors.grapeDark3} !important`
+        : 'auto',
   });
 
   render() {
@@ -162,6 +174,7 @@ class ElementsRowAttribute extends PureComponent<{
   value: string,
   matchingSearchQuery: ?string,
   selected: boolean,
+  focused: boolean,
 }> {
   render() {
     const {name, value, matchingSearchQuery, selected} = this.props;
@@ -195,6 +208,7 @@ type ElementsRowProps = {
   id: ElementID,
   level: number,
   selected: boolean,
+  focused: boolean,
   matchingSearchQuery: ?string,
   element: Element,
   even: boolean,
@@ -268,6 +282,7 @@ class ElementsRow extends PureComponent<ElementsRowProps, ElementsRowState> {
       id,
       level,
       selected,
+      focused,
       style,
       even,
       matchingSearchQuery,
@@ -295,6 +310,7 @@ class ElementsRow extends PureComponent<ElementsRowProps, ElementsRowState> {
             value={attr.value}
             matchingSearchQuery={matchingSearchQuery}
             selected={selected}
+            focused={focused}
           />
         ))
       : [];
@@ -327,6 +343,7 @@ class ElementsRow extends PureComponent<ElementsRowProps, ElementsRowState> {
         key={id}
         level={level}
         selected={selected}
+        focused={focused}
         matchingSearchQuery={matchingSearchQuery}
         even={even}
         onClick={this.onClick}
@@ -368,6 +385,7 @@ const ElementsBox = FlexColumn.extends({
 type ElementsProps = {|
   root: ?ElementID,
   selected: ?ElementID,
+  focused?: ?ElementID,
   searchResults: ?ElementSearchResultSet,
   elements: {[key: ElementID]: Element},
   onElementSelected: (key: ElementID) => void,
@@ -532,6 +550,7 @@ export class Elements extends PureComponent<ElementsProps, ElementsState> {
       onElementHovered,
       onElementSelected,
       selected,
+      focused,
       searchResults,
     } = this.props;
     const {flatElements} = this.state;
@@ -557,6 +576,7 @@ export class Elements extends PureComponent<ElementsProps, ElementsState> {
         onElementHovered={onElementHovered}
         onElementSelected={onElementSelected}
         selected={selected === row.key}
+        focused={focused === row.key}
         matchingSearchQuery={
           searchResults && searchResults.matches.has(row.key)
             ? searchResults.query