Click to inspect works with TalkBack

Summary: Allow click to inspect to work when Talkback is running (normal tree expands as usual although ax tree doesn't expand yet). Override onHover of Touchoverlay and onRequestSendAccessibilityEvent in viewRoot to prevent talkback focus when in Click to Inspect mode. Also update layout inspector to persist isSearchActive to prevent build up of TouchOverlayViews when navigating between plugins (which happened if leaving the layout inspector while in Click to Inspect mode).

Reviewed By: danielbuechele

Differential Revision: D9153446

fbshipit-source-id: f76982e8f8cea1e7b7e4c6b9bf73632d101222ef

android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java

@@ -15,6 +15,7 @@ import android.view.accessibility.AccessibilityEvent;
 import android.view.MotionEvent;
 import android.view.View;
 import android.view.ViewGroup;
+import android.view.accessibility.AccessibilityManager;
 import com.facebook.sonar.core.ErrorReportingRunnable;
 import com.facebook.sonar.core.SonarArray;
 import com.facebook.sonar.core.SonarConnection;
@@ -28,6 +29,7 @@ import com.facebook.sonar.plugins.console.iface.ConsoleCommandReceiver;
 import com.facebook.sonar.plugins.console.iface.NullScriptingEnvironment;
 import com.facebook.sonar.plugins.console.iface.ScriptingEnvironment;
 import com.facebook.sonar.plugins.inspector.descriptors.ApplicationDescriptor;
+import com.facebook.sonar.plugins.inspector.descriptors.utils.AccessibilityUtil;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -136,6 +138,7 @@ public class InspectorSonarPlugin implements SonarPlugin {
     connection.receive("setData", mSetData);
     connection.receive("setHighlighted", mSetHighlighted);
     connection.receive("setSearchActive", mSetSearchActive);
+    connection.receive("isSearchActive", mIsSearchActive);
     connection.receive("getSearchResults", mGetSearchResults);
     connection.receive("getAXRoot", mGetAXRoot);
     connection.receive("getAXNodes", mGetAXNodes);
@@ -155,7 +158,7 @@ public class InspectorSonarPlugin implements SonarPlugin {
       mHighlightedId = null;
     }
 
-    // remove any added accessibility delegates
+    // remove any added accessibility delegates, leave isSearchActive untouched
     ApplicationDescriptor.clearEditedDelegates();
 
     mObjectTracker.clear();
@@ -234,7 +237,7 @@ public class InspectorSonarPlugin implements SonarPlugin {
 
               responder.error(
                       new SonarObject.Builder()
-                              .put("message", "No node with given id")
+                              .put("message", "No accessibility node with given id")
                               .put("id", id)
                               .build());
               return;
@@ -311,6 +314,7 @@ public class InspectorSonarPlugin implements SonarPlugin {
         public void onReceiveOnMainThread(final SonarObject params, SonarResponder responder)
             throws Exception {
           final boolean active = params.getBoolean("active");
+          ApplicationDescriptor.setSearchActive(active);
           final List<View> roots = mApplication.getViewRoots();
 
           ViewGroup root = null;
@@ -334,6 +338,15 @@ public class InspectorSonarPlugin implements SonarPlugin {
         }
       };
 
+  final SonarReceiver mIsSearchActive =
+          new MainThreadSonarReceiver(mConnection) {
+            @Override
+            public void onReceiveOnMainThread(final SonarObject params, SonarResponder responder)
+                    throws Exception {
+              responder.success(new SonarObject.Builder().put("isSearchActive", ApplicationDescriptor.getSearchActive()).build());
+            }
+          };
+
   final SonarReceiver mGetSearchResults =
       new MainThreadSonarReceiver(mConnection) {
         @Override
@@ -354,6 +367,30 @@ public class InspectorSonarPlugin implements SonarPlugin {
       setBackgroundColor(BoundsDrawable.COLOR_HIGHLIGHT_CONTENT);
     }
 
+    @Override
+    public boolean onHoverEvent(MotionEvent event) {
+
+      // if in layout inspector and talkback is running, override the first click to locate the clicked view
+      if (mConnection != null && AccessibilityUtil.isTalkbackEnabled(getContext()) && event.getPointerCount() == 1) {
+        final int action = event.getAction();
+        switch (action) {
+          case MotionEvent.ACTION_HOVER_ENTER: {
+            event.setAction(MotionEvent.ACTION_DOWN);
+          } break;
+          case MotionEvent.ACTION_HOVER_MOVE: {
+            event.setAction(MotionEvent.ACTION_MOVE);
+          } break;
+          case MotionEvent.ACTION_HOVER_EXIT: {
+            event.setAction(MotionEvent.ACTION_UP);
+          } break;
+        }
+        return onTouchEvent(event);
+      }
+
+      // otherwise use the default
+      return super.onHoverEvent(event);
+    }
+
     @Override
     public boolean onTouchEvent(final MotionEvent event) {
       if (event.getAction() != MotionEvent.ACTION_UP) {

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ApplicationDescriptor.java

@@ -60,6 +60,15 @@ public class ApplicationDescriptor extends NodeDescriptor<ApplicationWrapper> {
   }
 
   private static List<ViewGroup> editedDelegates = new ArrayList<>();
+  private static boolean searchActive;
+
+  public static void setSearchActive(boolean active) {
+    searchActive = active;
+  }
+
+  public static boolean getSearchActive() {
+    return searchActive;
+  }
 
   private void setDelegates(ApplicationWrapper node) {
     clearEditedDelegates();
@@ -70,11 +79,16 @@ public class ApplicationDescriptor extends NodeDescriptor<ApplicationWrapper> {
 
         // add delegate to root to catch accessibility events so we can update focus in sonar
         view.setAccessibilityDelegate(new View.AccessibilityDelegate() {
-
           @Override
           public boolean onRequestSendAccessibilityEvent(ViewGroup host, View child, AccessibilityEvent event) {
             if (mConnection != null) {
 
+              // the touchOverlay will handle the event in this case
+              if (searchActive) {
+                return false;
+              }
+
+              // otherwise send the necessary focus event to the plugin
               int eventType = event.getEventType();
               if (eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) {
                 mConnection.send("axFocusEvent",
@@ -87,6 +101,7 @@ public class ApplicationDescriptor extends NodeDescriptor<ApplicationWrapper> {
                                 .put("isFocus", false)
                                 .build());
               }
+
             }
             return super.onRequestSendAccessibilityEvent(host, child, event);
           }

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java

@@ -90,6 +90,16 @@ public final class AccessibilityUtil {
     return ((AccessibilityManager) context.getSystemService(ACCESSIBILITY_SERVICE)).isEnabled();
   }
 
+  /**
+   * Given a {@link Context}, determine if an accessibility touch exploration service (TalkBack) is running.
+   *
+   * @param context The {@link Context} used to get the {@link AccessibilityManager}.
+   * @return {@code true} if an accessibility touch exploration service is currently running.
+   */
+  public static boolean isTalkbackEnabled(Context context) {
+    return ((AccessibilityManager) context.getSystemService(ACCESSIBILITY_SERVICE)).isTouchExplorationEnabled();
+  }
+
   /**
    * Returns a sentence describing why a given {@link View} will be ignored by Google's TalkBack
    * screen reader.

src/plugins/layout/index.js

@@ -485,6 +485,12 @@ export default class Layout extends SonarPlugin<InspectorState> {
   }
 
   init() {
+    // persist searchActive state when moving between plugins to prevent multiple
+    // TouchOverlayViews since we can't edit the view heirarchy in onDisconnect
+    this.client.call('isSearchActive').then(({isSearchActive}) => {
+      this.dispatchAction({type: 'SetSearchActive', isSearchActive});
+    });
+
     performance.mark('LayoutInspectorInitialize');
     this.client.call('getRoot').then((element: Element) => {
       this.dispatchAction({elements: [element], type: 'UpdateElements'});
@@ -510,9 +516,10 @@ export default class Layout extends SonarPlugin<InspectorState> {
       this.getNodesAndDirectChildren(path, false).then(
         (elements: Array<Element>) => {
           const selected = path[path.length - 1];
+          const {key, AXkey} = this.getKeysFromSelected(selected);
 
           this.dispatchAction({elements, type: 'UpdateElements'});
-          this.dispatchAction({key: selected, type: 'SelectElement'});
+          this.dispatchAction({key, AXkey, type: 'SelectElement'});
           this.dispatchAction({isSearchActive: false, type: 'SetSearchActive'});
 
           for (const key of path) {
@@ -719,51 +726,57 @@ export default class Layout extends SonarPlugin<InspectorState> {
     this.dispatchAction({isAlignmentMode, type: 'SetAlignmentActive'});
   };
 
-  onElementSelected = debounce((key: ElementID) => {
+  getKeysFromSelected(selectedKey: ElementID) {
-    let finalKey = key;
+    let key = selectedKey;
-    let finalAXkey = null;
+    let AXkey = null;
 
     if (this.axEnabled()) {
       const linkedAXNode =
-        this.state.elements[key] &&
+        this.state.elements[selectedKey] &&
-        this.state.elements[key].extraInfo &&
+        this.state.elements[selectedKey].extraInfo &&
-        this.state.elements[key].extraInfo.linkedAXNode;
+        this.state.elements[selectedKey].extraInfo.linkedAXNode;
 
       // element only in main tree with linkedAXNode selected
       if (linkedAXNode) {
-        finalAXkey = linkedAXNode;
+        AXkey = linkedAXNode;
 
         // element only in AX tree with linked nonAX (litho) element selected
       } else if (
-        (!this.state.elements[key] ||
+        (!this.state.elements[selectedKey] ||
-          this.state.elements[key].name === 'ComponentHost') &&
+          this.state.elements[selectedKey].name === 'ComponentHost') &&
-        this.state.AXtoNonAXMapping[key]
+        this.state.AXtoNonAXMapping[selectedKey]
       ) {
-        finalKey = this.state.AXtoNonAXMapping[key];
+        key = this.state.AXtoNonAXMapping[selectedKey];
-        finalAXkey = key;
+        AXkey = selectedKey;
 
         // keys are same for both trees or 'linked' element does not exist
       } else {
-        finalAXkey = key;
+        AXkey = selectedKey;
       }
     }
 
+    return {key, AXkey};
+  }
+
+  onElementSelected = debounce((selectedKey: ElementID) => {
+    const {key, AXkey} = this.getKeysFromSelected(selectedKey);
+
     this.dispatchAction({
-      key: finalKey,
+      key: key,
-      AXkey: finalAXkey,
+      AXkey: AXkey,
       type: 'SelectElement',
     });
     this.client.send('setHighlighted', {
-      id: key,
+      id: selectedKey,
       isAlignmentMode: this.state.isAlignmentMode,
     });
-    this.getNodes([finalKey], {force: true, ax: false}).then(
+    this.getNodes([key], {force: true, ax: false}).then(
       (elements: Array<Element>) => {
         this.dispatchAction({elements, type: 'UpdateElements'});
       },
     );
-    if (this.axEnabled() && finalAXkey) {
+    if (AXkey) {
-      this.getNodes([finalAXkey], {force: true, ax: true}).then(
+      this.getNodes([AXkey], {force: true, ax: true}).then(
         (elements: Array<Element>) => {
           this.dispatchAction({elements, type: 'UpdateAXElements'});
         },