Expand AX tree when clicking-to-inspect

Summary: Adds expanding functionality to ax tree when click-to-inspect is used. Highlight/selection priority is given to the ax tree if in ax mode and the main tree otherwise.

Reviewed By: danielbuechele

Differential Revision: D9206028

fbshipit-source-id: 364b3cfc2e68dbce6006c36002353295aa7cdd07

android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java

@@ -427,19 +427,19 @@ public class InspectorSonarPlugin implements SonarPlugin {
     }
   }
 
-  void hitTest(final int touchX, final int touchY) throws Exception {
+  private Touch createTouch(final int touchX, final int touchY, final boolean ax) throws Exception {
     final SonarArray.Builder path = new SonarArray.Builder();
     path.put(trackObject(mApplication));
 
-    final Touch touch =
+    return new Touch() {
-        new Touch() {
       int x = touchX;
       int y = touchY;
       Object node = mApplication;
 
+
       @Override
       public void finish() {
-            mConnection.send("select", new SonarObject.Builder().put("path", path).build());
+        mConnection.send(ax ? "selectAX" : "select", new SonarObject.Builder().put("path", path).build());
       }
 
       @Override
@@ -453,12 +453,21 @@ public class InspectorSonarPlugin implements SonarPlugin {
             x -= offsetX;
             y -= offsetY;
 
+            if (ax) {
+              node = assertNotNull(descriptorForObject(node).getAXChildAt(node, childIndex));
+            } else {
               node = assertNotNull(descriptorForObject(node).getChildAt(node, childIndex));
-                path.put(trackObject(node));
+            }
 
+            path.put(trackObject(node));
             final NodeDescriptor<Object> descriptor = descriptorForObject(node);
+
+            if (ax) {
+              descriptor.axHitTest(node, touch);
+            } else {
               descriptor.hitTest(node, touch);
             }
+          }
         }.run();
       }
 
@@ -467,9 +476,12 @@ public class InspectorSonarPlugin implements SonarPlugin {
         return x >= l && x <= r && y >= t && y <= b;
       }
     };
+  }
 
+  void hitTest(final int touchX, final int touchY) throws Exception {
     final NodeDescriptor<Object> descriptor = descriptorForObject(mApplication);
-    descriptor.hitTest(mApplication, touch);
+    descriptor.hitTest(mApplication, createTouch(touchX, touchY, false));
+    descriptor.axHitTest(mApplication, createTouch(touchX, touchY, true));
   }
 
   private void setHighlighted(final String id, final boolean highlighted, final boolean isAlignmentMode) throws Exception {

android/src/main/java/com/facebook/sonar/plugins/inspector/NodeDescriptor.java

@@ -177,6 +177,15 @@ public abstract class NodeDescriptor<T> {
    */
   public abstract void hitTest(T node, Touch touch) throws Exception;
 
+  /**
+   * Perform hit testing on the given ax node. Either continue the search in an ax child with {@link
+   * Touch#continueWithOffset(int, int, int, boolean)} or finish the hit testing on this ax node with {@link
+   * Touch#finish()}
+   */
+  public void axHitTest(T node, Touch touch) throws Exception {
+    touch.finish();
+  }
+
   /**
    * @return A string indicating how this element should be decorated. Check with the Sonar desktop
    *     app to see what values are supported.

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ApplicationDescriptor.java

@@ -210,12 +210,11 @@ public class ApplicationDescriptor extends NodeDescriptor<ApplicationWrapper> {
     }
   }
 
-  @Override
+  private void runHitTest(ApplicationWrapper node, Touch touch, boolean ax) throws Exception {
-  public void hitTest(ApplicationWrapper node, Touch touch) {
     final int childCount = getChildCount(node);
 
     for (int i = childCount - 1; i >= 0; i--) {
-      final Object child = getChildAt(node, i);
+      final Object child = ax ? getAXChildAt(node, i) : getChildAt(node, i);
       if (child instanceof Activity || child instanceof ViewGroup) {
         touch.continueWithOffset(i, 0, 0);
         return;
@@ -225,6 +224,16 @@ public class ApplicationDescriptor extends NodeDescriptor<ApplicationWrapper> {
     touch.finish();
   }
 
+  @Override
+  public void hitTest(ApplicationWrapper node, Touch touch) throws Exception{
+    runHitTest(node, touch, false);
+  }
+
+  @Override
+  public void axHitTest(ApplicationWrapper node, Touch touch) throws Exception {
+    runHitTest(node, touch, true);
+  }
+
   @Override
   public @Nullable String getDecoration(ApplicationWrapper obj) {
     return null;

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/TextViewDescriptor.java

@@ -149,6 +149,12 @@ public class TextViewDescriptor extends NodeDescriptor<TextView> {
     descriptor.hitTest(node, touch);
   }
 
+  @Override
+  public void axHitTest(TextView node, Touch touch) throws Exception {
+    final NodeDescriptor descriptor = descriptorForClass(View.class);
+    descriptor.axHitTest(node, touch);
+  }
+
   @Override
   public @Nullable String getDecoration(TextView node) throws Exception {
     final NodeDescriptor descriptor = descriptorForClass(View.class);

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewDescriptor.java

@@ -534,6 +534,11 @@ public class ViewDescriptor extends NodeDescriptor<View> {
     touch.finish();
   }
 
+  @Override
+  public void axHitTest(View node, Touch touch) {
+    touch.finish();
+  }
+
   @Override
   public @Nullable String getDecoration(View obj) {
     return null;

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewGroupDescriptor.java

@@ -248,8 +248,7 @@ public class ViewGroupDescriptor extends NodeDescriptor<ViewGroup> {
     descriptor.setHighlighted(node, selected, isAlignmentMode);
   }
 
-  @Override
+  private void runHitTest(ViewGroup node, Touch touch) {
-  public void hitTest(ViewGroup node, Touch touch) {
     for (int i = node.getChildCount() - 1; i >= 0; i--) {
       final View child = node.getChildAt(i);
       if (child instanceof HiddenNode
@@ -277,6 +276,16 @@ public class ViewGroupDescriptor extends NodeDescriptor<ViewGroup> {
     touch.finish();
   }
 
+  @Override
+  public void hitTest(ViewGroup node, Touch touch) {
+    runHitTest(node, touch);
+  }
+
+  @Override
+  public void axHitTest(ViewGroup node, Touch touch) {
+    runHitTest(node, touch);
+  }
+
   private static boolean shouldSkip(View view) {
     Object tag = view.getTag(R.id.sonar_skip_view_traversal);
     if (!(tag instanceof Boolean)) {

android/src/main/java/com/facebook/sonar/plugins/litho/LithoViewDescriptor.java

@@ -135,6 +135,12 @@ public class LithoViewDescriptor extends NodeDescriptor<LithoView> {
     touch.continueWithOffset(0, 0, 0);
   }
 
+  @Override
+  public void axHitTest(LithoView node, Touch touch) throws Exception {
+    final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class);
+    descriptor.axHitTest(node, touch);
+  }
+
   @Override
   public String getDecoration(LithoView node) throws Exception {
     final NodeDescriptor descriptor = descriptorForClass(ViewGroup.class);

src/plugins/layout/index.js

@@ -437,6 +437,48 @@ export default class Layout extends SonarPlugin<InspectorState> {
     return AXToggleButtonEnabled && this.realClient.query.os === 'Android';
   }
 
+  // expand tree and highlight click-to-inspect node that was found
+  onSelectResultsRecieved(path: Array<ElementID>, ax: boolean) {
+    this.getNodesAndDirectChildren(path, ax).then(
+      (elements: Array<Element>) => {
+        const selected = path[path.length - 1];
+
+        this.dispatchAction({
+          elements,
+          type: ax ? 'UpdateAXElements' : 'UpdateElements',
+        });
+
+        // select node from ax tree if in ax mode
+        // select node from main tree if not in ax mode
+        // (also selects corresponding node in other tree if it exists)
+        if ((ax && this.state.inAXMode) || (!ax && !this.state.inAXMode)) {
+          const {key, AXkey} = this.getKeysFromSelected(selected);
+          this.dispatchAction({key, AXkey, type: 'SelectElement'});
+        }
+
+        this.dispatchAction({
+          isSearchActive: false,
+          type: 'SetSearchActive',
+        });
+
+        for (const key of path) {
+          this.dispatchAction({
+            expand: true,
+            key,
+            type: ax ? 'ExpandAXElement' : 'ExpandElement',
+          });
+        }
+
+        this.client.send('setHighlighted', {
+          id: selected,
+          isAlignmentMode: this.state.isAlignmentMode,
+        });
+
+        this.client.send('setSearchActive', {active: false});
+      },
+    );
+  }
+
   initAX() {
     this.client.call('getAXRoot').then((element: Element) => {
       this.dispatchAction({elements: [element], type: 'UpdateAXElements'});
@@ -482,6 +524,10 @@ export default class Layout extends SonarPlugin<InspectorState> {
         );
       },
     );
+
+    this.client.subscribe('selectAX', ({path}: {path: Array<ElementID>}) => {
+      this.onSelectResultsRecieved(path, true);
+    });
   }
 
   init() {
@@ -513,26 +559,7 @@ export default class Layout extends SonarPlugin<InspectorState> {
     );
 
     this.client.subscribe('select', ({path}: {path: Array<ElementID>}) => {
-      this.getNodesAndDirectChildren(path, false).then(
+      this.onSelectResultsRecieved(path, false);
-        (elements: Array<Element>) => {
-          const selected = path[path.length - 1];
-          const {key, AXkey} = this.getKeysFromSelected(selected);
-
-          this.dispatchAction({elements, type: 'UpdateElements'});
-          this.dispatchAction({key, AXkey, type: 'SelectElement'});
-          this.dispatchAction({isSearchActive: false, type: 'SetSearchActive'});
-
-          for (const key of path) {
-            this.dispatchAction({expand: true, key, type: 'ExpandElement'});
-          }
-
-          this.client.send('setHighlighted', {
-            id: selected,
-            isAlignmentMode: this.state.isAlignmentMode,
-          });
-          this.client.send('setSearchActive', {active: false});
-        },
-      );
     });
 
     if (this.axEnabled()) {
@@ -568,7 +595,7 @@ export default class Layout extends SonarPlugin<InspectorState> {
 
   getNodesAndDirectChildren(
     ids: Array<ElementID>,
-    ax: boolean, // always false at the moment bc only used for select
+    ax: boolean,
   ): Promise<Array<Element>> {
     return this.getNodes(ids, {force: false, ax}).then(
       (elements: Array<Element>) => {
@@ -698,13 +725,22 @@ export default class Layout extends SonarPlugin<InspectorState> {
   };
 
   onElementExpanded = (key: ElementID, deep: boolean) => {
+    if (this.state.elements[key]) {
       if (deep) {
         this.deepExpandElement(key, false);
-      this.deepExpandElement(key, true);
       } else {
         this.expandElement(key, false);
+      }
+    }
+
+    if (this.state.AXelements[key]) {
+      if (deep) {
+        this.deepExpandElement(key, true);
+      } else {
         this.expandElement(key, true);
       }
+    }
+
     this.props.logger.track('usage', 'layout:element-expanded', {
       id: key,
       deep: deep,