From ff0b045bded82563e4886fd205b88ec62a86e00c Mon Sep 17 00:00:00 2001 From: Sara Valderrama Date: Thu, 2 Aug 2018 09:28:22 -0700 Subject: [PATCH] Allow for multiple view roots, include accessibility focus changing between view roots Summary: Ax mode now works with multiple view roots/windows, accessibility focus is also updated when new windows are opened. Reviewed By: danielbuechele Differential Revision: D9121844 fbshipit-source-id: 1da9327f5d6a784793db8076c2ad2d84e860ac1c --- .../inspector/InspectorSonarPlugin.java | 79 +++++++++---------- .../plugins/inspector/NodeDescriptor.java | 22 +++++- .../descriptors/ApplicationDescriptor.java | 62 +++++++++++++++ .../inspector/descriptors/ViewDescriptor.java | 20 +++-- .../descriptors/ViewGroupDescriptor.java | 1 + .../plugins/litho/LithoViewDescriptor.java | 1 + src/plugins/layout/index.js | 49 +++++++----- 7 files changed, 164 insertions(+), 70 deletions(-) diff --git a/android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java b/android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java index e9499a71b..6e2768f53 100644 --- a/android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java +++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java @@ -27,6 +27,8 @@ import com.facebook.sonar.plugins.common.MainThreadSonarReceiver; import com.facebook.sonar.plugins.console.iface.ConsoleCommandReceiver; import com.facebook.sonar.plugins.console.iface.NullScriptingEnvironment; import com.facebook.sonar.plugins.console.iface.ScriptingEnvironment; +import com.facebook.sonar.plugins.inspector.descriptors.ApplicationDescriptor; + import java.util.ArrayList; import java.util.List; import javax.annotation.Nullable; @@ -153,6 +155,9 @@ public class InspectorSonarPlugin implements SonarPlugin { mHighlightedId = null; } + // remove any added accessibility delegates + ApplicationDescriptor.clearEditedDelegates(); + mObjectTracker.clear(); mDescriptorMapping.onDisconnect(); mConnection = null; @@ -172,41 +177,8 @@ public class InspectorSonarPlugin implements SonarPlugin { @Override public void onReceiveOnMainThread(SonarObject params, SonarResponder responder) throws Exception { - final List viewRoots = mApplication.getViewRoots(); - - ViewGroup root = null; - for (int i = viewRoots.size() - 1; i >= 0; i--) { - if (viewRoots.get(i) instanceof ViewGroup) { - root = (ViewGroup) viewRoots.get(i); - break; - } - } - - if (root != null) { - - // unlikely, but check to make sure accessibility functionality doesn't change - if (!ViewCompat.hasAccessibilityDelegate(root)) { - - // add delegate to root to catch accessibility events so we can update focus in sonar - root.setAccessibilityDelegate(new View.AccessibilityDelegate() { - - @Override - public boolean onRequestSendAccessibilityEvent(ViewGroup host, View child, AccessibilityEvent event) { - int eventType = event.getEventType(); - if (eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED || eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED) { - mConnection.send("axFocusEvent", - new SonarObject.Builder() - .put("isFocus", eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) - .build()); - } - - return super.onRequestSendAccessibilityEvent(host, child, event); - } - }); - } - - responder.success(getAXNode(trackObject(root))); - } + // applicationWrapper is not used by accessibility, but is a common ancestor for multiple view roots + responder.success(getAXNode(trackObject(mApplication))); } }; @@ -245,21 +217,42 @@ public class InspectorSonarPlugin implements SonarPlugin { final SonarArray ids = params.getArray("ids"); final SonarArray.Builder result = new SonarArray.Builder(); + // getNodes called to refresh accessibility focus + final boolean forFocusEvent = params.getBoolean("forFocusEvent"); + for (int i = 0, count = ids.length(); i < count; i++) { final String id = ids.getString(i); final SonarObject node = getAXNode(id); - if (node != null) { - result.put(node); - } else { + + // sent request for non-existent node, potentially in error + if (node == null) { + + // some nodes may be null since we are searching through all current and previous known nodes + if (forFocusEvent) { + continue; + } + responder.error( - new SonarObject.Builder() - .put("message", "No node with given id") - .put("id", id) - .build()); + new SonarObject.Builder() + .put("message", "No node with given id") + .put("id", id) + .build()); return; + } else { + + // only need to get the focused node in this case + if (forFocusEvent) { + if (node.getObject("extraInfo").getBoolean("focused")) { + result.put(node); + break; + } + + // normal getNodes call, put any nodes in result + } else { + result.put(node); + } } } - responder.success(new SonarObject.Builder().put("elements", result).build()); } }; diff --git a/android/src/main/java/com/facebook/sonar/plugins/inspector/NodeDescriptor.java b/android/src/main/java/com/facebook/sonar/plugins/inspector/NodeDescriptor.java index c3b1690e3..dbda90cda 100644 --- a/android/src/main/java/com/facebook/sonar/plugins/inspector/NodeDescriptor.java +++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/NodeDescriptor.java @@ -22,7 +22,7 @@ import javax.annotation.Nullable; * data can be exposed to the inspector. */ public abstract class NodeDescriptor { - private SonarConnection mConnection; + protected SonarConnection mConnection; private DescriptorMapping mDescriptorMapping; void setConnection(SonarConnection connection) { @@ -63,6 +63,26 @@ public abstract class NodeDescriptor { } } + /** + * Invalidate a node in the ax tree. This tells Sonar that this node is no longer valid and its properties and/or + * children have changed. This will trigger Sonar to re-query this node getting any new data. + */ + protected final void invalidateAX(final T node) { + if (mConnection != null) { + new ErrorReportingRunnable() { + @Override + protected void runOrThrow() throws Exception { + SonarArray array = + new SonarArray.Builder() + .put(new SonarObject.Builder().put("id", getId(node)).build()) + .build(); + SonarObject params = new SonarObject.Builder().put("nodes", array).build(); + mConnection.send("invalidateAX", params); + } + }.run(); + } + } + protected final boolean connected() { return mConnection != null; } diff --git a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ApplicationDescriptor.java b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ApplicationDescriptor.java index 00f83be34..0284ad4a9 100644 --- a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ApplicationDescriptor.java +++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ApplicationDescriptor.java @@ -9,14 +9,18 @@ package com.facebook.sonar.plugins.inspector.descriptors; import android.app.Activity; +import android.support.v4.view.ViewCompat; import android.view.View; import android.view.ViewGroup; +import android.view.accessibility.AccessibilityEvent; import com.facebook.sonar.core.SonarDynamic; import com.facebook.sonar.core.SonarObject; import com.facebook.sonar.plugins.inspector.ApplicationWrapper; import com.facebook.sonar.plugins.inspector.Named; import com.facebook.sonar.plugins.inspector.NodeDescriptor; import com.facebook.sonar.plugins.inspector.Touch; + +import java.util.ArrayList; import java.util.Collections; import java.util.List; import javax.annotation.Nullable; @@ -55,6 +59,50 @@ public class ApplicationDescriptor extends NodeDescriptor { } } + private static List editedDelegates = new ArrayList<>(); + + private void setDelegates(ApplicationWrapper node) { + clearEditedDelegates(); + + for (View view : node.getViewRoots()) { + // unlikely, but check to make sure accessibility functionality doesn't change + if (view instanceof ViewGroup && !ViewCompat.hasAccessibilityDelegate(view)) { + + // add delegate to root to catch accessibility events so we can update focus in sonar + view.setAccessibilityDelegate(new View.AccessibilityDelegate() { + + @Override + public boolean onRequestSendAccessibilityEvent(ViewGroup host, View child, AccessibilityEvent event) { + if (mConnection != null) { + + int eventType = event.getEventType(); + if (eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) { + mConnection.send("axFocusEvent", + new SonarObject.Builder() + .put("isFocus", true) + .build()); + } else if (eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED) { + mConnection.send("axFocusEvent", + new SonarObject.Builder() + .put("isFocus", false) + .build()); + } + } + return super.onRequestSendAccessibilityEvent(host, child, event); + } + }); + editedDelegates.add((ViewGroup) view); + } + } + } + + public static void clearEditedDelegates() { + for (ViewGroup viewGroup : editedDelegates) { + viewGroup.setAccessibilityDelegate(null); + } + editedDelegates.clear(); + } + @Override public void init(final ApplicationWrapper node) { node.setListener( @@ -62,6 +110,8 @@ public class ApplicationDescriptor extends NodeDescriptor { @Override public void onActivityStackChanged(List stack) { invalidate(node); + invalidateAX(node); + setDelegates(node); } }); @@ -73,6 +123,8 @@ public class ApplicationDescriptor extends NodeDescriptor { if (connected()) { if (key.set(node)) { invalidate(node); + invalidateAX(node); + setDelegates(node); } node.postDelayed(this, 1000); } @@ -92,6 +144,11 @@ public class ApplicationDescriptor extends NodeDescriptor { return node.getApplication().getPackageName(); } + @Override + public String getAXName(ApplicationWrapper node) throws Exception { + return "Application"; + } + @Override public int getChildCount(ApplicationWrapper node) { return node.getViewRoots().size(); @@ -110,6 +167,11 @@ public class ApplicationDescriptor extends NodeDescriptor { return view; } + @Override + public Object getAXChildAt(ApplicationWrapper node, int index) { + return node.getViewRoots().get(index); + } + @Override public List> getData(ApplicationWrapper node) { return Collections.EMPTY_LIST; diff --git a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewDescriptor.java b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewDescriptor.java index 153cb072c..6f87bc608 100644 --- a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewDescriptor.java +++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewDescriptor.java @@ -19,6 +19,7 @@ import android.os.Build; import android.support.v4.view.MarginLayoutParamsCompat; import android.support.v4.view.ViewCompat; import android.support.v4.view.accessibility.AccessibilityNodeInfoCompat; +import android.util.Log; import android.util.SparseArray; import android.view.Gravity; import android.view.View; @@ -85,14 +86,19 @@ public class ViewDescriptor extends NodeDescriptor { } @Override - public String getAXName(View node) { + public String getAXName(View node) throws Exception { AccessibilityNodeInfoCompat nodeInfo = ViewAccessibilityHelper.createNodeInfoFromView(node); - if (nodeInfo == null) { - return "NULL NODEINFO"; + if (nodeInfo != null) { + + CharSequence name = nodeInfo.getClassName(); + nodeInfo.recycle(); + + if (name != null) { + return name.toString(); + } } - String name = nodeInfo.getClassName().toString(); - nodeInfo.recycle(); - return name; + return "NULL NODEINFO OR CLASSNAME"; + } @Override @@ -418,7 +424,7 @@ public class ViewDescriptor extends NodeDescriptor { node.setSelected(value.asBoolean()); break; } - invalidate(node); + invalidateAX(node); } @Override diff --git a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewGroupDescriptor.java b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewGroupDescriptor.java index a4bf3ee57..d3c5d8207 100644 --- a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewGroupDescriptor.java +++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewGroupDescriptor.java @@ -73,6 +73,7 @@ public class ViewGroupDescriptor extends NodeDescriptor { if (connected()) { if (key.set(node)) { invalidate(node); + invalidateAX(node); } final boolean hasAttachedToWindow = diff --git a/android/src/main/java/com/facebook/sonar/plugins/litho/LithoViewDescriptor.java b/android/src/main/java/com/facebook/sonar/plugins/litho/LithoViewDescriptor.java index 9bdf834e3..a69774511 100644 --- a/android/src/main/java/com/facebook/sonar/plugins/litho/LithoViewDescriptor.java +++ b/android/src/main/java/com/facebook/sonar/plugins/litho/LithoViewDescriptor.java @@ -24,6 +24,7 @@ public class LithoViewDescriptor extends NodeDescriptor { @Override public void onDirtyMount(LithoView view) { invalidate(view); + invalidateAX(view); } }); } diff --git a/src/plugins/layout/index.js b/src/plugins/layout/index.js index 801b2b5fd..1574b4b72 100644 --- a/src/plugins/layout/index.js +++ b/src/plugins/layout/index.js @@ -89,6 +89,7 @@ type GetNodesResult = {| type GetNodesOptions = {| force: boolean, ax: boolean, + forFocusEvent?: boolean, |}; type SearchResultTree = {| @@ -436,22 +437,22 @@ export default class Layout extends SonarPlugin { }); }); - this.client.subscribe('axFocusEvent', (focusEvent: AXFocusEventResult) => { + this.client.subscribe('axFocusEvent', ({isFocus}: AXFocusEventResult) => { // if focusing, need to update all elements in the tree because // we don't know which one now has focus - const keys = focusEvent.isFocus ? Object.keys(this.state.AXelements) : []; + const keys = isFocus ? Object.keys(this.state.AXelements) : []; // if unfocusing and currently focused element exists, update only the - // focused element (and only if it is loaded in tree) + // focused element (and only if it is/was loaded in tree) if ( - !focusEvent.isFocus && + !isFocus && this.state.AXfocused && this.state.AXelements[this.state.AXfocused] ) { keys.push(this.state.AXfocused); } - this.getNodes(keys, {force: true, ax: true}).then( + this.getNodes(keys, {force: true, ax: true, forFocusEvent: true}).then( (elements: Array) => { this.dispatchAction({ elements, @@ -461,6 +462,17 @@ export default class Layout extends SonarPlugin { }, ); }); + + this.client.subscribe( + 'invalidateAX', + ({nodes}: {nodes: Array<{id: ElementID}>}) => { + this.invalidate(nodes.map(node => node.id), true).then( + (elements: Array) => { + this.dispatchAction({elements, type: 'UpdateAXElements'}); + }, + ); + }, + ); } init() { @@ -477,14 +489,9 @@ export default class Layout extends SonarPlugin { this.client.subscribe( 'invalidate', ({nodes}: {nodes: Array<{id: ElementID}>}) => { - this.invalidate(nodes.map(node => node.id)).then( + this.invalidate(nodes.map(node => node.id), false).then( (elements: Array) => { - if (this.state.inAXMode) { - // to be removed once trees are separate - will have own invalidate - this.dispatchAction({elements, type: 'UpdateAXElements'}); - } else { - this.dispatchAction({elements, type: 'UpdateElements'}); - } + this.dispatchAction({elements, type: 'UpdateElements'}); }, ); }, @@ -514,12 +521,11 @@ export default class Layout extends SonarPlugin { } } - invalidate(ids: Array): Promise> { + invalidate(ids: Array, ax: boolean): Promise> { if (ids.length === 0) { return Promise.resolve([]); } - const ax = this.state.inAXMode; return this.getNodes(ids, {force: true, ax}).then( (elements: Array) => { const children = elements @@ -532,9 +538,11 @@ export default class Layout extends SonarPlugin { .map(element => element.children) .reduce((acc, val) => acc.concat(val), []); - return Promise.all([elements, this.invalidate(children)]).then(arr => { - return arr.reduce((acc, val) => acc.concat(val), []); - }); + return Promise.all([elements, this.invalidate(children, ax)]).then( + arr => { + return arr.reduce((acc, val) => acc.concat(val), []); + }, + ); }, ); } @@ -570,7 +578,7 @@ export default class Layout extends SonarPlugin { ids: Array = [], options: GetNodesOptions, ): Promise> { - const {force, ax} = options; + const {force, ax, forFocusEvent} = options; if (!force) { ids = ids.filter(id => { return ( @@ -582,7 +590,10 @@ export default class Layout extends SonarPlugin { if (ids.length > 0) { performance.mark('LayoutInspectorGetNodes'); return this.client - .call(ax ? 'getAXNodes' : 'getNodes', {ids}) + .call(ax ? 'getAXNodes' : 'getNodes', { + ids, + forFocusEvent, + }) .then(({elements}: GetNodesResult) => { this.props.logger.trackTimeSince('LayoutInspectorGetNodes'); return Promise.resolve(elements);