From fc7f949dafdb5f853ea979d08d33e5157836ba57 Mon Sep 17 00:00:00 2001 From: Sara Valderrama Date: Mon, 6 Aug 2018 16:33:23 -0700 Subject: [PATCH] Click to inspect works with TalkBack Summary: Allow click to inspect to work when Talkback is running (normal tree expands as usual although ax tree doesn't expand yet). Override onHover of Touchoverlay and onRequestSendAccessibilityEvent in viewRoot to prevent talkback focus when in Click to Inspect mode. Also update layout inspector to persist isSearchActive to prevent build up of TouchOverlayViews when navigating between plugins (which happened if leaving the layout inspector while in Click to Inspect mode). Reviewed By: danielbuechele Differential Revision: D9153446 fbshipit-source-id: f76982e8f8cea1e7b7e4c6b9bf73632d101222ef --- .../inspector/InspectorSonarPlugin.java | 41 +++++++++++++- .../descriptors/ApplicationDescriptor.java | 17 +++++- .../descriptors/utils/AccessibilityUtil.java | 10 ++++ src/plugins/layout/index.js | 53 ++++++++++++------- 4 files changed, 98 insertions(+), 23 deletions(-) diff --git a/android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java b/android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java index faad04430..265da760c 100644 --- a/android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java +++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/InspectorSonarPlugin.java @@ -15,6 +15,7 @@ import android.view.accessibility.AccessibilityEvent; import android.view.MotionEvent; import android.view.View; import android.view.ViewGroup; +import android.view.accessibility.AccessibilityManager; import com.facebook.sonar.core.ErrorReportingRunnable; import com.facebook.sonar.core.SonarArray; import com.facebook.sonar.core.SonarConnection; @@ -28,6 +29,7 @@ import com.facebook.sonar.plugins.console.iface.ConsoleCommandReceiver; import com.facebook.sonar.plugins.console.iface.NullScriptingEnvironment; import com.facebook.sonar.plugins.console.iface.ScriptingEnvironment; import com.facebook.sonar.plugins.inspector.descriptors.ApplicationDescriptor; +import com.facebook.sonar.plugins.inspector.descriptors.utils.AccessibilityUtil; import java.util.ArrayList; import java.util.List; @@ -136,6 +138,7 @@ public class InspectorSonarPlugin implements SonarPlugin { connection.receive("setData", mSetData); connection.receive("setHighlighted", mSetHighlighted); connection.receive("setSearchActive", mSetSearchActive); + connection.receive("isSearchActive", mIsSearchActive); connection.receive("getSearchResults", mGetSearchResults); connection.receive("getAXRoot", mGetAXRoot); connection.receive("getAXNodes", mGetAXNodes); @@ -155,7 +158,7 @@ public class InspectorSonarPlugin implements SonarPlugin { mHighlightedId = null; } - // remove any added accessibility delegates + // remove any added accessibility delegates, leave isSearchActive untouched ApplicationDescriptor.clearEditedDelegates(); mObjectTracker.clear(); @@ -234,7 +237,7 @@ public class InspectorSonarPlugin implements SonarPlugin { responder.error( new SonarObject.Builder() - .put("message", "No node with given id") + .put("message", "No accessibility node with given id") .put("id", id) .build()); return; @@ -311,6 +314,7 @@ public class InspectorSonarPlugin implements SonarPlugin { public void onReceiveOnMainThread(final SonarObject params, SonarResponder responder) throws Exception { final boolean active = params.getBoolean("active"); + ApplicationDescriptor.setSearchActive(active); final List roots = mApplication.getViewRoots(); ViewGroup root = null; @@ -334,6 +338,15 @@ public class InspectorSonarPlugin implements SonarPlugin { } }; + final SonarReceiver mIsSearchActive = + new MainThreadSonarReceiver(mConnection) { + @Override + public void onReceiveOnMainThread(final SonarObject params, SonarResponder responder) + throws Exception { + responder.success(new SonarObject.Builder().put("isSearchActive", ApplicationDescriptor.getSearchActive()).build()); + } + }; + final SonarReceiver mGetSearchResults = new MainThreadSonarReceiver(mConnection) { @Override @@ -354,6 +367,30 @@ public class InspectorSonarPlugin implements SonarPlugin { setBackgroundColor(BoundsDrawable.COLOR_HIGHLIGHT_CONTENT); } + @Override + public boolean onHoverEvent(MotionEvent event) { + + // if in layout inspector and talkback is running, override the first click to locate the clicked view + if (mConnection != null && AccessibilityUtil.isTalkbackEnabled(getContext()) && event.getPointerCount() == 1) { + final int action = event.getAction(); + switch (action) { + case MotionEvent.ACTION_HOVER_ENTER: { + event.setAction(MotionEvent.ACTION_DOWN); + } break; + case MotionEvent.ACTION_HOVER_MOVE: { + event.setAction(MotionEvent.ACTION_MOVE); + } break; + case MotionEvent.ACTION_HOVER_EXIT: { + event.setAction(MotionEvent.ACTION_UP); + } break; + } + return onTouchEvent(event); + } + + // otherwise use the default + return super.onHoverEvent(event); + } + @Override public boolean onTouchEvent(final MotionEvent event) { if (event.getAction() != MotionEvent.ACTION_UP) { diff --git a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ApplicationDescriptor.java b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ApplicationDescriptor.java index 56507d5ed..feca8219b 100644 --- a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ApplicationDescriptor.java +++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ApplicationDescriptor.java @@ -60,6 +60,15 @@ public class ApplicationDescriptor extends NodeDescriptor { } private static List editedDelegates = new ArrayList<>(); + private static boolean searchActive; + + public static void setSearchActive(boolean active) { + searchActive = active; + } + + public static boolean getSearchActive() { + return searchActive; + } private void setDelegates(ApplicationWrapper node) { clearEditedDelegates(); @@ -70,11 +79,16 @@ public class ApplicationDescriptor extends NodeDescriptor { // add delegate to root to catch accessibility events so we can update focus in sonar view.setAccessibilityDelegate(new View.AccessibilityDelegate() { - @Override public boolean onRequestSendAccessibilityEvent(ViewGroup host, View child, AccessibilityEvent event) { if (mConnection != null) { + // the touchOverlay will handle the event in this case + if (searchActive) { + return false; + } + + // otherwise send the necessary focus event to the plugin int eventType = event.getEventType(); if (eventType == AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED) { mConnection.send("axFocusEvent", @@ -87,6 +101,7 @@ public class ApplicationDescriptor extends NodeDescriptor { .put("isFocus", false) .build()); } + } return super.onRequestSendAccessibilityEvent(host, child, event); } diff --git a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java index 4e440021f..2302f7930 100644 --- a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java +++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java @@ -90,6 +90,16 @@ public final class AccessibilityUtil { return ((AccessibilityManager) context.getSystemService(ACCESSIBILITY_SERVICE)).isEnabled(); } + /** + * Given a {@link Context}, determine if an accessibility touch exploration service (TalkBack) is running. + * + * @param context The {@link Context} used to get the {@link AccessibilityManager}. + * @return {@code true} if an accessibility touch exploration service is currently running. + */ + public static boolean isTalkbackEnabled(Context context) { + return ((AccessibilityManager) context.getSystemService(ACCESSIBILITY_SERVICE)).isTouchExplorationEnabled(); + } + /** * Returns a sentence describing why a given {@link View} will be ignored by Google's TalkBack * screen reader. diff --git a/src/plugins/layout/index.js b/src/plugins/layout/index.js index 5fe047f68..59625e1a0 100644 --- a/src/plugins/layout/index.js +++ b/src/plugins/layout/index.js @@ -485,6 +485,12 @@ export default class Layout extends SonarPlugin { } init() { + // persist searchActive state when moving between plugins to prevent multiple + // TouchOverlayViews since we can't edit the view heirarchy in onDisconnect + this.client.call('isSearchActive').then(({isSearchActive}) => { + this.dispatchAction({type: 'SetSearchActive', isSearchActive}); + }); + performance.mark('LayoutInspectorInitialize'); this.client.call('getRoot').then((element: Element) => { this.dispatchAction({elements: [element], type: 'UpdateElements'}); @@ -510,9 +516,10 @@ export default class Layout extends SonarPlugin { this.getNodesAndDirectChildren(path, false).then( (elements: Array) => { const selected = path[path.length - 1]; + const {key, AXkey} = this.getKeysFromSelected(selected); this.dispatchAction({elements, type: 'UpdateElements'}); - this.dispatchAction({key: selected, type: 'SelectElement'}); + this.dispatchAction({key, AXkey, type: 'SelectElement'}); this.dispatchAction({isSearchActive: false, type: 'SetSearchActive'}); for (const key of path) { @@ -719,51 +726,57 @@ export default class Layout extends SonarPlugin { this.dispatchAction({isAlignmentMode, type: 'SetAlignmentActive'}); }; - onElementSelected = debounce((key: ElementID) => { - let finalKey = key; - let finalAXkey = null; + getKeysFromSelected(selectedKey: ElementID) { + let key = selectedKey; + let AXkey = null; if (this.axEnabled()) { const linkedAXNode = - this.state.elements[key] && - this.state.elements[key].extraInfo && - this.state.elements[key].extraInfo.linkedAXNode; + this.state.elements[selectedKey] && + this.state.elements[selectedKey].extraInfo && + this.state.elements[selectedKey].extraInfo.linkedAXNode; // element only in main tree with linkedAXNode selected if (linkedAXNode) { - finalAXkey = linkedAXNode; + AXkey = linkedAXNode; // element only in AX tree with linked nonAX (litho) element selected } else if ( - (!this.state.elements[key] || - this.state.elements[key].name === 'ComponentHost') && - this.state.AXtoNonAXMapping[key] + (!this.state.elements[selectedKey] || + this.state.elements[selectedKey].name === 'ComponentHost') && + this.state.AXtoNonAXMapping[selectedKey] ) { - finalKey = this.state.AXtoNonAXMapping[key]; - finalAXkey = key; + key = this.state.AXtoNonAXMapping[selectedKey]; + AXkey = selectedKey; // keys are same for both trees or 'linked' element does not exist } else { - finalAXkey = key; + AXkey = selectedKey; } } + return {key, AXkey}; + } + + onElementSelected = debounce((selectedKey: ElementID) => { + const {key, AXkey} = this.getKeysFromSelected(selectedKey); + this.dispatchAction({ - key: finalKey, - AXkey: finalAXkey, + key: key, + AXkey: AXkey, type: 'SelectElement', }); this.client.send('setHighlighted', { - id: key, + id: selectedKey, isAlignmentMode: this.state.isAlignmentMode, }); - this.getNodes([finalKey], {force: true, ax: false}).then( + this.getNodes([key], {force: true, ax: false}).then( (elements: Array) => { this.dispatchAction({elements, type: 'UpdateElements'}); }, ); - if (this.axEnabled() && finalAXkey) { - this.getNodes([finalAXkey], {force: true, ax: true}).then( + if (AXkey) { + this.getNodes([AXkey], {force: true, ax: true}).then( (elements: Array) => { this.dispatchAction({elements, type: 'UpdateAXElements'}); },