Talkback properties update for AX sidebar

Summary: Talkback-description now called talkback-output to avoid confusion about content-description. Talkback-output now includes the state and role of the view as well so it is more accurate to Talkback's actual output. Talkback-hint also added as a property to show what talkback will append to the description if a user has hints turned on. This may not always be 100% accurate because talkback changes what it says based on certain events and global settings as well but is correct for most general-use cases.

Reviewed By: blavalla

Differential Revision: D9317270

fbshipit-source-id: df9b9b5ebef19f583cbf997e0cd3fac6450170bb

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/ViewDescriptor.java

@@ -19,7 +19,6 @@ import android.os.Build;
 import android.support.v4.view.MarginLayoutParamsCompat;
 import android.support.v4.view.ViewCompat;
 import android.support.v4.view.accessibility.AccessibilityNodeInfoCompat;
-import android.util.Log;
 import android.util.SparseArray;
 import android.view.Gravity;
 import android.view.View;

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityRoleUtil.java

@@ -25,45 +25,52 @@ public class AccessibilityRoleUtil {
    * date with their implementation. Details can be seen in their source code here:
    *
    * <p>https://github.com/google/talkback/blob/master/utils/src/main/java/Role.java
+   *
+   * The roles spoken by Talkback (roleStrings) should also be kept up to date and are found here:
+   *
+   * <p>https://github.com/google/talkback/compositor/src/main/res/values/strings.xml
+   * <p>https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json
    */
   public enum AccessibilityRole {
-    NONE(null),
+    NONE(null, ""),
-    BUTTON("android.widget.Button"),
+    BUTTON("android.widget.Button", "Button"),
-    CHECK_BOX("android.widget.CompoundButton"),
+    CHECK_BOX("android.widget.CompoundButton", "Check box"),
-    DROP_DOWN_LIST("android.widget.Spinner"),
+    DROP_DOWN_LIST("android.widget.Spinner", "Drop down list"),
-    EDIT_TEXT("android.widget.EditText"),
+    EDIT_TEXT("android.widget.EditText", "Edit box"),
-    GRID("android.widget.GridView"),
+    GRID("android.widget.GridView", "Grid"),
-    IMAGE("android.widget.ImageView"),
+    IMAGE("android.widget.ImageView", "Image"),
-    IMAGE_BUTTON("android.widget.ImageView"),
+    IMAGE_BUTTON("android.widget.ImageView", "Button"),
-    LIST("android.widget.AbsListView"),
+    LIST("android.widget.AbsListView", "List"),
-    PAGER("android.support.v4.view.ViewPager"),
+    PAGER("android.support.v4.view.ViewPager", "Multi-page view"),
-    RADIO_BUTTON("android.widget.RadioButton"),
+    RADIO_BUTTON("android.widget.RadioButton", "Radio button"),
-    SEEK_CONTROL("android.widget.SeekBar"),
+    SEEK_CONTROL("android.widget.SeekBar", "Seek control"),
-    SWITCH("android.widget.Switch"),
+    SWITCH("android.widget.Switch", "Switch"),
-    TAB_BAR("android.widget.TabWidget"),
+    TAB_BAR("android.widget.TabWidget", "Tab bar"),
-    TOGGLE_BUTTON("android.widget.ToggleButton"),
+    TOGGLE_BUTTON("android.widget.ToggleButton", "Switch"),
-    VIEW_GROUP("android.view.ViewGroup"),
+    VIEW_GROUP("android.view.ViewGroup", ""),
-    WEB_VIEW("android.webkit.WebView"),
+    WEB_VIEW("android.webkit.WebView", "Webview"),
-    CHECKED_TEXT_VIEW("android.widget.CheckedTextView"),
+    CHECKED_TEXT_VIEW("android.widget.CheckedTextView", ""),
-    PROGRESS_BAR("android.widget.ProgressBar"),
+    PROGRESS_BAR("android.widget.ProgressBar", "Progress bar"),
-    ACTION_BAR_TAB("android.app.ActionBar$Tab"),
+    ACTION_BAR_TAB("android.app.ActionBar$Tab", ""),
-    DRAWER_LAYOUT("android.support.v4.widget.DrawerLayout"),
+    DRAWER_LAYOUT("android.support.v4.widget.DrawerLayout", ""),
-    SLIDING_DRAWER("android.widget.SlidingDrawer"),
+    SLIDING_DRAWER("android.widget.SlidingDrawer", ""),
-    ICON_MENU("com.android.internal.view.menu.IconMenuView"),
+    ICON_MENU("com.android.internal.view.menu.IconMenuView", ""),
-    TOAST("android.widget.Toast$TN"),
+    TOAST("android.widget.Toast$TN", ""),
-    DATE_PICKER_DIALOG("android.app.DatePickerDialog"),
+    DATE_PICKER_DIALOG("android.app.DatePickerDialog", ""),
-    TIME_PICKER_DIALOG("android.app.TimePickerDialog"),
+    TIME_PICKER_DIALOG("android.app.TimePickerDialog", ""),
-    DATE_PICKER("android.widget.DatePicker"),
+    DATE_PICKER("android.widget.DatePicker", ""),
-    TIME_PICKER("android.widget.TimePicker"),
+    TIME_PICKER("android.widget.TimePicker", ""),
-    NUMBER_PICKER("android.widget.NumberPicker"),
+    NUMBER_PICKER("android.widget.NumberPicker", ""),
-    SCROLL_VIEW("android.widget.ScrollView"),
+    SCROLL_VIEW("android.widget.ScrollView", ""),
-    HORIZONTAL_SCROLL_VIEW("android.widget.HorizontalScrollView"),
+    HORIZONTAL_SCROLL_VIEW("android.widget.HorizontalScrollView", ""),
-    KEYBOARD_KEY("android.inputmethodservice.Keyboard$Key");
+    KEYBOARD_KEY("android.inputmethodservice.Keyboard$Key", "");
 
     @Nullable private final String mValue;
+    private final String mRoleString;
 
-    AccessibilityRole(String type) {
+    AccessibilityRole(String type, String roleString) {
       mValue = type;
+      mRoleString = roleString;
     }
 
     @Nullable
@@ -71,6 +78,8 @@ public class AccessibilityRoleUtil {
       return mValue;
     }
 
+    public String getRoleString() { return mRoleString; }
+
     public static AccessibilityRole fromValue(String value) {
       for (AccessibilityRole role : AccessibilityRole.values()) {
         if (role.getValue() != null && role.getValue().equals(value)) {
@@ -120,4 +129,11 @@ public class AccessibilityRoleUtil {
 
     return role;
   }
+
+  public static String getTalkbackRoleString(View view) {
+    if (view == null) {
+      return "";
+    }
+    return getRole(view).getRoleString();
+  }
 }

android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java

@@ -33,6 +33,9 @@ import javax.annotation.Nullable;
  * are unnecessary here.
  */
 public final class AccessibilityUtil {
+  private static final String delimiter = ", ";
+  private static final int delimiterLength = delimiter.length();
+
   private AccessibilityUtil() {}
 
   public static final EnumMapping sAccessibilityActionMapping =
@@ -207,16 +210,225 @@ public final class AccessibilityUtil {
     }
   }
 
+  private static boolean supportsAction(AccessibilityNodeInfoCompat node, int action) {
+    if (node != null) {
+      final int supportedActions = node.getActions();
+
+      if ((supportedActions & action) == action) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
   /**
-   * Creates the text that Gogole's TalkBack screen reader will read aloud for a given {@link View}.
+   * Adds the state segments of Talkback's response to a given list. This should be kept up to date as
+   * much as necessary. Details can be seen in the source code here :
+   *
+   * https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json
+   *  - search for "description_for_tree_status", "get_switch_state"
+   *
+   * https://github.com/google/talkback/compositor/src/main/res/values/strings.xml
+   */
+  private static void addStateSegments(StringBuilder talkbackSegments, AccessibilityNodeInfoCompat node, AccessibilityRoleUtil.AccessibilityRole role) {
+    // selected status is always prepended
+    if (node.isSelected()) {
+      talkbackSegments.append("selected" + delimiter);
+    }
+
+    // next check collapse/expand/checked status
+    if (supportsAction(node, AccessibilityNodeInfoCompat.ACTION_EXPAND)) {
+      talkbackSegments.append("collapsed" + delimiter);
+    }
+
+    if (supportsAction(node, AccessibilityNodeInfoCompat.ACTION_COLLAPSE)) {
+      talkbackSegments.append("expanded" + delimiter);
+    }
+
+    String roleString = role.getRoleString();
+    if (node.isCheckable() && !roleString.equals("Switch") &&
+            (!role.equals(AccessibilityRoleUtil.AccessibilityRole.CHECKED_TEXT_VIEW) || node.isChecked())) {
+      talkbackSegments.append((node.isChecked() ? "checked" : "not checked") + delimiter);
+    }
+
+    if (roleString.equals("Switch")) {
+      CharSequence switchState = node.getText();
+      if (TextUtils.isEmpty(switchState) || role == AccessibilityRoleUtil.AccessibilityRole.TOGGLE_BUTTON) {
+        talkbackSegments.append((node.isChecked() ? "checked" : "not checked") + delimiter);
+      } else {
+        talkbackSegments.append(switchState + delimiter);
+      }
+    }
+  }
+
+  private static String removeFinalDelimiter(StringBuilder builder) {
+    int end = builder.length();
+    if (end > 0) {
+      builder.delete(end - delimiterLength, end);
+    }
+    return builder.toString();
+  }
+
+  private static final int SYSTEM_ACTION_MAX = 0x01FFFFFF;
+  private static String getHintForCustomActions(AccessibilityNodeInfoCompat node) {
+    StringBuilder customActions = new StringBuilder();
+    for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action : node.getActionList()) {
+      int id = action.getId();
+      CharSequence label = action.getLabel();
+      if (id > SYSTEM_ACTION_MAX) {
+        // don't include custom actions that don't have a label
+        if (!TextUtils.isEmpty(label)) {
+          customActions.append(label + delimiter);
+        }
+      } else if (id == AccessibilityNodeInfoCompat.ACTION_DISMISS) {
+        customActions.append("Dismiss" + delimiter);
+      } else if (id == AccessibilityNodeInfoCompat.ACTION_EXPAND) {
+        customActions.append("Expand" + delimiter);
+      } else if (id == AccessibilityNodeInfoCompat.ACTION_COLLAPSE) {
+        customActions.append("Collapse" + delimiter);
+      }
+    }
+
+    String actions = removeFinalDelimiter(customActions);
+    return actions.length() > 0 ? "Actions: " + actions : "";
+  }
+
+  // currently this is not used because the Talkback source logic seems erroneous resulting in get_hint_for_actions never
+  // returning any strings - see the TO DO in getTalkbackHint below once source is fixed
+  private static String getHintForActions(AccessibilityNodeInfoCompat node) {
+    StringBuilder actions = new StringBuilder();
+    for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action : node.getActionList()) {
+      int id = action.getId();
+      CharSequence label = action.getLabel();
+      if (id != AccessibilityNodeInfoCompat.ACTION_CLICK && id != AccessibilityNodeInfoCompat.ACTION_LONG_CLICK &&
+              !TextUtils.isEmpty(label) && id <= SYSTEM_ACTION_MAX) {
+        actions.append(label + delimiter);
+      }
+    }
+
+    return removeFinalDelimiter(actions);
+  }
+
+  private static String getHintForClick(AccessibilityNodeInfoCompat node) {
+    for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action : node.getActionList()) {
+      int id = action.getId();
+      CharSequence label = action.getLabel();
+      if (id == AccessibilityNodeInfoCompat.ACTION_CLICK && !TextUtils.isEmpty(label)) {
+        return "Double tap to " + label;
+      }
+    }
+
+    if (node.isCheckable()) {
+      return "Double tap to toggle";
+    }
+
+    if (node.isClickable()) {
+      return "Double tap to activate";
+    }
+
+    return "";
+  }
+
+  private static String getHintForLongClick(AccessibilityNodeInfoCompat node) {
+    for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action : node.getActionList()) {
+      int id = action.getId();
+      CharSequence label = action.getLabel();
+      if (id == AccessibilityNodeInfoCompat.ACTION_LONG_CLICK && !TextUtils.isEmpty(label)) {
+        return "Double tap and hold to " + label;
+      }
+    }
+
+    if (node.isLongClickable()) {
+      return "Double tap and hold to long press";
+    }
+
+    return "";
+  }
+
+
+  /**
+   * Creates the text that Google's TalkBack screen reader will read aloud for a given {@link View}'s hint.
+   * This hint is generally ported over from Google's TalkBack screen reader, and this should be kept up to
+   * date with their implementation (as much as necessary).  Hints can be turned off by user, so it may not
+   * actually be spoken and this method assumes the selection style is double tapping (it can also be set to
+   * keyboard or single tap but the general idea for the hint is the same). Details can be seen in their
+   * source code here:
+   *
+   * https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json
+   *  - search for "get_hint_from_node"
+   *
+   * https://github.com/google/talkback/compositor/src/main/res/values/strings.xml
+   *
+   * @param view The {@link View} to evaluate for a hint.
+   * @return {@code String} representing the hint talkback will say when a {@link View} is focused.
+   */
+  public static CharSequence getTalkbackHint(View view) {
+
+    final AccessibilityNodeInfoCompat node = ViewAccessibilityHelper.createNodeInfoFromView(view);
+    if (node == null) {
+      return "";
+    }
+
+    StringBuilder hint = new StringBuilder();
+    if (node.isEnabled()) {
+      AccessibilityRoleUtil.AccessibilityRole role = AccessibilityRoleUtil.getRole(view);
+
+      // special cases for spinners, pagers, and seek bars
+      if (role == AccessibilityRoleUtil.AccessibilityRole.DROP_DOWN_LIST) {
+        return "Double tap to change";
+      } else if (role == AccessibilityRoleUtil.AccessibilityRole.PAGER) {
+        if (supportsAction(node, AccessibilityNodeInfoCompat.ACTION_SCROLL_FORWARD) || supportsAction(node, AccessibilityNodeInfoCompat.ACTION_SCROLL_BACKWARD)) {
+          return "Swipe with two fingers to switch pages";
+        } else {
+          return "No more pages";
+        }
+      } else if (role == AccessibilityRoleUtil.AccessibilityRole.SEEK_CONTROL &&
+              (supportsAction(node, AccessibilityNodeInfoCompat.ACTION_SCROLL_FORWARD) || supportsAction(node, AccessibilityNodeInfoCompat.ACTION_SCROLL_BACKWARD))) {
+        return "Use volume keys to adjust";
+      } else {
+
+        // first custom actions
+        String segmentToAdd = getHintForCustomActions(node);
+        if (segmentToAdd.length() > 0) { hint.append(segmentToAdd + delimiter); }
+
+        // TODO: add getHintForActions(node) here if Talkback source gets fixed.
+        // Currently the "get_hint_for_actions" in the compositor source never adds to Talkback output
+        // because of a mismatched if condition/body. If this changes, we should also add a getHintForActions
+        // method here. Source at https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json
+
+        // then normal tap (special case for EditText)
+        if (role == AccessibilityRoleUtil.AccessibilityRole.EDIT_TEXT) {
+          if (!node.isFocused()) {
+            hint.append("Double tap to enter text" + delimiter);
+          }
+        } else {
+          segmentToAdd = getHintForClick(node);
+          if (segmentToAdd.length() > 0) { hint.append(segmentToAdd + delimiter); }
+        }
+
+        // then long press
+        segmentToAdd = getHintForLongClick(node);
+        if (segmentToAdd.length() > 0) { hint.append(segmentToAdd + delimiter); }
+      }
+    }
+    node.recycle();
+    return removeFinalDelimiter(hint);
+  }
+
+  /**
+   * Creates the text that Google's TalkBack screen reader will read aloud for a given {@link View}.
    * This may be any combination of the {@link View}'s {@code text}, {@code contentDescription}, and
    * the {@code text} and {@code contentDescription} of any ancestor {@link View}.
    *
-   * <p>Note: This string does not include any additional semantic information that Talkback will
+   * This description is generally ported over from Google's TalkBack screen reader, and this should be kept up to
-   * read, such as "Button", or "disabled".
+   * date with their implementation (as much as necessary). Details can be seen in their source code here:
+   *
+   * https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json
+   *  - search for "get_description_for_tree", "append_description_for_tree", "description_for_tree_nodes"
    *
    * @param view The {@link View} to evaluate.
-   * @return {@code String} describing why a {@link View} is focusable.
+   * @return {@code String} representing what talkback will say when a {@link View} is focused.
    */
   @Nullable
   public static CharSequence getTalkbackDescription(View view) {
@@ -231,20 +443,46 @@ public final class AccessibilityUtil {
       final boolean hasNodeText = !TextUtils.isEmpty(nodeText);
       final boolean isEditText = view instanceof EditText;
 
-      // EditText's prioritize their own text content over a contentDescription
+      StringBuilder talkbackSegments = new StringBuilder();
+      AccessibilityRoleUtil.AccessibilityRole role = AccessibilityRoleUtil.getRole(view);
+      String roleString = (String) node.getRoleDescription();
+      if (roleString == null) {
+        roleString = role.getRoleString();
+      }
+      boolean disabled = AccessibilityEvaluationUtil.isActionableForAccessibility(node) && !node.isEnabled();
+
+      // EditText's prioritize their own text content over a contentDescription so skip this
       if (!TextUtils.isEmpty(contentDescription) && (!isEditText || !hasNodeText)) {
-        return contentDescription;
+
+        // first prepend any status modifiers
+        addStateSegments(talkbackSegments, node, role);
+
+        // next add content description
+        talkbackSegments.append(contentDescription + delimiter);
+
+        // then role
+        if (roleString.length() > 0) { talkbackSegments.append(roleString + delimiter); }
+
+        // lastly disabled is appended if applicable
+        if (disabled) { talkbackSegments.append("disabled" + delimiter); }
+
+        return removeFinalDelimiter(talkbackSegments);
       }
 
+      // EditText
       if (hasNodeText) {
-        return nodeText;
+        // skip status checks for EditText, but description, role, and disabled are included
+        talkbackSegments.append(nodeText + delimiter);
+        if (roleString.length() > 0) { talkbackSegments.append(roleString + delimiter); }
+        if (disabled) { talkbackSegments.append("disabled" + delimiter); }
+
+        return removeFinalDelimiter(talkbackSegments);
       }
 
       // If there are child views and no contentDescription the text of all non-focusable children,
       // comma separated, becomes the description.
       if (view instanceof ViewGroup) {
         final StringBuilder concatChildDescription = new StringBuilder();
-        final String separator = ", ";
         final ViewGroup viewGroup = (ViewGroup) view;
 
         for (int i = 0, count = viewGroup.getChildCount(); i < count; i++) {
@@ -253,22 +491,17 @@ public final class AccessibilityUtil {
           final AccessibilityNodeInfoCompat childNodeInfo = AccessibilityNodeInfoCompat.obtain();
           ViewCompat.onInitializeAccessibilityNodeInfo(child, childNodeInfo);
 
-          CharSequence childNodeDescription = null;
           if (AccessibilityEvaluationUtil.isSpeakingNode(childNodeInfo, child)
               && !AccessibilityEvaluationUtil.isAccessibilityFocusable(childNodeInfo, child)) {
-            childNodeDescription = getTalkbackDescription(child);
+            CharSequence childNodeDescription = getTalkbackDescription(child);
-          }
+            if (!TextUtils.isEmpty(childNodeDescription)) {
-
+              concatChildDescription.append(childNodeDescription + delimiter);
-          if (!TextUtils.isEmpty(childNodeDescription)) {
-            if (concatChildDescription.length() > 0) {
-              concatChildDescription.append(separator);
             }
-            concatChildDescription.append(childNodeDescription);
           }
           childNodeInfo.recycle();
         }
 
-        return concatChildDescription.length() > 0 ? concatChildDescription.toString() : null;
+        return removeFinalDelimiter(concatChildDescription);
       }
 
       return null;
@@ -458,7 +691,8 @@ public final class AccessibilityUtil {
       props
           .put("talkback-focusable", true)
           .put("talkback-focusable-reasons", getTalkbackFocusableReasons(view))
-          .put("talkback-description", getTalkbackDescription(view));
+          .put("talkback-output", getTalkbackDescription(view))
+          .put("talkback-hint", getTalkbackHint(view));
     }
   }
 
@@ -500,10 +734,12 @@ public final class AccessibilityUtil {
     } else {
       String reason = getTalkbackFocusableReasons(view);
       CharSequence description = getTalkbackDescription(view);
+      CharSequence hint = getTalkbackHint(view);
       props
           .put("talkback-focusable", true)
-          .put("talkback-focusable-reasons", reason == null ? "" : reason)
+          .put("talkback-focusable-reasons", reason)
-          .put("talkback-description", description == null ? "" : description);
+          .put("talkback-output", description)
+          .put("talkback-hint", hint);
     }
 
     SonarObject axNodeInfo = getAXNodeInfoProperties(view);