https://github.com/google/talkback/blob/master/utils/src/main/java/Role.java + * + * The roles spoken by Talkback (roleStrings) should also be kept up to date and are found here: + * + *
https://github.com/google/talkback/compositor/src/main/res/values/strings.xml + *
https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json */ public enum AccessibilityRole { - NONE(null), - BUTTON("android.widget.Button"), - CHECK_BOX("android.widget.CompoundButton"), - DROP_DOWN_LIST("android.widget.Spinner"), - EDIT_TEXT("android.widget.EditText"), - GRID("android.widget.GridView"), - IMAGE("android.widget.ImageView"), - IMAGE_BUTTON("android.widget.ImageView"), - LIST("android.widget.AbsListView"), - PAGER("android.support.v4.view.ViewPager"), - RADIO_BUTTON("android.widget.RadioButton"), - SEEK_CONTROL("android.widget.SeekBar"), - SWITCH("android.widget.Switch"), - TAB_BAR("android.widget.TabWidget"), - TOGGLE_BUTTON("android.widget.ToggleButton"), - VIEW_GROUP("android.view.ViewGroup"), - WEB_VIEW("android.webkit.WebView"), - CHECKED_TEXT_VIEW("android.widget.CheckedTextView"), - PROGRESS_BAR("android.widget.ProgressBar"), - ACTION_BAR_TAB("android.app.ActionBar$Tab"), - DRAWER_LAYOUT("android.support.v4.widget.DrawerLayout"), - SLIDING_DRAWER("android.widget.SlidingDrawer"), - ICON_MENU("com.android.internal.view.menu.IconMenuView"), - TOAST("android.widget.Toast$TN"), - DATE_PICKER_DIALOG("android.app.DatePickerDialog"), - TIME_PICKER_DIALOG("android.app.TimePickerDialog"), - DATE_PICKER("android.widget.DatePicker"), - TIME_PICKER("android.widget.TimePicker"), - NUMBER_PICKER("android.widget.NumberPicker"), - SCROLL_VIEW("android.widget.ScrollView"), - HORIZONTAL_SCROLL_VIEW("android.widget.HorizontalScrollView"), - KEYBOARD_KEY("android.inputmethodservice.Keyboard$Key"); + NONE(null, ""), + BUTTON("android.widget.Button", "Button"), + CHECK_BOX("android.widget.CompoundButton", "Check box"), + DROP_DOWN_LIST("android.widget.Spinner", "Drop down list"), + EDIT_TEXT("android.widget.EditText", "Edit box"), + GRID("android.widget.GridView", "Grid"), + IMAGE("android.widget.ImageView", "Image"), + IMAGE_BUTTON("android.widget.ImageView", "Button"), + LIST("android.widget.AbsListView", "List"), + PAGER("android.support.v4.view.ViewPager", "Multi-page view"), + RADIO_BUTTON("android.widget.RadioButton", "Radio button"), + SEEK_CONTROL("android.widget.SeekBar", "Seek control"), + SWITCH("android.widget.Switch", "Switch"), + TAB_BAR("android.widget.TabWidget", "Tab bar"), + TOGGLE_BUTTON("android.widget.ToggleButton", "Switch"), + VIEW_GROUP("android.view.ViewGroup", ""), + WEB_VIEW("android.webkit.WebView", "Webview"), + CHECKED_TEXT_VIEW("android.widget.CheckedTextView", ""), + PROGRESS_BAR("android.widget.ProgressBar", "Progress bar"), + ACTION_BAR_TAB("android.app.ActionBar$Tab", ""), + DRAWER_LAYOUT("android.support.v4.widget.DrawerLayout", ""), + SLIDING_DRAWER("android.widget.SlidingDrawer", ""), + ICON_MENU("com.android.internal.view.menu.IconMenuView", ""), + TOAST("android.widget.Toast$TN", ""), + DATE_PICKER_DIALOG("android.app.DatePickerDialog", ""), + TIME_PICKER_DIALOG("android.app.TimePickerDialog", ""), + DATE_PICKER("android.widget.DatePicker", ""), + TIME_PICKER("android.widget.TimePicker", ""), + NUMBER_PICKER("android.widget.NumberPicker", ""), + SCROLL_VIEW("android.widget.ScrollView", ""), + HORIZONTAL_SCROLL_VIEW("android.widget.HorizontalScrollView", ""), + KEYBOARD_KEY("android.inputmethodservice.Keyboard$Key", ""); @Nullable private final String mValue; + private final String mRoleString; - AccessibilityRole(String type) { + AccessibilityRole(String type, String roleString) { mValue = type; + mRoleString = roleString; } @Nullable @@ -71,6 +78,8 @@ public class AccessibilityRoleUtil { return mValue; } + public String getRoleString() { return mRoleString; } + public static AccessibilityRole fromValue(String value) { for (AccessibilityRole role : AccessibilityRole.values()) { if (role.getValue() != null && role.getValue().equals(value)) { @@ -120,4 +129,11 @@ public class AccessibilityRoleUtil { return role; } + + public static String getTalkbackRoleString(View view) { + if (view == null) { + return ""; + } + return getRole(view).getRoleString(); + } } diff --git a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java index 2302f7930..6bb437f6f 100644 --- a/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java +++ b/android/src/main/java/com/facebook/sonar/plugins/inspector/descriptors/utils/AccessibilityUtil.java @@ -33,6 +33,9 @@ import javax.annotation.Nullable; * are unnecessary here. */ public final class AccessibilityUtil { + private static final String delimiter = ", "; + private static final int delimiterLength = delimiter.length(); + private AccessibilityUtil() {} public static final EnumMapping sAccessibilityActionMapping = @@ -207,16 +210,225 @@ public final class AccessibilityUtil { } } + private static boolean supportsAction(AccessibilityNodeInfoCompat node, int action) { + if (node != null) { + final int supportedActions = node.getActions(); + + if ((supportedActions & action) == action) { + return true; + } + } + + return false; + } + /** - * Creates the text that Gogole's TalkBack screen reader will read aloud for a given {@link View}. + * Adds the state segments of Talkback's response to a given list. This should be kept up to date as + * much as necessary. Details can be seen in the source code here : + * + * https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json + * - search for "description_for_tree_status", "get_switch_state" + * + * https://github.com/google/talkback/compositor/src/main/res/values/strings.xml + */ + private static void addStateSegments(StringBuilder talkbackSegments, AccessibilityNodeInfoCompat node, AccessibilityRoleUtil.AccessibilityRole role) { + // selected status is always prepended + if (node.isSelected()) { + talkbackSegments.append("selected" + delimiter); + } + + // next check collapse/expand/checked status + if (supportsAction(node, AccessibilityNodeInfoCompat.ACTION_EXPAND)) { + talkbackSegments.append("collapsed" + delimiter); + } + + if (supportsAction(node, AccessibilityNodeInfoCompat.ACTION_COLLAPSE)) { + talkbackSegments.append("expanded" + delimiter); + } + + String roleString = role.getRoleString(); + if (node.isCheckable() && !roleString.equals("Switch") && + (!role.equals(AccessibilityRoleUtil.AccessibilityRole.CHECKED_TEXT_VIEW) || node.isChecked())) { + talkbackSegments.append((node.isChecked() ? "checked" : "not checked") + delimiter); + } + + if (roleString.equals("Switch")) { + CharSequence switchState = node.getText(); + if (TextUtils.isEmpty(switchState) || role == AccessibilityRoleUtil.AccessibilityRole.TOGGLE_BUTTON) { + talkbackSegments.append((node.isChecked() ? "checked" : "not checked") + delimiter); + } else { + talkbackSegments.append(switchState + delimiter); + } + } + } + + private static String removeFinalDelimiter(StringBuilder builder) { + int end = builder.length(); + if (end > 0) { + builder.delete(end - delimiterLength, end); + } + return builder.toString(); + } + + private static final int SYSTEM_ACTION_MAX = 0x01FFFFFF; + private static String getHintForCustomActions(AccessibilityNodeInfoCompat node) { + StringBuilder customActions = new StringBuilder(); + for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action : node.getActionList()) { + int id = action.getId(); + CharSequence label = action.getLabel(); + if (id > SYSTEM_ACTION_MAX) { + // don't include custom actions that don't have a label + if (!TextUtils.isEmpty(label)) { + customActions.append(label + delimiter); + } + } else if (id == AccessibilityNodeInfoCompat.ACTION_DISMISS) { + customActions.append("Dismiss" + delimiter); + } else if (id == AccessibilityNodeInfoCompat.ACTION_EXPAND) { + customActions.append("Expand" + delimiter); + } else if (id == AccessibilityNodeInfoCompat.ACTION_COLLAPSE) { + customActions.append("Collapse" + delimiter); + } + } + + String actions = removeFinalDelimiter(customActions); + return actions.length() > 0 ? "Actions: " + actions : ""; + } + + // currently this is not used because the Talkback source logic seems erroneous resulting in get_hint_for_actions never + // returning any strings - see the TO DO in getTalkbackHint below once source is fixed + private static String getHintForActions(AccessibilityNodeInfoCompat node) { + StringBuilder actions = new StringBuilder(); + for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action : node.getActionList()) { + int id = action.getId(); + CharSequence label = action.getLabel(); + if (id != AccessibilityNodeInfoCompat.ACTION_CLICK && id != AccessibilityNodeInfoCompat.ACTION_LONG_CLICK && + !TextUtils.isEmpty(label) && id <= SYSTEM_ACTION_MAX) { + actions.append(label + delimiter); + } + } + + return removeFinalDelimiter(actions); + } + + private static String getHintForClick(AccessibilityNodeInfoCompat node) { + for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action : node.getActionList()) { + int id = action.getId(); + CharSequence label = action.getLabel(); + if (id == AccessibilityNodeInfoCompat.ACTION_CLICK && !TextUtils.isEmpty(label)) { + return "Double tap to " + label; + } + } + + if (node.isCheckable()) { + return "Double tap to toggle"; + } + + if (node.isClickable()) { + return "Double tap to activate"; + } + + return ""; + } + + private static String getHintForLongClick(AccessibilityNodeInfoCompat node) { + for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action : node.getActionList()) { + int id = action.getId(); + CharSequence label = action.getLabel(); + if (id == AccessibilityNodeInfoCompat.ACTION_LONG_CLICK && !TextUtils.isEmpty(label)) { + return "Double tap and hold to " + label; + } + } + + if (node.isLongClickable()) { + return "Double tap and hold to long press"; + } + + return ""; + } + + + /** + * Creates the text that Google's TalkBack screen reader will read aloud for a given {@link View}'s hint. + * This hint is generally ported over from Google's TalkBack screen reader, and this should be kept up to + * date with their implementation (as much as necessary). Hints can be turned off by user, so it may not + * actually be spoken and this method assumes the selection style is double tapping (it can also be set to + * keyboard or single tap but the general idea for the hint is the same). Details can be seen in their + * source code here: + * + * https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json + * - search for "get_hint_from_node" + * + * https://github.com/google/talkback/compositor/src/main/res/values/strings.xml + * + * @param view The {@link View} to evaluate for a hint. + * @return {@code String} representing the hint talkback will say when a {@link View} is focused. + */ + public static CharSequence getTalkbackHint(View view) { + + final AccessibilityNodeInfoCompat node = ViewAccessibilityHelper.createNodeInfoFromView(view); + if (node == null) { + return ""; + } + + StringBuilder hint = new StringBuilder(); + if (node.isEnabled()) { + AccessibilityRoleUtil.AccessibilityRole role = AccessibilityRoleUtil.getRole(view); + + // special cases for spinners, pagers, and seek bars + if (role == AccessibilityRoleUtil.AccessibilityRole.DROP_DOWN_LIST) { + return "Double tap to change"; + } else if (role == AccessibilityRoleUtil.AccessibilityRole.PAGER) { + if (supportsAction(node, AccessibilityNodeInfoCompat.ACTION_SCROLL_FORWARD) || supportsAction(node, AccessibilityNodeInfoCompat.ACTION_SCROLL_BACKWARD)) { + return "Swipe with two fingers to switch pages"; + } else { + return "No more pages"; + } + } else if (role == AccessibilityRoleUtil.AccessibilityRole.SEEK_CONTROL && + (supportsAction(node, AccessibilityNodeInfoCompat.ACTION_SCROLL_FORWARD) || supportsAction(node, AccessibilityNodeInfoCompat.ACTION_SCROLL_BACKWARD))) { + return "Use volume keys to adjust"; + } else { + + // first custom actions + String segmentToAdd = getHintForCustomActions(node); + if (segmentToAdd.length() > 0) { hint.append(segmentToAdd + delimiter); } + + // TODO: add getHintForActions(node) here if Talkback source gets fixed. + // Currently the "get_hint_for_actions" in the compositor source never adds to Talkback output + // because of a mismatched if condition/body. If this changes, we should also add a getHintForActions + // method here. Source at https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json + + // then normal tap (special case for EditText) + if (role == AccessibilityRoleUtil.AccessibilityRole.EDIT_TEXT) { + if (!node.isFocused()) { + hint.append("Double tap to enter text" + delimiter); + } + } else { + segmentToAdd = getHintForClick(node); + if (segmentToAdd.length() > 0) { hint.append(segmentToAdd + delimiter); } + } + + // then long press + segmentToAdd = getHintForLongClick(node); + if (segmentToAdd.length() > 0) { hint.append(segmentToAdd + delimiter); } + } + } + node.recycle(); + return removeFinalDelimiter(hint); + } + + /** + * Creates the text that Google's TalkBack screen reader will read aloud for a given {@link View}. * This may be any combination of the {@link View}'s {@code text}, {@code contentDescription}, and * the {@code text} and {@code contentDescription} of any ancestor {@link View}. * - *
Note: This string does not include any additional semantic information that Talkback will - * read, such as "Button", or "disabled". + * This description is generally ported over from Google's TalkBack screen reader, and this should be kept up to + * date with their implementation (as much as necessary). Details can be seen in their source code here: + * + * https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json + * - search for "get_description_for_tree", "append_description_for_tree", "description_for_tree_nodes" * * @param view The {@link View} to evaluate. - * @return {@code String} describing why a {@link View} is focusable. + * @return {@code String} representing what talkback will say when a {@link View} is focused. */ @Nullable public static CharSequence getTalkbackDescription(View view) { @@ -231,20 +443,46 @@ public final class AccessibilityUtil { final boolean hasNodeText = !TextUtils.isEmpty(nodeText); final boolean isEditText = view instanceof EditText; - // EditText's prioritize their own text content over a contentDescription + StringBuilder talkbackSegments = new StringBuilder(); + AccessibilityRoleUtil.AccessibilityRole role = AccessibilityRoleUtil.getRole(view); + String roleString = (String) node.getRoleDescription(); + if (roleString == null) { + roleString = role.getRoleString(); + } + boolean disabled = AccessibilityEvaluationUtil.isActionableForAccessibility(node) && !node.isEnabled(); + + // EditText's prioritize their own text content over a contentDescription so skip this if (!TextUtils.isEmpty(contentDescription) && (!isEditText || !hasNodeText)) { - return contentDescription; + + // first prepend any status modifiers + addStateSegments(talkbackSegments, node, role); + + // next add content description + talkbackSegments.append(contentDescription + delimiter); + + // then role + if (roleString.length() > 0) { talkbackSegments.append(roleString + delimiter); } + + // lastly disabled is appended if applicable + if (disabled) { talkbackSegments.append("disabled" + delimiter); } + + return removeFinalDelimiter(talkbackSegments); } + // EditText if (hasNodeText) { - return nodeText; + // skip status checks for EditText, but description, role, and disabled are included + talkbackSegments.append(nodeText + delimiter); + if (roleString.length() > 0) { talkbackSegments.append(roleString + delimiter); } + if (disabled) { talkbackSegments.append("disabled" + delimiter); } + + return removeFinalDelimiter(talkbackSegments); } // If there are child views and no contentDescription the text of all non-focusable children, // comma separated, becomes the description. if (view instanceof ViewGroup) { final StringBuilder concatChildDescription = new StringBuilder(); - final String separator = ", "; final ViewGroup viewGroup = (ViewGroup) view; for (int i = 0, count = viewGroup.getChildCount(); i < count; i++) { @@ -253,22 +491,17 @@ public final class AccessibilityUtil { final AccessibilityNodeInfoCompat childNodeInfo = AccessibilityNodeInfoCompat.obtain(); ViewCompat.onInitializeAccessibilityNodeInfo(child, childNodeInfo); - CharSequence childNodeDescription = null; if (AccessibilityEvaluationUtil.isSpeakingNode(childNodeInfo, child) && !AccessibilityEvaluationUtil.isAccessibilityFocusable(childNodeInfo, child)) { - childNodeDescription = getTalkbackDescription(child); - } - - if (!TextUtils.isEmpty(childNodeDescription)) { - if (concatChildDescription.length() > 0) { - concatChildDescription.append(separator); + CharSequence childNodeDescription = getTalkbackDescription(child); + if (!TextUtils.isEmpty(childNodeDescription)) { + concatChildDescription.append(childNodeDescription + delimiter); } - concatChildDescription.append(childNodeDescription); } childNodeInfo.recycle(); } - return concatChildDescription.length() > 0 ? concatChildDescription.toString() : null; + return removeFinalDelimiter(concatChildDescription); } return null; @@ -458,7 +691,8 @@ public final class AccessibilityUtil { props .put("talkback-focusable", true) .put("talkback-focusable-reasons", getTalkbackFocusableReasons(view)) - .put("talkback-description", getTalkbackDescription(view)); + .put("talkback-output", getTalkbackDescription(view)) + .put("talkback-hint", getTalkbackHint(view)); } } @@ -500,10 +734,12 @@ public final class AccessibilityUtil { } else { String reason = getTalkbackFocusableReasons(view); CharSequence description = getTalkbackDescription(view); + CharSequence hint = getTalkbackHint(view); props .put("talkback-focusable", true) - .put("talkback-focusable-reasons", reason == null ? "" : reason) - .put("talkback-description", description == null ? "" : description); + .put("talkback-focusable-reasons", reason) + .put("talkback-output", description) + .put("talkback-hint", hint); } SonarObject axNodeInfo = getAXNodeInfoProperties(view);