Talkback properties update for AX sidebar

Summary: Talkback-description now called talkback-output to avoid confusion about content-description. Talkback-output now includes the state and role of the view as well so it is more accurate to Talkback's actual output. Talkback-hint also added as a property to show what talkback will append to the description if a user has hints turned on. This may not always be 100% accurate because talkback changes what it says based on certain events and global settings as well but is correct for most general-use cases.

Reviewed By: blavalla

Differential Revision: D9317270

fbshipit-source-id: df9b9b5ebef19f583cbf997e0cd3fac6450170bb
This commit is contained in:
Sara Valderrama
2018-08-15 20:35:16 -07:00
committed by Facebook Github Bot
parent e76a6ef529
commit e82808da6f
3 changed files with 305 additions and 54 deletions

View File

@@ -19,7 +19,6 @@ import android.os.Build;
import android.support.v4.view.MarginLayoutParamsCompat;
import android.support.v4.view.ViewCompat;
import android.support.v4.view.accessibility.AccessibilityNodeInfoCompat;
import android.util.Log;
import android.util.SparseArray;
import android.view.Gravity;
import android.view.View;

View File

@@ -25,45 +25,52 @@ public class AccessibilityRoleUtil {
* date with their implementation. Details can be seen in their source code here:
*
* <p>https://github.com/google/talkback/blob/master/utils/src/main/java/Role.java
*
* The roles spoken by Talkback (roleStrings) should also be kept up to date and are found here:
*
* <p>https://github.com/google/talkback/compositor/src/main/res/values/strings.xml
* <p>https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json
*/
public enum AccessibilityRole {
NONE(null),
BUTTON("android.widget.Button"),
CHECK_BOX("android.widget.CompoundButton"),
DROP_DOWN_LIST("android.widget.Spinner"),
EDIT_TEXT("android.widget.EditText"),
GRID("android.widget.GridView"),
IMAGE("android.widget.ImageView"),
IMAGE_BUTTON("android.widget.ImageView"),
LIST("android.widget.AbsListView"),
PAGER("android.support.v4.view.ViewPager"),
RADIO_BUTTON("android.widget.RadioButton"),
SEEK_CONTROL("android.widget.SeekBar"),
SWITCH("android.widget.Switch"),
TAB_BAR("android.widget.TabWidget"),
TOGGLE_BUTTON("android.widget.ToggleButton"),
VIEW_GROUP("android.view.ViewGroup"),
WEB_VIEW("android.webkit.WebView"),
CHECKED_TEXT_VIEW("android.widget.CheckedTextView"),
PROGRESS_BAR("android.widget.ProgressBar"),
ACTION_BAR_TAB("android.app.ActionBar$Tab"),
DRAWER_LAYOUT("android.support.v4.widget.DrawerLayout"),
SLIDING_DRAWER("android.widget.SlidingDrawer"),
ICON_MENU("com.android.internal.view.menu.IconMenuView"),
TOAST("android.widget.Toast$TN"),
DATE_PICKER_DIALOG("android.app.DatePickerDialog"),
TIME_PICKER_DIALOG("android.app.TimePickerDialog"),
DATE_PICKER("android.widget.DatePicker"),
TIME_PICKER("android.widget.TimePicker"),
NUMBER_PICKER("android.widget.NumberPicker"),
SCROLL_VIEW("android.widget.ScrollView"),
HORIZONTAL_SCROLL_VIEW("android.widget.HorizontalScrollView"),
KEYBOARD_KEY("android.inputmethodservice.Keyboard$Key");
NONE(null, ""),
BUTTON("android.widget.Button", "Button"),
CHECK_BOX("android.widget.CompoundButton", "Check box"),
DROP_DOWN_LIST("android.widget.Spinner", "Drop down list"),
EDIT_TEXT("android.widget.EditText", "Edit box"),
GRID("android.widget.GridView", "Grid"),
IMAGE("android.widget.ImageView", "Image"),
IMAGE_BUTTON("android.widget.ImageView", "Button"),
LIST("android.widget.AbsListView", "List"),
PAGER("android.support.v4.view.ViewPager", "Multi-page view"),
RADIO_BUTTON("android.widget.RadioButton", "Radio button"),
SEEK_CONTROL("android.widget.SeekBar", "Seek control"),
SWITCH("android.widget.Switch", "Switch"),
TAB_BAR("android.widget.TabWidget", "Tab bar"),
TOGGLE_BUTTON("android.widget.ToggleButton", "Switch"),
VIEW_GROUP("android.view.ViewGroup", ""),
WEB_VIEW("android.webkit.WebView", "Webview"),
CHECKED_TEXT_VIEW("android.widget.CheckedTextView", ""),
PROGRESS_BAR("android.widget.ProgressBar", "Progress bar"),
ACTION_BAR_TAB("android.app.ActionBar$Tab", ""),
DRAWER_LAYOUT("android.support.v4.widget.DrawerLayout", ""),
SLIDING_DRAWER("android.widget.SlidingDrawer", ""),
ICON_MENU("com.android.internal.view.menu.IconMenuView", ""),
TOAST("android.widget.Toast$TN", ""),
DATE_PICKER_DIALOG("android.app.DatePickerDialog", ""),
TIME_PICKER_DIALOG("android.app.TimePickerDialog", ""),
DATE_PICKER("android.widget.DatePicker", ""),
TIME_PICKER("android.widget.TimePicker", ""),
NUMBER_PICKER("android.widget.NumberPicker", ""),
SCROLL_VIEW("android.widget.ScrollView", ""),
HORIZONTAL_SCROLL_VIEW("android.widget.HorizontalScrollView", ""),
KEYBOARD_KEY("android.inputmethodservice.Keyboard$Key", "");
@Nullable private final String mValue;
private final String mRoleString;
AccessibilityRole(String type) {
AccessibilityRole(String type, String roleString) {
mValue = type;
mRoleString = roleString;
}
@Nullable
@@ -71,6 +78,8 @@ public class AccessibilityRoleUtil {
return mValue;
}
public String getRoleString() { return mRoleString; }
public static AccessibilityRole fromValue(String value) {
for (AccessibilityRole role : AccessibilityRole.values()) {
if (role.getValue() != null && role.getValue().equals(value)) {
@@ -120,4 +129,11 @@ public class AccessibilityRoleUtil {
return role;
}
public static String getTalkbackRoleString(View view) {
if (view == null) {
return "";
}
return getRole(view).getRoleString();
}
}

View File

@@ -33,6 +33,9 @@ import javax.annotation.Nullable;
* are unnecessary here.
*/
public final class AccessibilityUtil {
private static final String delimiter = ", ";
private static final int delimiterLength = delimiter.length();
private AccessibilityUtil() {}
public static final EnumMapping sAccessibilityActionMapping =
@@ -207,16 +210,225 @@ public final class AccessibilityUtil {
}
}
private static boolean supportsAction(AccessibilityNodeInfoCompat node, int action) {
if (node != null) {
final int supportedActions = node.getActions();
if ((supportedActions & action) == action) {
return true;
}
}
return false;
}
/**
* Creates the text that Gogole's TalkBack screen reader will read aloud for a given {@link View}.
* Adds the state segments of Talkback's response to a given list. This should be kept up to date as
* much as necessary. Details can be seen in the source code here :
*
* https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json
* - search for "description_for_tree_status", "get_switch_state"
*
* https://github.com/google/talkback/compositor/src/main/res/values/strings.xml
*/
private static void addStateSegments(StringBuilder talkbackSegments, AccessibilityNodeInfoCompat node, AccessibilityRoleUtil.AccessibilityRole role) {
// selected status is always prepended
if (node.isSelected()) {
talkbackSegments.append("selected" + delimiter);
}
// next check collapse/expand/checked status
if (supportsAction(node, AccessibilityNodeInfoCompat.ACTION_EXPAND)) {
talkbackSegments.append("collapsed" + delimiter);
}
if (supportsAction(node, AccessibilityNodeInfoCompat.ACTION_COLLAPSE)) {
talkbackSegments.append("expanded" + delimiter);
}
String roleString = role.getRoleString();
if (node.isCheckable() && !roleString.equals("Switch") &&
(!role.equals(AccessibilityRoleUtil.AccessibilityRole.CHECKED_TEXT_VIEW) || node.isChecked())) {
talkbackSegments.append((node.isChecked() ? "checked" : "not checked") + delimiter);
}
if (roleString.equals("Switch")) {
CharSequence switchState = node.getText();
if (TextUtils.isEmpty(switchState) || role == AccessibilityRoleUtil.AccessibilityRole.TOGGLE_BUTTON) {
talkbackSegments.append((node.isChecked() ? "checked" : "not checked") + delimiter);
} else {
talkbackSegments.append(switchState + delimiter);
}
}
}
private static String removeFinalDelimiter(StringBuilder builder) {
int end = builder.length();
if (end > 0) {
builder.delete(end - delimiterLength, end);
}
return builder.toString();
}
private static final int SYSTEM_ACTION_MAX = 0x01FFFFFF;
private static String getHintForCustomActions(AccessibilityNodeInfoCompat node) {
StringBuilder customActions = new StringBuilder();
for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action : node.getActionList()) {
int id = action.getId();
CharSequence label = action.getLabel();
if (id > SYSTEM_ACTION_MAX) {
// don't include custom actions that don't have a label
if (!TextUtils.isEmpty(label)) {
customActions.append(label + delimiter);
}
} else if (id == AccessibilityNodeInfoCompat.ACTION_DISMISS) {
customActions.append("Dismiss" + delimiter);
} else if (id == AccessibilityNodeInfoCompat.ACTION_EXPAND) {
customActions.append("Expand" + delimiter);
} else if (id == AccessibilityNodeInfoCompat.ACTION_COLLAPSE) {
customActions.append("Collapse" + delimiter);
}
}
String actions = removeFinalDelimiter(customActions);
return actions.length() > 0 ? "Actions: " + actions : "";
}
// currently this is not used because the Talkback source logic seems erroneous resulting in get_hint_for_actions never
// returning any strings - see the TO DO in getTalkbackHint below once source is fixed
private static String getHintForActions(AccessibilityNodeInfoCompat node) {
StringBuilder actions = new StringBuilder();
for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action : node.getActionList()) {
int id = action.getId();
CharSequence label = action.getLabel();
if (id != AccessibilityNodeInfoCompat.ACTION_CLICK && id != AccessibilityNodeInfoCompat.ACTION_LONG_CLICK &&
!TextUtils.isEmpty(label) && id <= SYSTEM_ACTION_MAX) {
actions.append(label + delimiter);
}
}
return removeFinalDelimiter(actions);
}
private static String getHintForClick(AccessibilityNodeInfoCompat node) {
for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action : node.getActionList()) {
int id = action.getId();
CharSequence label = action.getLabel();
if (id == AccessibilityNodeInfoCompat.ACTION_CLICK && !TextUtils.isEmpty(label)) {
return "Double tap to " + label;
}
}
if (node.isCheckable()) {
return "Double tap to toggle";
}
if (node.isClickable()) {
return "Double tap to activate";
}
return "";
}
private static String getHintForLongClick(AccessibilityNodeInfoCompat node) {
for (AccessibilityNodeInfoCompat.AccessibilityActionCompat action : node.getActionList()) {
int id = action.getId();
CharSequence label = action.getLabel();
if (id == AccessibilityNodeInfoCompat.ACTION_LONG_CLICK && !TextUtils.isEmpty(label)) {
return "Double tap and hold to " + label;
}
}
if (node.isLongClickable()) {
return "Double tap and hold to long press";
}
return "";
}
/**
* Creates the text that Google's TalkBack screen reader will read aloud for a given {@link View}'s hint.
* This hint is generally ported over from Google's TalkBack screen reader, and this should be kept up to
* date with their implementation (as much as necessary). Hints can be turned off by user, so it may not
* actually be spoken and this method assumes the selection style is double tapping (it can also be set to
* keyboard or single tap but the general idea for the hint is the same). Details can be seen in their
* source code here:
*
* https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json
* - search for "get_hint_from_node"
*
* https://github.com/google/talkback/compositor/src/main/res/values/strings.xml
*
* @param view The {@link View} to evaluate for a hint.
* @return {@code String} representing the hint talkback will say when a {@link View} is focused.
*/
public static CharSequence getTalkbackHint(View view) {
final AccessibilityNodeInfoCompat node = ViewAccessibilityHelper.createNodeInfoFromView(view);
if (node == null) {
return "";
}
StringBuilder hint = new StringBuilder();
if (node.isEnabled()) {
AccessibilityRoleUtil.AccessibilityRole role = AccessibilityRoleUtil.getRole(view);
// special cases for spinners, pagers, and seek bars
if (role == AccessibilityRoleUtil.AccessibilityRole.DROP_DOWN_LIST) {
return "Double tap to change";
} else if (role == AccessibilityRoleUtil.AccessibilityRole.PAGER) {
if (supportsAction(node, AccessibilityNodeInfoCompat.ACTION_SCROLL_FORWARD) || supportsAction(node, AccessibilityNodeInfoCompat.ACTION_SCROLL_BACKWARD)) {
return "Swipe with two fingers to switch pages";
} else {
return "No more pages";
}
} else if (role == AccessibilityRoleUtil.AccessibilityRole.SEEK_CONTROL &&
(supportsAction(node, AccessibilityNodeInfoCompat.ACTION_SCROLL_FORWARD) || supportsAction(node, AccessibilityNodeInfoCompat.ACTION_SCROLL_BACKWARD))) {
return "Use volume keys to adjust";
} else {
// first custom actions
String segmentToAdd = getHintForCustomActions(node);
if (segmentToAdd.length() > 0) { hint.append(segmentToAdd + delimiter); }
// TODO: add getHintForActions(node) here if Talkback source gets fixed.
// Currently the "get_hint_for_actions" in the compositor source never adds to Talkback output
// because of a mismatched if condition/body. If this changes, we should also add a getHintForActions
// method here. Source at https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json
// then normal tap (special case for EditText)
if (role == AccessibilityRoleUtil.AccessibilityRole.EDIT_TEXT) {
if (!node.isFocused()) {
hint.append("Double tap to enter text" + delimiter);
}
} else {
segmentToAdd = getHintForClick(node);
if (segmentToAdd.length() > 0) { hint.append(segmentToAdd + delimiter); }
}
// then long press
segmentToAdd = getHintForLongClick(node);
if (segmentToAdd.length() > 0) { hint.append(segmentToAdd + delimiter); }
}
}
node.recycle();
return removeFinalDelimiter(hint);
}
/**
* Creates the text that Google's TalkBack screen reader will read aloud for a given {@link View}.
* This may be any combination of the {@link View}'s {@code text}, {@code contentDescription}, and
* the {@code text} and {@code contentDescription} of any ancestor {@link View}.
*
* <p>Note: This string does not include any additional semantic information that Talkback will
* read, such as "Button", or "disabled".
* This description is generally ported over from Google's TalkBack screen reader, and this should be kept up to
* date with their implementation (as much as necessary). Details can be seen in their source code here:
*
* https://github.com/google/talkback/compositor/src/main/res/raw/compositor.json
* - search for "get_description_for_tree", "append_description_for_tree", "description_for_tree_nodes"
*
* @param view The {@link View} to evaluate.
* @return {@code String} describing why a {@link View} is focusable.
* @return {@code String} representing what talkback will say when a {@link View} is focused.
*/
@Nullable
public static CharSequence getTalkbackDescription(View view) {
@@ -231,20 +443,46 @@ public final class AccessibilityUtil {
final boolean hasNodeText = !TextUtils.isEmpty(nodeText);
final boolean isEditText = view instanceof EditText;
// EditText's prioritize their own text content over a contentDescription
StringBuilder talkbackSegments = new StringBuilder();
AccessibilityRoleUtil.AccessibilityRole role = AccessibilityRoleUtil.getRole(view);
String roleString = (String) node.getRoleDescription();
if (roleString == null) {
roleString = role.getRoleString();
}
boolean disabled = AccessibilityEvaluationUtil.isActionableForAccessibility(node) && !node.isEnabled();
// EditText's prioritize their own text content over a contentDescription so skip this
if (!TextUtils.isEmpty(contentDescription) && (!isEditText || !hasNodeText)) {
return contentDescription;
// first prepend any status modifiers
addStateSegments(talkbackSegments, node, role);
// next add content description
talkbackSegments.append(contentDescription + delimiter);
// then role
if (roleString.length() > 0) { talkbackSegments.append(roleString + delimiter); }
// lastly disabled is appended if applicable
if (disabled) { talkbackSegments.append("disabled" + delimiter); }
return removeFinalDelimiter(talkbackSegments);
}
// EditText
if (hasNodeText) {
return nodeText;
// skip status checks for EditText, but description, role, and disabled are included
talkbackSegments.append(nodeText + delimiter);
if (roleString.length() > 0) { talkbackSegments.append(roleString + delimiter); }
if (disabled) { talkbackSegments.append("disabled" + delimiter); }
return removeFinalDelimiter(talkbackSegments);
}
// If there are child views and no contentDescription the text of all non-focusable children,
// comma separated, becomes the description.
if (view instanceof ViewGroup) {
final StringBuilder concatChildDescription = new StringBuilder();
final String separator = ", ";
final ViewGroup viewGroup = (ViewGroup) view;
for (int i = 0, count = viewGroup.getChildCount(); i < count; i++) {
@@ -253,22 +491,17 @@ public final class AccessibilityUtil {
final AccessibilityNodeInfoCompat childNodeInfo = AccessibilityNodeInfoCompat.obtain();
ViewCompat.onInitializeAccessibilityNodeInfo(child, childNodeInfo);
CharSequence childNodeDescription = null;
if (AccessibilityEvaluationUtil.isSpeakingNode(childNodeInfo, child)
&& !AccessibilityEvaluationUtil.isAccessibilityFocusable(childNodeInfo, child)) {
childNodeDescription = getTalkbackDescription(child);
}
if (!TextUtils.isEmpty(childNodeDescription)) {
if (concatChildDescription.length() > 0) {
concatChildDescription.append(separator);
CharSequence childNodeDescription = getTalkbackDescription(child);
if (!TextUtils.isEmpty(childNodeDescription)) {
concatChildDescription.append(childNodeDescription + delimiter);
}
concatChildDescription.append(childNodeDescription);
}
childNodeInfo.recycle();
}
return concatChildDescription.length() > 0 ? concatChildDescription.toString() : null;
return removeFinalDelimiter(concatChildDescription);
}
return null;
@@ -458,7 +691,8 @@ public final class AccessibilityUtil {
props
.put("talkback-focusable", true)
.put("talkback-focusable-reasons", getTalkbackFocusableReasons(view))
.put("talkback-description", getTalkbackDescription(view));
.put("talkback-output", getTalkbackDescription(view))
.put("talkback-hint", getTalkbackHint(view));
}
}
@@ -500,10 +734,12 @@ public final class AccessibilityUtil {
} else {
String reason = getTalkbackFocusableReasons(view);
CharSequence description = getTalkbackDescription(view);
CharSequence hint = getTalkbackHint(view);
props
.put("talkback-focusable", true)
.put("talkback-focusable-reasons", reason == null ? "" : reason)
.put("talkback-description", description == null ? "" : description);
.put("talkback-focusable-reasons", reason)
.put("talkback-output", description)
.put("talkback-hint", hint);
}
SonarObject axNodeInfo = getAXNodeInfoProperties(view);