jens/flipper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Capture accessibility hierarchy and package it for the sonar plugin

Browse Source 
Summary: In this diff we load and call a private API enabling voiceover hierarchy and pass it over via existing channel when client is in the accessibility mode

Reviewed By: lblasa

Differential Revision: D49393813

fbshipit-source-id: 437af1131547218cd52f4a56797707411787d7cf
This commit is contained in:
Sash Zats
2023-09-20 12:41:38 -07:00
committed by Facebook GitHub Bot
parent 3f0e1f76d5
commit 550b49e690
9 changed files with 436 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
iOS/Plugins/FlipperKitUIDebuggerPlugin/FlipperKitUIDebuggerPlugin/Descriptors/UIDDescriptorRegister.m
View File

@@ -10,6 +10,7 @@
#import "UIDDescriptorRegister.h"
#import <objc/runtime.h>
#import "UIDChainedDescriptor.h"
#import "UIDUIAccessibilityElementDescriptor.h"
#import "UIDUIApplicationDescriptor.h"
#import "UIDUILabelDescriptor.h"
#import "UIDUINavigationControllerDescriptor.h"
@@ -53,6 +54,9 @@
forClass:[UIView class]];
[defaultRegister registerDescriptor:[UIDUILabelDescriptor new]
forClass:[UILabel class]];
[defaultRegister
registerDescriptor:[UIDUIAccessibilityElementDescriptor new]
forClass:[UIAccessibilityElement class]];
});
return defaultRegister;

22
iOS/Plugins/FlipperKitUIDebuggerPlugin/FlipperKitUIDebuggerPlugin/Descriptors/UIDUIAccessibilityElementDescriptor.h Normal file
View File

@@ -0,0 +1,22 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#if FB_SONARKIT_ENABLED
#import <UIKit/UIKit.h>
#import "UIDNodeDescriptor.h"
NS_ASSUME_NONNULL_BEGIN
@interface UIDUIAccessibilityElementDescriptor
: UIDNodeDescriptor<UIAccessibilityElement*>
@end
NS_ASSUME_NONNULL_END
#endif

22
iOS/Plugins/FlipperKitUIDebuggerPlugin/FlipperKitUIDebuggerPlugin/Descriptors/UIDUIAccessibilityElementDescriptor.m Normal file
View File

@@ -0,0 +1,22 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#if FB_SONARKIT_ENABLED
#import "UIDUIAccessibilityElementDescriptor.h"
#import "UIDBounds.h"
#import "UIDSnapshot.h"
@implementation UIDUIAccessibilityElementDescriptor
- (UIDBounds*)boundsForNode:(UIAccessibilityElement*)node {
return [UIDBounds fromRect:node.accessibilityFrame];
}
@end
#endif

3
iOS/Plugins/FlipperKitUIDebuggerPlugin/FlipperKitUIDebuggerPlugin/Observer/UIDTreeObserver.h
View File

@@ -7,6 +7,7 @@
#if FB_SONARKIT_ENABLED
#import <FlipperKitUIDebuggerPlugin/UIDTraversalMode.h>
#import <UIKit/UIKit.h>
NS_ASSUME_NONNULL_BEGIN
@@ -19,6 +20,8 @@ NS_ASSUME_NONNULL_BEGIN
NSMutableDictionary<NSNumber*, UIDTreeObserver<T>*>* children;
@property(nonatomic, strong) NSString* type;
@property(nonatomic, assign) UIDTraversalMode traversalMode;
- (void)subscribe:(nullable T)node;
- (void)unsubscribe;
- (void)processNode:(id)node withContext:(UIDContext*)context;

23
iOS/Plugins/FlipperKitUIDebuggerPlugin/FlipperKitUIDebuggerPlugin/Observer/UIDTreeObserver.m
View File

@@ -8,6 +8,7 @@
#if FB_SONARKIT_ENABLED
#import "UIDTreeObserver.h"
#import "UIDAllyTraversal.h"
#import "UIDContext.h"
#import "UIDHierarchyTraversal.h"
#import "UIDTimeUtilities.h"
@@ -39,15 +40,29 @@
uint64_t t0 = UIDPerformanceNow();
UIDHierarchyTraversal* traversal = [UIDHierarchyTraversal
createWithDescriptorRegister:context.descriptorRegister];
UIDNodeDescriptor* descriptor =
[context.descriptorRegister descriptorForClass:[node class]];
UIDNodeDescriptor* rootDescriptor = [context.descriptorRegister
descriptorForClass:[context.application class]];
NSArray* nodes = [traversal traverse:node];
NSArray* nodes;
switch (_traversalMode) {
case UIDTraversalModeViewHierarchy: {
UIDHierarchyTraversal* const traversal = [UIDHierarchyTraversal
createWithDescriptorRegister:context.descriptorRegister];
nodes = [traversal traverse:node];
break;
}
case UIDTraversalModeAccessibilityHierarchy: {
UIDAllyTraversal* allyTraversal = [[UIDAllyTraversal alloc]
initWithDescriptorRegister:context.descriptorRegister];
nodes = [allyTraversal traverse:context.application root:node];
break;
}
default:
// Unexpected value, abort
return;
}
uint64_t t1 = UIDPerformanceNow();

2
iOS/Plugins/FlipperKitUIDebuggerPlugin/FlipperKitUIDebuggerPlugin/Observer/UIDTreeObserverManager.m
View File

@@ -60,6 +60,8 @@
return;
}
_rootObserver.traversalMode = _traversalMode = traversalMode;
// trigger another pass
dispatch_async(dispatch_get_main_queue(), ^{
[self->_rootObserver processNode:self->_context.application

6
iOS/Plugins/FlipperKitUIDebuggerPlugin/FlipperKitUIDebuggerPlugin/Serializers/UIDNode+Foundation.m
View File

@@ -17,11 +17,11 @@ FB_LINKABLE(UIDNode_Foundation)
- (id)toFoundation {
NSMutableDictionary* data = [NSMutableDictionary dictionaryWithDictionary:@{
@"id" : [NSNumber numberWithUnsignedInt:self.identifier],
@"qualifiedName" : self.qualifiedName,
@"qualifiedName" : self.qualifiedName ?: @"",
@"name" : self.name,
@"bounds" : [self.bounds toFoundation],
@"tags" : self.tags.allObjects,
@"inlineAttributes" : self.inlineAttributes,
@"tags" : self.tags ? self.tags.allObjects : @[],
@"inlineAttributes" : self.inlineAttributes ?: @{},
@"children" : self.children,
}];

29
iOS/Plugins/FlipperKitUIDebuggerPlugin/FlipperKitUIDebuggerPlugin/Traversal/UIDAllyTraversal.h Normal file
View File

@@ -0,0 +1,29 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#if FB_SONARKIT_ENABLED
#import <Foundation/Foundation.h>
NS_ASSUME_NONNULL_BEGIN
@class UIDNode;
@class UIApplication;
@class UIDDescriptorRegister;
@interface UIDAllyTraversal : NSObject
- (instancetype)initWithDescriptorRegister:
(UIDDescriptorRegister*)descriptorRegister;
- (NSArray<UIDNode*>*)traverse:(UIApplication*)application root:(id)root;
@end
NS_ASSUME_NONNULL_END
#endif

332
iOS/Plugins/FlipperKitUIDebuggerPlugin/FlipperKitUIDebuggerPlugin/Traversal/UIDAllyTraversal.m Normal file
View File

@@ -0,0 +1,332 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#if FB_SONARKIT_ENABLED
#import "UIDAllyTraversal.h"
#import "UIDDescriptorRegister.h"
#import "UIDMetadataRegister.h"
#import "UIDNode.h"
@interface UIAccessibilityElementTraversalOptions : NSObject
+ (instancetype)defaultVoiceOverOptions;
+ (instancetype)voiceOverOptionsIncludingElementsFromOpaqueProviders:(BOOL)arg1
honorsGroups:(BOOL)arg2;
@end
@interface UIApplication (Ally)
- (NSArray*)_accessibilityLeafDescendantsWithOptions:
(UIAccessibilityElementTraversalOptions*)option;
@end
@implementation UIDAllyTraversal {
UIDDescriptorRegister* _descriptorRegister;
}
- (instancetype)initWithDescriptorRegister:
(UIDDescriptorRegister*)descriptorRegister {
self = [super init];
if (self) {
_descriptorRegister = descriptorRegister;
}
return self;
}
- (NSArray<UIDNode*>*)traverse:(UIApplication*)application root:(id)root {
if (!root) {
return @[];
}
// create voice over representation of the app
id options = [NSClassFromString(@"UIAccessibilityElementTraversalOptions")
voiceOverOptionsIncludingElementsFromOpaqueProviders:YES
honorsGroups:NO];
NSArray<NSObject*>* const allyNodes = [[application
_accessibilityLeafDescendantsWithOptions:options] mutableCopy];
UIDNode* rootNode = [self _uidNodeForNode:root];
NSInteger rootIdentifier = rootNode.identifier;
NSMutableArray<UIDNode*>* nodes = [NSMutableArray new];
NSMutableArray* childrenIds = [NSMutableArray new];
for (NSObject* node in allyNodes) {
UIDNode* uidNode = [self _uidNodeForNode:node];
uidNode.parent = @(rootIdentifier);
[nodes addObject:uidNode];
[childrenIds addObject:[NSNumber numberWithUnsignedInt:uidNode.identifier]];
}
rootNode.children = childrenIds;
[nodes insertObject:rootNode atIndex:0];
return nodes;
}
- (UIDNode*)_uidNodeForNode:(NSObject*)node {
UIDNodeDescriptor* descriptor =
[_descriptorRegister descriptorForClass:[node class]];
NSUInteger nodeIdentifier = [descriptor identifierForNode:node];
UIDNode* uidNode = [[UIDNode alloc]
initWithIdentifier:nodeIdentifier
qualifiedName:[descriptor nameForNode:node]
name:_nameForNode(node)
bounds:[UIDBounds fromRect:node.accessibilityFrame]
tags:[descriptor tagsForNode:node]];
uidNode.attributes = _atrtibutesForNode(node);
return uidNode;
}
static NSString* _nameForNode(NSObject* node) {
NSMutableArray* const parts = [NSMutableArray new];
if (node.accessibilityLabel.length > 0) {
[parts addObject:node.accessibilityLabel];
}
if (node.accessibilityValue.length > 0) {
[parts addObject:node.accessibilityValue];
}
if (parts.count == 0) {
return @"[No accessibility label]";
}
NSString* const emojiTraits =
_descriptionFromTraits(node.accessibilityTraits, YES);
NSString* const fullNodeName = [parts componentsJoinedByString:@", "];
return emojiTraits
? [NSString stringWithFormat:@"%@: %@", emojiTraits, fullNodeName]
: fullNodeName;
}
static UIDAttributes* _atrtibutesForNode(NSObject* node) {
static UIDMetadataId ClassAttributeId;
static UIDMetadataId AddressAttributeId;
static UIDMetadataId AccessibilityAttributeId;
static UIDMetadataId IsAccessibilityElementAttributeId;
static UIDMetadataId AccessibilityLabelAttributeId;
static UIDMetadataId AccessibilityIdentifierAttributeId;
static UIDMetadataId AccessibilityValueAttributeId;
static UIDMetadataId AccessibilityHintAttributeId;
static UIDMetadataId AccessibilityTraitsAttributeId;
static UIDMetadataId AccessibilityViewIsModalAttributeId;
static UIDMetadataId ShouldGroupAccessibilityChildrenAttributeId;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
UIDMetadataRegister* const metadataRegister = [UIDMetadataRegister shared];
AccessibilityAttributeId = [metadataRegister
registerMetadataWithType:UIDEBUGGER_METADATA_TYPE_ATTRIBUTE
name:@"Accessibility"];
AddressAttributeId = [metadataRegister
registerMetadataWithType:UIDEBUGGER_METADATA_TYPE_ATTRIBUTE
name:@"Address"
isMutable:NO
definedBy:AccessibilityAttributeId];
ClassAttributeId = [metadataRegister
registerMetadataWithType:UIDEBUGGER_METADATA_TYPE_ATTRIBUTE
name:@"Class"
isMutable:NO
definedBy:AccessibilityAttributeId];
IsAccessibilityElementAttributeId = [metadataRegister
registerMetadataWithType:UIDEBUGGER_METADATA_TYPE_ATTRIBUTE
name:@"isAccessibilityElement"
isMutable:NO
definedBy:AccessibilityAttributeId];
AccessibilityLabelAttributeId = [[UIDMetadataRegister shared]
registerMetadataWithType:UIDEBUGGER_METADATA_TYPE_ATTRIBUTE
name:@"accessibilityLabel"
isMutable:false
definedBy:AccessibilityAttributeId];
AccessibilityIdentifierAttributeId = [[UIDMetadataRegister shared]
registerMetadataWithType:UIDEBUGGER_METADATA_TYPE_ATTRIBUTE
name:@"accessibilityIdentifier"
isMutable:false
definedBy:AccessibilityAttributeId];
AccessibilityValueAttributeId = [[UIDMetadataRegister shared]
registerMetadataWithType:UIDEBUGGER_METADATA_TYPE_ATTRIBUTE
name:@"accessibilityValue"
isMutable:false
definedBy:AccessibilityAttributeId];
AccessibilityHintAttributeId = [[UIDMetadataRegister shared]
registerMetadataWithType:UIDEBUGGER_METADATA_TYPE_ATTRIBUTE
name:@"accessibilityHint"
isMutable:false
definedBy:AccessibilityAttributeId];
AccessibilityTraitsAttributeId = [[UIDMetadataRegister shared]
registerMetadataWithType:UIDEBUGGER_METADATA_TYPE_ATTRIBUTE
name:@"accessibilityTraits"
isMutable:false
definedBy:AccessibilityAttributeId];
AccessibilityViewIsModalAttributeId = [[UIDMetadataRegister shared]
registerMetadataWithType:UIDEBUGGER_METADATA_TYPE_ATTRIBUTE
name:@"accessibilityViewIsModal"
isMutable:false
definedBy:AccessibilityAttributeId];
ShouldGroupAccessibilityChildrenAttributeId = [[UIDMetadataRegister shared]
registerMetadataWithType:UIDEBUGGER_METADATA_TYPE_ATTRIBUTE
name:@"shouldGroupAccessibilityChildren"
isMutable:false
definedBy:AccessibilityAttributeId];
});
NSMutableDictionary* const accessibilityAttributes =
[NSMutableDictionary new];
accessibilityAttributes[ClassAttributeId] =
[UIDInspectableText fromText:NSStringFromClass(node.class)];
accessibilityAttributes[AddressAttributeId] =
[UIDInspectableText fromText:[NSString stringWithFormat:@"%p", node]];
accessibilityAttributes[IsAccessibilityElementAttributeId] =
[UIDInspectableBoolean fromBoolean:node.isAccessibilityElement];
accessibilityAttributes[IsAccessibilityElementAttributeId] =
[UIDInspectableBoolean fromBoolean:node.isAccessibilityElement];
accessibilityAttributes[AccessibilityLabelAttributeId] =
[UIDInspectableText fromText:node.accessibilityLabel];
if ([node conformsToProtocol:@protocol(UIAccessibilityIdentification)]) {
accessibilityAttributes[AccessibilityIdentifierAttributeId] =
[UIDInspectableText fromText:((id<UIAccessibilityIdentification>)node)
.accessibilityIdentifier];
}
accessibilityAttributes[AccessibilityValueAttributeId] =
[UIDInspectableText fromText:node.accessibilityValue];
if (node.accessibilityHint != nil) {
accessibilityAttributes[AccessibilityHintAttributeId] =
[UIDInspectableText fromText:node.accessibilityHint];
}
accessibilityAttributes[AccessibilityViewIsModalAttributeId] =
[UIDInspectableBoolean fromBoolean:node.accessibilityViewIsModal];
accessibilityAttributes[ShouldGroupAccessibilityChildrenAttributeId] =
[UIDInspectableBoolean fromBoolean:node.shouldGroupAccessibilityChildren];
accessibilityAttributes[AccessibilityTraitsAttributeId] = [UIDInspectableText
fromText:_descriptionFromTraits(node.accessibilityTraits, NO)];
return @{
AccessibilityAttributeId :
[UIDInspectableObject fromFields:accessibilityAttributes]
};
}
static NSString* _Nullable _descriptionFromTraits(
UIAccessibilityTraits traits,
BOOL emojiOnly) {
if (traits == UIAccessibilityTraitNone) {
return emojiOnly ? nil : @"None";
}
static NSArray* allTraits;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
allTraits = @[
@(UIAccessibilityTraitButton),
@(UIAccessibilityTraitLink),
@(UIAccessibilityTraitHeader),
@(UIAccessibilityTraitSearchField),
@(UIAccessibilityTraitImage),
@(UIAccessibilityTraitSelected),
@(UIAccessibilityTraitPlaysSound),
@(UIAccessibilityTraitKeyboardKey),
@(UIAccessibilityTraitStaticText),
@(UIAccessibilityTraitSummaryElement),
@(UIAccessibilityTraitNotEnabled),
@(UIAccessibilityTraitUpdatesFrequently),
@(UIAccessibilityTraitStartsMediaSession),
@(UIAccessibilityTraitAdjustable),
@(UIAccessibilityTraitAllowsDirectInteraction),
@(UIAccessibilityTraitCausesPageTurn),
@(UIAccessibilityTraitTabBar),
];
});
NSMutableArray* descriptionComponents = [NSMutableArray new];
for (NSNumber* wrappedTrait in allTraits) {
UIAccessibilityTraits trait = wrappedTrait.unsignedIntegerValue;
if ((traits & trait) == trait) {
NSString* traitDescription = emojiOnly
? _emojiFromTrait(trait)
: [NSString stringWithFormat:@"%@ - %@",
_emojiFromTrait(trait),
_descriptionFromTrait(trait)];
if (traitDescription) {
[descriptionComponents addObject:traitDescription];
}
}
}
return descriptionComponents.count > 0
? [descriptionComponents componentsJoinedByString:@", "]
: (emojiOnly ? nil : @"None");
}
static NSString* _Nullable _descriptionFromTrait(UIAccessibilityTraits trait) {
if (trait == UIAccessibilityTraitButton) {
return @"button";
} else if (trait == UIAccessibilityTraitLink) {
return @"link";
} else if (trait == UIAccessibilityTraitHeader) {
return @"header";
} else if (trait == UIAccessibilityTraitSearchField) {
return @"search field";
} else if (trait == UIAccessibilityTraitImage) {
return @"image";
} else if (trait == UIAccessibilityTraitSelected) {
return @"selected";
} else if (trait == UIAccessibilityTraitPlaysSound) {
return @"plays sound";
} else if (trait == UIAccessibilityTraitKeyboardKey) {
return @"keyboard key";
} else if (trait == UIAccessibilityTraitStaticText) {
return @"static text";
} else if (trait == UIAccessibilityTraitSummaryElement) {
return @"summary element";
} else if (trait == UIAccessibilityTraitNotEnabled) {
return @"not enabled";
} else if (trait == UIAccessibilityTraitUpdatesFrequently) {
return @"updates frequently";
} else if (trait == UIAccessibilityTraitStartsMediaSession) {
return @"starts media session";
} else if (trait == UIAccessibilityTraitAdjustable) {
return @"adjustable";
} else if (trait == UIAccessibilityTraitAllowsDirectInteraction) {
return @"allows direct interaction";
} else if (trait == UIAccessibilityTraitCausesPageTurn) {
return @"causes page turn";
} else if (trait == UIAccessibilityTraitTabBar) {
return @"tab tar";
}
return nil;
}
static NSString* _Nullable _emojiFromTrait(UIAccessibilityTraits trait) {
if (trait == UIAccessibilityTraitButton) {
return @"🆗";
} else if (trait == UIAccessibilityTraitLink) {
return @"🔗";
} else if (trait == UIAccessibilityTraitHeader) {
return @"🏷️";
} else if (trait == UIAccessibilityTraitSearchField) {
return @"🔍";
} else if (trait == UIAccessibilityTraitImage) {
return @"🖼️";
} else if (trait == UIAccessibilityTraitSelected) {
return @"✅";
} else if (trait == UIAccessibilityTraitPlaysSound) {
return @"🔊";
} else if (trait == UIAccessibilityTraitKeyboardKey) {
return @"⌨️";
} else if (trait == UIAccessibilityTraitStaticText) {
return @"📄";
} else if (trait == UIAccessibilityTraitSummaryElement) {
return @"ℹ️";
} else if (trait == UIAccessibilityTraitNotEnabled) {
return @"❌";
} else if (trait == UIAccessibilityTraitUpdatesFrequently) {
return @"🔄";
} else if (trait == UIAccessibilityTraitStartsMediaSession) {
return @"🎬";
} else if (trait == UIAccessibilityTraitAdjustable) {
return @"🔧";
} else if (trait == UIAccessibilityTraitAllowsDirectInteraction) {
return @"👆";
} else if (trait == UIAccessibilityTraitCausesPageTurn) {
return @"📖";
} else if (trait == UIAccessibilityTraitTabBar) {
return @"🗂️";
}
return nil;
}
@end
#endif