| // Copyright 2013 The Flutter Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| #include "flutter/shell/platform/fuchsia/flutter/accessibility_bridge.h" | |
| #include <lib/inspect/cpp/inspector.h> | |
| #include <lib/zx/process.h> | |
| #include <zircon/status.h> | |
| #include <zircon/types.h> | |
| #include <algorithm> | |
| #include <deque> | |
| #include "flutter/fml/logging.h" | |
| #include "flutter/lib/ui/semantics/semantics_node.h" | |
| #include "../runtime/dart/utils/root_inspect_node.h" | |
| namespace flutter_runner { | |
| namespace { | |
| #if !FLUTTER_RELEASE | |
| static constexpr char kTreeDumpInspectRootName[] = "semantic_tree_root"; | |
| // Converts flutter semantic node flags to a string representation. | |
| std::string NodeFlagsToString(const flutter::SemanticsNode& node) { | |
| std::string output; | |
| if (node.HasFlag(flutter::SemanticsFlags::kHasCheckedState)) { | |
| output += "kHasCheckedState|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kHasEnabledState)) { | |
| output += "kHasEnabledState|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kHasImplicitScrolling)) { | |
| output += "kHasImplicitScrolling|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kHasToggledState)) { | |
| output += "kHasToggledState|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsButton)) { | |
| output += "kIsButton|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsChecked)) { | |
| output += "kIsChecked|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsEnabled)) { | |
| output += "kIsEnabled|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsFocusable)) { | |
| output += "kIsFocusable|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsFocused)) { | |
| output += "kIsFocused|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsHeader)) { | |
| output += "kIsHeader|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsHidden)) { | |
| output += "kIsHidden|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsImage)) { | |
| output += "kIsImage|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsInMutuallyExclusiveGroup)) { | |
| output += "kIsInMutuallyExclusiveGroup|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsKeyboardKey)) { | |
| output += "kIsKeyboardKey|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsLink)) { | |
| output += "kIsLink|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsLiveRegion)) { | |
| output += "kIsLiveRegion|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsObscured)) { | |
| output += "kIsObscured|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsReadOnly)) { | |
| output += "kIsReadOnly|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsSelected)) { | |
| output += "kIsSelected|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsSlider)) { | |
| output += "kIsSlider|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsTextField)) { | |
| output += "kIsTextField|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsToggled)) { | |
| output += "kIsToggled|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kNamesRoute)) { | |
| output += "kNamesRoute|"; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kScopesRoute)) { | |
| output += "kScopesRoute|"; | |
| } | |
| return output; | |
| } | |
| // Converts flutter semantic node actions to a string representation. | |
| std::string NodeActionsToString(const flutter::SemanticsNode& node) { | |
| std::string output; | |
| if (node.HasAction(flutter::SemanticsAction::kCopy)) { | |
| output += "kCopy|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kCustomAction)) { | |
| output += "kCustomAction|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kCut)) { | |
| output += "kCut|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kDecrease)) { | |
| output += "kDecrease|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kDidGainAccessibilityFocus)) { | |
| output += "kDidGainAccessibilityFocus|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kDidLoseAccessibilityFocus)) { | |
| output += "kDidLoseAccessibilityFocus|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kDismiss)) { | |
| output += "kDismiss|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kIncrease)) { | |
| output += "kIncrease|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kLongPress)) { | |
| output += "kLongPress|"; | |
| } | |
| if (node.HasAction( | |
| flutter::SemanticsAction::kMoveCursorBackwardByCharacter)) { | |
| output += "kMoveCursorBackwardByCharacter|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kMoveCursorBackwardByWord)) { | |
| output += "kMoveCursorBackwardByWord|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kMoveCursorForwardByCharacter)) { | |
| output += "kMoveCursorForwardByCharacter|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kMoveCursorForwardByWord)) { | |
| output += "kMoveCursorForwardByWord|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kPaste)) { | |
| output += "kPaste|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kScrollDown)) { | |
| output += "kScrollDown|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kScrollLeft)) { | |
| output += "kScrollLeft|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kScrollRight)) { | |
| output += "kScrollRight|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kScrollUp)) { | |
| output += "kScrollUp|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kSetSelection)) { | |
| output += "kSetSelection|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kSetText)) { | |
| output += "kSetText|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kShowOnScreen)) { | |
| output += "kShowOnScreen|"; | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kTap)) { | |
| output += "kTap|"; | |
| } | |
| return output; | |
| } | |
| // Returns a string representation of the flutter semantic node absolut | |
| // location. | |
| std::string NodeLocationToString(const SkRect& rect) { | |
| auto min_x = rect.fLeft; | |
| auto min_y = rect.fTop; | |
| auto max_x = rect.fRight; | |
| auto max_y = rect.fBottom; | |
| std::string location = | |
| "min(" + std::to_string(min_x) + ", " + std::to_string(min_y) + ") max(" + | |
| std::to_string(max_x) + ", " + std::to_string(max_y) + ")"; | |
| return location; | |
| } | |
| // Returns a string representation of the node's different types of children. | |
| std::string NodeChildrenToString(const flutter::SemanticsNode& node) { | |
| std::stringstream output; | |
| if (!node.childrenInTraversalOrder.empty()) { | |
| output << "children in traversal order:["; | |
| for (const auto child_id : node.childrenInTraversalOrder) { | |
| output << child_id << ", "; | |
| } | |
| output << "]\n"; | |
| } | |
| if (!node.childrenInHitTestOrder.empty()) { | |
| output << "children in hit test order:["; | |
| for (const auto child_id : node.childrenInHitTestOrder) { | |
| output << child_id << ", "; | |
| } | |
| output << ']'; | |
| } | |
| return output.str(); | |
| } | |
| #endif // !FLUTTER_RELEASE | |
| } // namespace | |
| AccessibilityBridge::AccessibilityBridge( | |
| SetSemanticsEnabledCallback set_semantics_enabled_callback, | |
| DispatchSemanticsActionCallback dispatch_semantics_action_callback, | |
| fuchsia::accessibility::semantics::SemanticsManagerHandle semantics_manager, | |
| fuchsia::ui::views::ViewRef view_ref, | |
| inspect::Node inspect_node) | |
| : set_semantics_enabled_callback_( | |
| std::move(set_semantics_enabled_callback)), | |
| dispatch_semantics_action_callback_( | |
| std::move(dispatch_semantics_action_callback)), | |
| binding_(this), | |
| fuchsia_semantics_manager_(semantics_manager.Bind()), | |
| atomic_updates_(std::make_shared<std::queue<FuchsiaAtomicUpdate>>()), | |
| inspect_node_(std::move(inspect_node)) { | |
| fuchsia_semantics_manager_.set_error_handler([](zx_status_t status) { | |
| FML_LOG(ERROR) << "Flutter cannot connect to SemanticsManager with status: " | |
| << zx_status_get_string(status) << "."; | |
| }); | |
| fuchsia_semantics_manager_->RegisterViewForSemantics( | |
| std::move(view_ref), binding_.NewBinding(), tree_ptr_.NewRequest()); | |
| #if !FLUTTER_RELEASE | |
| // The first argument to |CreateLazyValues| is the name of the lazy node, and | |
| // will only be displayed if the callback used to generate the node's content | |
| // fails. Therefore, we use an error message for this node name. | |
| inspect_node_tree_dump_ = | |
| inspect_node_.CreateLazyValues("dump_fail", [this]() { | |
| inspect::Inspector inspector; | |
| if (auto it = nodes_.find(kRootNodeId); it == nodes_.end()) { | |
| inspector.GetRoot().CreateString( | |
| "empty_tree", "this semantic tree is empty", &inspector); | |
| } else { | |
| FillInspectTree( | |
| kRootNodeId, /*current_level=*/1, | |
| inspector.GetRoot().CreateChild(kTreeDumpInspectRootName), | |
| &inspector); | |
| } | |
| return fpromise::make_ok_promise(std::move(inspector)); | |
| }); | |
| #endif // !FLUTTER_RELEASE | |
| } | |
| bool AccessibilityBridge::GetSemanticsEnabled() const { | |
| return semantics_enabled_; | |
| } | |
| void AccessibilityBridge::SetSemanticsEnabled(bool enabled) { | |
| semantics_enabled_ = enabled; | |
| if (!enabled) { | |
| nodes_.clear(); | |
| } | |
| } | |
| fuchsia::ui::gfx::BoundingBox AccessibilityBridge::GetNodeLocation( | |
| const flutter::SemanticsNode& node) const { | |
| fuchsia::ui::gfx::BoundingBox box; | |
| box.min.x = node.rect.fLeft; | |
| box.min.y = node.rect.fTop; | |
| box.min.z = static_cast<float>(node.elevation); | |
| box.max.x = node.rect.fRight; | |
| box.max.y = node.rect.fBottom; | |
| box.max.z = static_cast<float>(node.thickness); | |
| return box; | |
| } | |
| fuchsia::ui::gfx::mat4 AccessibilityBridge::GetNodeTransform( | |
| const flutter::SemanticsNode& node) const { | |
| return ConvertSkiaTransformToMat4(node.transform); | |
| } | |
| fuchsia::ui::gfx::mat4 AccessibilityBridge::ConvertSkiaTransformToMat4( | |
| const SkM44 transform) const { | |
| fuchsia::ui::gfx::mat4 value; | |
| float* m = value.matrix.data(); | |
| transform.getColMajor(m); | |
| return value; | |
| } | |
| fuchsia::accessibility::semantics::Attributes | |
| AccessibilityBridge::GetNodeAttributes(const flutter::SemanticsNode& node, | |
| size_t* added_size) const { | |
| fuchsia::accessibility::semantics::Attributes attributes; | |
| // TODO(MI4-2531): Don't truncate. | |
| if (node.label.size() > fuchsia::accessibility::semantics::MAX_LABEL_SIZE) { | |
| attributes.set_label(node.label.substr( | |
| 0, fuchsia::accessibility::semantics::MAX_LABEL_SIZE)); | |
| *added_size += fuchsia::accessibility::semantics::MAX_LABEL_SIZE; | |
| } else { | |
| attributes.set_label(node.label); | |
| *added_size += node.label.size(); | |
| } | |
| if (node.tooltip.size() > fuchsia::accessibility::semantics::MAX_LABEL_SIZE) { | |
| attributes.set_secondary_label(node.tooltip.substr( | |
| 0, fuchsia::accessibility::semantics::MAX_LABEL_SIZE)); | |
| *added_size += fuchsia::accessibility::semantics::MAX_LABEL_SIZE; | |
| } else { | |
| attributes.set_secondary_label(node.tooltip); | |
| *added_size += node.tooltip.size(); | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsKeyboardKey)) { | |
| attributes.set_is_keyboard_key(true); | |
| } | |
| return attributes; | |
| } | |
| fuchsia::accessibility::semantics::States AccessibilityBridge::GetNodeStates( | |
| const flutter::SemanticsNode& node, | |
| size_t* additional_size) const { | |
| fuchsia::accessibility::semantics::States states; | |
| (*additional_size) += sizeof(fuchsia::accessibility::semantics::States); | |
| // Set checked state. | |
| if (!node.HasFlag(flutter::SemanticsFlags::kHasCheckedState)) { | |
| states.set_checked_state( | |
| fuchsia::accessibility::semantics::CheckedState::NONE); | |
| } else { | |
| states.set_checked_state( | |
| node.HasFlag(flutter::SemanticsFlags::kIsChecked) | |
| ? fuchsia::accessibility::semantics::CheckedState::CHECKED | |
| : fuchsia::accessibility::semantics::CheckedState::UNCHECKED); | |
| } | |
| // Set selected state. | |
| states.set_selected(node.HasFlag(flutter::SemanticsFlags::kIsSelected)); | |
| // Flutter's definition of a hidden node is different from Fuchsia, so it must | |
| // not be set here. | |
| // Set value. | |
| if (node.value.size() > fuchsia::accessibility::semantics::MAX_VALUE_SIZE) { | |
| states.set_value(node.value.substr( | |
| 0, fuchsia::accessibility::semantics::MAX_VALUE_SIZE)); | |
| (*additional_size) += fuchsia::accessibility::semantics::MAX_VALUE_SIZE; | |
| } else { | |
| states.set_value(node.value); | |
| (*additional_size) += node.value.size(); | |
| } | |
| // Set toggled state. | |
| if (node.HasFlag(flutter::SemanticsFlags::kHasToggledState)) { | |
| states.set_toggled_state( | |
| node.HasFlag(flutter::SemanticsFlags::kIsToggled) | |
| ? fuchsia::accessibility::semantics::ToggledState::ON | |
| : fuchsia::accessibility::semantics::ToggledState::OFF); | |
| } | |
| return states; | |
| } | |
| std::vector<fuchsia::accessibility::semantics::Action> | |
| AccessibilityBridge::GetNodeActions(const flutter::SemanticsNode& node, | |
| size_t* additional_size) const { | |
| std::vector<fuchsia::accessibility::semantics::Action> node_actions; | |
| if (node.HasAction(flutter::SemanticsAction::kTap)) { | |
| node_actions.push_back(fuchsia::accessibility::semantics::Action::DEFAULT); | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kLongPress)) { | |
| node_actions.push_back( | |
| fuchsia::accessibility::semantics::Action::SECONDARY); | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kShowOnScreen)) { | |
| node_actions.push_back( | |
| fuchsia::accessibility::semantics::Action::SHOW_ON_SCREEN); | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kIncrease)) { | |
| node_actions.push_back( | |
| fuchsia::accessibility::semantics::Action::INCREMENT); | |
| } | |
| if (node.HasAction(flutter::SemanticsAction::kDecrease)) { | |
| node_actions.push_back( | |
| fuchsia::accessibility::semantics::Action::DECREMENT); | |
| } | |
| *additional_size += | |
| node_actions.size() * sizeof(fuchsia::accessibility::semantics::Action); | |
| return node_actions; | |
| } | |
| fuchsia::accessibility::semantics::Role AccessibilityBridge::GetNodeRole( | |
| const flutter::SemanticsNode& node) const { | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsButton)) { | |
| return fuchsia::accessibility::semantics::Role::BUTTON; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsTextField)) { | |
| return fuchsia::accessibility::semantics::Role::TEXT_FIELD; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsLink)) { | |
| return fuchsia::accessibility::semantics::Role::LINK; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsSlider)) { | |
| return fuchsia::accessibility::semantics::Role::SLIDER; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsHeader)) { | |
| return fuchsia::accessibility::semantics::Role::HEADER; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsImage)) { | |
| return fuchsia::accessibility::semantics::Role::IMAGE; | |
| } | |
| // If a flutter node supports the kIncrease or kDecrease actions, it can be | |
| // treated as a slider control by assistive technology. This is important | |
| // because users have special gestures to deal with sliders, and Fuchsia API | |
| // requires nodes that can receive this kind of action to be a slider control. | |
| if (node.HasAction(flutter::SemanticsAction::kIncrease) || | |
| node.HasAction(flutter::SemanticsAction::kDecrease)) { | |
| return fuchsia::accessibility::semantics::Role::SLIDER; | |
| } | |
| // If a flutter node has a checked state, then we assume it is either a | |
| // checkbox or a radio button. We distinguish between checkboxes and | |
| // radio buttons based on membership in a mutually exclusive group. | |
| if (node.HasFlag(flutter::SemanticsFlags::kHasCheckedState)) { | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsInMutuallyExclusiveGroup)) { | |
| return fuchsia::accessibility::semantics::Role::RADIO_BUTTON; | |
| } else { | |
| return fuchsia::accessibility::semantics::Role::CHECK_BOX; | |
| } | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kHasToggledState)) { | |
| return fuchsia::accessibility::semantics::Role::TOGGLE_SWITCH; | |
| } | |
| return fuchsia::accessibility::semantics::Role::UNKNOWN; | |
| } | |
| std::unordered_set<int32_t> AccessibilityBridge::GetDescendants( | |
| int32_t node_id) const { | |
| std::unordered_set<int32_t> descendents; | |
| std::deque<int32_t> to_process = {node_id}; | |
| while (!to_process.empty()) { | |
| int32_t id = to_process.front(); | |
| to_process.pop_front(); | |
| descendents.emplace(id); | |
| auto it = nodes_.find(id); | |
| if (it != nodes_.end()) { | |
| const auto& node = it->second.data; | |
| for (const auto& child : node.childrenInHitTestOrder) { | |
| if (descendents.find(child) == descendents.end()) { | |
| to_process.push_back(child); | |
| } else { | |
| // This indicates either a cycle or a child with multiple parents. | |
| // Flutter should never let this happen, but the engine API does not | |
| // explicitly forbid it right now. | |
| // TODO(http://fxbug.dev/75905): Crash flutter accessibility bridge | |
| // when a cycle in the tree is found. | |
| FML_LOG(ERROR) << "Semantics Node " << child | |
| << " has already been listed as a child of another " | |
| "node, ignoring for parent " | |
| << id << "."; | |
| } | |
| } | |
| } | |
| } | |
| return descendents; | |
| } | |
| // The only known usage of a negative number for a node ID is in the embedder | |
| // API as a sentinel value, which is not expected here. No valid producer of | |
| // nodes should give us a negative ID. | |
| static uint32_t FlutterIdToFuchsiaId(int32_t flutter_node_id) { | |
| FML_DCHECK(flutter_node_id >= 0) | |
| << "Unexpectedly received a negative semantics node ID."; | |
| return static_cast<uint32_t>(flutter_node_id); | |
| } | |
| void AccessibilityBridge::PruneUnreachableNodes( | |
| FuchsiaAtomicUpdate* atomic_update) { | |
| const auto& reachable_nodes = GetDescendants(kRootNodeId); | |
| auto iter = nodes_.begin(); | |
| while (iter != nodes_.end()) { | |
| int32_t id = iter->first; | |
| if (reachable_nodes.find(id) == reachable_nodes.end()) { | |
| atomic_update->AddNodeDeletion(FlutterIdToFuchsiaId(id)); | |
| iter = nodes_.erase(iter); | |
| } else { | |
| iter++; | |
| } | |
| } | |
| } | |
| // TODO(FIDL-718) - remove this, handle the error instead in something like | |
| // set_error_handler. | |
| static void PrintNodeSizeError(uint32_t node_id) { | |
| FML_LOG(ERROR) << "Semantics node with ID " << node_id | |
| << " exceeded the maximum FIDL message size and may not " | |
| "be delivered to the accessibility manager service."; | |
| } | |
| void AccessibilityBridge::AddSemanticsNodeUpdate( | |
| const flutter::SemanticsNodeUpdates update, | |
| float view_pixel_ratio) { | |
| if (update.empty()) { | |
| return; | |
| } | |
| FML_DCHECK(nodes_.find(kRootNodeId) != nodes_.end() || | |
| update.find(kRootNodeId) != update.end()) | |
| << "AccessibilityBridge received an update with out ever getting a root " | |
| "node."; | |
| FuchsiaAtomicUpdate atomic_update; | |
| bool has_root_node_update = false; | |
| // TODO(MI4-2498): Actions, Roles, hit test children, additional | |
| // flags/states/attr | |
| // TODO(MI4-1478): Support for partial updates for nodes > 64kb | |
| // e.g. if a node has a long label or more than 64k children. | |
| for (const auto& [flutter_node_id, flutter_node] : update) { | |
| size_t this_node_size = sizeof(fuchsia::accessibility::semantics::Node); | |
| // We handle root update separately in GetRootNodeUpdate. | |
| // TODO(chunhtai): remove this special case after we remove the inverse | |
| // view pixel ratio transformation in scenic view. | |
| // TODO(http://fxbug.dev/75908): Investigate flutter a11y bridge refactor | |
| // after removal of the inverse view pixel ratio transformation in scenic | |
| // view). | |
| if (flutter_node.id == kRootNodeId) { | |
| root_flutter_semantics_node_ = flutter_node; | |
| has_root_node_update = true; | |
| continue; | |
| } | |
| // Store the nodes for later hit testing and logging. | |
| nodes_[flutter_node.id].data = flutter_node; | |
| fuchsia::accessibility::semantics::Node fuchsia_node; | |
| std::vector<uint32_t> child_ids; | |
| // Send the nodes in traversal order, so the manager can figure out | |
| // traversal. | |
| for (int32_t flutter_child_id : flutter_node.childrenInTraversalOrder) { | |
| child_ids.push_back(FlutterIdToFuchsiaId(flutter_child_id)); | |
| } | |
| // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in | |
| // the flutter accessibility bridge. | |
| fuchsia_node.set_node_id(flutter_node.id) | |
| .set_role(GetNodeRole(flutter_node)) | |
| .set_location(GetNodeLocation(flutter_node)) | |
| .set_transform(GetNodeTransform(flutter_node)) | |
| .set_attributes(GetNodeAttributes(flutter_node, &this_node_size)) | |
| .set_states(GetNodeStates(flutter_node, &this_node_size)) | |
| .set_actions(GetNodeActions(flutter_node, &this_node_size)) | |
| .set_child_ids(child_ids); | |
| this_node_size += | |
| kNodeIdSize * flutter_node.childrenInTraversalOrder.size(); | |
| atomic_update.AddNodeUpdate(std::move(fuchsia_node), this_node_size); | |
| } | |
| // Handles root node update. | |
| if (has_root_node_update || last_seen_view_pixel_ratio_ != view_pixel_ratio) { | |
| last_seen_view_pixel_ratio_ = view_pixel_ratio; | |
| size_t root_node_size; | |
| fuchsia::accessibility::semantics::Node root_update = | |
| GetRootNodeUpdate(root_node_size); | |
| atomic_update.AddNodeUpdate(std::move(root_update), root_node_size); | |
| } | |
| PruneUnreachableNodes(&atomic_update); | |
| UpdateScreenRects(); | |
| atomic_updates_->push(std::move(atomic_update)); | |
| if (atomic_updates_->size() == 1) { | |
| // There were no commits in the queue, so send this one. | |
| Apply(&atomic_updates_->front()); | |
| } | |
| } | |
| fuchsia::accessibility::semantics::Node AccessibilityBridge::GetRootNodeUpdate( | |
| size_t& node_size) { | |
| fuchsia::accessibility::semantics::Node root_fuchsia_node; | |
| std::vector<uint32_t> child_ids; | |
| node_size = sizeof(fuchsia::accessibility::semantics::Node); | |
| for (int32_t flutter_child_id : | |
| root_flutter_semantics_node_.childrenInTraversalOrder) { | |
| child_ids.push_back(FlutterIdToFuchsiaId(flutter_child_id)); | |
| } | |
| // Applies the inverse view pixel ratio transformation to the root node. | |
| float inverse_view_pixel_ratio = 1.f / last_seen_view_pixel_ratio_; | |
| SkM44 inverse_view_pixel_ratio_transform; | |
| inverse_view_pixel_ratio_transform.setScale(inverse_view_pixel_ratio, | |
| inverse_view_pixel_ratio, 1.f); | |
| SkM44 result = root_flutter_semantics_node_.transform * | |
| inverse_view_pixel_ratio_transform; | |
| nodes_[root_flutter_semantics_node_.id].data = root_flutter_semantics_node_; | |
| // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in | |
| // the flutter accessibility bridge. | |
| root_fuchsia_node.set_node_id(root_flutter_semantics_node_.id) | |
| .set_role(GetNodeRole(root_flutter_semantics_node_)) | |
| .set_location(GetNodeLocation(root_flutter_semantics_node_)) | |
| .set_transform(ConvertSkiaTransformToMat4(result)) | |
| .set_attributes( | |
| GetNodeAttributes(root_flutter_semantics_node_, &node_size)) | |
| .set_states(GetNodeStates(root_flutter_semantics_node_, &node_size)) | |
| .set_actions(GetNodeActions(root_flutter_semantics_node_, &node_size)) | |
| .set_child_ids(child_ids); | |
| node_size += kNodeIdSize * | |
| root_flutter_semantics_node_.childrenInTraversalOrder.size(); | |
| return root_fuchsia_node; | |
| } | |
| void AccessibilityBridge::RequestAnnounce(const std::string message) { | |
| fuchsia::accessibility::semantics::SemanticEvent semantic_event; | |
| fuchsia::accessibility::semantics::AnnounceEvent announce_event; | |
| announce_event.set_message(message); | |
| semantic_event.set_announce(std::move(announce_event)); | |
| tree_ptr_->SendSemanticEvent(std::move(semantic_event), []() {}); | |
| } | |
| void AccessibilityBridge::UpdateScreenRects() { | |
| std::unordered_set<int32_t> visited_nodes; | |
| // The embedder applies a special pixel ratio transform to the root of the | |
| // view, and the accessibility bridge applies the inverse of this transform | |
| // to the root node. However, this transform is not persisted in the flutter | |
| // representation of the root node, so we need to account for it explicitly | |
| // here. | |
| float inverse_view_pixel_ratio = 1.f / last_seen_view_pixel_ratio_; | |
| SkM44 inverse_view_pixel_ratio_transform; | |
| inverse_view_pixel_ratio_transform.setScale(inverse_view_pixel_ratio, | |
| inverse_view_pixel_ratio, 1.f); | |
| UpdateScreenRects(kRootNodeId, inverse_view_pixel_ratio_transform, | |
| &visited_nodes); | |
| } | |
| void AccessibilityBridge::UpdateScreenRects( | |
| int32_t node_id, | |
| SkM44 parent_transform, | |
| std::unordered_set<int32_t>* visited_nodes) { | |
| auto it = nodes_.find(node_id); | |
| if (it == nodes_.end()) { | |
| FML_LOG(ERROR) << "UpdateScreenRects called on unknown node"; | |
| return; | |
| } | |
| auto& node = it->second; | |
| const auto& current_transform = parent_transform * node.data.transform; | |
| const auto& rect = node.data.rect; | |
| SkV4 dst[2] = { | |
| current_transform.map(rect.left(), rect.top(), 0, 1), | |
| current_transform.map(rect.right(), rect.bottom(), 0, 1), | |
| }; | |
| node.screen_rect.setLTRB(dst[0].x, dst[0].y, dst[1].x, dst[1].y); | |
| node.screen_rect.sort(); | |
| visited_nodes->emplace(node_id); | |
| for (uint32_t child_id : node.data.childrenInHitTestOrder) { | |
| if (visited_nodes->find(child_id) == visited_nodes->end()) { | |
| UpdateScreenRects(child_id, current_transform, visited_nodes); | |
| } | |
| } | |
| } | |
| std::optional<flutter::SemanticsAction> | |
| AccessibilityBridge::GetFlutterSemanticsAction( | |
| fuchsia::accessibility::semantics::Action fuchsia_action, | |
| uint32_t node_id) { | |
| switch (fuchsia_action) { | |
| // The default action associated with the element. | |
| case fuchsia::accessibility::semantics::Action::DEFAULT: | |
| return flutter::SemanticsAction::kTap; | |
| // The secondary action associated with the element. This may correspond to | |
| // a long press (touchscreens) or right click (mouse). | |
| case fuchsia::accessibility::semantics::Action::SECONDARY: | |
| return flutter::SemanticsAction::kLongPress; | |
| // Set (input/non-accessibility) focus on this element. | |
| case fuchsia::accessibility::semantics::Action::SET_FOCUS: | |
| FML_LOG(WARNING) | |
| << "Unsupported action SET_FOCUS sent for accessibility node " | |
| << node_id; | |
| return {}; | |
| // Set the element's value. | |
| case fuchsia::accessibility::semantics::Action::SET_VALUE: | |
| FML_LOG(WARNING) | |
| << "Unsupported action SET_VALUE sent for accessibility node " | |
| << node_id; | |
| return {}; | |
| // Scroll node to make it visible. | |
| case fuchsia::accessibility::semantics::Action::SHOW_ON_SCREEN: | |
| return flutter::SemanticsAction::kShowOnScreen; | |
| case fuchsia::accessibility::semantics::Action::INCREMENT: | |
| return flutter::SemanticsAction::kIncrease; | |
| case fuchsia::accessibility::semantics::Action::DECREMENT: | |
| return flutter::SemanticsAction::kDecrease; | |
| default: | |
| FML_LOG(WARNING) << "Unexpected action " | |
| << static_cast<int32_t>(fuchsia_action) | |
| << " sent for accessibility node " << node_id; | |
| return {}; | |
| } | |
| } | |
| // |fuchsia::accessibility::semantics::SemanticListener| | |
| void AccessibilityBridge::OnAccessibilityActionRequested( | |
| uint32_t node_id, | |
| fuchsia::accessibility::semantics::Action action, | |
| fuchsia::accessibility::semantics::SemanticListener:: | |
| OnAccessibilityActionRequestedCallback callback) { | |
| // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in | |
| // the flutter accessibility bridge. | |
| if (nodes_.find(node_id) == nodes_.end()) { | |
| FML_LOG(ERROR) << "Attempted to send accessibility action " | |
| << static_cast<int32_t>(action) | |
| << " to unknown node id: " << node_id; | |
| callback(false); | |
| return; | |
| } | |
| std::optional<flutter::SemanticsAction> flutter_action = | |
| GetFlutterSemanticsAction(action, node_id); | |
| if (!flutter_action.has_value()) { | |
| callback(false); | |
| return; | |
| } | |
| dispatch_semantics_action_callback_(static_cast<int32_t>(node_id), | |
| flutter_action.value()); | |
| callback(true); | |
| } | |
| // |fuchsia::accessibility::semantics::SemanticListener| | |
| void AccessibilityBridge::HitTest( | |
| fuchsia::math::PointF local_point, | |
| fuchsia::accessibility::semantics::SemanticListener::HitTestCallback | |
| callback) { | |
| auto hit_node_id = GetHitNode(kRootNodeId, local_point.x, local_point.y); | |
| FML_DCHECK(hit_node_id.has_value()); | |
| fuchsia::accessibility::semantics::Hit hit; | |
| // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in | |
| // the flutter accessibility bridge. | |
| hit.set_node_id(hit_node_id.value_or(kRootNodeId)); | |
| callback(std::move(hit)); | |
| } | |
| std::optional<int32_t> AccessibilityBridge::GetHitNode(int32_t node_id, | |
| float x, | |
| float y) { | |
| auto it = nodes_.find(node_id); | |
| if (it == nodes_.end()) { | |
| FML_LOG(ERROR) << "Attempted to hit test unknown node id: " << node_id; | |
| return {}; | |
| } | |
| auto const& node = it->second; | |
| if (node.data.flags & | |
| static_cast<int32_t>(flutter::SemanticsFlags::kIsHidden) || // | |
| !node.screen_rect.contains(x, y)) { | |
| return {}; | |
| } | |
| for (int32_t child_id : node.data.childrenInHitTestOrder) { | |
| auto candidate = GetHitNode(child_id, x, y); | |
| if (candidate) { | |
| return candidate; | |
| } | |
| } | |
| if (IsFocusable(node.data)) { | |
| return node_id; | |
| } | |
| return {}; | |
| } | |
| bool AccessibilityBridge::IsFocusable( | |
| const flutter::SemanticsNode& node) const { | |
| if (node.HasFlag(flutter::SemanticsFlags::kScopesRoute)) { | |
| return false; | |
| } | |
| if (node.HasFlag(flutter::SemanticsFlags::kIsFocusable)) { | |
| return true; | |
| } | |
| // Always consider platform views focusable. | |
| if (node.IsPlatformViewNode()) { | |
| return true; | |
| } | |
| // Always consider actionable nodes focusable. | |
| if (node.actions != 0) { | |
| return true; | |
| } | |
| // Consider text nodes focusable. | |
| return !node.label.empty() || !node.value.empty() || !node.hint.empty(); | |
| } | |
| // |fuchsia::accessibility::semantics::SemanticListener| | |
| void AccessibilityBridge::OnSemanticsModeChanged( | |
| bool enabled, | |
| OnSemanticsModeChangedCallback callback) { | |
| set_semantics_enabled_callback_(enabled); | |
| } | |
| #if !FLUTTER_RELEASE | |
| void AccessibilityBridge::FillInspectTree(int32_t flutter_node_id, | |
| int32_t current_level, | |
| inspect::Node inspect_node, | |
| inspect::Inspector* inspector) const { | |
| const auto it = nodes_.find(flutter_node_id); | |
| if (it == nodes_.end()) { | |
| inspect_node.CreateString( | |
| "missing_child", | |
| "This node has a parent in the semantic tree but has no value", | |
| inspector); | |
| inspector->emplace(std::move(inspect_node)); | |
| return; | |
| } | |
| const auto& semantic_node = it->second; | |
| const auto& data = semantic_node.data; | |
| inspect_node.CreateInt("id", data.id, inspector); | |
| // Even with an empty label, we still want to create the property to | |
| // explicetly show that it is empty. | |
| inspect_node.CreateString("label", data.label, inspector); | |
| if (!data.hint.empty()) { | |
| inspect_node.CreateString("hint", data.hint, inspector); | |
| } | |
| if (!data.value.empty()) { | |
| inspect_node.CreateString("value", data.value, inspector); | |
| } | |
| if (!data.increasedValue.empty()) { | |
| inspect_node.CreateString("increased_value", data.increasedValue, | |
| inspector); | |
| } | |
| if (!data.decreasedValue.empty()) { | |
| inspect_node.CreateString("decreased_value", data.decreasedValue, | |
| inspector); | |
| } | |
| if (data.textDirection) { | |
| inspect_node.CreateString( | |
| "text_direction", data.textDirection == 1 ? "RTL" : "LTR", inspector); | |
| } | |
| if (data.flags) { | |
| inspect_node.CreateString("flags", NodeFlagsToString(data), inspector); | |
| } | |
| if (data.actions) { | |
| inspect_node.CreateString("actions", NodeActionsToString(data), inspector); | |
| } | |
| inspect_node.CreateString( | |
| "location", NodeLocationToString(semantic_node.screen_rect), inspector); | |
| if (!data.childrenInTraversalOrder.empty() || | |
| !data.childrenInHitTestOrder.empty()) { | |
| inspect_node.CreateString("children", NodeChildrenToString(data), | |
| inspector); | |
| } | |
| inspect_node.CreateInt("current_level", current_level, inspector); | |
| for (int32_t flutter_child_id : semantic_node.data.childrenInTraversalOrder) { | |
| const auto inspect_name = "node_" + std::to_string(flutter_child_id); | |
| FillInspectTree(flutter_child_id, current_level + 1, | |
| inspect_node.CreateChild(inspect_name), inspector); | |
| } | |
| inspector->emplace(std::move(inspect_node)); | |
| } | |
| #endif // !FLUTTER_RELEASE | |
| void AccessibilityBridge::Apply(FuchsiaAtomicUpdate* atomic_update) { | |
| size_t begin = 0; | |
| auto it = atomic_update->deletions.begin(); | |
| // Process up to kMaxDeletionsPerUpdate deletions at a time. | |
| while (it != atomic_update->deletions.end()) { | |
| std::vector<uint32_t> to_delete; | |
| size_t end = std::min(atomic_update->deletions.size() - begin, | |
| kMaxDeletionsPerUpdate); | |
| std::copy(std::make_move_iterator(it), std::make_move_iterator(it + end), | |
| std::back_inserter(to_delete)); | |
| tree_ptr_->DeleteSemanticNodes(std::move(to_delete)); | |
| begin = end; | |
| it += end; | |
| } | |
| std::vector<fuchsia::accessibility::semantics::Node> to_update; | |
| size_t current_size = 0; | |
| for (auto& node_and_size : atomic_update->updates) { | |
| if (current_size + node_and_size.second > kMaxMessageSize) { | |
| tree_ptr_->UpdateSemanticNodes(std::move(to_update)); | |
| current_size = 0; | |
| to_update.clear(); | |
| } | |
| current_size += node_and_size.second; | |
| to_update.push_back(std::move(node_and_size.first)); | |
| } | |
| if (!to_update.empty()) { | |
| tree_ptr_->UpdateSemanticNodes(std::move(to_update)); | |
| } | |
| // Commit this update and subsequent ones; for flow control wait for a | |
| // response between each commit. | |
| tree_ptr_->CommitUpdates( | |
| [this, atomic_updates = std::weak_ptr<std::queue<FuchsiaAtomicUpdate>>( | |
| atomic_updates_)]() { | |
| auto atomic_updates_ptr = atomic_updates.lock(); | |
| if (!atomic_updates_ptr) { | |
| // The queue no longer exists, which means that is no longer | |
| // necessary. | |
| return; | |
| } | |
| // Removes the update that just went through. | |
| atomic_updates_ptr->pop(); | |
| if (!atomic_updates_ptr->empty()) { | |
| Apply(&atomic_updates_ptr->front()); | |
| } | |
| }); | |
| atomic_update->deletions.clear(); | |
| atomic_update->updates.clear(); | |
| } | |
| void AccessibilityBridge::FuchsiaAtomicUpdate::AddNodeUpdate( | |
| fuchsia::accessibility::semantics::Node node, | |
| size_t size) { | |
| if (size > kMaxMessageSize) { | |
| // TODO(MI4-2531, FIDL-718): Remove this | |
| // This is defensive. If, despite our best efforts, we ended up with a node | |
| // that is larger than the max fidl size, we send no updates. | |
| PrintNodeSizeError(node.node_id()); | |
| return; | |
| } | |
| updates.emplace_back(std::move(node), size); | |
| } | |
| void AccessibilityBridge::FuchsiaAtomicUpdate::AddNodeDeletion(uint32_t id) { | |
| deletions.push_back(id); | |
| } | |
| } // namespace flutter_runner |