blob: d28fbb927e31ef2df8ad6f2bf46d838f96ce4d11 [file] [log] [blame]
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "pointer_delegate.h"
#include <lib/trace/event.h>
#include <limits>
#include "flutter/fml/logging.h"
#include "flutter/fml/trace_event.h"
// TODO(fxbug.dev/87076): Add MouseSource tests.
namespace fuchsia::ui::pointer {
// For using TouchInteractionId as a map key.
bool operator==(const fuchsia::ui::pointer::TouchInteractionId& a,
const fuchsia::ui::pointer::TouchInteractionId& b) {
return a.device_id == b.device_id && a.pointer_id == b.pointer_id &&
a.interaction_id == b.interaction_id;
}
} // namespace fuchsia::ui::pointer
namespace flutter_runner {
using fup_EventPhase = fuchsia::ui::pointer::EventPhase;
using fup_MouseDeviceInfo = fuchsia::ui::pointer::MouseDeviceInfo;
using fup_MouseEvent = fuchsia::ui::pointer::MouseEvent;
using fup_TouchEvent = fuchsia::ui::pointer::TouchEvent;
using fup_TouchIxnStatus = fuchsia::ui::pointer::TouchInteractionStatus;
using fup_TouchResponse = fuchsia::ui::pointer::TouchResponse;
using fup_TouchResponseType = fuchsia::ui::pointer::TouchResponseType;
using fup_ViewParameters = fuchsia::ui::pointer::ViewParameters;
namespace {
void IssueTouchTraceEvent(const fup_TouchEvent& event) {
FML_DCHECK(event.has_trace_flow_id()) << "API guarantee";
TRACE_FLOW_END("input", "dispatch_event_to_client", event.trace_flow_id());
}
void IssueMouseTraceEvent(const fup_MouseEvent& event) {
FML_DCHECK(event.has_trace_flow_id()) << "API guarantee";
TRACE_FLOW_END("input", "dispatch_event_to_client", event.trace_flow_id());
}
bool HasValidatedTouchSample(const fup_TouchEvent& event) {
if (!event.has_pointer_sample()) {
return false;
}
FML_DCHECK(event.pointer_sample().has_interaction()) << "API guarantee";
FML_DCHECK(event.pointer_sample().has_phase()) << "API guarantee";
FML_DCHECK(event.pointer_sample().has_position_in_viewport())
<< "API guarantee";
return true;
}
bool HasValidatedMouseSample(const fup_MouseEvent& event) {
if (!event.has_pointer_sample()) {
return false;
}
const auto& sample = event.pointer_sample();
FML_DCHECK(sample.has_device_id()) << "API guarantee";
FML_DCHECK(sample.has_position_in_viewport()) << "API guarantee";
FML_DCHECK(!sample.has_pressed_buttons() ||
sample.pressed_buttons().size() > 0)
<< "API guarantee";
return true;
}
std::array<float, 2> ViewportToViewCoordinates(
std::array<float, 2> viewport_coordinates,
const std::array<float, 9>& viewport_to_view_transform) {
// The transform matrix is a FIDL array with matrix data in column-major
// order. For a matrix with data [a b c d e f g h i], and with the viewport
// coordinates expressed as homogeneous coordinates, the logical view
// coordinates are obtained with the following formula:
// |a d g| |x| |x'|
// |b e h| * |y| = |y'|
// |c f i| |1| |w'|
// which we then normalize based on the w component:
// if z' not zero: (x'/w', y'/w')
// else (x', y')
const auto& M = viewport_to_view_transform;
const float x = viewport_coordinates[0];
const float y = viewport_coordinates[1];
const float xp = M[0] * x + M[3] * y + M[6];
const float yp = M[1] * x + M[4] * y + M[7];
const float wp = M[2] * x + M[5] * y + M[8];
if (wp != 0) {
return {xp / wp, yp / wp};
} else {
return {xp, yp};
}
}
flutter::PointerData::Change GetChangeFromTouchEventPhase(
fup_EventPhase phase) {
switch (phase) {
case fup_EventPhase::ADD:
return flutter::PointerData::Change::kAdd;
case fup_EventPhase::CHANGE:
return flutter::PointerData::Change::kMove;
case fup_EventPhase::REMOVE:
return flutter::PointerData::Change::kRemove;
case fup_EventPhase::CANCEL:
return flutter::PointerData::Change::kCancel;
default:
return flutter::PointerData::Change::kCancel;
}
}
std::array<float, 2> ClampToViewSpace(const float x,
const float y,
const fup_ViewParameters& p) {
const float min_x = p.view.min[0];
const float min_y = p.view.min[1];
const float max_x = p.view.max[0];
const float max_y = p.view.max[1];
if (min_x <= x && x < max_x && min_y <= y && y < max_y) {
return {x, y}; // No clamping to perform.
}
// View boundary is [min_x, max_x) x [min_y, max_y). Note that min is
// inclusive, but max is exclusive - so we subtract epsilon.
const float max_x_inclusive = max_x - std::numeric_limits<float>::epsilon();
const float max_y_inclusive = max_y - std::numeric_limits<float>::epsilon();
const float& clamped_x = std::clamp(x, min_x, max_x_inclusive);
const float& clamped_y = std::clamp(y, min_y, max_y_inclusive);
FML_LOG(INFO) << "Clamped (" << x << ", " << y << ") to (" << clamped_x
<< ", " << clamped_y << ").";
return {clamped_x, clamped_y};
}
flutter::PointerData::Change ComputePhase(
bool any_button_down,
std::unordered_set<uint32_t>& mouse_down,
uint32_t id) {
if (!mouse_down.count(id) && !any_button_down) {
return flutter::PointerData::Change::kHover;
} else if (!mouse_down.count(id) && any_button_down) {
mouse_down.insert(id);
return flutter::PointerData::Change::kDown;
} else if (mouse_down.count(id) && any_button_down) {
return flutter::PointerData::Change::kMove;
} else if (mouse_down.count(id) && !any_button_down) {
mouse_down.erase(id);
return flutter::PointerData::Change::kUp;
}
FML_UNREACHABLE();
return flutter::PointerData::Change::kCancel;
}
// Flutter's PointerData.device field is 64 bits and is expected to be unique
// for each pointer. We pack Fuchsia's device ID (hi) and pointer ID (lo) into
// 64 bits to retain uniqueness across multiple touch devices.
uint64_t PackFuchsiaDeviceIdAndPointerId(uint32_t fuchsia_device_id,
uint32_t fuchsia_pointer_id) {
return (((uint64_t)fuchsia_device_id) << 32) | fuchsia_pointer_id;
}
// It returns a "draft" because the coordinates are logical. Later, view pixel
// ratio is applied to obtain physical coordinates.
//
// The flutter pointerdata state machine has extra phases, which this function
// synthesizes on the fly. Hence the return data is a flutter pointerdata, and
// optionally a second one.
// For example: <ADD, DOWN>, <MOVE, nullopt>, <UP, REMOVE>.
// TODO(fxbug.dev/87074): Let PointerDataPacketConverter synthesize events.
//
// Flutter gestures expect a gesture to start within the logical view space, and
// is not tolerant of floating point drift. This function coerces just the DOWN
// event's coordinate to start within the logical view.
std::pair<flutter::PointerData, std::optional<flutter::PointerData>>
CreateTouchDraft(const fup_TouchEvent& event,
const fup_ViewParameters& view_parameters) {
FML_DCHECK(HasValidatedTouchSample(event)) << "precondition";
const auto& sample = event.pointer_sample();
const auto& ixn = sample.interaction();
flutter::PointerData ptr;
ptr.Clear();
ptr.time_stamp = event.timestamp() / 1000; // in microseconds
ptr.change = GetChangeFromTouchEventPhase(sample.phase());
ptr.kind = flutter::PointerData::DeviceKind::kTouch;
// Load Fuchsia's pointer ID onto Flutter's |device| field, and not the
// |pointer_identifier| field. The latter is written by
// PointerDataPacketConverter, to track individual gesture interactions.
ptr.device = PackFuchsiaDeviceIdAndPointerId(ixn.device_id, ixn.pointer_id);
// View parameters can change mid-interaction; apply transform on the fly.
auto logical =
ViewportToViewCoordinates(sample.position_in_viewport(),
view_parameters.viewport_to_view_transform);
ptr.physical_x = logical[0]; // Not yet physical; adjusted in PlatformView.
ptr.physical_y = logical[1]; // Not yet physical; adjusted in PlatformView.
// Match Flutter pointer's state machine with synthesized events.
if (ptr.change == flutter::PointerData::Change::kAdd) {
flutter::PointerData down;
memcpy(&down, &ptr, sizeof(flutter::PointerData));
down.change = flutter::PointerData::Change::kDown;
{ // Ensure gesture recognition: DOWN starts in the logical view space.
auto [x, y] =
ClampToViewSpace(down.physical_x, down.physical_y, view_parameters);
down.physical_x = x;
down.physical_y = y;
}
return {std::move(ptr), std::move(down)};
} else if (ptr.change == flutter::PointerData::Change::kRemove) {
flutter::PointerData up;
memcpy(&up, &ptr, sizeof(flutter::PointerData));
up.change = flutter::PointerData::Change::kUp;
return {std::move(up), std::move(ptr)};
} else {
return {std::move(ptr), std::nullopt};
}
}
// It returns a "draft" because the coordinates are logical. Later, view pixel
// ratio is applied to obtain physical coordinates.
//
// Phase data is computed before this call; it involves state tracking based on
// button-down state.
//
// Button data, if available, gets packed into the |buttons| field, in flutter
// button order (kMousePrimaryButton, etc). The device-assigned button IDs are
// provided in priority order in MouseEvent.device_info (at the start of channel
// connection), and maps from device button ID (given in fup_MouseEvent) to
// flutter button ID (flutter::PointerData).
//
// Scroll data, if available, gets packed into the |scroll_delta_x| or
// |scroll_delta_y| fields, and the |signal_kind| field is set to kScroll.
// The PointerDataPacketConverter reads this field to synthesize events to match
// Flutter's expected pointer stream.
// TODO(fxbug.dev/87073): PointerDataPacketConverter should synthesize a
// discrete scroll event on kDown or kUp, to match engine expectations.
//
// Flutter gestures expect a gesture to start within the logical view space, and
// is not tolerant of floating point drift. This function coerces just the DOWN
// event's coordinate to start within the logical view.
flutter::PointerData CreateMouseDraft(const fup_MouseEvent& event,
const flutter::PointerData::Change phase,
const fup_ViewParameters& view_parameters,
const fup_MouseDeviceInfo& device_info) {
FML_DCHECK(HasValidatedMouseSample(event)) << "precondition";
const auto& sample = event.pointer_sample();
flutter::PointerData ptr;
ptr.Clear();
ptr.time_stamp = event.timestamp() / 1000; // in microseconds
ptr.change = phase;
ptr.kind = flutter::PointerData::DeviceKind::kMouse;
ptr.device = sample.device_id();
// View parameters can change mid-interaction; apply transform on the fly.
auto logical =
ViewportToViewCoordinates(sample.position_in_viewport(),
view_parameters.viewport_to_view_transform);
ptr.physical_x = logical[0]; // Not yet physical; adjusted in PlatformView.
ptr.physical_y = logical[1]; // Not yet physical; adjusted in PlatformView.
// Ensure gesture recognition: DOWN starts in the logical view space.
if (ptr.change == flutter::PointerData::Change::kDown) {
auto [x, y] =
ClampToViewSpace(ptr.physical_x, ptr.physical_y, view_parameters);
ptr.physical_x = x;
ptr.physical_y = y;
}
if (sample.has_pressed_buttons()) {
int64_t flutter_buttons = 0;
const auto& pressed = sample.pressed_buttons();
for (size_t idx = 0; idx < pressed.size(); ++idx) {
const uint8_t button_id = pressed[idx];
FML_DCHECK(device_info.has_buttons()) << "API guarantee";
// Priority 0 maps to kPrimaryButton, and so on.
for (uint8_t prio = 0; prio < device_info.buttons().size(); ++prio) {
if (button_id == device_info.buttons()[prio]) {
flutter_buttons |= (1 << prio);
}
}
}
FML_DCHECK(flutter_buttons != 0);
ptr.buttons = flutter_buttons;
}
// Fuchsia currently provides scroll data in "ticks", not physical pixels.
// However, Flutter expects scroll data in physical pixels. To compensate for
// lack of guidance, we make up a "reasonable amount".
// TODO(fxbug.dev/85388): Replace with physical pixel scroll.
const int kScrollOffsetMultiplier = 20;
if (sample.has_scroll_v()) {
ptr.signal_kind = flutter::PointerData::SignalKind::kScroll;
double dy = -sample.scroll_v() * kScrollOffsetMultiplier; // logical amount
ptr.scroll_delta_y = dy; // Not yet physical; adjusted in Platform View.
}
if (sample.has_scroll_h()) {
ptr.signal_kind = flutter::PointerData::SignalKind::kScroll;
double dx = sample.scroll_h() * kScrollOffsetMultiplier; // logical amount
ptr.scroll_delta_x = dx; // Not yet physical; adjusted in Platform View.
}
return ptr;
}
// Helper to insert one or two events into a vector buffer.
void InsertIntoBuffer(
std::pair<flutter::PointerData, std::optional<flutter::PointerData>> events,
std::vector<flutter::PointerData>* buffer) {
FML_DCHECK(buffer);
buffer->emplace_back(std::move(events.first));
if (events.second.has_value()) {
buffer->emplace_back(std::move(events.second.value()));
}
}
} // namespace
// Core logic of this class.
// Aim to keep state management in this function.
void PointerDelegate::WatchLoop(
std::function<void(std::vector<flutter::PointerData>)> callback) {
FML_LOG(INFO) << "Flutter - PointerDelegate started.";
if (touch_responder_) {
FML_LOG(ERROR) << "PointerDelegate::WatchLoop() must be called once.";
return;
}
touch_responder_ = [this, callback](std::vector<fup_TouchEvent> events) {
TRACE_EVENT0("flutter", "PointerDelegate::TouchHandler");
FML_DCHECK(touch_responses_.empty()) << "precondition";
std::vector<flutter::PointerData> to_client;
for (const fup_TouchEvent& event : events) {
IssueTouchTraceEvent(event);
fup_TouchResponse
response; // Response per event, matched on event's index.
if (event.has_view_parameters()) {
touch_view_parameters_ = std::move(event.view_parameters());
}
if (HasValidatedTouchSample(event)) {
const auto& sample = event.pointer_sample();
const auto& ixn = sample.interaction();
if (sample.phase() == fup_EventPhase::ADD &&
!event.has_interaction_result()) {
touch_buffer_.emplace(ixn, std::vector<flutter::PointerData>());
}
FML_DCHECK(touch_view_parameters_.has_value()) << "API guarantee";
auto events = CreateTouchDraft(event, touch_view_parameters_.value());
if (touch_buffer_.count(ixn) > 0) {
InsertIntoBuffer(std::move(events), &touch_buffer_[ixn]);
} else {
InsertIntoBuffer(std::move(events), &to_client);
}
// For this simple client, always claim we want the gesture.
response.set_response_type(fup_TouchResponseType::YES_PRIORITIZE);
}
if (event.has_interaction_result()) {
const auto& result = event.interaction_result();
const auto& ixn = result.interaction;
if (result.status == fup_TouchIxnStatus::GRANTED &&
touch_buffer_.count(ixn) > 0) {
FML_DCHECK(to_client.empty()) << "invariant";
to_client.insert(to_client.end(), touch_buffer_[ixn].begin(),
touch_buffer_[ixn].end());
}
touch_buffer_.erase(ixn); // Result seen, delete the buffer.
}
touch_responses_.push_back(std::move(response));
}
callback(std::move(to_client)); // Notify client of touch events, if any.
touch_source_->Watch(std::move(touch_responses_),
/*copy*/ touch_responder_);
touch_responses_.clear();
};
mouse_responder_ = [this, callback](std::vector<fup_MouseEvent> events) {
TRACE_EVENT0("flutter", "PointerDelegate::MouseHandler");
std::vector<flutter::PointerData> to_client;
for (fup_MouseEvent& event : events) {
IssueMouseTraceEvent(event);
if (event.has_device_info()) {
const auto& id = event.device_info().id();
mouse_device_info_[id] = std::move(*event.mutable_device_info());
}
if (event.has_view_parameters()) {
mouse_view_parameters_ = std::move(event.view_parameters());
}
if (HasValidatedMouseSample(event)) {
const auto& sample = event.pointer_sample();
const auto& id = sample.device_id();
const bool any_button_down = sample.has_pressed_buttons();
FML_DCHECK(mouse_view_parameters_.has_value()) << "API guarantee";
FML_DCHECK(mouse_device_info_.count(id) > 0) << "API guarantee";
const auto phase = ComputePhase(any_button_down, mouse_down_, id);
flutter::PointerData data =
CreateMouseDraft(event, phase, mouse_view_parameters_.value(),
mouse_device_info_[id]);
to_client.emplace_back(std::move(data));
}
}
callback(std::move(to_client));
mouse_source_->Watch(/*copy*/ mouse_responder_);
};
// Start watching both channels.
touch_source_->Watch(std::move(touch_responses_), /*copy*/ touch_responder_);
touch_responses_.clear();
mouse_source_->Watch(/*copy*/ mouse_responder_);
}
} // namespace flutter_runner