diff --git a/src/nvhttp.cpp b/src/nvhttp.cpp index e260141a7d7..30e9d49cc91 100644 --- a/src/nvhttp.cpp +++ b/src/nvhttp.cpp @@ -28,6 +28,9 @@ #include "logging.h" #include "network.h" #include "nvhttp.h" +#ifdef __APPLE__ + #include "src/platform/macos/misc.h" +#endif #include "platform/common.h" #include "process.h" #include "rtsp.h" @@ -981,6 +984,59 @@ namespace nvhttp { } } + /** + * @brief Report where the focused application is expecting text. + * + * A phone's on-screen keyboard covers half the picture, and the client has no way of knowing + * which half matters. The host does. Coordinates are fractions of the streamed display so the + * client needs to know nothing about resolutions, and "source" says whether the answer is the + * insertion point itself or the pointer standing in for it. An empty body means neither was + * available, and the client should leave the picture where it is. + * + * Served only over HTTPS, so it reaches paired and enabled clients alone. Where someone is + * typing, and the pointer position it falls back to, describe what the user is doing closely + * enough that they belong behind the same verification as the rest of the session. + * + * @param response HTTP response object to populate. + * @param request HTTP request data from the client. + */ + void caret(resp_https_t response, req_https_t request) { + print_req(request); + + SimpleWeb::CaseInsensitiveMultimap headers; + headers.emplace("Content-Type", "application/json"); + + // The caret when the focused application will say where it is, the pointer when it will not, + // which is most of them. Accessibility is the only interface that reports an insertion point + // and it is macOS-only, so elsewhere the pointer is the whole answer. +#ifdef __APPLE__ + if (const auto rect = platf::focused_caret()) { + response->write( + SimpleWeb::StatusCode::success_ok, + std::format(R"({{"x":{},"y":{},"w":{},"h":{},"source":"caret"}})", + (*rect)[0], (*rect)[1], (*rect)[2], (*rect)[3]), + headers + ); + return; + } +#endif + + // Where you clicked to start typing, so it is close enough to the field to be worth moving + // the picture for, and in trackpad mode it is the only thing the client cannot work out for + // itself: it sends relative motion and never learns where the pointer ended up. + if (const auto point = platf::pointer_location()) { + response->write( + SimpleWeb::StatusCode::success_ok, + std::format(R"({{"x":{},"y":{},"w":0,"h":0,"source":"pointer"}})", + (*point)[0], (*point)[1]), + headers + ); + return; + } + + response->write(SimpleWeb::StatusCode::success_ok, "{}", headers); + } + /** * @brief Launch the requested application for a GameStream session. * @@ -1362,6 +1418,7 @@ namespace nvhttp { pair(add_cert, resp, req); }; https_server.resource["^/applist$"]["GET"] = applist; + https_server.resource["^/caret$"]["GET"] = caret; https_server.resource["^/appasset$"]["GET"] = appasset; https_server.resource["^/launch$"]["GET"] = [&host_audio](auto resp, auto req) { launch(host_audio, resp, req); diff --git a/src/platform/common.h b/src/platform/common.h index 3625abf965d..5c72db85b8c 100644 --- a/src/platform/common.h +++ b/src/platform/common.h @@ -5,6 +5,7 @@ #pragma once // standard includes +#include #include #include #include @@ -1123,6 +1124,23 @@ namespace platf { * @examples_end */ std::optional get_mouse_loc(input_t &input); + + /** + * @brief Where the pointer is, as a fraction of the display being streamed. + * + * Unlike `get_mouse_loc()` this takes no input backend and is meant to be read outside a + * session, and it answers in fractions rather than screen coordinates so a caller can use it + * without knowing the host's resolution. + * + * Every implementation measures against the primary display rather than resolving the one a + * session was configured to capture. That is the default in every case, and a pointer on any + * other display reports nothing rather than a number the client would misplace. + * + * @return `{x, y}` in 0..1, or `std::nullopt` when the pointer is on another display or the + * platform cannot observe it. + */ + std::optional> pointer_location(); + /** * @brief Move mouse using the backend coordinate system. * diff --git a/src/platform/linux/input/virtualhid.cpp b/src/platform/linux/input/virtualhid.cpp index f18233d8e79..ca626484b35 100644 --- a/src/platform/linux/input/virtualhid.cpp +++ b/src/platform/linux/input/virtualhid.cpp @@ -66,6 +66,47 @@ namespace platf { #endif } + // Kept beside get_mouse_loc() rather than in misc.cpp, which is where the other two platforms + // put it, because this is the only Linux translation unit with an X11 connection already set up. + std::optional> pointer_location() { +#ifdef SUNSHINE_BUILD_X11 + auto *display = XOpenDisplay(nullptr); + if (!display) { + return std::nullopt; + } + + const auto screen = DefaultScreen(display); + const auto width = static_cast(DisplayWidth(display, screen)); + const auto height = static_cast(DisplayHeight(display, screen)); + + const auto root = DefaultRootWindow(display); + Window root_return {}; + Window child_return {}; + int root_x = 0; + int root_y = 0; + int window_x = 0; + int window_y = 0; + unsigned int mask = 0; + const auto queried = XQueryPointer(display, root, &root_return, &child_return, &root_x, &root_y, &window_x, &window_y, &mask); + XCloseDisplay(display); + + if (!queried || width <= 0 || height <= 0) { + return std::nullopt; + } + + // The X screen, which spans every output. That is also what x11grab captures, so the two + // agree by default; a session capturing one output of several would need the pointer placed + // against that output instead. + return std::array { + root_x / width, + root_y / height + }; +#else + // Wayland gives no way to ask, so the client falls back to leaving the picture alone. + return std::nullopt; +#endif + } + std::vector &supported_gamepads(input_t *input) { static std::vector gamepads; if (!input || !input->get()) { diff --git a/src/platform/macos/misc.h b/src/platform/macos/misc.h index 0af5d1ae252..e6373bc7055 100644 --- a/src/platform/macos/misc.h +++ b/src/platform/macos/misc.h @@ -5,6 +5,8 @@ #pragma once // standard includes +#include +#include #include // platform includes @@ -17,6 +19,21 @@ namespace platf { * @return True when Sunshine can capture the screen. */ bool is_screen_capture_allowed(); + + /** + * @brief Where the focused application is expecting text, as a fraction of the streamed display. + * + * A client whose on-screen keyboard covers half the picture has no way of knowing which half + * matters. The host does: the focused element knows where its insertion point is, and + * Accessibility will say so. Normalised to 0..1 of the display so the client needs to know + * nothing about resolutions. + * + * Empty when the focused application does not report an insertion point — which is most of + * them — or when the caret is on a display other than the one being streamed. + * + * @return {x, y, width, height} in 0..1 of the streamed display, or nothing. + */ + std::optional> focused_caret(); } // namespace platf namespace dyn { diff --git a/src/platform/macos/misc.mm b/src/platform/macos/misc.mm index fcfe1dc86a9..c3a2e082244 100644 --- a/src/platform/macos/misc.mm +++ b/src/platform/macos/misc.mm @@ -19,6 +19,8 @@ // platform includes #include #include +#include +#include #include #include #include @@ -32,6 +34,7 @@ // local includes #include "misc.h" +#include "src/utility.h" #include "src/entry_handler.h" #include "src/logging.h" #include "src/platform/common.h" @@ -69,6 +72,119 @@ /** * @brief Check whether screen capture allowed. */ + namespace { + /// Reads an Accessibility attribute, returning nothing rather than an error code. + CFTypeRef copy_attribute(AXUIElementRef element, CFStringRef name) { + CFTypeRef value = nullptr; + if (AXUIElementCopyAttributeValue(element, name, &value) != kAXErrorSuccess) { + return nullptr; + } + return value; + } + } // namespace + + std::optional> focused_caret() { + if (!AXIsProcessTrusted()) { + return std::nullopt; + } + + NSRunningApplication *front = [[NSWorkspace sharedWorkspace] frontmostApplication]; + if (front == nil) { + return std::nullopt; + } + + // Ask the application, not the system. AXUIElementCreateSystemWide() with + // kAXFocusedUIElementAttribute comes back empty for applications that answer perfectly well + // when asked directly — iTerm2 among them, which is the case this exists for. + AXUIElementRef app = AXUIElementCreateApplication(front.processIdentifier); + if (!app) { + return std::nullopt; + } + // A busy or hung application must not stall the request that asked for this. + AXUIElementSetMessagingTimeout(app, 0.1f); + + const auto release_app = util::fail_guard([app]() { + CFRelease(app); + }); + + CFTypeRef focused = copy_attribute(app, kAXFocusedUIElementAttribute); + if (!focused) { + return std::nullopt; + } + const auto release_focused = util::fail_guard([focused]() { + CFRelease(focused); + }); + + CFTypeRef range = copy_attribute(static_cast(focused), kAXSelectedTextRangeAttribute); + if (!range) { + return std::nullopt; + } + const auto release_range = util::fail_guard([range]() { + CFRelease(range); + }); + + CFTypeRef bounds = nullptr; + if (AXUIElementCopyParameterizedAttributeValue( + static_cast(focused), + kAXBoundsForRangeParameterizedAttribute, + range, + &bounds + ) != kAXErrorSuccess || + !bounds) { + return std::nullopt; + } + const auto release_bounds = util::fail_guard([bounds]() { + CFRelease(bounds); + }); + + CGRect caret = CGRectZero; + if (!AXValueGetValue(static_cast(bounds), kAXValueTypeCGRect, &caret)) { + return std::nullopt; + } + // An element that does not really have an insertion point answers with an empty rect at the + // origin rather than with an error. + if (caret.size.width == 0 && caret.size.height == 0) { + return std::nullopt; + } + + // Accessibility works in the coordinates of the whole desktop arrangement, which on a second + // display can be negative or larger than the streamed display. Only a caret on the display + // being streamed means anything to the client. + const CGRect display = CGDisplayBounds(CGMainDisplayID()); + const CGPoint anchor = CGPointMake(CGRectGetMidX(caret), CGRectGetMidY(caret)); + if (!CGRectContainsPoint(display, anchor)) { + return std::nullopt; + } + + return std::array { + (caret.origin.x - display.origin.x) / display.size.width, + (caret.origin.y - display.origin.y) / display.size.height, + caret.size.width / display.size.width, + caret.size.height / display.size.height + }; + } + + std::optional> pointer_location() { + // A fresh event every time rather than a reused one, as get_mouse_loc() does: the location + // on a reused event is whatever it was when the event was made. + CGEventRef snapshot = CGEventCreate(nullptr); + if (!snapshot) { + return std::nullopt; + } + const CGPoint location = CGEventGetLocation(snapshot); + CFRelease(snapshot); + + const CGRect display = CGDisplayBounds(CGMainDisplayID()); + if (!CGRectContainsPoint(display, location)) { + return std::nullopt; + } + + return std::array { + (location.x - display.origin.x) / display.size.width, + (location.y - display.origin.y) / display.size.height + }; + } + bool is_screen_capture_allowed() { return screen_capture_allowed; } diff --git a/src/platform/windows/misc.cpp b/src/platform/windows/misc.cpp index 01065d05c2a..8690f361488 100644 --- a/src/platform/windows/misc.cpp +++ b/src/platform/windows/misc.cpp @@ -1877,4 +1877,37 @@ namespace platf { std::string resolve_render_device() { return {}; } + + std::optional> pointer_location() { + POINT cursor {}; + if (!GetCursorPos(&cursor)) { + return std::nullopt; + } + + // GetCursorPos answers in virtual-screen coordinates, which span every monitor, so a pointer + // on a second one lands outside the streamed display rather than nowhere. Only a pointer on + // the display being streamed means anything to the client. + MONITORINFO primary {}; + primary.cbSize = sizeof(primary); + if (const auto monitor = MonitorFromPoint(POINT {0, 0}, MONITOR_DEFAULTTOPRIMARY); + !monitor || !GetMonitorInfo(monitor, &primary)) { + return std::nullopt; + } + + const auto &bounds = primary.rcMonitor; + const auto width = static_cast(bounds.right - bounds.left); + const auto height = static_cast(bounds.bottom - bounds.top); + if (width <= 0 || height <= 0) { + return std::nullopt; + } + + if (cursor.x < bounds.left || cursor.x >= bounds.right || cursor.y < bounds.top || cursor.y >= bounds.bottom) { + return std::nullopt; + } + + return std::array { + (cursor.x - bounds.left) / width, + (cursor.y - bounds.top) / height + }; + } } // namespace platf