feat: report where the user is typing, so a client can keep it on screen

src/nvhttp.cpp

@@ -28,6 +28,9 @@
 #include "logging.h"
 #include "network.h"
 #include "nvhttp.h"
+#ifdef __APPLE__
+  #include "src/platform/macos/misc.h"
+#endif
 #include "platform/common.h"
 #include "process.h"
 #include "rtsp.h"
@@ -981,6 +984,59 @@ namespace nvhttp {
     }
   }
 
+  /**
+   * @brief Report where the focused application is expecting text.
+   *
+   * A phone's on-screen keyboard covers half the picture, and the client has no way of knowing
+   * which half matters. The host does. Coordinates are fractions of the streamed display so the
+   * client needs to know nothing about resolutions, and "source" says whether the answer is the
+   * insertion point itself or the pointer standing in for it. An empty body means neither was
+   * available, and the client should leave the picture where it is.
+   *
+   * Served only over HTTPS, so it reaches paired and enabled clients alone. Where someone is
+   * typing, and the pointer position it falls back to, describe what the user is doing closely
+   * enough that they belong behind the same verification as the rest of the session.
+   *
+   * @param response HTTP response object to populate.
+   * @param request HTTP request data from the client.
+   */
+  void caret(resp_https_t response, req_https_t request) {
+    print_req<SunshineHTTPS>(request);
+
+    SimpleWeb::CaseInsensitiveMultimap headers;
+    headers.emplace("Content-Type", "application/json");
+
+    // The caret when the focused application will say where it is, the pointer when it will not,
+    // which is most of them. Accessibility is the only interface that reports an insertion point
+    // and it is macOS-only, so elsewhere the pointer is the whole answer.
+#ifdef __APPLE__
+    if (const auto rect = platf::focused_caret()) {
+      response->write(
+        SimpleWeb::StatusCode::success_ok,
+        std::format(R"({{"x":{},"y":{},"w":{},"h":{},"source":"caret"}})",
+                    (*rect)[0], (*rect)[1], (*rect)[2], (*rect)[3]),
+        headers
+      );
+      return;
+    }
+#endif
+
+    // Where you clicked to start typing, so it is close enough to the field to be worth moving
+    // the picture for, and in trackpad mode it is the only thing the client cannot work out for
+    // itself: it sends relative motion and never learns where the pointer ended up.
+    if (const auto point = platf::pointer_location()) {
+      response->write(
+        SimpleWeb::StatusCode::success_ok,
+        std::format(R"({{"x":{},"y":{},"w":0,"h":0,"source":"pointer"}})",
+                    (*point)[0], (*point)[1]),
+        headers
+      );
+      return;
+    }
+
+    response->write(SimpleWeb::StatusCode::success_ok, "{}", headers);
+  }
+
   /**
    * @brief Launch the requested application for a GameStream session.
    *
@@ -1362,6 +1418,7 @@ namespace nvhttp {
       pair<SunshineHTTPS>(add_cert, resp, req);
     };
     https_server.resource["^/applist$"]["GET"] = applist;
+    https_server.resource["^/caret$"]["GET"] = caret;
     https_server.resource["^/appasset$"]["GET"] = appasset;
     https_server.resource["^/launch$"]["GET"] = [&host_audio](auto resp, auto req) {
       launch(host_audio, resp, req);

src/platform/common.h

@@ -5,6 +5,7 @@
 #pragma once
 
 // standard includes
+#include <array>
 #include <bitset>
 #include <filesystem>
 #include <functional>
@@ -1123,6 +1124,23 @@ namespace platf {
    * @examples_end
    */
   std::optional<util::point_t> get_mouse_loc(input_t &input);
+
+  /**
+   * @brief Where the pointer is, as a fraction of the display being streamed.
+   *
+   * Unlike `get_mouse_loc()` this takes no input backend and is meant to be read outside a
+   * session, and it answers in fractions rather than screen coordinates so a caller can use it
+   * without knowing the host's resolution.
+   *
+   * Every implementation measures against the primary display rather than resolving the one a
+   * session was configured to capture. That is the default in every case, and a pointer on any
+   * other display reports nothing rather than a number the client would misplace.
+   *
+   * @return `{x, y}` in 0..1, or `std::nullopt` when the pointer is on another display or the
+   *         platform cannot observe it.
+   */
+  std::optional<std::array<double, 2>> pointer_location();
+
   /**
    * @brief Move mouse using the backend coordinate system.
    *

src/platform/linux/input/virtualhid.cpp

@@ -66,6 +66,47 @@ namespace platf {
 #endif
   }
 
+  // Kept beside get_mouse_loc() rather than in misc.cpp, which is where the other two platforms
+  // put it, because this is the only Linux translation unit with an X11 connection already set up.
+  std::optional<std::array<double, 2>> pointer_location() {
+#ifdef SUNSHINE_BUILD_X11
+    auto *display = XOpenDisplay(nullptr);
+    if (!display) {
+      return std::nullopt;
+    }
+
+    const auto screen = DefaultScreen(display);
+    const auto width = static_cast<double>(DisplayWidth(display, screen));
+    const auto height = static_cast<double>(DisplayHeight(display, screen));
+
+    const auto root = DefaultRootWindow(display);
+    Window root_return {};
+    Window child_return {};
+    int root_x = 0;
+    int root_y = 0;
+    int window_x = 0;
+    int window_y = 0;
+    unsigned int mask = 0;
+    const auto queried = XQueryPointer(display, root, &root_return, &child_return, &root_x, &root_y, &window_x, &window_y, &mask);
+    XCloseDisplay(display);
+
+    if (!queried || width <= 0 || height <= 0) {
+      return std::nullopt;
+    }
+
+    // The X screen, which spans every output. That is also what x11grab captures, so the two
+    // agree by default; a session capturing one output of several would need the pointer placed
+    // against that output instead.
+    return std::array<double, 2> {
+      root_x / width,
+      root_y / height
+    };
+#else
+    // Wayland gives no way to ask, so the client falls back to leaving the picture alone.
+    return std::nullopt;
+#endif
+  }
+
   std::vector<supported_gamepad_t> &supported_gamepads(input_t *input) {
     static std::vector<supported_gamepad_t> gamepads;
     if (!input || !input->get()) {

src/platform/macos/misc.h

@@ -5,6 +5,8 @@
 #pragma once
 
 // standard includes
+#include <array>
+#include <optional>
 #include <vector>
 
 // platform includes
@@ -17,6 +19,21 @@ namespace platf {
    * @return True when Sunshine can capture the screen.
    */
   bool is_screen_capture_allowed();
+
+  /**
+   * @brief Where the focused application is expecting text, as a fraction of the streamed display.
+   *
+   * A client whose on-screen keyboard covers half the picture has no way of knowing which half
+   * matters. The host does: the focused element knows where its insertion point is, and
+   * Accessibility will say so. Normalised to 0..1 of the display so the client needs to know
+   * nothing about resolutions.
+   *
+   * Empty when the focused application does not report an insertion point — which is most of
+   * them — or when the caret is on a display other than the one being streamed.
+   *
+   * @return {x, y, width, height} in 0..1 of the streamed display, or nothing.
+   */
+  std::optional<std::array<double, 4>> focused_caret();
 }  // namespace platf
 
 namespace dyn {

src/platform/macos/misc.mm

@@ -19,6 +19,8 @@
 // platform includes
 #include <arpa/inet.h>
 #include <dlfcn.h>
+#include <ApplicationServices/ApplicationServices.h>
+#include <AppKit/AppKit.h>
 #include <Foundation/Foundation.h>
 #include <mach-o/dyld.h>
 #include <net/if_dl.h>
@@ -32,6 +34,7 @@
 
 // local includes
 #include "misc.h"
+#include "src/utility.h"
 #include "src/entry_handler.h"
 #include "src/logging.h"
 #include "src/platform/common.h"
@@ -69,6 +72,119 @@
   /**
    * @brief Check whether screen capture allowed.
    */
+  namespace {
+    /// Reads an Accessibility attribute, returning nothing rather than an error code.
+    CFTypeRef copy_attribute(AXUIElementRef element, CFStringRef name) {
+      CFTypeRef value = nullptr;
+      if (AXUIElementCopyAttributeValue(element, name, &value) != kAXErrorSuccess) {
+        return nullptr;
+      }
+      return value;
+    }
+  }  // namespace
+
+  std::optional<std::array<double, 4>> focused_caret() {
+    if (!AXIsProcessTrusted()) {
+      return std::nullopt;
+    }
+
+    NSRunningApplication *front = [[NSWorkspace sharedWorkspace] frontmostApplication];
+    if (front == nil) {
+      return std::nullopt;
+    }
+
+    // Ask the application, not the system. AXUIElementCreateSystemWide() with
+    // kAXFocusedUIElementAttribute comes back empty for applications that answer perfectly well
+    // when asked directly — iTerm2 among them, which is the case this exists for.
+    AXUIElementRef app = AXUIElementCreateApplication(front.processIdentifier);
+    if (!app) {
+      return std::nullopt;
+    }
+    // A busy or hung application must not stall the request that asked for this.
+    AXUIElementSetMessagingTimeout(app, 0.1f);
+
+    const auto release_app = util::fail_guard([app]() {
+      CFRelease(app);
+    });
+
+    CFTypeRef focused = copy_attribute(app, kAXFocusedUIElementAttribute);
+    if (!focused) {
+      return std::nullopt;
+    }
+    const auto release_focused = util::fail_guard([focused]() {
+      CFRelease(focused);
+    });
+
+    CFTypeRef range = copy_attribute(static_cast<AXUIElementRef>(focused), kAXSelectedTextRangeAttribute);
+    if (!range) {
+      return std::nullopt;
+    }
+    const auto release_range = util::fail_guard([range]() {
+      CFRelease(range);
+    });
+
+    CFTypeRef bounds = nullptr;
+    if (AXUIElementCopyParameterizedAttributeValue(
+          static_cast<AXUIElementRef>(focused),
+          kAXBoundsForRangeParameterizedAttribute,
+          range,
+          &bounds
+        ) != kAXErrorSuccess ||
+        !bounds) {
+      return std::nullopt;
+    }
+    const auto release_bounds = util::fail_guard([bounds]() {
+      CFRelease(bounds);
+    });
+
+    CGRect caret = CGRectZero;
+    if (!AXValueGetValue(static_cast<AXValueRef>(bounds), kAXValueTypeCGRect, &caret)) {
+      return std::nullopt;
+    }
+    // An element that does not really have an insertion point answers with an empty rect at the
+    // origin rather than with an error.
+    if (caret.size.width == 0 && caret.size.height == 0) {
+      return std::nullopt;
+    }
+
+    // Accessibility works in the coordinates of the whole desktop arrangement, which on a second
+    // display can be negative or larger than the streamed display. Only a caret on the display
+    // being streamed means anything to the client.
+    const CGRect display = CGDisplayBounds(CGMainDisplayID());
+    const CGPoint anchor = CGPointMake(CGRectGetMidX(caret), CGRectGetMidY(caret));
+    if (!CGRectContainsPoint(display, anchor)) {
+      return std::nullopt;
+    }
+
+    return std::array<double, 4> {
+      (caret.origin.x - display.origin.x) / display.size.width,
+      (caret.origin.y - display.origin.y) / display.size.height,
+      caret.size.width / display.size.width,
+      caret.size.height / display.size.height
+    };
+  }
+
+  std::optional<std::array<double, 2>> pointer_location() {
+    // A fresh event every time rather than a reused one, as get_mouse_loc() does: the location
+    // on a reused event is whatever it was when the event was made.
+    CGEventRef snapshot = CGEventCreate(nullptr);
+    if (!snapshot) {
+      return std::nullopt;
+    }
+    const CGPoint location = CGEventGetLocation(snapshot);
+    CFRelease(snapshot);
+
+    const CGRect display = CGDisplayBounds(CGMainDisplayID());
+    if (!CGRectContainsPoint(display, location)) {
+      return std::nullopt;
+    }
+
+    return std::array<double, 2> {
+      (location.x - display.origin.x) / display.size.width,
+      (location.y - display.origin.y) / display.size.height
+    };
+  }
+
   bool is_screen_capture_allowed() {
     return screen_capture_allowed;
   }

src/platform/windows/misc.cpp

@@ -1877,4 +1877,37 @@ namespace platf {
   std::string resolve_render_device() {
     return {};
   }
+
+  std::optional<std::array<double, 2>> pointer_location() {
+    POINT cursor {};
+    if (!GetCursorPos(&cursor)) {
+      return std::nullopt;
+    }
+
+    // GetCursorPos answers in virtual-screen coordinates, which span every monitor, so a pointer
+    // on a second one lands outside the streamed display rather than nowhere. Only a pointer on
+    // the display being streamed means anything to the client.
+    MONITORINFO primary {};
+    primary.cbSize = sizeof(primary);
+    if (const auto monitor = MonitorFromPoint(POINT {0, 0}, MONITOR_DEFAULTTOPRIMARY);
+        !monitor || !GetMonitorInfo(monitor, &primary)) {
+      return std::nullopt;
+    }
+
+    const auto &bounds = primary.rcMonitor;
+    const auto width = static_cast<double>(bounds.right - bounds.left);
+    const auto height = static_cast<double>(bounds.bottom - bounds.top);
+    if (width <= 0 || height <= 0) {
+      return std::nullopt;
+    }
+
+    if (cursor.x < bounds.left || cursor.x >= bounds.right || cursor.y < bounds.top || cursor.y >= bounds.bottom) {
+      return std::nullopt;
+    }
+
+    return std::array<double, 2> {
+      (cursor.x - bounds.left) / width,
+      (cursor.y - bounds.top) / height
+    };
+  }
 }  // namespace platf