Feature/vnc 393 touch device support

app/styles/base.css

@@ -977,6 +977,7 @@ body {
   right: 5%;
   z-index: 100000;
   cursor: pointer;
+  pointer-events: none;
 }
 
 .keyboard-controls .buttons {
@@ -1053,6 +1054,7 @@ body {
 .keyboard-controls.is-visible {
   display: flex;
   flex-direction: column;
+  pointer-events: auto;
 }
 
 .keyboard-controls.is-open .button.handle {

app/ui.js

@@ -326,6 +326,7 @@ const UI = {
         UI.initSetting('enable_threading', true);
         UI.initSetting('virtual_keyboard_visible', false);
         UI.initSetting('enable_ime', false);
+        UI.initSetting('touch_mode', 'native');
         UI.initSetting('enable_webrtc', false);
         UI.initSetting('enable_hidpi', false);
         UI.initSetting('fallback_image_mode', false);
@@ -693,6 +694,8 @@ const UI = {
         UI.addSettingChangeHandler('virtual_keyboard_visible', UI.toggleKeyboardControls);
         UI.addSettingChangeHandler('enable_ime');
         UI.addSettingChangeHandler('enable_ime', UI.toggleIMEMode);
+        UI.addSettingChangeHandler('touch_mode');
+        UI.addSettingChangeHandler('touch_mode', UI.updateTouchMode);
         UI.addSettingChangeHandler('enable_webrtc');
         UI.addSettingChangeHandler('enable_webrtc', UI.toggleWebRTC);
         UI.addSettingChangeHandler('enable_hidpi');
@@ -3132,6 +3135,12 @@ const UI = {
         }
     },
 
+    updateTouchMode() {
+        if (UI.rfb) {
+            UI.rfb.touchMode = UI.getSetting('touch_mode');
+        }
+    },
+
     toggleWebRTC() {
         if (UI.rfb) {
             if (typeof RTCPeerConnection === 'undefined') {

core/encodings.js

@@ -40,6 +40,7 @@ export const encodings = {
     pseudoEncodingQOI: -1886,
     pseudoEncodingKasmDisconnectNotify: -1885,
     pseudoEncodingDirectMouse: -1884,
+    pseudoEncodingTouch: -1883,
 
     pseudoEncodingHardwareProfile0: -1170,
     pseudoEncodingHardwareProfile4: -1166,

core/messages.js

@@ -4,6 +4,8 @@ export const messages = {
     msgTypeServerDisconnect: 186,
     msgTypeForceGameMode: 187,
     msgTypeDirectMouseEvent: 188,
+    msgTypeTouchSupported: 189,
+    msgTypeTouchEvent: 190,
     msgTypeUserAddedToSession: 253,
     msgTypeUserRemovedFromSession: 254
 };

core/rfb.js

@@ -175,6 +175,7 @@ export default class RFB extends EventTargetMixin {
         this._clipboardServerCapabilitiesFormats = {};
 
         this._threading = true;
+        this._touchMode = 'native';
 
         // Internal objects
         this._sock = null;              // Websock object
@@ -249,6 +250,7 @@ export default class RFB extends EventTargetMixin {
             handlePointerLockError: this._handlePointerLockError.bind(this),
             handleWheel: this._handleWheel.bind(this),
             handleGesture: this._handleGesture.bind(this),
+            handleNativeTouch: this._handleNativeTouch.bind(this),
             handleFocusChange: this._handleFocusChange.bind(this),
             handleMouseOut: this._handleMouseOut.bind(this),
             handleVisibilityChange: this._handleVisibilityChange.bind(this),
@@ -768,6 +770,13 @@ export default class RFB extends EventTargetMixin {
         }
     }
 
+    get touchMode() { return this._touchMode; }
+    set touchMode(value) {
+        if (value !== this._touchMode) {
+            this._touchMode = value;
+        }
+    }
+
     get hwEncoderProfile() { return this._hwEncoderProfile; }
     set hwEncoderProfile(value) {
         if (value !== this._hwEncoderProfile) {
@@ -1391,6 +1400,11 @@ export default class RFB extends EventTargetMixin {
         this._canvas.addEventListener("gesturemove", this._eventHandlers.handleGesture);
         this._canvas.addEventListener("gestureend", this._eventHandlers.handleGesture);
 
+        this._canvas.addEventListener("touchstart", this._eventHandlers.handleNativeTouch, true);
+        this._canvas.addEventListener("touchmove", this._eventHandlers.handleNativeTouch, true);
+        this._canvas.addEventListener("touchend", this._eventHandlers.handleNativeTouch, true);
+        this._canvas.addEventListener("touchcancel", this._eventHandlers.handleNativeTouch, true);
+
         this._resendClipboardNextUserDrivenEvent = true;
 
         // WebRTC UDP datachannel inits
@@ -1512,6 +1526,10 @@ export default class RFB extends EventTargetMixin {
         this._canvas.removeEventListener("gesturestart", this._eventHandlers.handleGesture);
         this._canvas.removeEventListener("gesturemove", this._eventHandlers.handleGesture);
         this._canvas.removeEventListener("gestureend", this._eventHandlers.handleGesture);
+        this._canvas.removeEventListener("touchstart", this._eventHandlers.handleNativeTouch, true);
+        this._canvas.removeEventListener("touchmove", this._eventHandlers.handleNativeTouch, true);
+        this._canvas.removeEventListener("touchend", this._eventHandlers.handleNativeTouch, true);
+        this._canvas.removeEventListener("touchcancel", this._eventHandlers.handleNativeTouch, true);
         this._canvas.removeEventListener("wheel", this._eventHandlers.handleWheel);
         this._canvas.removeEventListener('mousedown', this._eventHandlers.handleMouse);
         this._canvas.removeEventListener('mouseup', this._eventHandlers.handleMouse);
@@ -2450,9 +2468,8 @@ export default class RFB extends EventTargetMixin {
     }
 
     _sendMouse(x, y, mask) {
-        if (this._rfbConnectionState !== 'connected') { return; }
-        if (this._viewOnly) { return; } // View only, skip mouse events
-        if (!this._isPrimaryDisplay) { return; }
+        if (!this.isConnected || this._viewOnly || !this._isPrimaryDisplay)
+            return;
 
         if (this._pointerLock && this._directMouseEnabled) {
             // Direct drive: button state changes only (movement is sent raw from _handleMouse)
@@ -2464,8 +2481,8 @@ export default class RFB extends EventTargetMixin {
     }
 
     _sendScroll(x, y, dX, dY) {
-        if (this._rfbConnectionState !== 'connected') { return; }
-        if (this._viewOnly) { return; } // View only, skip mouse events
+        if (!this.isConnected || this._viewOnly)  // View only, skip mouse events
+            return;
 
         if (this._pointerLock && this._directMouseEnabled) {
             this._sendDirectMouse(0, 0, this._mouseButtonMask, dX, dY);
@@ -2477,15 +2494,15 @@ export default class RFB extends EventTargetMixin {
     }
 
     _sendDirectMouse(dx, dy, buttonMask, scrollDX, scrollDY) {
-        if (this._rfbConnectionState !== 'connected') { return; }
-        if (this._viewOnly) { return; }
-        if (!this._isPrimaryDisplay) { return; }
+        if (!this.isConnected || this._viewOnly || !this._isPrimaryDisplay)
+            return;
+
         RFB.messages.directMouseEvent(this._sock, dx, dy, buttonMask, scrollDX, scrollDY);
     }
 
     _handleWheel(ev) {
-        if (this._rfbConnectionState !== 'connected') { return; }
-        if (this._viewOnly) { return; } // View only, skip mouse events
+        if (!this.isConnected || this._viewOnly)  // View only, skip mouse events
+            return;
 
         ev.stopPropagation();
         ev.preventDefault();
@@ -2539,6 +2556,44 @@ export default class RFB extends EventTargetMixin {
         this._sendScroll(pointer.x, pointer.y, dX, dY);
     }
 
+    _handleNativeTouch(ev) {
+        if (this._touchMode !== 'native') {
+            return;
+        }
+
+        if (!this.isConnected || this._viewOnly || !this._isPrimaryDisplay)
+            return;
+
+        ev.preventDefault();
+
+        const changed = ev.changedTouches;
+        for (let i = 0; i < changed.length; i++) {
+            const touch = changed[i];
+            const pos = clientToElement(touch.clientX, touch.clientY, this._canvas);
+            const x = this._display.absX(pos.x);
+            const y = this._display.absY(pos.y);
+            // Browser touch identifiers can be large or negative; coerce to an
+            // unsigned 32-bit value to match the wire format.
+            const id = touch.identifier >>> 0;
+
+            let state;
+            if (ev.type === 'touchstart') {
+                state = 0; // begin
+            } else if (ev.type === 'touchmove') {
+                state = 1; // update
+            } else {
+                // touchend / touchcancel
+                state = 2; // end
+            }
+
+            this._sendTouch(id, state, x, y);
+        }
+    }
+
+    _sendTouch(id, state, x, y) {
+        RFB.messages.touchEvent(this._sock, id, state, x, y);
+    }
+
     _fakeMouseMove(ev, elementX, elementY) {
         if (this._isPrimaryDisplay) {
             this._handleMouseMove(elementX, elementY, false, true);
@@ -2594,6 +2649,10 @@ export default class RFB extends EventTargetMixin {
     }
 
     _handleGesture(ev) {
+        if (this._touchMode === 'native') {
+            return;
+        }
+
         let magnitude;
 
         let pos = clientToElement(ev.detail.clientX, ev.detail.clientY,
@@ -3310,6 +3369,8 @@ export default class RFB extends EventTargetMixin {
         encs.push(encodings.pseudoEncodingExtendedClipboard);
         encs.push(encodings.pseudoEncodingKasmDisconnectNotify);
         encs.push(encodings.pseudoEncodingDirectMouse);
+        if (isTouchDevice)
+            encs.push(encodings.pseudoEncodingTouch);
         if (this._hasWebp())
             encs.push(encodings.pseudoEncodingWEBP);
         if (this._enableQOI)
@@ -3850,6 +3911,9 @@ export default class RFB extends EventTargetMixin {
             case messages.msgTypeForceGameMode:
                 return this._handleForceGameMode();
 
+            case messages.msgTypeTouchSupported:
+                return this._handleTouchSupported();
+
             case messages.msgTypeVideoEncoders:
                 return this._handleServerVideoEncoders();
 
@@ -4067,6 +4131,8 @@ export default class RFB extends EventTargetMixin {
                 configurations: codecConfigurations
             }
         }));
+
+        return true;
     }
 
     _handleForceGameMode() {
@@ -4076,6 +4142,14 @@ export default class RFB extends EventTargetMixin {
         return true;
     }
 
+    _handleTouchSupported() {
+        const result = isTouchDevice;
+        if (result)
+            Log.Info("Server supports touch; using phone-style touch gestures.");
+
+        return result;
+    }
+
     _handleDisconnectNotify() {
         if (this._sock.rQwait("DisconnectNotify header", 8, 1)) { return false; }
         const flags = this._sock.rQshift8();
@@ -4704,6 +4778,26 @@ RFB.messages = {
         sock.flush();
     },
 
+    touchEvent(sock, id, type, x, y) {
+        const buff = sock._sQ;
+        const offset = sock._sQlen;
+
+        // Wire format: type(1) + id(4, BE) + x(2, BE) + y(2, BE)
+        buff[offset]     = messages.msgTypeTouchEvent;
+        buff[offset + 1] = type & 0xff;
+        buff[offset + 2] = (id >>> 24) & 0xff;
+        buff[offset + 3] = (id >>> 16) & 0xff;
+        buff[offset + 4] = (id >>> 8) & 0xff;
+        buff[offset + 5] = id & 0xff;
+        buff[offset + 6] = (x >> 8) & 0xff;
+        buff[offset + 7] = x & 0xff;
+        buff[offset + 8] = (y >> 8) & 0xff;
+        buff[offset + 9] = y & 0xff;
+
+        sock._sQlen += 10;
+        sock.flush();
+    },
+
     keepAlive(sock) {
         const buff = sock._sQ;
         const offset = sock._sQlen;

index.html

@@ -299,6 +299,13 @@ <h1 class="noVNC_logo">
                                         <span class="slider-label">IME Input Mode</span>
                                     </label>
                                 </li>
+                                <li>
+                                    <label for="noVNC_setting_touch_mode">Touch Mode:</label>
+                                    <select id="noVNC_setting_touch_mode" name="vncTouchMode">
+                                        <option value="native">Native Touch (Direct)</option>
+                                        <option value="gesture">Gesture (Enulation)</option>
+                                    </select>
+                                </li>
                                 <li>
                                     <label class="switch"><input id="noVNC_setting_virtual_keyboard_visible" type="checkbox" />
                                         <span class="slider round"></span>

package.json

@@ -38,11 +38,10 @@
   },
   "homepage": "https://github.com/kasmtech/noVNC",
   "devDependencies": {
-    "@babel/cli": "*",
-    "@babel/core": "*",
-    "@babel/plugin-syntax-dynamic-import": "*",
-    "@babel/plugin-transform-modules-commonjs": "*",
-    "@babel/preset-env": "*",
+    "@babel/cli": "^7.0.0",
+    "@babel/core": "^7.0.0",
+    "@babel/plugin-transform-modules-commonjs": "^7.0.0",
+    "@babel/preset-env": "^7.0.0",
     "@chiragrupani/karma-chromium-edge-launcher": "*",
     "@interactjs/actions": "^1.10.27",
     "@interactjs/interact": "^1.10.27",