Hide cursor while dragging number inputs in Safari to approximate PointerLock, and disable it on desktop (#3425)

* Make pointerlock conditional and opt out on Safari and desktop

* Add Safari workaround

Author: Keavon

frontend/src/components/Editor.svelte

@@ -223,6 +223,10 @@
 		user-select: none;
 	}
 
+	body.cursor-hidden * {
+		cursor: none !important;
+	}
+
 	// Needed for the viewport hole punch on desktop
 	html:has(body > .viewport-hole-punch),
 	body:has(> .viewport-hole-punch) {

frontend/src/components/widgets/inputs/NumberInput.svelte

@@ -4,6 +4,7 @@
 	import { evaluateMathExpression } from "@graphite/../wasm/pkg/graphite_wasm.js";
 	import { PRESS_REPEAT_DELAY_MS, PRESS_REPEAT_INTERVAL_MS } from "@graphite/io-managers/input";
 	import { type NumberInputMode, type NumberInputIncrementBehavior } from "@graphite/messages";
+	import { browserVersion, isDesktop } from "@graphite/utility-functions/platform";
 
 	import { preventEscapeClosingParentFloatingMenu } from "@graphite/components/layout/FloatingMenu.svelte";
 	import FieldInput from "@graphite/components/widgets/inputs/FieldInput.svelte";
@@ -315,10 +316,10 @@
 		// Remove the text entry cursor from any other selected text field
 		if (document.activeElement instanceof HTMLElement) document.activeElement.blur();
 
-		// Don't drag the text value from is input element
+		// Don't drag the text value from its input element
 		e.preventDefault();
 
-		// Now we need to wait and see if the user follows this up with a mousemove or mouseup.
+		// Now we need to wait and see if the user follows this up with a mousemove or mouseup...
 
 		// For some reason, both events can get fired before their event listeners are removed, so we need to guard against both running.
 		let alreadyActedGuard = false;
@@ -327,16 +328,22 @@
 		const onMove = () => {
 			if (alreadyActedGuard) return;
 			alreadyActedGuard = true;
+
 			isDragging = true;
 			beginDrag(e);
+
 			removeEventListener("pointermove", onMove);
+			removeEventListener("pointerup", onUp);
 		};
 		// If it's a mouseup, we'll begin editing the text field.
 		const onUp = () => {
 			if (alreadyActedGuard) return;
 			alreadyActedGuard = true;
+
 			isDragging = false;
 			self?.focus();
+
+			removeEventListener("pointermove", onMove);
 			removeEventListener("pointerup", onUp);
 		};
 		addEventListener("pointermove", onMove);
@@ -348,8 +355,25 @@
 		const target = e.target || undefined;
 		if (!(target instanceof HTMLElement)) return;
 
+		// Default to using pointer lock except on unsupported platforms (Safari and the native desktop app).
+		// Even though Safari supposedly supports the PointerLock API, it implements an old (2016) version of the spec that requires an "engagement
+		// gesture" (<https://www.w3.org/TR/2016/REC-pointerlock-20161027/#glossary>) to initiate pointer lock, which the newer spec doesn't require.
+		// Because "mousemove" (and similarly, the "pointermove" event we use) is defined as not being a user-initiated "engagement gesture" event,
+		// Safari never lets us to enter pointer lock while the mouse button is held down and we are awaiting movement to begin dragging the slider.
+		const isSafari = browserVersion().toLowerCase().includes("safari");
+		const usePointerLock = !isSafari && !isDesktop();
+
+		// On Safari, we use a workaround involving an alternative strategy where we hide the cursor while it's within the web page
+		// (but we can't hide it when it ventures outside the page), taking advantage of a separate (helpful) Safari bug where it
+		// keeps reporting deltas to the pointer position even as it pushes up against edges of the screen. Like the PointerLock API,
+		// this allows infinite movement in each direction, but the downside is that the cursor remains visible outside the page and
+		// it ends up in that location when the drag ends. It isn't possible to take advantage of the PointerCapture API (yes, that's
+		// a different API than PointerLock) which is supposed to allow the CSS cursor style (such as `cursor: none`) to persist even
+		// while dragging it outside the browser window, because Safari has another bug where the cursor icon is unaffected by that API.
+		if (isSafari) document.body.classList.add("cursor-hidden");
+
 		// Enter dragging state
-		target.requestPointerLock();
+		if (usePointerLock) target.requestPointerLock();
 		initialValueBeforeDragging = value;
 		cumulativeDragDelta = 0;
 
@@ -359,6 +383,9 @@
 		// We ignore the first event invocation's `e.movementX` value because it's unreliable.
 		// In both Chrome and Firefox (tested on Windows 10), the first `e.movementX` value is occasionally a very large number
 		// (around positive 1000, even if movement was in the negative direction). This seems to happen more often if the movement is rapid.
+		// TODO: On rarer occasions, it isn't sufficient to ignore just the first event, so this solution is imperfect.
+		// TODO: Using a counter to ignore more frames helps progressively decrease—but not eliminate—the issue, but it makes drag initiation feel delayed so we don't do that.
+		// TODO: A better solution will need to discard outlier movement values across multiple frames by basically implementing a time-series data analysis filtering algorithm.
 		let ignoredFirstMovement = false;
 
 		const pointerUp = () => {
@@ -367,24 +394,24 @@
 			initialValueBeforeDragging = value;
 			cumulativeDragDelta = 0;
 
-			document.exitPointerLock();
-
-			// Fallback for Safari in case pointerlockchange never fires
-			setTimeout(() => {
-				if (!document.pointerLockElement) pointerLockChange();
-			}, 0);
+			if (usePointerLock) document.exitPointerLock();
+			else pointerLockChange();
 		};
 		const pointerMove = (e: PointerEvent) => {
+			// TODO: Display a fake cursor over the top of the page which wraps around the edges of the editor.
+
 			// Abort the drag if right click is down. This works here because a "pointermove" event is fired when right clicking even if the cursor didn't move.
 			if (e.buttons & BUTTONS_RIGHT) {
-				document.exitPointerLock();
+				if (usePointerLock) document.exitPointerLock();
+				else pointerLockChange();
 				return;
 			}
 
-			// If no buttons are down, we are stuck in the drag state after having released the mouse, so we should exit.
-			// For some reason on firefox in wayland the button is -1 and the buttons is 0.
+			// If no buttons are down, that means we are stuck in the drag state after having released the mouse, so we should exit.
+			// For some reason on Firefox in Wayland, `e.buttons` can be 0 while `e.button` is -1, but we don't want to exit in that state.
 			if (e.buttons === 0 && e.button !== -1) {
-				document.exitPointerLock();
+				if (usePointerLock) document.exitPointerLock();
+				else pointerLockChange();
 				return;
 			}
 
@@ -408,7 +435,10 @@
 		};
 		const pointerLockChange = () => {
 			// Do nothing if we just entered, rather than exited, pointer lock.
-			if (document.pointerLockElement) return;
+			if (usePointerLock && document.pointerLockElement) return;
+
+			// Un-hide the cursor if we're using the Safari workaround.
+			if (isSafari) document.body.classList.remove("cursor-hidden");
 
 			// Reset the value to the initial value if the drag was aborted, or to the current value if it was just confirmed by changing the initial value to the current value.
 			updateValue(initialValueBeforeDragging);
@@ -418,12 +448,12 @@
 			// Clean up the event listeners.
 			removeEventListener("pointerup", pointerUp);
 			removeEventListener("pointermove", pointerMove);
-			document.removeEventListener("pointerlockchange", pointerLockChange);
+			if (usePointerLock) document.removeEventListener("pointerlockchange", pointerLockChange);
 		};
 
 		addEventListener("pointerup", pointerUp);
 		addEventListener("pointermove", pointerMove);
-		document.addEventListener("pointerlockchange", pointerLockChange);
+		if (usePointerLock) document.addEventListener("pointerlockchange", pointerLockChange);
 	}
 
 	// ===============================

frontend/src/utility-functions/platform.ts

@@ -1,3 +1,5 @@
+import { isPlatformNative } from "@graphite/../wasm/pkg/graphite_wasm.js";
+
 export function browserVersion(): string {
 	const agent = window.navigator.userAgent;
 	let match = agent.match(/(opera|chrome|safari|firefox|msie|trident(?=\/))\/?\s*(\d+)/i) || [];
@@ -37,6 +39,10 @@ export function operatingSystem(): OperatingSystem {
 	return osTable[userAgentOS || "Unknown"];
 }
 
+export function isDesktop(): boolean {
+	return isPlatformNative();
+}
+
 export function isEventSupported(eventName: string) {
 	const onEventName = `on${eventName}`;
 

frontend/wasm/src/editor_api.rs

@@ -55,72 +55,15 @@ pub fn wasm_memory() -> JsValue {
 	wasm_bindgen::memory()
 }
 
-fn render_image_data_to_canvases(image_data: &[(u64, Image<Color>)]) {
-	let window = match window() {
-		Some(window) => window,
-		None => {
-			error!("Cannot render canvas: window object not found");
-			return;
-		}
-	};
-	let document = window.document().expect("window should have a document");
-	let window_obj = Object::from(window);
-	let image_canvases_key = JsValue::from_str("imageCanvases");
-
-	let canvases_obj = match Reflect::get(&window_obj, &image_canvases_key) {
-		Ok(obj) if !obj.is_undefined() && !obj.is_null() => obj,
-		_ => {
-			let new_obj = Object::new();
-			if Reflect::set(&window_obj, &image_canvases_key, &new_obj).is_err() {
-				error!("Failed to create and set imageCanvases object on window");
-				return;
-			}
-			new_obj.into()
-		}
-	};
-	let canvases_obj = Object::from(canvases_obj);
-
-	for (placeholder_id, image) in image_data.iter() {
-		let canvas_name = placeholder_id.to_string();
-		let js_key = JsValue::from_str(&canvas_name);
-
-		if Reflect::has(&canvases_obj, &js_key).unwrap_or(false) || image.width == 0 || image.height == 0 {
-			continue;
-		}
-
-		let canvas: HtmlCanvasElement = document
-			.create_element("canvas")
-			.expect("Failed to create canvas element")
-			.dyn_into::<HtmlCanvasElement>()
-			.expect("Failed to cast element to HtmlCanvasElement");
-
-		canvas.set_width(image.width);
-		canvas.set_height(image.height);
-
-		let context: CanvasRenderingContext2d = canvas
-			.get_context("2d")
-			.expect("Failed to get 2d context")
-			.expect("2d context was not found")
-			.dyn_into::<CanvasRenderingContext2d>()
-			.expect("Failed to cast context to CanvasRenderingContext2d");
-		let u8_data: Vec<u8> = image.data.iter().flat_map(|color| color.to_rgba8_srgb()).collect();
-		let clamped_u8_data = wasm_bindgen::Clamped(&u8_data[..]);
-		match ImageData::new_with_u8_clamped_array_and_sh(clamped_u8_data, image.width, image.height) {
-			Ok(image_data_obj) => {
-				if context.put_image_data(&image_data_obj, 0., 0.).is_err() {
-					error!("Failed to put image data on canvas for id: {placeholder_id}");
-				}
-			}
-			Err(e) => {
-				error!("Failed to create ImageData for id: {placeholder_id}: {e:?}");
-			}
-		}
-
-		let js_value = JsValue::from(canvas);
-
-		if Reflect::set(&canvases_obj, &js_key, &js_value).is_err() {
-			error!("Failed to set canvas '{canvas_name}' on imageCanvases object");
-		}
+#[wasm_bindgen(js_name = isPlatformNative)]
+pub fn is_platform_native() -> bool {
+	#[cfg(feature = "native")]
+	{
+		true
+	}
+	#[cfg(not(feature = "native"))]
+	{
+		false
 	}
 }
 
@@ -1061,3 +1004,72 @@ fn auto_save_all_documents() {
 		handle.dispatch(PortfolioMessage::AutoSaveAllDocuments);
 	});
 }
+
+fn render_image_data_to_canvases(image_data: &[(u64, Image<Color>)]) {
+	let window = match window() {
+		Some(window) => window,
+		None => {
+			error!("Cannot render canvas: window object not found");
+			return;
+		}
+	};
+	let document = window.document().expect("window should have a document");
+	let window_obj = Object::from(window);
+	let image_canvases_key = JsValue::from_str("imageCanvases");
+
+	let canvases_obj = match Reflect::get(&window_obj, &image_canvases_key) {
+		Ok(obj) if !obj.is_undefined() && !obj.is_null() => obj,
+		_ => {
+			let new_obj = Object::new();
+			if Reflect::set(&window_obj, &image_canvases_key, &new_obj).is_err() {
+				error!("Failed to create and set imageCanvases object on window");
+				return;
+			}
+			new_obj.into()
+		}
+	};
+	let canvases_obj = Object::from(canvases_obj);
+
+	for (placeholder_id, image) in image_data.iter() {
+		let canvas_name = placeholder_id.to_string();
+		let js_key = JsValue::from_str(&canvas_name);
+
+		if Reflect::has(&canvases_obj, &js_key).unwrap_or(false) || image.width == 0 || image.height == 0 {
+			continue;
+		}
+
+		let canvas: HtmlCanvasElement = document
+			.create_element("canvas")
+			.expect("Failed to create canvas element")
+			.dyn_into::<HtmlCanvasElement>()
+			.expect("Failed to cast element to HtmlCanvasElement");
+
+		canvas.set_width(image.width);
+		canvas.set_height(image.height);
+
+		let context: CanvasRenderingContext2d = canvas
+			.get_context("2d")
+			.expect("Failed to get 2d context")
+			.expect("2d context was not found")
+			.dyn_into::<CanvasRenderingContext2d>()
+			.expect("Failed to cast context to CanvasRenderingContext2d");
+		let u8_data: Vec<u8> = image.data.iter().flat_map(|color| color.to_rgba8_srgb()).collect();
+		let clamped_u8_data = wasm_bindgen::Clamped(&u8_data[..]);
+		match ImageData::new_with_u8_clamped_array_and_sh(clamped_u8_data, image.width, image.height) {
+			Ok(image_data_obj) => {
+				if context.put_image_data(&image_data_obj, 0., 0.).is_err() {
+					error!("Failed to put image data on canvas for id: {placeholder_id}");
+				}
+			}
+			Err(e) => {
+				error!("Failed to create ImageData for id: {placeholder_id}: {e:?}");
+			}
+		}
+
+		let js_value = JsValue::from(canvas);
+
+		if Reflect::set(&canvases_obj, &js_key, &js_value).is_err() {
+			error!("Failed to set canvas '{canvas_name}' on imageCanvases object");
+		}
+	}
+}