minecraft characters.

bun.lock

@@ -9,12 +9,14 @@
         "@tauri-apps/api": "^2",
         "@tauri-apps/plugin-opener": "^2",
         "tailwindcss": "^4.3.3",
+        "three": "^0.185.1",
       },
       "devDependencies": {
         "@sveltejs/adapter-static": "^3.0.6",
         "@sveltejs/kit": "^2.9.0",
         "@sveltejs/vite-plugin-svelte": "^5.0.0",
         "@tauri-apps/cli": "^2",
+        "@types/three": "^0.185.4",
         "daisyui": "^5.7.21",
         "svelte": "^5.0.0",
         "svelte-check": "^4.0.0",
@@ -24,6 +26,8 @@
     },
   },
   "packages": {
+    "@dimforge/rapier3d-compat": ["@dimforge/rapier3d-compat@0.12.0", "", {}, "sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow=="],
+
     "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="],
 
     "@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="],
@@ -210,12 +214,20 @@
 
     "@tauri-apps/plugin-opener": ["@tauri-apps/plugin-opener@2.5.4", "", { "dependencies": { "@tauri-apps/api": "^2.11.0" } }, "sha512-1HnPkb+AmgO29HBazm4uPLKB+r7zzcTBW1d0fyYp1uP+jwtpoiNDGKMMzz58SFp49nOIrxdE3aUJtT57lfO9CQ=="],
 
+    "@tweenjs/tween.js": ["@tweenjs/tween.js@23.1.3", "", {}, "sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA=="],
+
     "@types/cookie": ["@types/cookie@0.6.0", "", {}, "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA=="],
 
     "@types/estree": ["@types/estree@1.0.9", "", {}, "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg=="],
 
+    "@types/stats.js": ["@types/stats.js@0.17.4", "", {}, "sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA=="],
+
+    "@types/three": ["@types/three@0.185.4", "", { "dependencies": { "@dimforge/rapier3d-compat": "~0.12.0", "@tweenjs/tween.js": "~23.1.3", "@types/stats.js": "*", "@types/webxr": ">=0.5.17", "fflate": "~0.8.2", "meshoptimizer": "~1.1.1" } }, "sha512-gAsBIC07NIFrxjbf7tH2t71c38uulFfk/RFoC7FNBSjMRAQ8J1x/RBvusX0N5PJouaYFJawXQqfCQ0RKUx/1nA=="],
+
     "@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="],
 
+    "@types/webxr": ["@types/webxr@0.5.24", "", {}, "sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg=="],
+
     "acorn": ["acorn@8.16.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="],
 
     "aria-query": ["aria-query@5.3.1", "", {}, "sha512-Z/ZeOgVl7bcSYZ/u/rh0fOpvEpq//LZmdbkXyc7syVzjPAhfOa9ebsdTSjEBDU4vs5nC98Kfduj1uFo0qyET3g=="],
@@ -248,6 +260,8 @@
 
     "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="],
 
+    "fflate": ["fflate@0.8.3", "", {}, "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA=="],
+
     "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
 
     "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="],
@@ -286,6 +300,8 @@
 
     "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="],
 
+    "meshoptimizer": ["meshoptimizer@1.1.1", "", {}, "sha512-oRFNWJRDA/WTrVj7NWvqa5HqE1t9MYDj2VaWirQCzCCrAd2GHrqR/sQezCxiWATPNlKTcRaPRHPJwIRoPBAp5g=="],
+
     "mri": ["mri@1.2.0", "", {}, "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA=="],
 
     "mrmime": ["mrmime@2.0.1", "", {}, "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ=="],
@@ -320,6 +336,8 @@
 
     "tapable": ["tapable@2.3.3", "", {}, "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A=="],
 
+    "three": ["three@0.185.1", "", {}, "sha512-5aojFCXKwnjBRZvUnt3WFfEcvUJgkN5LlijRFN95hMy8WVkG4I0QNcJE+OuWvuJ0bOdStrbfXn0pkd6/QyiAlg=="],
+
     "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="],
 
     "totalist": ["totalist@3.0.1", "", {}, "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ=="],

package.json

@@ -16,13 +16,15 @@
     "@tailwindcss/vite": "^4.3.3",
     "@tauri-apps/api": "^2",
     "@tauri-apps/plugin-opener": "^2",
-    "tailwindcss": "^4.3.3"
+    "tailwindcss": "^4.3.3",
+    "three": "^0.185.1"
   },
   "devDependencies": {
     "@sveltejs/adapter-static": "^3.0.6",
     "@sveltejs/kit": "^2.9.0",
     "@sveltejs/vite-plugin-svelte": "^5.0.0",
     "@tauri-apps/cli": "^2",
+    "@types/three": "^0.185.4",
     "daisyui": "^5.7.21",
     "svelte": "^5.0.0",
     "svelte-check": "^4.0.0",

src-tauri/src/cursor/mod.rs

@@ -11,14 +11,14 @@ use tauri_specta::Event;
 use tokio::sync::mpsc;
 use tokio::sync::watch;
 
-#[derive(Debug, Clone, Serialize, Deserialize, Default, Type)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Type)]
 #[serde(rename_all = "camelCase")]
 pub struct CursorPosition {
     pub x: f64,
     pub y: f64,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, Default, Type)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Type)]
 #[serde(rename_all = "camelCase")]
 pub struct CursorPositions {
     /// Absolute cursor coordinates in physical pixels.
@@ -56,6 +56,13 @@ impl CursorState {
         positions_by_user.retain(|user_id, _| !user_ids.contains(user_id));
         Ok((positions_by_user.len() != previous_len).then(|| positions_by_user.clone()))
     }
+
+    pub(crate) fn snapshot(&self) -> Result<HashMap<String, CursorPositions>, String> {
+        self.0
+            .read()
+            .map(|positions| positions.clone())
+            .map_err(|error| error.to_string())
+    }
 }
 
 // Was private, but for some reason LSP

src-tauri/src/lib.rs

@@ -7,6 +7,7 @@ mod keypair;
 mod live_data;
 mod network;
 mod profile;
+mod puppet;
 mod remotes;
 mod ufa;
 mod ui;
@@ -22,8 +23,10 @@ async fn launch_app(app: &tauri::App) -> Result<(), Box<dyn std::error::Error>>
     network::init(handle).await?;
     app.manage(ufa::ForegroundAppState::default());
     app.manage(cursor::CursorState::default());
+    app.manage(puppet::PuppetStateStore::default());
     ufa::init(handle);
     cursor::init(handle);
+    puppet::init(handle)?;
     ui::init(handle);
     Ok(())
 }
@@ -76,6 +79,7 @@ fn specta_builder() -> tauri_specta::Builder<tauri::Wry> {
             images::pick_and_send_image,
             images::send_image_bytes,
             interactions::send_interaction,
+            puppet::list_puppet_states,
             ui::control_panel::open_action_window,
             ui::scene::update_scene_hitboxes,
         ])
@@ -89,6 +93,7 @@ fn specta_builder() -> tauri_specta::Builder<tauri::Wry> {
             ufa::ForegroundAppChanged,
             ufa::FriendForegroundAppChanged,
             interactions::FriendInteractionReceived,
+            puppet::PuppetStatesChanged,
         ])
 }
 

src-tauri/src/puppet/mod.rs

@@ -0,0 +1,332 @@
+use std::{
+    collections::HashMap,
+    sync::RwLock,
+    time::{Duration, Instant},
+};
+
+use serde::{Deserialize, Serialize};
+use specta::Type;
+use tauri::{AppHandle, Manager, State};
+use tauri_specta::Event;
+use tokio::time::MissedTickBehavior;
+
+use crate::cursor::{CursorPosition, CursorPositions, CursorState};
+
+const TICK_INTERVAL: Duration = Duration::from_millis(125);
+const SPEED_LOGICAL_PIXELS_PER_SECOND: f64 = 80.0;
+const FOLLOW_RADIUS_LOGICAL_PIXELS: f64 = 48.0;
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Type)]
+#[serde(rename_all = "camelCase")]
+pub struct PuppetState {
+    pub id: String,
+    pub position: CursorPosition,
+    pub is_moving: bool,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, Type, Event)]
+#[serde(rename_all = "camelCase")]
+pub struct PuppetStatesChanged {
+    pub puppets: Vec<PuppetState>,
+}
+
+#[derive(Default)]
+pub struct PuppetStateStore(RwLock<HashMap<String, PuppetState>>);
+
+impl PuppetStateStore {
+    fn update(
+        &self,
+        cursor_positions: &HashMap<String, CursorPositions>,
+        viewport: Viewport,
+        elapsed: Duration,
+    ) -> Result<Option<Vec<PuppetState>>, String> {
+        let mut puppets = self.0.write().map_err(|error| error.to_string())?;
+        let changed = advance_puppets(&mut puppets, cursor_positions, viewport, elapsed);
+        Ok(changed.then(|| sorted_snapshot(&puppets)))
+    }
+
+    fn snapshot(&self) -> Result<Vec<PuppetState>, String> {
+        self.0
+            .read()
+            .map(|puppets| sorted_snapshot(&puppets))
+            .map_err(|error| error.to_string())
+    }
+}
+
+#[derive(Clone, Copy)]
+struct Viewport {
+    width: f64,
+    height: f64,
+}
+
+#[tauri::command]
+#[specta::specta]
+pub fn list_puppet_states(state: State<'_, PuppetStateStore>) -> Result<Vec<PuppetState>, String> {
+    state.snapshot()
+}
+
+pub fn init(handle: &AppHandle) -> Result<(), String> {
+    let monitor = handle
+        .primary_monitor()
+        .map_err(|error| error.to_string())?
+        .ok_or_else(|| "primary monitor is unavailable".to_owned())?;
+    let scale_factor = monitor.scale_factor();
+    let viewport = Viewport {
+        width: monitor.size().width as f64 / scale_factor,
+        height: monitor.size().height as f64 / scale_factor,
+    };
+    let handle = handle.clone();
+
+    tauri::async_runtime::spawn(async move {
+        let mut ticker = tokio::time::interval(TICK_INTERVAL);
+        ticker.set_missed_tick_behavior(MissedTickBehavior::Skip);
+        let mut previous_tick = Instant::now();
+
+        loop {
+            ticker.tick().await;
+            let now = Instant::now();
+            let elapsed = now.duration_since(previous_tick);
+            previous_tick = now;
+
+            let cursor_positions = match handle.state::<CursorState>().snapshot() {
+                Ok(positions) => positions,
+                Err(error) => {
+                    eprintln!("failed to read cursor positions for puppet motion: {error}");
+                    continue;
+                }
+            };
+            let puppets = match handle.state::<PuppetStateStore>().update(
+                &cursor_positions,
+                viewport,
+                elapsed,
+            ) {
+                Ok(Some(puppets)) => puppets,
+                Ok(None) => continue,
+                Err(error) => {
+                    eprintln!("failed to update puppet motion: {error}");
+                    continue;
+                }
+            };
+
+            if let Err(error) = emit_changed(&handle, puppets) {
+                eprintln!("failed to emit puppet states: {error}");
+            }
+        }
+    });
+
+    Ok(())
+}
+
+fn emit_changed(handle: &AppHandle, puppets: Vec<PuppetState>) -> Result<(), String> {
+    (PuppetStatesChanged { puppets })
+        .emit(handle)
+        .map_err(|error| error.to_string())
+}
+
+fn advance_puppets(
+    puppets: &mut HashMap<String, PuppetState>,
+    cursor_positions: &HashMap<String, CursorPositions>,
+    viewport: Viewport,
+    elapsed: Duration,
+) -> bool {
+    let previous_len = puppets.len();
+    puppets.retain(|id, _| cursor_positions.contains_key(id));
+    let mut changed = puppets.len() != previous_len;
+
+    for (id, cursor) in cursor_positions {
+        let Some(puppet) = puppets.get_mut(id) else {
+            puppets.insert(
+                id.clone(),
+                PuppetState {
+                    id: id.clone(),
+                    position: cursor.mapped.clone(),
+                    is_moving: false,
+                },
+            );
+            changed = true;
+            continue;
+        };
+
+        changed |= advance_puppet(puppet, &cursor.mapped, viewport, elapsed.as_secs_f64());
+    }
+
+    changed
+}
+
+fn advance_puppet(
+    puppet: &mut PuppetState,
+    target: &CursorPosition,
+    viewport: Viewport,
+    elapsed_seconds: f64,
+) -> bool {
+    let delta_x = (target.x - puppet.position.x) * viewport.width;
+    let delta_y = (target.y - puppet.position.y) * viewport.height;
+    let distance = delta_x.hypot(delta_y);
+    let available_distance = (distance - FOLLOW_RADIUS_LOGICAL_PIXELS).max(0.0);
+    let step = (SPEED_LOGICAL_PIXELS_PER_SECOND * elapsed_seconds).min(available_distance);
+    let is_moving = step > 0.0;
+
+    if !is_moving {
+        if puppet.is_moving {
+            puppet.is_moving = false;
+            return true;
+        }
+        return false;
+    }
+
+    puppet.position.x += delta_x / distance * step / viewport.width;
+    puppet.position.y += delta_y / distance * step / viewport.height;
+    puppet.is_moving = true;
+    true
+}
+
+fn sorted_snapshot(puppets: &HashMap<String, PuppetState>) -> Vec<PuppetState> {
+    let mut snapshot = puppets.values().cloned().collect::<Vec<_>>();
+    snapshot.sort_unstable_by(|left, right| left.id.cmp(&right.id));
+    snapshot
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn viewport() -> Viewport {
+        Viewport {
+            width: 1_000.0,
+            height: 500.0,
+        }
+    }
+
+    fn cursors(entries: &[(&str, f64, f64)]) -> HashMap<String, CursorPositions> {
+        entries
+            .iter()
+            .map(|(id, x, y)| {
+                (
+                    (*id).to_owned(),
+                    CursorPositions {
+                        raw: CursorPosition::default(),
+                        mapped: CursorPosition { x: *x, y: *y },
+                    },
+                )
+            })
+            .collect()
+    }
+
+    #[test]
+    fn new_puppet_starts_at_first_cursor_target() {
+        let mut puppets = HashMap::new();
+        let changed = advance_puppets(
+            &mut puppets,
+            &cursors(&[("friend", 0.75, 0.25)]),
+            viewport(),
+            TICK_INTERVAL,
+        );
+
+        assert!(changed);
+        assert_eq!(
+            puppets["friend"].position,
+            CursorPosition { x: 0.75, y: 0.25 }
+        );
+        assert!(!puppets["friend"].is_moving);
+    }
+
+    #[test]
+    fn movement_is_bounded_by_speed_and_elapsed_time() {
+        let mut puppet = PuppetState {
+            id: "friend".to_owned(),
+            position: CursorPosition { x: 0.0, y: 0.5 },
+            is_moving: false,
+        };
+
+        assert!(advance_puppet(
+            &mut puppet,
+            &CursorPosition { x: 1.0, y: 0.5 },
+            viewport(),
+            0.125,
+        ));
+        assert_eq!(puppet.position.x, 0.01);
+        assert_eq!(puppet.position.y, 0.5);
+        assert!(puppet.is_moving);
+    }
+
+    #[test]
+    fn puppet_stops_at_follow_radius_without_overshooting() {
+        let mut puppet = PuppetState {
+            id: "friend".to_owned(),
+            position: CursorPosition { x: 0.0, y: 0.5 },
+            is_moving: true,
+        };
+
+        advance_puppet(
+            &mut puppet,
+            &CursorPosition { x: 0.05, y: 0.5 },
+            viewport(),
+            0.125,
+        );
+
+        assert_eq!(puppet.position.x, 0.002);
+        assert!(advance_puppet(
+            &mut puppet,
+            &CursorPosition { x: 0.05, y: 0.5 },
+            viewport(),
+            0.125,
+        ));
+        assert!(!puppet.is_moving);
+        assert!(!advance_puppet(
+            &mut puppet,
+            &CursorPosition { x: 0.05, y: 0.5 },
+            viewport(),
+            0.125,
+        ));
+    }
+
+    #[test]
+    fn removed_cursor_removes_puppet() {
+        let mut puppets = HashMap::from([(
+            "friend".to_owned(),
+            PuppetState {
+                id: "friend".to_owned(),
+                position: CursorPosition::default(),
+                is_moving: false,
+            },
+        )]);
+
+        assert!(advance_puppets(
+            &mut puppets,
+            &HashMap::new(),
+            viewport(),
+            TICK_INTERVAL,
+        ));
+        assert!(puppets.is_empty());
+    }
+
+    #[test]
+    fn normalized_step_accounts_for_viewport_aspect_ratio() {
+        let mut horizontal = PuppetState {
+            id: "horizontal".to_owned(),
+            position: CursorPosition::default(),
+            is_moving: false,
+        };
+        let mut vertical = PuppetState {
+            id: "vertical".to_owned(),
+            position: CursorPosition::default(),
+            is_moving: false,
+        };
+
+        advance_puppet(
+            &mut horizontal,
+            &CursorPosition { x: 1.0, y: 0.0 },
+            viewport(),
+            0.125,
+        );
+        advance_puppet(
+            &mut vertical,
+            &CursorPosition { x: 0.0, y: 1.0 },
+            viewport(),
+            0.125,
+        );
+
+        assert_eq!(horizontal.position.x, 0.01);
+        assert_eq!(vertical.position.y, 0.02);
+    }
+}

src/app.css

@@ -77,6 +77,10 @@
   background-color: transparent;
 }
 
+.scene-hitbox {
+  pointer-events: auto;
+}
+
 button,
 input {
   @apply btn-sm;

src/lib/bindings.ts

@@ -59,6 +59,9 @@ async sendImageBytes(recipientId: string, bytes: number[]) : Promise<null> {
 async sendInteraction(recipientId: string, content: InteractionContent) : Promise<null> {
     return await TAURI_INVOKE("send_interaction", { recipientId, content });
 },
+async listPuppetStates() : Promise<PuppetState[]> {
+    return await TAURI_INVOKE("list_puppet_states");
+},
 async openActionWindow(name: string, title: string, pageUrl: string) : Promise<null> {
     return await TAURI_INVOKE("open_action_window", { name, title, pageUrl });
 },
@@ -79,6 +82,7 @@ friendStatusesChanged: FriendStatusesChanged,
 friendsChanged: FriendsChanged,
 networkStatusChanged: NetworkStatusChanged,
 profileChanged: ProfileChanged,
+puppetStatesChanged: PuppetStatesChanged,
 remotesChanged: RemotesChanged
 }>({
 cursorPositionChanged: "cursor-position-changed",
@@ -89,6 +93,7 @@ friendStatusesChanged: "friend-statuses-changed",
 friendsChanged: "friends-changed",
 networkStatusChanged: "network-status-changed",
 profileChanged: "profile-changed",
+puppetStatesChanged: "puppet-states-changed",
 remotesChanged: "remotes-changed"
 })
 
@@ -128,6 +133,8 @@ export type FriendsChanged = { friends: Friend[] }
 export type InteractionContent = { type: "text"; text: string } | { type: "wave" } | { type: "image"; mediaType: string; data: string }
 export type NetworkStatusChanged = { statuses: ConnectionStatus[] }
 export type ProfileChanged = { profile: User }
+export type PuppetState = { id: string; position: CursorPosition; isMoving: boolean }
+export type PuppetStatesChanged = { puppets: PuppetState[] }
 export type Remote = { id: string; address: string; name: string | null; port: number | null }
 export type RemoteInput = { address: string; name: string | null; port: number | null }
 export type RemotesChanged = { remotes: Remote[] }

src/lib/listeners/index.ts

@@ -4,6 +4,7 @@ import { initFriendsListener } from "./friends";
 import { initInteractionListener } from "./interactions";
 import { initLiveMetadataListeners } from "./live-metadata";
 import { initProfileListener } from "./profile";
+import { initPuppetStatesListener } from "./puppets";
 import { initRemotesListener } from "./remotes";
 
 type Unlisten = () => void;
@@ -20,6 +21,7 @@ export function initAppListeners(): Unlisten {
     initFriendStatusesListener(),
     initLiveMetadataListeners(),
     initInteractionListener(),
+    initPuppetStatesListener(),
   ])
     .then((results) => {
       const listeners = results.flatMap((result) =>

src/lib/listeners/puppets.ts

@@ -0,0 +1,19 @@
+import { writable } from "svelte/store";
+import { commands, events, type PuppetState } from "$lib/bindings";
+
+export const puppetStates = writable<PuppetState[]>([]);
+export const puppetStatesListenerError = writable("");
+
+export async function initPuppetStatesListener() {
+  const unlisten = await events.puppetStatesChanged.listen((event) => {
+    puppetStates.set(event.payload.puppets);
+  });
+
+  try {
+    puppetStates.set(await commands.listPuppetStates());
+  } catch (error) {
+    puppetStatesListenerError.set(String(error));
+  }
+
+  return unlisten;
+}

src/routes/scene/+page.svelte

@@ -6,15 +6,27 @@
   import { incomingInteraction } from "$lib/listeners/interactions";
   import { liveMetadata } from "$lib/listeners/live-metadata";
   import { profile } from "$lib/listeners/profile";
+  import { puppetStates } from "$lib/listeners/puppets";
+  import Renderer from "./components/renderer.svelte";
+  import type { PuppetScreenBounds } from "./components/renderer/types";
+  import { startHitboxSync } from "./hitboxes";
   import SceneImageViewer from "./popovers/image-viewer.svelte";
   import SceneInteractionBubble from "./popovers/interaction-bubble.svelte";
   import SceneUserPopoverContent from "./popovers/user-interaction.svelte";
-  import { startHitboxSync } from "./hitboxes";
 
   let selectedUserId = $state<string | null>(null);
-  let selectedUserX = $state<number | null>(null);
   let lockedPopoverUserId = $state<string | null>(null);
   let viewedImage = $state<{ source: string; senderName: string } | null>(null);
+  let puppetBounds = $state<PuppetScreenBounds[]>([]);
+  let puppetBoundsById = $derived(
+    new Map(puppetBounds.map((bounds) => [bounds.id, bounds])),
+  );
+  let visiblePuppets = $derived(
+    $puppetStates.filter(
+      ({ id }) =>
+        id === $liveMetadata.localId || $onlineFriendIds.has(id),
+    ),
+  );
 
   onMount(startHitboxSync);
 
@@ -32,12 +44,9 @@
   $effect(() => {
     if (
       selectedUserId &&
-      selectedUserId !== $liveMetadata.localId &&
+      !visiblePuppets.some((puppet) => puppet.id === selectedUserId)
-      !$onlineFriendIds.has(selectedUserId)
     ) {
-      selectedUserId = null;
+      dismissPopover();
-      selectedUserX = null;
-      lockedPopoverUserId = null;
     }
   });
 
@@ -56,36 +65,52 @@
     );
   }
 
-  function setPopoverOpen(
+  function setPopoverOpen(userId: string, open: boolean) {
-    userId: string,
-    cursorX: number,
-    open: boolean,
-    trigger: HTMLButtonElement,
-  ) {
     if (open && lockedPopoverUserId && lockedPopoverUserId !== userId) return;
-    selectedUserId = open ? userId : null;
+    if (open) {
-    if (!open && lockedPopoverUserId === userId) lockedPopoverUserId = null;
+      selectedUserId = userId;
-    selectedUserX = open
+    } else {
-      ? window.innerWidth > 0
+      dismissPopover();
-        ? trigger.getBoundingClientRect().left / window.innerWidth
+    }
-        : cursorX
+  }
-      : null;
+
+  function dismissPopover() {
+    selectedUserId = null;
+    lockedPopoverUserId = null;
+  }
+
+  function popoverPosition(bounds: PuppetScreenBounds) {
+    const centerX = bounds.x + bounds.width / 2;
+    const horizontal = `position: fixed; left: clamp(0.5rem, calc(${centerX}px - 8.5rem), calc(100vw - 17.5rem));`;
+
+    return bounds.y > window.innerHeight / 2
+      ? `${horizontal} bottom: calc(100vh - ${bounds.y}px + 0.75rem);`
+      : `${horizontal} top: ${bounds.y + bounds.height + 12}px;`;
   }
 </script>
 
-<div class="flex size-full flex-col justify-end">
+<div class="relative size-full">
-  <div role="banner" class="relative my-2 h-8 w-full">
+  <Renderer
-    {#each Object.entries($liveMetadata.cursorPositions) as [userId, cursor], index (userId)}
+    puppets={visiblePuppets}
+    frozenPuppetId={selectedUserId}
+    onBoundsChange={(bounds) => (puppetBounds = bounds)}
+  />
+
+  <div role="banner" class="pointer-events-none fixed inset-0 z-10">
+    {#each visiblePuppets as puppet, index (puppet.id)}
+      {@const userId = puppet.id}
+      {@const bounds = puppetBoundsById.get(userId)}
       {@const foregroundApp = $liveMetadata.foregroundApps.get(userId)}
       {@const popoverId = `scene-user-${index}`}
       {@const interaction =
         $incomingInteraction?.friendId === userId ? $incomingInteraction : null}
-      {#if cursor && (userId === $liveMetadata.localId || $onlineFriendIds.has(userId))}
+      {#if bounds}
-        {@const renderedX =
-          selectedUserId === userId ? selectedUserX : cursor.mapped.x}
         <div
-          class="absolute bottom-0 left-0 flex flex-col items-center gap-1 transition-transform duration-1000 ease-linear"
+          class="pointer-events-none fixed"
-          style:transform={`translateX(${(renderedX ?? cursor.mapped.x) * 100}vw)`}
+          style:left={`${bounds.x}px`}
+          style:top={`${bounds.y}px`}
+          style:width={`${bounds.width}px`}
+          style:height={`${bounds.height}px`}
         >
           {#if interaction}
             <SceneInteractionBubble
@@ -103,7 +128,7 @@
             <img
               src={`data:image/png;base64,${foregroundApp.ico}`}
               alt=""
-              class="size-4 object-contain"
+              class="pointer-events-none absolute -top-5 left-1/2 size-4 -translate-x-1/2 object-contain"
             />
           {/if}
 
@@ -112,21 +137,17 @@
             label="Show live user information"
             labelledBy={`${popoverId}-title`}
             open={selectedUserId === userId}
-            onOpenChange={(open, trigger) =>
+            onOpenChange={(open) => setPopoverOpen(userId, open)}
-              setPopoverOpen(userId, cursor.mapped.x, open, trigger)}
+            class="size-full"
-            triggerClass="scene-hitbox"
+            triggerClass="scene-hitbox size-full min-h-0"
-            panelClass="scene-hitbox bottom-[calc(100%+0.75rem)] w-68"
+            panelClass="scene-hitbox fixed w-68"
-            panelStyle={`left: clamp(calc(0.5rem - ${(renderedX ?? cursor.mapped.x) * 100}vw), -8rem, calc(100vw - ${(renderedX ?? cursor.mapped.x) * 100}vw - 17.5rem))`}
+            panelStyle={popoverPosition(bounds)}
             closeOnOutsidePointer={lockedPopoverUserId !== userId}
             closeOnWindowBlur={lockedPopoverUserId !== userId}
             closeOnFocusOut={lockedPopoverUserId !== userId}
           >
             {#snippet trigger()}
-              <img
+              <span class="block size-full"></span>
-                src="/fa.png"
-                alt=""
-                class="size-6 sepia saturate-200 hue-rotate-[100deg]"
-              />
             {/snippet}
 
             <SceneUserPopoverContent
@@ -136,16 +157,8 @@
               {foregroundApp}
               onModeChange={(active) =>
                 (lockedPopoverUserId = active ? userId : null)}
-              onDismiss={() => {
+              onDismiss={dismissPopover}
-                selectedUserId = null;
+              onSent={dismissPopover}
-                selectedUserX = null;
-                lockedPopoverUserId = null;
-              }}
-              onSent={() => {
-                selectedUserId = null;
-                selectedUserX = null;
-                lockedPopoverUserId = null;
-              }}
             />
           </Popover>
         </div>

src/routes/scene/components/renderer.svelte

@@ -0,0 +1,99 @@
+<script lang="ts">
+  import type { PuppetState } from "$lib/bindings";
+  import { onDestroy, onMount } from "svelte";
+  import * as THREE from "three";
+  import { PuppetManager } from "./renderer/puppet/manager";
+  import type { PuppetScreenBounds } from "./renderer/types";
+  import { World } from "./renderer/world";
+
+  let {
+    puppets,
+    frozenPuppetId,
+    onBoundsChange,
+  }: {
+    puppets: readonly PuppetState[];
+    frozenPuppetId: string | null;
+    onBoundsChange: (bounds: PuppetScreenBounds[]) => void;
+  } = $props();
+
+  let renderDiv = $state<HTMLDivElement | null>(null);
+
+  let world: World | null = null;
+  let puppetManager: PuppetManager | null = null;
+  let resizeObserver: ResizeObserver | null = null;
+  let clock: THREE.Timer | null = null;
+  let animationFrameId = 0;
+  let lastBoundsUpdate = 0;
+  let previousBounds: PuppetScreenBounds[] = [];
+
+  function animate() {
+    if (!world || !puppetManager || !clock) return;
+
+    animationFrameId = requestAnimationFrame(animate);
+
+    clock.update();
+    puppetManager.update(
+      puppets,
+      frozenPuppetId,
+      clock.getDelta(),
+      clock.getElapsed(),
+    );
+    world.render();
+
+    const elapsed = clock.getElapsed();
+    if (elapsed - lastBoundsUpdate >= 1 / 30) {
+      lastBoundsUpdate = elapsed;
+      const nextBounds = puppetManager.screenBounds();
+      if (!sameBounds(previousBounds, nextBounds)) {
+        previousBounds = nextBounds;
+        onBoundsChange(nextBounds);
+      }
+    }
+  }
+
+  function sameBounds(
+    previous: PuppetScreenBounds[],
+    next: PuppetScreenBounds[],
+  ) {
+    if (previous.length !== next.length) return false;
+    return previous.every((bounds, index) => {
+      const candidate = next[index];
+      return (
+        candidate?.id === bounds.id &&
+        Math.abs(candidate.x - bounds.x) < 0.25 &&
+        Math.abs(candidate.y - bounds.y) < 0.25 &&
+        Math.abs(candidate.width - bounds.width) < 0.25 &&
+        Math.abs(candidate.height - bounds.height) < 0.25
+      );
+    });
+  }
+
+  onMount(() => {
+    if (!renderDiv) return;
+
+    world = new World(renderDiv);
+    puppetManager = new PuppetManager(world);
+    clock = new THREE.Timer();
+    clock.connect(document);
+
+    resizeObserver = new ResizeObserver(world.resizeWorld);
+    resizeObserver.observe(renderDiv);
+
+    animate();
+  });
+
+  onDestroy(() => {
+    cancelAnimationFrame(animationFrameId);
+    resizeObserver?.disconnect();
+    clock?.dispose();
+    puppetManager?.dispose();
+    world?.dispose();
+    onBoundsChange([]);
+  });
+</script>
+
+<div
+  class="pointer-events-none absolute inset-0 overflow-hidden"
+  aria-hidden="true"
+  bind:this={renderDiv}
+></div>

src/routes/scene/components/renderer/puppet/animation.ts

@@ -0,0 +1,83 @@
+import * as THREE from "three";
+import type { PuppetRig } from ".";
+
+export class PuppetAnimation {
+  private readonly smoothedGroundPosition = new THREE.Vector3();
+  private readonly movementDelta = new THREE.Vector3();
+  private hasSmoothedPosition = false;
+
+  constructor(private readonly rig: PuppetRig) {}
+
+  update(
+    targetGroundPosition: THREE.Vector3,
+    isMoving: boolean,
+    deltaSeconds: number,
+    elapsedSeconds: number,
+  ) {
+    if (!this.hasSmoothedPosition) {
+      this.smoothedGroundPosition.copy(targetGroundPosition);
+      this.rig.root.position.copy(targetGroundPosition);
+      this.hasSmoothedPosition = true;
+    }
+
+    this.movementDelta
+      .copy(targetGroundPosition)
+      .sub(this.smoothedGroundPosition);
+
+    if (this.movementDelta.lengthSq() <= 0.01) {
+      this.pause();
+      this.rig.root.position.copy(targetGroundPosition);
+      return;
+    }
+
+    const alpha = 1 - Math.exp(-12 * deltaSeconds);
+    this.smoothedGroundPosition.lerp(targetGroundPosition, alpha);
+    this.rig.root.position.copy(this.smoothedGroundPosition);
+
+    const targetRotationY = Math.atan2(
+      this.movementDelta.x,
+      this.movementDelta.z,
+    );
+    this.rig.root.rotation.y = this.dampAngle(
+      this.rig.root.rotation.y,
+      targetRotationY,
+      14,
+      deltaSeconds,
+    );
+
+    if (isMoving) {
+      this.animateWalkCycle(elapsedSeconds);
+    } else {
+      this.pause();
+    }
+  }
+
+  pause() {
+    this.rig.leftLeg.rotation.x *= 0.8;
+    this.rig.rightLeg.rotation.x *= 0.8;
+    this.rig.leftArm.rotation.x *= 0.8;
+    this.rig.rightArm.rotation.x *= 0.8;
+  }
+
+  private dampAngle(
+    currentRadians: number,
+    targetRadians: number,
+    smoothingSpeed: number,
+    deltaSeconds: number,
+  ) {
+    const deltaRadians = Math.atan2(
+      Math.sin(targetRadians - currentRadians),
+      Math.cos(targetRadians - currentRadians),
+    );
+    const alpha = 1 - Math.exp(-smoothingSpeed * deltaSeconds);
+    return currentRadians + deltaRadians * alpha;
+  }
+
+  private animateWalkCycle(elapsedSeconds: number) {
+    const swing = Math.sin(elapsedSeconds * 8) * 0.5;
+    this.rig.leftLeg.rotation.x = swing * 2;
+    this.rig.rightLeg.rotation.x = -swing * 2;
+    this.rig.leftArm.rotation.x = -swing * 2;
+    this.rig.rightArm.rotation.x = swing * 2;
+  }
+}

src/routes/scene/components/renderer/puppet/index.ts

@@ -0,0 +1,121 @@
+import type { PuppetState } from "$lib/bindings";
+import * as THREE from "three";
+import type { World } from "../world";
+import { PuppetAnimation } from "./animation";
+
+export type PuppetRig = {
+  root: THREE.Group;
+  leftArm: THREE.Group;
+  rightArm: THREE.Group;
+  leftLeg: THREE.Group;
+  rightLeg: THREE.Group;
+};
+
+export class Puppet {
+  readonly root: THREE.Group;
+
+  private readonly rig: PuppetRig;
+  private readonly animation: PuppetAnimation;
+  private readonly targetGroundPosition = new THREE.Vector3();
+
+  constructor(readonly id: string) {
+    this.rig = this.createRig();
+    this.root = this.rig.root;
+    this.animation = new PuppetAnimation(this.rig);
+  }
+
+  update(
+    state: PuppetState,
+    world: World,
+    frozen: boolean,
+    deltaSeconds: number,
+    elapsedSeconds: number,
+  ) {
+    if (frozen) {
+      this.animation.pause();
+      return;
+    }
+
+    world.normalizedPointToGroundPoint(
+      state.position.x,
+      state.position.y,
+      this.targetGroundPosition,
+    );
+    this.animation.update(
+      this.targetGroundPosition,
+      state.isMoving,
+      deltaSeconds,
+      elapsedSeconds,
+    );
+  }
+
+  dispose() {
+    const disposedMaterials = new Set<THREE.Material>();
+
+    this.root.traverse((object) => {
+      if (!(object instanceof THREE.Mesh)) return;
+
+      object.geometry.dispose();
+      const materials = Array.isArray(object.material)
+        ? object.material
+        : [object.material];
+      for (const material of materials) {
+        if (disposedMaterials.has(material)) continue;
+        material.dispose();
+        disposedMaterials.add(material);
+      }
+    });
+  }
+
+  private createRig(): PuppetRig {
+    const root = new THREE.Group();
+    const material = new THREE.MeshStandardMaterial({
+      color: this.colorFromId(),
+    });
+
+    const body = new THREE.Mesh(new THREE.BoxGeometry(16, 24, 8), material);
+    body.position.y = 36;
+
+    const head = new THREE.Mesh(new THREE.BoxGeometry(16, 16, 16), material);
+    head.position.y = 56;
+
+    const leftArm = this.createLimbPivot(-12, 48, 8, 24, 8, material);
+    const rightArm = this.createLimbPivot(12, 48, 8, 24, 8, material);
+    const leftLeg = this.createLimbPivot(-4, 24, 8, 24, 8, material);
+    const rightLeg = this.createLimbPivot(4, 24, 8, 24, 8, material);
+
+    root.add(body, head, leftArm, rightArm, leftLeg, rightLeg);
+
+    return { root, leftArm, rightArm, leftLeg, rightLeg };
+  }
+
+  private createLimbPivot(
+    x: number,
+    y: number,
+    width: number,
+    height: number,
+    depth: number,
+    material: THREE.Material,
+  ) {
+    const pivot = new THREE.Group();
+    pivot.position.set(x, y, 0);
+
+    const limb = new THREE.Mesh(
+      new THREE.BoxGeometry(width, height, depth),
+      material,
+    );
+    limb.position.y = -height / 2;
+    pivot.add(limb);
+
+    return pivot;
+  }
+
+  private colorFromId() {
+    let hash = 0;
+    for (const character of this.id) {
+      hash = (hash * 31 + character.charCodeAt(0)) >>> 0;
+    }
+
+    return new THREE.Color().setHSL((hash % 360) / 360, 0.55, 0.62);
+  }
+}

src/routes/scene/components/renderer/puppet/manager.ts

@@ -0,0 +1,66 @@
+import type { PuppetState } from "$lib/bindings";
+import type { PuppetScreenBounds } from "../types";
+import type { World } from "../world";
+import { Puppet } from ".";
+
+export class PuppetManager {
+  private readonly puppets = new Map<string, Puppet>();
+
+  constructor(private readonly world: World) {}
+
+  update(
+    states: readonly PuppetState[],
+    frozenPuppetId: string | null,
+    deltaSeconds: number,
+    elapsedSeconds: number,
+  ) {
+    this.syncPuppets(states);
+
+    for (const state of states) {
+      this.puppets
+        .get(state.id)
+        ?.update(
+          state,
+          this.world,
+          state.id === frozenPuppetId,
+          deltaSeconds,
+          elapsedSeconds,
+        );
+    }
+  }
+
+  screenBounds(): PuppetScreenBounds[] {
+    return [...this.puppets.values()].map((puppet) => ({
+      id: puppet.id,
+      ...this.world.objectScreenBounds(puppet.root),
+    }));
+  }
+
+  dispose() {
+    for (const puppet of this.puppets.values()) {
+      this.world.removeObject(puppet.root);
+      puppet.dispose();
+    }
+    this.puppets.clear();
+  }
+
+  private syncPuppets(states: readonly PuppetState[]) {
+    const nextPuppetIds = new Set(states.map((state) => state.id));
+
+    for (const state of states) {
+      if (this.puppets.has(state.id)) continue;
+
+      const puppet = new Puppet(state.id);
+      this.puppets.set(puppet.id, puppet);
+      this.world.addObject(puppet.root);
+    }
+
+    for (const [puppetId, puppet] of this.puppets) {
+      if (nextPuppetIds.has(puppetId)) continue;
+
+      this.world.removeObject(puppet.root);
+      puppet.dispose();
+      this.puppets.delete(puppetId);
+    }
+  }
+}

src/routes/scene/components/renderer/types.ts

@@ -0,0 +1,7 @@
+export type PuppetScreenBounds = {
+  id: string;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+};

src/routes/scene/components/renderer/world.ts

@@ -0,0 +1,127 @@
+import * as THREE from "three";
+
+export class World {
+  private readonly renderer: THREE.WebGLRenderer;
+  private readonly scene: THREE.Scene;
+  private readonly camera: THREE.OrthographicCamera;
+  private readonly objectBounds = new THREE.Box3();
+  private readonly projectedCorner = new THREE.Vector3();
+
+  constructor(private readonly container: HTMLDivElement) {
+    const { width, height } = this.getRenderSize();
+
+    this.renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
+    this.renderer.setPixelRatio(window.devicePixelRatio);
+    this.renderer.setSize(width, height);
+
+    this.camera = new THREE.OrthographicCamera(
+      width / -2,
+      width / 2,
+      height / 2,
+      height / -2,
+      0.1,
+      5_000,
+    );
+    this.camera.position.set(2_000, 1_000, 2_000);
+    this.camera.lookAt(0, 0, 0);
+    this.camera.updateProjectionMatrix();
+
+    this.scene = new THREE.Scene();
+    this.scene.background = null;
+
+    const ambientLight = new THREE.AmbientLight("white", 0.8);
+    const directionalLight = new THREE.DirectionalLight("white", 2);
+    directionalLight.position.set(2_000, 1_500, 1_500);
+    this.scene.add(ambientLight, directionalLight);
+
+    this.container.appendChild(this.renderer.domElement);
+  }
+
+  addObject(object: THREE.Object3D) {
+    this.scene.add(object);
+  }
+
+  removeObject(object: THREE.Object3D) {
+    this.scene.remove(object);
+  }
+
+  normalizedPointToGroundPoint(
+    normalizedX: number,
+    normalizedY: number,
+    target: THREE.Vector3,
+  ) {
+    const ndcX = normalizedX * 2 - 1;
+    const ndcY = -normalizedY * 2 + 1;
+    const rayOrigin = new THREE.Vector3(ndcX, ndcY, -1).unproject(this.camera);
+    const rayDirection = new THREE.Vector3(0, 0, -1)
+      .transformDirection(this.camera.matrixWorld)
+      .normalize();
+    const distanceToGround = -rayOrigin.y / rayDirection.y;
+
+    target.copy(rayOrigin).addScaledVector(rayDirection, distanceToGround);
+  }
+
+  objectScreenBounds(object: THREE.Object3D) {
+    object.updateWorldMatrix(true, true);
+    this.objectBounds.setFromObject(object);
+
+    const { min, max } = this.objectBounds;
+    let minX = Number.POSITIVE_INFINITY;
+    let minY = Number.POSITIVE_INFINITY;
+    let maxX = Number.NEGATIVE_INFINITY;
+    let maxY = Number.NEGATIVE_INFINITY;
+
+    for (const x of [min.x, max.x]) {
+      for (const y of [min.y, max.y]) {
+        for (const z of [min.z, max.z]) {
+          this.projectedCorner.set(x, y, z).project(this.camera);
+          minX = Math.min(minX, this.projectedCorner.x);
+          maxX = Math.max(maxX, this.projectedCorner.x);
+          minY = Math.min(minY, this.projectedCorner.y);
+          maxY = Math.max(maxY, this.projectedCorner.y);
+        }
+      }
+    }
+
+    const rect = this.container.getBoundingClientRect();
+    const left = rect.left + ((minX + 1) / 2) * rect.width;
+    const right = rect.left + ((maxX + 1) / 2) * rect.width;
+    const top = rect.top + ((1 - maxY) / 2) * rect.height;
+    const bottom = rect.top + ((1 - minY) / 2) * rect.height;
+
+    return {
+      x: left,
+      y: top,
+      width: right - left,
+      height: bottom - top,
+    };
+  }
+
+  render() {
+    this.renderer.render(this.scene, this.camera);
+  }
+
+  resizeWorld = () => {
+    const { width, height } = this.getRenderSize();
+
+    this.renderer.setPixelRatio(window.devicePixelRatio);
+    this.renderer.setSize(width, height, false);
+    this.camera.left = width / -2;
+    this.camera.right = width / 2;
+    this.camera.top = height / 2;
+    this.camera.bottom = height / -2;
+    this.camera.updateProjectionMatrix();
+  };
+
+  dispose() {
+    this.renderer.dispose();
+    this.renderer.domElement.remove();
+  }
+
+  private getRenderSize() {
+    return {
+      width: this.container.clientWidth,
+      height: this.container.clientHeight,
+    };
+  }
+}