minecraft characters.

src/routes/scene/+page.svelte

@@ -6,15 +6,27 @@
   import { incomingInteraction } from "$lib/listeners/interactions";
   import { liveMetadata } from "$lib/listeners/live-metadata";
   import { profile } from "$lib/listeners/profile";
+  import { puppetStates } from "$lib/listeners/puppets";
+  import Renderer from "./components/renderer.svelte";
+  import type { PuppetScreenBounds } from "./components/renderer/types";
+  import { startHitboxSync } from "./hitboxes";
   import SceneImageViewer from "./popovers/image-viewer.svelte";
   import SceneInteractionBubble from "./popovers/interaction-bubble.svelte";
   import SceneUserPopoverContent from "./popovers/user-interaction.svelte";
-  import { startHitboxSync } from "./hitboxes";
 
   let selectedUserId = $state<string | null>(null);
-  let selectedUserX = $state<number | null>(null);
   let lockedPopoverUserId = $state<string | null>(null);
   let viewedImage = $state<{ source: string; senderName: string } | null>(null);
+  let puppetBounds = $state<PuppetScreenBounds[]>([]);
+  let puppetBoundsById = $derived(
+    new Map(puppetBounds.map((bounds) => [bounds.id, bounds])),
+  );
+  let visiblePuppets = $derived(
+    $puppetStates.filter(
+      ({ id }) =>
+        id === $liveMetadata.localId || $onlineFriendIds.has(id),
+    ),
+  );
 
   onMount(startHitboxSync);
 
@@ -32,12 +44,9 @@
   $effect(() => {
     if (
       selectedUserId &&
-      selectedUserId !== $liveMetadata.localId &&
-      !$onlineFriendIds.has(selectedUserId)
+      !visiblePuppets.some((puppet) => puppet.id === selectedUserId)
     ) {
-      selectedUserId = null;
-      selectedUserX = null;
-      lockedPopoverUserId = null;
+      dismissPopover();
     }
   });
 
@@ -56,36 +65,52 @@
     );
   }
 
-  function setPopoverOpen(
-    userId: string,
-    cursorX: number,
-    open: boolean,
-    trigger: HTMLButtonElement,
-  ) {
+  function setPopoverOpen(userId: string, open: boolean) {
     if (open && lockedPopoverUserId && lockedPopoverUserId !== userId) return;
-    selectedUserId = open ? userId : null;
-    if (!open && lockedPopoverUserId === userId) lockedPopoverUserId = null;
-    selectedUserX = open
-      ? window.innerWidth > 0
-        ? trigger.getBoundingClientRect().left / window.innerWidth
-        : cursorX
-      : null;
+    if (open) {
+      selectedUserId = userId;
+    } else {
+      dismissPopover();
+    }
+  }
+
+  function dismissPopover() {
+    selectedUserId = null;
+    lockedPopoverUserId = null;
+  }
+
+  function popoverPosition(bounds: PuppetScreenBounds) {
+    const centerX = bounds.x + bounds.width / 2;
+    const horizontal = `position: fixed; left: clamp(0.5rem, calc(${centerX}px - 8.5rem), calc(100vw - 17.5rem));`;
+
+    return bounds.y > window.innerHeight / 2
+      ? `${horizontal} bottom: calc(100vh - ${bounds.y}px + 0.75rem);`
+      : `${horizontal} top: ${bounds.y + bounds.height + 12}px;`;
   }
 </script>
 
-<div class="flex size-full flex-col justify-end">
-  <div role="banner" class="relative my-2 h-8 w-full">
-    {#each Object.entries($liveMetadata.cursorPositions) as [userId, cursor], index (userId)}
+<div class="relative size-full">
+  <Renderer
+    puppets={visiblePuppets}
+    frozenPuppetId={selectedUserId}
+    onBoundsChange={(bounds) => (puppetBounds = bounds)}
+  />
+
+  <div role="banner" class="pointer-events-none fixed inset-0 z-10">
+    {#each visiblePuppets as puppet, index (puppet.id)}
+      {@const userId = puppet.id}
+      {@const bounds = puppetBoundsById.get(userId)}
       {@const foregroundApp = $liveMetadata.foregroundApps.get(userId)}
       {@const popoverId = `scene-user-${index}`}
       {@const interaction =
         $incomingInteraction?.friendId === userId ? $incomingInteraction : null}
-      {#if cursor && (userId === $liveMetadata.localId || $onlineFriendIds.has(userId))}
-        {@const renderedX =
-          selectedUserId === userId ? selectedUserX : cursor.mapped.x}
+      {#if bounds}
         <div
-          class="absolute bottom-0 left-0 flex flex-col items-center gap-1 transition-transform duration-1000 ease-linear"
-          style:transform={`translateX(${(renderedX ?? cursor.mapped.x) * 100}vw)`}
+          class="pointer-events-none fixed"
+          style:left={`${bounds.x}px`}
+          style:top={`${bounds.y}px`}
+          style:width={`${bounds.width}px`}
+          style:height={`${bounds.height}px`}
         >
           {#if interaction}
             <SceneInteractionBubble
@@ -103,7 +128,7 @@
             <img
               src={`data:image/png;base64,${foregroundApp.ico}`}
               alt=""
-              class="size-4 object-contain"
+              class="pointer-events-none absolute -top-5 left-1/2 size-4 -translate-x-1/2 object-contain"
             />
           {/if}
 
@@ -112,21 +137,17 @@
             label="Show live user information"
             labelledBy={`${popoverId}-title`}
             open={selectedUserId === userId}
-            onOpenChange={(open, trigger) =>
-              setPopoverOpen(userId, cursor.mapped.x, open, trigger)}
-            triggerClass="scene-hitbox"
-            panelClass="scene-hitbox bottom-[calc(100%+0.75rem)] w-68"
-            panelStyle={`left: clamp(calc(0.5rem - ${(renderedX ?? cursor.mapped.x) * 100}vw), -8rem, calc(100vw - ${(renderedX ?? cursor.mapped.x) * 100}vw - 17.5rem))`}
+            onOpenChange={(open) => setPopoverOpen(userId, open)}
+            class="size-full"
+            triggerClass="scene-hitbox size-full min-h-0"
+            panelClass="scene-hitbox fixed w-68"
+            panelStyle={popoverPosition(bounds)}
             closeOnOutsidePointer={lockedPopoverUserId !== userId}
             closeOnWindowBlur={lockedPopoverUserId !== userId}
             closeOnFocusOut={lockedPopoverUserId !== userId}
           >
             {#snippet trigger()}
-              <img
-                src="/fa.png"
-                alt=""
-                class="size-6 sepia saturate-200 hue-rotate-[100deg]"
-              />
+              <span class="block size-full"></span>
             {/snippet}
 
             <SceneUserPopoverContent
@@ -136,16 +157,8 @@
               {foregroundApp}
               onModeChange={(active) =>
                 (lockedPopoverUserId = active ? userId : null)}
-              onDismiss={() => {
-                selectedUserId = null;
-                selectedUserX = null;
-                lockedPopoverUserId = null;
-              }}
-              onSent={() => {
-                selectedUserId = null;
-                selectedUserX = null;
-                lockedPopoverUserId = null;
-              }}
+              onDismiss={dismissPopover}
+              onSent={dismissPopover}
             />
           </Popover>
         </div>

src/routes/scene/components/renderer.svelte

@@ -0,0 +1,99 @@
+<script lang="ts">
+  import type { PuppetState } from "$lib/bindings";
+  import { onDestroy, onMount } from "svelte";
+  import * as THREE from "three";
+  import { PuppetManager } from "./renderer/puppet/manager";
+  import type { PuppetScreenBounds } from "./renderer/types";
+  import { World } from "./renderer/world";
+
+  let {
+    puppets,
+    frozenPuppetId,
+    onBoundsChange,
+  }: {
+    puppets: readonly PuppetState[];
+    frozenPuppetId: string | null;
+    onBoundsChange: (bounds: PuppetScreenBounds[]) => void;
+  } = $props();
+
+  let renderDiv = $state<HTMLDivElement | null>(null);
+
+  let world: World | null = null;
+  let puppetManager: PuppetManager | null = null;
+  let resizeObserver: ResizeObserver | null = null;
+  let clock: THREE.Timer | null = null;
+  let animationFrameId = 0;
+  let lastBoundsUpdate = 0;
+  let previousBounds: PuppetScreenBounds[] = [];
+
+  function animate() {
+    if (!world || !puppetManager || !clock) return;
+
+    animationFrameId = requestAnimationFrame(animate);
+
+    clock.update();
+    puppetManager.update(
+      puppets,
+      frozenPuppetId,
+      clock.getDelta(),
+      clock.getElapsed(),
+    );
+    world.render();
+
+    const elapsed = clock.getElapsed();
+    if (elapsed - lastBoundsUpdate >= 1 / 30) {
+      lastBoundsUpdate = elapsed;
+      const nextBounds = puppetManager.screenBounds();
+      if (!sameBounds(previousBounds, nextBounds)) {
+        previousBounds = nextBounds;
+        onBoundsChange(nextBounds);
+      }
+    }
+  }
+
+  function sameBounds(
+    previous: PuppetScreenBounds[],
+    next: PuppetScreenBounds[],
+  ) {
+    if (previous.length !== next.length) return false;
+    return previous.every((bounds, index) => {
+      const candidate = next[index];
+      return (
+        candidate?.id === bounds.id &&
+        Math.abs(candidate.x - bounds.x) < 0.25 &&
+        Math.abs(candidate.y - bounds.y) < 0.25 &&
+        Math.abs(candidate.width - bounds.width) < 0.25 &&
+        Math.abs(candidate.height - bounds.height) < 0.25
+      );
+    });
+  }
+
+  onMount(() => {
+    if (!renderDiv) return;
+
+    world = new World(renderDiv);
+    puppetManager = new PuppetManager(world);
+    clock = new THREE.Timer();
+    clock.connect(document);
+
+    resizeObserver = new ResizeObserver(world.resizeWorld);
+    resizeObserver.observe(renderDiv);
+
+    animate();
+  });
+
+  onDestroy(() => {
+    cancelAnimationFrame(animationFrameId);
+    resizeObserver?.disconnect();
+    clock?.dispose();
+    puppetManager?.dispose();
+    world?.dispose();
+    onBoundsChange([]);
+  });
+</script>
+
+<div
+  class="pointer-events-none absolute inset-0 overflow-hidden"
+  aria-hidden="true"
+  bind:this={renderDiv}
+></div>

src/routes/scene/components/renderer/puppet/animation.ts

@@ -0,0 +1,83 @@
+import * as THREE from "three";
+import type { PuppetRig } from ".";
+
+export class PuppetAnimation {
+  private readonly smoothedGroundPosition = new THREE.Vector3();
+  private readonly movementDelta = new THREE.Vector3();
+  private hasSmoothedPosition = false;
+
+  constructor(private readonly rig: PuppetRig) {}
+
+  update(
+    targetGroundPosition: THREE.Vector3,
+    isMoving: boolean,
+    deltaSeconds: number,
+    elapsedSeconds: number,
+  ) {
+    if (!this.hasSmoothedPosition) {
+      this.smoothedGroundPosition.copy(targetGroundPosition);
+      this.rig.root.position.copy(targetGroundPosition);
+      this.hasSmoothedPosition = true;
+    }
+
+    this.movementDelta
+      .copy(targetGroundPosition)
+      .sub(this.smoothedGroundPosition);
+
+    if (this.movementDelta.lengthSq() <= 0.01) {
+      this.pause();
+      this.rig.root.position.copy(targetGroundPosition);
+      return;
+    }
+
+    const alpha = 1 - Math.exp(-12 * deltaSeconds);
+    this.smoothedGroundPosition.lerp(targetGroundPosition, alpha);
+    this.rig.root.position.copy(this.smoothedGroundPosition);
+
+    const targetRotationY = Math.atan2(
+      this.movementDelta.x,
+      this.movementDelta.z,
+    );
+    this.rig.root.rotation.y = this.dampAngle(
+      this.rig.root.rotation.y,
+      targetRotationY,
+      14,
+      deltaSeconds,
+    );
+
+    if (isMoving) {
+      this.animateWalkCycle(elapsedSeconds);
+    } else {
+      this.pause();
+    }
+  }
+
+  pause() {
+    this.rig.leftLeg.rotation.x *= 0.8;
+    this.rig.rightLeg.rotation.x *= 0.8;
+    this.rig.leftArm.rotation.x *= 0.8;
+    this.rig.rightArm.rotation.x *= 0.8;
+  }
+
+  private dampAngle(
+    currentRadians: number,
+    targetRadians: number,
+    smoothingSpeed: number,
+    deltaSeconds: number,
+  ) {
+    const deltaRadians = Math.atan2(
+      Math.sin(targetRadians - currentRadians),
+      Math.cos(targetRadians - currentRadians),
+    );
+    const alpha = 1 - Math.exp(-smoothingSpeed * deltaSeconds);
+    return currentRadians + deltaRadians * alpha;
+  }
+
+  private animateWalkCycle(elapsedSeconds: number) {
+    const swing = Math.sin(elapsedSeconds * 8) * 0.5;
+    this.rig.leftLeg.rotation.x = swing * 2;
+    this.rig.rightLeg.rotation.x = -swing * 2;
+    this.rig.leftArm.rotation.x = -swing * 2;
+    this.rig.rightArm.rotation.x = swing * 2;
+  }
+}

src/routes/scene/components/renderer/puppet/index.ts

@@ -0,0 +1,121 @@
+import type { PuppetState } from "$lib/bindings";
+import * as THREE from "three";
+import type { World } from "../world";
+import { PuppetAnimation } from "./animation";
+
+export type PuppetRig = {
+  root: THREE.Group;
+  leftArm: THREE.Group;
+  rightArm: THREE.Group;
+  leftLeg: THREE.Group;
+  rightLeg: THREE.Group;
+};
+
+export class Puppet {
+  readonly root: THREE.Group;
+
+  private readonly rig: PuppetRig;
+  private readonly animation: PuppetAnimation;
+  private readonly targetGroundPosition = new THREE.Vector3();
+
+  constructor(readonly id: string) {
+    this.rig = this.createRig();
+    this.root = this.rig.root;
+    this.animation = new PuppetAnimation(this.rig);
+  }
+
+  update(
+    state: PuppetState,
+    world: World,
+    frozen: boolean,
+    deltaSeconds: number,
+    elapsedSeconds: number,
+  ) {
+    if (frozen) {
+      this.animation.pause();
+      return;
+    }
+
+    world.normalizedPointToGroundPoint(
+      state.position.x,
+      state.position.y,
+      this.targetGroundPosition,
+    );
+    this.animation.update(
+      this.targetGroundPosition,
+      state.isMoving,
+      deltaSeconds,
+      elapsedSeconds,
+    );
+  }
+
+  dispose() {
+    const disposedMaterials = new Set<THREE.Material>();
+
+    this.root.traverse((object) => {
+      if (!(object instanceof THREE.Mesh)) return;
+
+      object.geometry.dispose();
+      const materials = Array.isArray(object.material)
+        ? object.material
+        : [object.material];
+      for (const material of materials) {
+        if (disposedMaterials.has(material)) continue;
+        material.dispose();
+        disposedMaterials.add(material);
+      }
+    });
+  }
+
+  private createRig(): PuppetRig {
+    const root = new THREE.Group();
+    const material = new THREE.MeshStandardMaterial({
+      color: this.colorFromId(),
+    });
+
+    const body = new THREE.Mesh(new THREE.BoxGeometry(16, 24, 8), material);
+    body.position.y = 36;
+
+    const head = new THREE.Mesh(new THREE.BoxGeometry(16, 16, 16), material);
+    head.position.y = 56;
+
+    const leftArm = this.createLimbPivot(-12, 48, 8, 24, 8, material);
+    const rightArm = this.createLimbPivot(12, 48, 8, 24, 8, material);
+    const leftLeg = this.createLimbPivot(-4, 24, 8, 24, 8, material);
+    const rightLeg = this.createLimbPivot(4, 24, 8, 24, 8, material);
+
+    root.add(body, head, leftArm, rightArm, leftLeg, rightLeg);
+
+    return { root, leftArm, rightArm, leftLeg, rightLeg };
+  }
+
+  private createLimbPivot(
+    x: number,
+    y: number,
+    width: number,
+    height: number,
+    depth: number,
+    material: THREE.Material,
+  ) {
+    const pivot = new THREE.Group();
+    pivot.position.set(x, y, 0);
+
+    const limb = new THREE.Mesh(
+      new THREE.BoxGeometry(width, height, depth),
+      material,
+    );
+    limb.position.y = -height / 2;
+    pivot.add(limb);
+
+    return pivot;
+  }
+
+  private colorFromId() {
+    let hash = 0;
+    for (const character of this.id) {
+      hash = (hash * 31 + character.charCodeAt(0)) >>> 0;
+    }
+
+    return new THREE.Color().setHSL((hash % 360) / 360, 0.55, 0.62);
+  }
+}

src/routes/scene/components/renderer/puppet/manager.ts

@@ -0,0 +1,66 @@
+import type { PuppetState } from "$lib/bindings";
+import type { PuppetScreenBounds } from "../types";
+import type { World } from "../world";
+import { Puppet } from ".";
+
+export class PuppetManager {
+  private readonly puppets = new Map<string, Puppet>();
+
+  constructor(private readonly world: World) {}
+
+  update(
+    states: readonly PuppetState[],
+    frozenPuppetId: string | null,
+    deltaSeconds: number,
+    elapsedSeconds: number,
+  ) {
+    this.syncPuppets(states);
+
+    for (const state of states) {
+      this.puppets
+        .get(state.id)
+        ?.update(
+          state,
+          this.world,
+          state.id === frozenPuppetId,
+          deltaSeconds,
+          elapsedSeconds,
+        );
+    }
+  }
+
+  screenBounds(): PuppetScreenBounds[] {
+    return [...this.puppets.values()].map((puppet) => ({
+      id: puppet.id,
+      ...this.world.objectScreenBounds(puppet.root),
+    }));
+  }
+
+  dispose() {
+    for (const puppet of this.puppets.values()) {
+      this.world.removeObject(puppet.root);
+      puppet.dispose();
+    }
+    this.puppets.clear();
+  }
+
+  private syncPuppets(states: readonly PuppetState[]) {
+    const nextPuppetIds = new Set(states.map((state) => state.id));
+
+    for (const state of states) {
+      if (this.puppets.has(state.id)) continue;
+
+      const puppet = new Puppet(state.id);
+      this.puppets.set(puppet.id, puppet);
+      this.world.addObject(puppet.root);
+    }
+
+    for (const [puppetId, puppet] of this.puppets) {
+      if (nextPuppetIds.has(puppetId)) continue;
+
+      this.world.removeObject(puppet.root);
+      puppet.dispose();
+      this.puppets.delete(puppetId);
+    }
+  }
+}

src/routes/scene/components/renderer/types.ts

@@ -0,0 +1,7 @@
+export type PuppetScreenBounds = {
+  id: string;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+};

src/routes/scene/components/renderer/world.ts

@@ -0,0 +1,127 @@
+import * as THREE from "three";
+
+export class World {
+  private readonly renderer: THREE.WebGLRenderer;
+  private readonly scene: THREE.Scene;
+  private readonly camera: THREE.OrthographicCamera;
+  private readonly objectBounds = new THREE.Box3();
+  private readonly projectedCorner = new THREE.Vector3();
+
+  constructor(private readonly container: HTMLDivElement) {
+    const { width, height } = this.getRenderSize();
+
+    this.renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
+    this.renderer.setPixelRatio(window.devicePixelRatio);
+    this.renderer.setSize(width, height);
+
+    this.camera = new THREE.OrthographicCamera(
+      width / -2,
+      width / 2,
+      height / 2,
+      height / -2,
+      0.1,
+      5_000,
+    );
+    this.camera.position.set(2_000, 1_000, 2_000);
+    this.camera.lookAt(0, 0, 0);
+    this.camera.updateProjectionMatrix();
+
+    this.scene = new THREE.Scene();
+    this.scene.background = null;
+
+    const ambientLight = new THREE.AmbientLight("white", 0.8);
+    const directionalLight = new THREE.DirectionalLight("white", 2);
+    directionalLight.position.set(2_000, 1_500, 1_500);
+    this.scene.add(ambientLight, directionalLight);
+
+    this.container.appendChild(this.renderer.domElement);
+  }
+
+  addObject(object: THREE.Object3D) {
+    this.scene.add(object);
+  }
+
+  removeObject(object: THREE.Object3D) {
+    this.scene.remove(object);
+  }
+
+  normalizedPointToGroundPoint(
+    normalizedX: number,
+    normalizedY: number,
+    target: THREE.Vector3,
+  ) {
+    const ndcX = normalizedX * 2 - 1;
+    const ndcY = -normalizedY * 2 + 1;
+    const rayOrigin = new THREE.Vector3(ndcX, ndcY, -1).unproject(this.camera);
+    const rayDirection = new THREE.Vector3(0, 0, -1)
+      .transformDirection(this.camera.matrixWorld)
+      .normalize();
+    const distanceToGround = -rayOrigin.y / rayDirection.y;
+
+    target.copy(rayOrigin).addScaledVector(rayDirection, distanceToGround);
+  }
+
+  objectScreenBounds(object: THREE.Object3D) {
+    object.updateWorldMatrix(true, true);
+    this.objectBounds.setFromObject(object);
+
+    const { min, max } = this.objectBounds;
+    let minX = Number.POSITIVE_INFINITY;
+    let minY = Number.POSITIVE_INFINITY;
+    let maxX = Number.NEGATIVE_INFINITY;
+    let maxY = Number.NEGATIVE_INFINITY;
+
+    for (const x of [min.x, max.x]) {
+      for (const y of [min.y, max.y]) {
+        for (const z of [min.z, max.z]) {
+          this.projectedCorner.set(x, y, z).project(this.camera);
+          minX = Math.min(minX, this.projectedCorner.x);
+          maxX = Math.max(maxX, this.projectedCorner.x);
+          minY = Math.min(minY, this.projectedCorner.y);
+          maxY = Math.max(maxY, this.projectedCorner.y);
+        }
+      }
+    }
+
+    const rect = this.container.getBoundingClientRect();
+    const left = rect.left + ((minX + 1) / 2) * rect.width;
+    const right = rect.left + ((maxX + 1) / 2) * rect.width;
+    const top = rect.top + ((1 - maxY) / 2) * rect.height;
+    const bottom = rect.top + ((1 - minY) / 2) * rect.height;
+
+    return {
+      x: left,
+      y: top,
+      width: right - left,
+      height: bottom - top,
+    };
+  }
+
+  render() {
+    this.renderer.render(this.scene, this.camera);
+  }
+
+  resizeWorld = () => {
+    const { width, height } = this.getRenderSize();
+
+    this.renderer.setPixelRatio(window.devicePixelRatio);
+    this.renderer.setSize(width, height, false);
+    this.camera.left = width / -2;
+    this.camera.right = width / 2;
+    this.camera.top = height / 2;
+    this.camera.bottom = height / -2;
+    this.camera.updateProjectionMatrix();
+  };
+
+  dispose() {
+    this.renderer.dispose();
+    this.renderer.domElement.remove();
+  }
+
+  private getRenderSize() {
+    return {
+      width: this.container.clientWidth,
+      height: this.container.clientHeight,
+    };
+  }
+}