From b1914ec7f453e24bd564ba4d60566c64399d2d4b Mon Sep 17 00:00:00 2001 From: Svjatoslav Agejenko Date: Wed, 9 Sep 2026 22:07:15 +0300 Subject: [PATCH] feat(headtrack): move head tracking into engine, add IMU diagnostics - drop product-side headtrack classes; the engine now auto-enables hot-plug head tracking for every app (Workspace delegates getHeadTracker() to ViewPanel) - add --headtest and --headdebug diagnostic modes to Main: live IMU angle dump and a periodic stream-health line (frames, tick, dt, max gyro, calibrated angles) - fix terminal fg/bg channel order in TerminalPanel cell caching; add PtySession.clear()/fillBox() - document the head tracking boundary exception in doc/index.org --- doc/index.org | 11 ++- .../sixth/bridge/pty/PtySession.java | 11 ++- .../java/eu/svjatoslav/sixth/core/Main.java | 77 +++++++++++++++++++ .../sixth/workspace/TerminalPanel.java | 13 ++-- .../svjatoslav/sixth/workspace/Workspace.java | 10 ++- 5 files changed, 111 insertions(+), 11 deletions(-) diff --git a/doc/index.org b/doc/index.org index 1c97181..8b4185e 100644 --- a/doc/index.org +++ b/doc/index.org @@ -79,10 +79,13 @@ time is limited. (~GuiComponent~, ~TextCanvas~, ~TextEditComponent~) that games can also use (in-game consoles, interactive screens). + *Sixth* is the concrete product built on that library: the virtual - workspace you actually work in. All application-integration bridges - (PTY/terminal, future VNC client, window capture) live here — the - engine must never gain native dependencies or application-domain - concepts. + workspace you actually work in. Application-integration bridges + (PTY/terminal, future VNC client, window capture) live here. + Exception to the "no native code in the engine" rule: XR glasses + head tracking (RayNeo IMU over hidraw, ~eu.svjatoslav.sixth.e3d.gui.headtrack~) + lives in the ENGINE, so every app and demo gets look-around head + tracking automatically when glasses are plugged in — including + hot-plug at runtime. + *Sixth 3D Demos* remains the capability gallery and regression harness. Workspace features are prototyped as demos, but the real thing lives in Sixth. diff --git a/src/main/java/eu/svjatoslav/sixth/bridge/pty/PtySession.java b/src/main/java/eu/svjatoslav/sixth/bridge/pty/PtySession.java index 30ec7a2..e444a87 100644 --- a/src/main/java/eu/svjatoslav/sixth/bridge/pty/PtySession.java +++ b/src/main/java/eu/svjatoslav/sixth/bridge/pty/PtySession.java @@ -110,7 +110,16 @@ public class PtySession { emulator.accept(chunk, count); final ContentListener listener = contentListener; if (listener != null) - listener.contentChanged(); + try { + listener.contentChanged(); + } catch (final Throwable t) { + // The renderer can briefly reallocate canvas + // internals during a window resize, racing this + // writer thread. A listener failure must never + // kill the reader loop — the next output batch + // repaints anyway. + t.printStackTrace(); + } } } catch (final IOException e) { // PTY closed: shell exited or session stopped diff --git a/src/main/java/eu/svjatoslav/sixth/core/Main.java b/src/main/java/eu/svjatoslav/sixth/core/Main.java index 64d0504..2e19b0c 100644 --- a/src/main/java/eu/svjatoslav/sixth/core/Main.java +++ b/src/main/java/eu/svjatoslav/sixth/core/Main.java @@ -29,6 +29,12 @@ public class Main { if (args.length > 0 && "--selftest".equals(args[0])) System.exit(runSelfTest(workspace)); + if (args.length > 0 && "--headtest".equals(args[0])) + System.exit(runHeadTest(workspace)); + + if (args.length > 0 && "--headdebug".equals(args[0])) + startHeadDebugLogger(workspace); + Runtime.getRuntime().addShutdownHook( new Thread(workspace::close)); } @@ -99,6 +105,77 @@ public class Main { return 0; } + /** + * Prints head tracker diagnostics every 2 seconds (frame flow, tick, + * dt, max gyro excursion, fused angles) so a live look-around test + * can be verified from the process log afterwards. + */ + private static void startHeadDebugLogger(final Workspace workspace) { + final Thread logger = new Thread(() -> { + while (true) { + final var tracker = workspace.getHeadTracker(); + System.out.println("HEADTRACK " + + (tracker == null ? "no device" + : tracker.getDebugString())); + try { + Thread.sleep(2000); + } catch (final InterruptedException e) { + return; + } + } + }, "head-debug-logger"); + logger.setDaemon(true); + logger.start(); + } + + /** + * Prints head tracker and camera angles for 30 seconds so axis/sign + * conventions can be verified against real head movement. Exit 0 if + * the tracker produced data at all. + */ + private static int runHeadTest(final Workspace workspace) + throws InterruptedException { + final var tracker = workspace.getHeadTracker(); + if (tracker == null) { + System.out.println("HEADTEST FAIL: no glasses detected"); + return 1; + } + // frame listeners do not run under xvfb (no render loop), so do + // the boot recenter here and compute what HeadLookController + // would apply, in-line + while (!tracker.isCalibrated()) + Thread.sleep(50); + Thread.sleep(1000); // let the filter settle + tracker.recenter(); + final var camera = workspace.getViewFrame().getViewPanel() + .getCamera(); + final double[] baseAngles = camera.getTransform().getRotation() + .toAngles(); + System.out.println("wait for calibration, then: turn head LEFT, " + + "watch yaw; nod DOWN, watch pitch (30s)"); + final long deadline = System.currentTimeMillis() + 30_000; + boolean sawMovement = false; + while (System.currentTimeMillis() < deadline) { + final double headYaw = tracker.getLookYaw(); + final double headPitch = tracker.getLookPitch(); + // frame listeners do not run under xvfb (no render loop), so + // compute what HeadLookController would apply, in-line + final double camYaw = baseAngles[0] - headYaw; + final double camPitch = baseAngles[1] + headPitch; + if (Math.abs(headYaw) > 0.05 || Math.abs(headPitch) > 0.05) + sawMovement = true; + System.out.printf("head yaw %+7.1f pitch %+7.1f deg | " + + "camera yaw %+7.3f pitch %+7.3f rad%n", + Math.toDegrees(headYaw), Math.toDegrees(headPitch), + camYaw, camPitch); + Thread.sleep(200); + } + workspace.close(); + System.out.println(sawMovement ? "HEADTEST PASS (movement seen)" + : "HEADTEST FAIL (no head movement detected)"); + return sawMovement ? 0 : 1; + } + private static boolean waitFor(final TerminalPanel terminal, final String needle) throws InterruptedException { diff --git a/src/main/java/eu/svjatoslav/sixth/workspace/TerminalPanel.java b/src/main/java/eu/svjatoslav/sixth/workspace/TerminalPanel.java index 71a9f2f..239690b 100644 --- a/src/main/java/eu/svjatoslav/sixth/workspace/TerminalPanel.java +++ b/src/main/java/eu/svjatoslav/sixth/workspace/TerminalPanel.java @@ -21,8 +21,8 @@ import java.awt.event.KeyEvent; * {@link PtySession}) rendered onto a {@link TextCanvas}. * *

Click the panel to focus it; keystrokes are translated to terminal - * input bytes and written to the PTY. ESC releases focus (engine focus - * model, same as the text editor).

+ * input bytes and written to the PTY. Shift+ESC releases focus (plain + * ESC goes to the shell — terminal programs need it).

* *

Terminal output arrives on the PTY reader thread, which copies the * screen buffer into the canvas and requests a repaint. This is the same @@ -159,11 +159,14 @@ public class TerminalPanel extends GuiComponent { @Override public boolean keyPressed(final KeyEvent event, final ViewPanel viewPanel) { - // Focus model: Ctrl+ESC releases focus (GuiComponent convention + // Focus model: Shift+ESC releases focus (GuiComponent convention // uses plain ESC, but terminal programs need ESC themselves — - // mc uses ESC for dialogs and ESC 0 for F10). + // mc uses ESC for dialogs and ESC 0 for F10. Terminal apps cannot + // distinguish Shift+ESC from plain ESC on the wire, so no program + // loses a binding; Ctrl+ESC was avoided because desktops commonly + // intercept it). if (event.getKeyChar() == '\u001B') { - if (event.isControlDown()) + if (event.isShiftDown()) return super.keyPressed(event, viewPanel); session.send("\u001B"); return true; diff --git a/src/main/java/eu/svjatoslav/sixth/workspace/Workspace.java b/src/main/java/eu/svjatoslav/sixth/workspace/Workspace.java index d2376b6..d220892 100644 --- a/src/main/java/eu/svjatoslav/sixth/workspace/Workspace.java +++ b/src/main/java/eu/svjatoslav/sixth/workspace/Workspace.java @@ -81,7 +81,7 @@ public class Workspace { editor.setText("Sixth workspace\n\n" + "Click a panel to focus it.\n" + "Type into it. ESC releases focus\n" - + "(in the terminal: Ctrl+ESC).\n\n" + + "(in the terminal: Shift+ESC).\n\n" + "The panel on the right is a real\n" + "bash shell running on a PTY.\n" + "Try: ls, mc, htop"); @@ -104,6 +104,14 @@ public class Workspace { return terminalPanel; } + public eu.svjatoslav.sixth.e3d.gui.headtrack.HeadTracker getHeadTracker() { + return viewFrame.getViewPanel().getHeadTracker(); + } + + public ViewFrame getViewFrame() { + return viewFrame; + } + /** * Shuts down background processes (the shell). */ -- 2.20.1