From: Svjatoslav Agejenko Date: Thu, 10 Sep 2026 15:30:42 +0000 (+0300) Subject: feat(workspace): embed live GUI apps via Xvfb capture with input forwarding X-Git-Url: http://www2.svjatoslav.eu/gitweb/?a=commitdiff_plain;h=7bf0044c670bafb7a46e86e6741a964ae2d1d165;p=sixth.git feat(workspace): embed live GUI apps via Xvfb capture with input forwarding Add a bridge/x11 package that runs GUI applications on a private, per-app Xvfb display and captures the screen into a texture: XvfbServer manages display lifecycle, X11Native reads the root window into caller-owned ARGB pixels over JNA, and GuiAppSession owns the app process (dbus-run-session isolation, generated Firefox profile that disables system extensions) plus a 10 Hz capture thread that repaints only on changed frames. FirefoxPanel renders the captured screen on a textured rectangle and, while focused, forwards input into the virtual display over XTEST: clicks at the perspective-correct texture coordinate (with an 80 ms button hold so Firefox synthesizes DOM clicks), keyboard events 1:1 through an AWT-to-X keysym table, scroll on both axes (X buttons 4-7), mouse back/forward (8/9, dispatched on press since AWT never fires CLICKED for extra buttons), and pointer hover motion. Shift+ESC or a middle click releases focus; focus loss releases server-side modifiers so Shift never sticks. Workspace now spawns a 4x4 grid of independent PTY terminals whose mouse wheel maps to arrow keys honoring DECCKM. UnixPty pads the posix_spawn structs to 512 bytes (the amd64 attribute block is 336) and destroys them after spawning. Main gains headless verification drivers: --guitest, --clicktest, --keytest, --scrolltest, --hovertest. TODO.org collects ideas for embodied agents in the world. --- diff --git a/TODO.org b/TODO.org new file mode 100644 index 0000000..6063160 --- /dev/null +++ b/TODO.org @@ -0,0 +1,22 @@ +* Put agents in 3D world + +- Agent has body that resembles human + - TODO: find good design for humanoid + +- Agent has ability to move around cyberspace + +- Agent has ability to see (project image from agent perspective) + +- Agent has ability to write and execute programs + +- Agent has ability to read and write documents in cyberspace + +- Agent can hear your voice (STT, whisper) + +- Agent can speak (TTS, Piper) + +- Local LLM as well as cloud based LLM can be used + - OpenRouter free LLMs + - Kimi K3 + + diff --git a/src/main/java/eu/svjatoslav/sixth/bridge/pty/UnixPty.java b/src/main/java/eu/svjatoslav/sixth/bridge/pty/UnixPty.java index 7ef4b70..40d8fbe 100644 --- a/src/main/java/eu/svjatoslav/sixth/bridge/pty/UnixPty.java +++ b/src/main/java/eu/svjatoslav/sixth/bridge/pty/UnixPty.java @@ -63,6 +63,10 @@ final class UnixPty { int posix_spawnattr_setflags(Pointer attributes, short flags); + int posix_spawn_file_actions_destroy(Pointer fileActions); + + int posix_spawnattr_destroy(Pointer attributes); + int close(int fd); int read(int fd, Pointer buffer, int count); @@ -129,8 +133,10 @@ final class UnixPty { // the master. POSIX_SPAWN_SETSID makes the child a session leader; // glibc applies it BEFORE file actions, so the addopen below (no // O_NOCTTY) also acquires the slave as the controlling terminal. - final Memory fileActions = new Memory(256); - final Memory attributes = new Memory(256); + // glibc's posix_spawnattr_t is 336 bytes on amd64; allocate a + // padded block for both structs rather than relying on ABI sizes. + final Memory fileActions = new Memory(512); + final Memory attributes = new Memory(512); CLib.INSTANCE.posix_spawn_file_actions_init(fileActions); CLib.INSTANCE.posix_spawnattr_init(attributes); CLib.INSTANCE.posix_spawnattr_setflags(attributes, @@ -163,6 +169,8 @@ final class UnixPty { final Memory pidResult = new Memory(4); final int rc = CLib.INSTANCE.posix_spawn(pidResult, "/bin/sh", fileActions, attributes, shellArgv, envp); + CLib.INSTANCE.posix_spawn_file_actions_destroy(fileActions); + CLib.INSTANCE.posix_spawnattr_destroy(attributes); if (rc != 0) throw new IllegalStateException("posix_spawn failed: " + rc); childPid = pidResult.getInt(0); diff --git a/src/main/java/eu/svjatoslav/sixth/bridge/x11/AwtKeysyms.java b/src/main/java/eu/svjatoslav/sixth/bridge/x11/AwtKeysyms.java new file mode 100644 index 0000000..cb83566 --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/bridge/x11/AwtKeysyms.java @@ -0,0 +1,159 @@ +/* Sixth spatial computing environment. Author: Svjatoslav Agejenko. This project is released under Creative Commons Zero (CC0) license. */ +package eu.svjatoslav.sixth.bridge.x11; + +import java.awt.event.KeyEvent; + +/** + * Translates AWT key events to X11 keysyms for injection into the virtual + * display via the X TEST extension. + * + *

Mapping is by {@code keyCode}, not {@code keyChar}: letters map to + * their unshifted keysyms and uppercase is produced by the real Shift key + * events, which are forwarded like any other key. This mirrors how a + * physical keyboard works and assumes the virtual server's keymap is a + * US-style layout (Xvfb default).

+ * + *

Keys with no obvious mapping fall back to the Unicode keysym scheme + * ({@code 0x01000000 | codepoint}) when the event carries a character — + * this covers characters like ä and ö when they exist in the keymap.

+ */ +final class AwtKeysyms { + + // X11 keysym values (from keysymdef.h) + static final long XK_BACK_SPACE = 0xFF08; + static final long XK_TAB = 0xFF09; + static final long XK_RETURN = 0xFF0D; + static final long XK_ESCAPE = 0xFF1B; + static final long XK_HOME = 0xFF50; + static final long XK_LEFT = 0xFF51; + static final long XK_UP = 0xFF52; + static final long XK_RIGHT = 0xFF53; + static final long XK_DOWN = 0xFF54; + static final long XK_PAGE_UP = 0xFF55; + static final long XK_PAGE_DOWN = 0xFF56; + static final long XK_END = 0xFF57; + static final long XK_INSERT = 0xFF63; + static final long XK_F1 = 0xFFBE; + static final long XK_KP_MULTIPLY = 0xFFAA; + static final long XK_KP_ADD = 0xFFAB; + static final long XK_KP_SUBTRACT = 0xFFAD; + static final long XK_KP_DECIMAL = 0xFFAE; + static final long XK_KP_DIVIDE = 0xFFAF; + static final long XK_KP_0 = 0xFFB0; + static final long XK_SHIFT_L = 0xFFE1; + static final long XK_CONTROL_L = 0xFFE3; + static final long XK_CAPS_LOCK = 0xFFE5; + static final long XK_ALT_L = 0xFFE9; + static final long XK_DELETE = 0xFFFF; + + private AwtKeysyms() { + } + + /** + * Maps an AWT key event to an X keysym. + * + * @return the keysym, or {@code -1} when the key cannot be represented + */ + static long keysymFor(final KeyEvent event) { + final int code = event.getKeyCode(); + + // letters: unshifted keysyms; shift state comes from real Shift + // key events + if (code >= KeyEvent.VK_A && code <= KeyEvent.VK_Z) + return 'a' + (code - KeyEvent.VK_A); + if (code >= KeyEvent.VK_0 && code <= KeyEvent.VK_9) + return '0' + (code - KeyEvent.VK_0); + if (code >= KeyEvent.VK_F1 && code <= KeyEvent.VK_F12) + return XK_F1 + (code - KeyEvent.VK_F1); + if (code >= KeyEvent.VK_NUMPAD0 && code <= KeyEvent.VK_NUMPAD9) + return XK_KP_0 + (code - KeyEvent.VK_NUMPAD0); + + switch (code) { + case KeyEvent.VK_BACK_SPACE: + return XK_BACK_SPACE; + case KeyEvent.VK_TAB: + return XK_TAB; + case KeyEvent.VK_ENTER: + return XK_RETURN; + case KeyEvent.VK_ESCAPE: + return XK_ESCAPE; + case KeyEvent.VK_INSERT: + return XK_INSERT; + case KeyEvent.VK_DELETE: + return XK_DELETE; + case KeyEvent.VK_HOME: + return XK_HOME; + case KeyEvent.VK_END: + return XK_END; + case KeyEvent.VK_PAGE_UP: + return XK_PAGE_UP; + case KeyEvent.VK_PAGE_DOWN: + return XK_PAGE_DOWN; + case KeyEvent.VK_LEFT: + return XK_LEFT; + case KeyEvent.VK_UP: + return XK_UP; + case KeyEvent.VK_RIGHT: + return XK_RIGHT; + case KeyEvent.VK_DOWN: + return XK_DOWN; + case KeyEvent.VK_SHIFT: + return XK_SHIFT_L; + case KeyEvent.VK_CONTROL: + return XK_CONTROL_L; + case KeyEvent.VK_ALT: + return XK_ALT_L; + case KeyEvent.VK_CAPS_LOCK: + return XK_CAPS_LOCK; + case KeyEvent.VK_MULTIPLY: + return XK_KP_MULTIPLY; + case KeyEvent.VK_ADD: + return XK_KP_ADD; + case KeyEvent.VK_SUBTRACT: + return XK_KP_SUBTRACT; + case KeyEvent.VK_DECIMAL: + return XK_KP_DECIMAL; + case KeyEvent.VK_DIVIDE: + return XK_KP_DIVIDE; + case KeyEvent.VK_SPACE: + return ' '; + case KeyEvent.VK_COMMA: + return ','; + case KeyEvent.VK_PERIOD: + return '.'; + case KeyEvent.VK_SLASH: + return '/'; + case KeyEvent.VK_BACK_SLASH: + return '\\'; + case KeyEvent.VK_SEMICOLON: + return ';'; + case KeyEvent.VK_QUOTE: + return '\''; + case KeyEvent.VK_OPEN_BRACKET: + return '['; + case KeyEvent.VK_CLOSE_BRACKET: + return ']'; + case KeyEvent.VK_BACK_QUOTE: + return '`'; + case KeyEvent.VK_MINUS: + return '-'; + case KeyEvent.VK_EQUALS: + return '='; + default: + return unicodeFallback(event.getKeyChar()); + } + } + + /** + * Unicode keysym fallback ({@code 0x01000000 | codepoint}) for + * characters outside the keyCode-based map, e.g. ä, ö, é. + */ + private static long unicodeFallback(final char keyChar) { + if (keyChar == KeyEvent.CHAR_UNDEFINED || keyChar < 0x20) + return -1; + if (keyChar < 0x7F) + // printable ASCII not covered above + return keyChar; + return 0x01000000L | keyChar; + } +} diff --git a/src/main/java/eu/svjatoslav/sixth/bridge/x11/GuiAppSession.java b/src/main/java/eu/svjatoslav/sixth/bridge/x11/GuiAppSession.java new file mode 100644 index 0000000..cc9019a --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/bridge/x11/GuiAppSession.java @@ -0,0 +1,298 @@ +/* + * Sixth spatial computing environment. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + */ +package eu.svjatoslav.sixth.bridge.x11; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import java.util.Map; + +/** + * A GUI application running on a private Xvfb display, with its screen + * continuously captured into a caller-owned ARGB pixel buffer. + * + *

The session owns three things: the Xvfb server, the application + * process, and a capture thread that grabs the root window at a fixed + * rate. The capture thread writes directly into the destination array + * (typically a texture's primary bitmap pixels) and invokes the frame + * listener only when the image actually changed, so a static screen costs + * no repaints.

+ */ +public final class GuiAppSession implements AutoCloseable { + + /** + * Called on the capture thread after the destination buffer was + * updated with a changed frame. + */ + public interface FrameListener { + void frameCaptured(); + } + + private static final long CAPTURE_INTERVAL_MS = 100; + + private final int width; + private final int height; + private final List command; + private final Map extraEnvironment; + private final boolean privateDbusSession; + private final File logFile; + + private XvfbServer server; + private Process application; + private X11Native.Connection connection; + private Thread captureThread; + private volatile boolean running; + + public GuiAppSession(final int width, final int height, + final List command, + final Map extraEnvironment) { + this(width, height, command, extraEnvironment, false); + } + + /** + * @param privateDbusSession wrap the app in {@code dbus-run-session} + * so it gets a private session bus. Needed + * because merely unsetting + * DBUS_SESSION_BUS_ADDRESS is not enough: + * GIO then falls back to the systemd user + * bus socket (/run/user/$UID/bus), where the + * app's desktop instance can still be found + * and command-line handling forwarded to it. + */ + public GuiAppSession(final int width, final int height, + final List command, + final Map extraEnvironment, + final boolean privateDbusSession) { + this.width = width; + this.height = height; + this.command = command; + this.extraEnvironment = extraEnvironment; + this.privateDbusSession = privateDbusSession; + logFile = new File("/tmp/sixth-xapp.log"); + } + + /** + * Starts Xvfb, launches the application onto it, and begins capturing. + * + * @param destination ARGB pixel buffer of exactly + * {@code width * height} ints; every captured frame + * is written into this array + * @param listener notified (on the capture thread) whenever a frame + * changed the buffer + */ + public void start(final int[] destination, final FrameListener listener) + throws IOException { + if (destination.length != width * height) + throw new IllegalArgumentException("destination buffer holds " + + destination.length + " pixels, expected " + + (width * height)); + + server = XvfbServer.start(width, height); + connection = new X11Native.Connection(server.getDisplayName()); + + final List effectiveCommand = privateDbusSession + ? java.util.stream.Stream.concat( + java.util.stream.Stream.of("dbus-run-session", "--"), + command.stream()).toList() + : command; + final ProcessBuilder builder = new ProcessBuilder(effectiveCommand); + final Map environment = builder.environment(); + environment.put("DISPLAY", server.getDisplayName()); + // Firefox/GTK apps locate an already-running instance through the + // SESSION D-Bus (GApplication), ignoring --no-remote and DISPLAY: + // the window then pops up on the user's desktop while our Xvfb + // screen stays black. Hide the session bus so the app is forced + // to be its own primary instance on the private display. + environment.remove("DBUS_SESSION_BUS_ADDRESS"); + environment.put("MOZ_NO_REMOTE", "1"); + // On Wayland desktops (GNOME/Mutter) Firefox ignores DISPLAY + // entirely and renders natively into the user's compositor — + // the window pops up on the real screen while the Xvfb display + // stays windowless. Force the X11 backend onto our display. + environment.remove("WAYLAND_DISPLAY"); + environment.put("MOZ_ENABLE_WAYLAND", "0"); + environment.putAll(extraEnvironment); + builder.redirectOutput(logFile); + builder.redirectError(logFile); + application = builder.start(); + + running = true; + captureThread = new Thread( + () -> captureLoop(destination, listener), "x11-capture"); + captureThread.setDaemon(true); + captureThread.start(); + } + + private void captureLoop(final int[] destination, + final FrameListener listener) { + final boolean debug = Boolean.getBoolean("sixth.xcapture.debug"); + int iterations = 0; + while (running) { + try { + final boolean changed = X11Native.captureRoot(connection, + width, height, destination); + if (debug && iterations++ % 10 == 0) { + final var distinct = new java.util.HashSet(); + for (int i = 0; i < destination.length; i += 97) + distinct.add(destination[i] & 0xFFFFFF); + System.out.println("XCAPTURE changed=" + changed + + " distinct=" + distinct.size() + + " appAlive=" + isApplicationRunning()); + } + if (changed) + listener.frameCaptured(); + } catch (final Throwable throwable) { + // capture failures (window closing, renderer racing the + // buffer) must never kill the capture thread + if (debug) + throwable.printStackTrace(); + } + try { + Thread.sleep(CAPTURE_INTERVAL_MS); + } catch (final InterruptedException e) { + return; + } + } + } + + /** + * Forwards a keyboard event (press or release) into the virtual + * display. Modifier keys are forwarded as ordinary keys, so the + * server-side modifier state tracks the user's real keyboard. + * + * @param event the AWT key event + * @param pressed {@code true} for key press, {@code false} for release + */ + public void sendKeyEvent(final java.awt.event.KeyEvent event, + final boolean pressed) { + if (connection == null || !isApplicationRunning()) + return; + final long keysym = AwtKeysyms.keysymFor(event); + if (keysym < 0) + return; + if (Boolean.getBoolean("sixth.xcapture.debug")) + System.out.println("XINPUT key keysym=0x" + + Long.toHexString(keysym) + (pressed ? " press" : " release") + + " on " + server.getDisplayName()); + X11Native.sendKeyEvent(connection, keysym, pressed); + } + + /** + * Releases Shift/Control/Alt server-side. Called when keyboard focus + * is lost mid-modifier (e.g. Shift+ESC pops focus while Shift is + * held), so the browser is not left with stuck modifiers. + */ + public void releaseModifiers() { + if (connection == null || !isApplicationRunning()) + return; + X11Native.sendKeyEvent(connection, AwtKeysyms.XK_SHIFT_L, false); + X11Native.sendKeyEvent(connection, AwtKeysyms.XK_CONTROL_L, false); + X11Native.sendKeyEvent(connection, AwtKeysyms.XK_ALT_L, false); + } + + /** + * Forwards scroll wheel input into the virtual display at the given + * screen coordinates (typically the current pointer hover position on + * the panel). + * + * @param verticalUnits positive = scroll down, negative = scroll up + * @param horizontalUnits positive = scroll right, negative = left + */ + public void sendScroll(final int verticalUnits, final int horizontalUnits, + final int x, final int y) { + if (connection == null || !isApplicationRunning()) + return; + final int clampedX = Math.max(0, Math.min(x, width - 1)); + final int clampedY = Math.max(0, Math.min(y, height - 1)); + X11Native.sendScroll(connection, clampedX, clampedY, verticalUnits, + horizontalUnits); + } + + /** + * Moves the pointer inside the virtual display without pressing + * buttons (hover forwarding). + */ + public void sendMouseMove(final int x, final int y) { + if (connection == null || !isApplicationRunning()) + return; + X11Native.sendMouseMove(connection, + Math.max(0, Math.min(x, width - 1)), + Math.max(0, Math.min(y, height - 1))); + } + + public boolean isApplicationRunning() { + return application != null && application.isAlive(); + } + + /** + * Forwards a mouse click into the virtual display at screen + * coordinates (x, y). The captured texture IS the virtual screen, so + * texture pixels map 1:1 onto these coordinates. The click is + * synthesized with the X TEST extension, which applications treat as + * real device input. + * + * @param x screen X coordinate (clamped into the screen) + * @param y screen Y coordinate (clamped into the screen) + * @param button X button number: 1 = left, 3 = right + */ + public void sendMouseClick(final int x, final int y, final int button) { + if (connection == null || !isApplicationRunning()) + return; + final int clampedX = Math.max(0, Math.min(x, width - 1)); + final int clampedY = Math.max(0, Math.min(y, height - 1)); + if (Boolean.getBoolean("sixth.xcapture.debug")) + System.out.println("XINPUT click at " + clampedX + "," + clampedY + + " button " + button + " on " + + server.getDisplayName()); + X11Native.sendMouseClick(connection, clampedX, clampedY, button); + } + + @Override + public void close() { + running = false; + if (captureThread != null) + captureThread.interrupt(); + if (application != null) + application.destroyForcibly(); + if (connection != null) + connection.close(); + if (server != null) + server.close(); + } + + /** + * Builds a session that runs Firefox on a fresh throwaway profile, + * sized to fill the whole virtual screen. + */ + public static GuiAppSession firefox(final int width, final int height, + final String url) + throws IOException { + final File profile = Files.createTempDirectory( + "sixth-firefox-profile").toFile(); + + // keep first-run noise out of the captured screen + final String userJs = """ + user_pref("browser.shell.checkDefaultBrowser", false); + user_pref("browser.aboutwelcome.enabled", false); + user_pref("datareporting.policy.dataSubmissionEnabled", false); + user_pref("datareporting.policy.firstRunURL", ""); + user_pref("trailhead.firstrun.didSeeAboutWelcome", true); + // stop Debian's system-wide extensions (Web eID) from + // hijacking the first-run session with an active tab + user_pref("extensions.autoDisableScopes", 15); + user_pref("extensions.shownSelectionUI", true); + """; + Files.writeString(new File(profile, "user.js").toPath(), userJs); + + return new GuiAppSession(width, height, + List.of("firefox", "--no-remote", "--new-instance", + "--profile", profile.getAbsolutePath(), + "--width", String.valueOf(width), + "--height", String.valueOf(height), + url), + Map.of(), true); + } +} diff --git a/src/main/java/eu/svjatoslav/sixth/bridge/x11/X11Native.java b/src/main/java/eu/svjatoslav/sixth/bridge/x11/X11Native.java new file mode 100644 index 0000000..4581ddf --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/bridge/x11/X11Native.java @@ -0,0 +1,339 @@ +/* + * Sixth spatial computing environment. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + */ +package eu.svjatoslav.sixth.bridge.x11; + +import com.sun.jna.Function; +import com.sun.jna.Library; +import com.sun.jna.Native; +import com.sun.jna.NativeLong; +import com.sun.jna.Pointer; + +/** + * Minimal libX11 bindings for screen capture via JNA. + * + *

Only what {@link X11Capture} needs: open/close a display, resolve the + * root window, and {@code XGetImage} a rectangular area into ARGB pixels. + * The XImage struct is read through manual field offsets (amd64 layout) + * instead of a JNA {@code Structure} — the struct embeds function pointers + * and exact alignment is easier to get right explicitly.

+ * + *

Linux-only by design; the workspace targets Linux.

+ */ +final class X11Native { + + /** ZPixmap format constant for XGetImage. */ + static final int Z_PIXMAP = 2; + + /** XImage field offsets (amd64 / LP64 layout). */ + private static final int IMAGE_WIDTH = 0; + private static final int IMAGE_HEIGHT = 4; + private static final int IMAGE_DATA = 16; + private static final int IMAGE_BYTE_ORDER = 24; + private static final int IMAGE_DEPTH = 40; + private static final int IMAGE_BYTES_PER_LINE = 44; + private static final int IMAGE_BITS_PER_PIXEL = 48; + private static final int IMAGE_RED_MASK = 56; + private static final int IMAGE_GREEN_MASK = 64; + private static final int IMAGE_BLUE_MASK = 72; + private static final int IMAGE_DESTROY_FUNCTION = 96; + + private static final int LSB_FIRST = 0; + + private interface X11Lib extends Library { + X11Lib INSTANCE = Native.load("X11", X11Lib.class); + + Pointer XOpenDisplay(String displayName); + + int XCloseDisplay(Pointer display); + + int XDefaultScreen(Pointer display); + + NativeLong XRootWindow(Pointer display, int screenNumber); + + Pointer XGetImage(Pointer display, NativeLong drawable, int x, int y, + int width, int height, NativeLong planeMask, + int format); + + int XFlush(Pointer display); + + /** + * Converts a keysym to a keycode in the server's current keymap. + * Returns 0 when the keysym is not present in the keymap. + */ + int XKeysymToKeycode(Pointer display, NativeLong keysym); + } + + /** + * libXtst (X TEST extension) bindings for synthesizing input. Fake + * input events are indistinguishable from real device events, unlike + * XSendEvent which many toolkits (GTK included) treat as untrusted. + */ + private interface XTestLib extends Library { + XTestLib INSTANCE = Native.load("Xtst", XTestLib.class); + + int XTestFakeMotionEvent(Pointer display, int screenNumber, + int x, int y, NativeLong delay); + + int XTestFakeButtonEvent(Pointer display, int button, + boolean isPress, NativeLong delay); + + /** + * Synthesizes a key press or release. + * + * @return nonzero on success + */ + int XTestFakeKeyEvent(Pointer display, int keycode, boolean isPress, + NativeLong delay); + } + + /** + * An open connection to an X server. + * + *

libX11 is not thread-safe unless XInitThreads was called; all + * requests on this connection are instead serialized through + * {@link #lock} (the capture loop and the input injector run on + * different threads).

+ */ + static final class Connection { + final Pointer display; + final NativeLong rootWindow; + final int screenNumber; + /** Serializes all X protocol traffic on this connection. */ + final Object lock = new Object(); + + Connection(final String displayName) { + display = X11Lib.INSTANCE.XOpenDisplay(displayName); + if (display == null) + throw new IllegalStateException( + "cannot open X display " + displayName); + screenNumber = X11Lib.INSTANCE.XDefaultScreen(display); + rootWindow = X11Lib.INSTANCE.XRootWindow(display, screenNumber); + } + + void close() { + synchronized (lock) { + if (display != null) + X11Lib.INSTANCE.XCloseDisplay(display); + } + } + } + + /** + * Grabs the top-left rectangle of the root window and converts it to + * ARGB, honoring the server's pixel masks, bits-per-pixel and byte + * order. Writes directly into {@code destination} and reports whether + * any pixel changed. + * + * @return true when at least one written pixel differs from the + * previous buffer content + */ + static boolean captureRoot(final Connection connection, + final int width, final int height, + final int[] destination) { + synchronized (connection.lock) { + final Pointer image = X11Lib.INSTANCE.XGetImage(connection.display, + connection.rootWindow, 0, 0, width, height, + new NativeLong(-1L), Z_PIXMAP); + if (image == null) + return false; + try { + return convert(image, width, height, destination); + } finally { + // XDestroyImage is a macro calling image->f.destroy_image + final Pointer destroyFunction = image.getPointer( + IMAGE_DESTROY_FUNCTION); + Function.getFunction(destroyFunction).invokeInt( + new Object[]{image}); + } + } + } + + /** + * Sends a single key press or release, identified by X keysym, to the + * virtual display. Modifier state is NOT synthesized here — real + * modifier key events (Shift/Ctrl/Alt) are forwarded like any other + * key, so the server-side modifier state tracks the user's keyboard. + * + * @param keysym the X keysym (see {@link AwtKeysyms}) + * @param press {@code true} for press, {@code false} for release + */ + static void sendKeyEvent(final Connection connection, final long keysym, + final boolean press) { + synchronized (connection.lock) { + final int keycode = X11Lib.INSTANCE.XKeysymToKeycode( + connection.display, new NativeLong(keysym)); + if (keycode == 0) + // keysym not present in the server keymap; nothing to send + return; + XTestLib.INSTANCE.XTestFakeKeyEvent(connection.display, keycode, + press, new NativeLong(0)); + X11Lib.INSTANCE.XFlush(connection.display); + } + } + + /** + * Sends scroll wheel input at the given root-window coordinates: moves + * the pointer there, then presses/releases the X scroll buttons + * (4 = up, 5 = down, 6 = left, 7 = right), one click per notch. + * + * @param verticalUnits positive = scroll down, negative = scroll up + * @param horizontalUnits positive = scroll right, negative = left + */ + static void sendScroll(final Connection connection, final int x, + final int y, final int verticalUnits, + final int horizontalUnits) { + synchronized (connection.lock) { + XTestLib.INSTANCE.XTestFakeMotionEvent(connection.display, + connection.screenNumber, x, y, new NativeLong(0)); + final int verticalButton = verticalUnits < 0 ? 4 : 5; + for (int i = 0; i < Math.abs(verticalUnits); i++) { + XTestLib.INSTANCE.XTestFakeButtonEvent(connection.display, + verticalButton, true, new NativeLong(0)); + XTestLib.INSTANCE.XTestFakeButtonEvent(connection.display, + verticalButton, false, new NativeLong(0)); + } + final int horizontalButton = horizontalUnits < 0 ? 6 : 7; + for (int i = 0; i < Math.abs(horizontalUnits); i++) { + XTestLib.INSTANCE.XTestFakeButtonEvent(connection.display, + horizontalButton, true, new NativeLong(0)); + XTestLib.INSTANCE.XTestFakeButtonEvent(connection.display, + horizontalButton, false, new NativeLong(0)); + } + X11Lib.INSTANCE.XFlush(connection.display); + } + } + + /** + * Moves the pointer to the given root-window coordinates without + * pressing any button (hover forwarding). + */ + static void sendMouseMove(final Connection conn, final int x, + final int y) { + synchronized (conn.lock) { + XTestLib.INSTANCE.XTestFakeMotionEvent(conn.display, + conn.screenNumber, x, y, new NativeLong(0)); + X11Lib.INSTANCE.XFlush(conn.display); + } + } + + /** + * Synthesizes a full mouse click (move pointer, press, release) at the + * given root-window coordinates. The pointer is moved first because a + * button press acts at the current pointer location. The button is + * held briefly because Firefox does not synthesize a DOM click from a + * zero-duration press+release. + * + * @param x root-window X coordinate + * @param y root-window Y coordinate + * @param button X button number (1 = left, 3 = right) + */ + static void sendMouseClick(final Connection connection, final int x, + final int y, final int button) { + synchronized (connection.lock) { + XTestLib.INSTANCE.XTestFakeMotionEvent(connection.display, + connection.screenNumber, x, y, new NativeLong(0)); + XTestLib.INSTANCE.XTestFakeButtonEvent(connection.display, button, + true, new NativeLong(0)); + X11Lib.INSTANCE.XFlush(connection.display); + // Firefox (EventStateManager) does not synthesize a DOM click + // from a zero-duration press+release; hold the button briefly + try { + Thread.sleep(80); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + XTestLib.INSTANCE.XTestFakeButtonEvent(connection.display, button, + false, new NativeLong(0)); + X11Lib.INSTANCE.XFlush(connection.display); + } + } + + private static boolean convert(final Pointer image, final int width, + final int height, final int[] destination) { + final int imageWidth = image.getInt(IMAGE_WIDTH); + final int imageHeight = image.getInt(IMAGE_HEIGHT); + final Pointer data = image.getPointer(IMAGE_DATA); + final int byteOrder = image.getInt(IMAGE_BYTE_ORDER); + final int bytesPerLine = image.getInt(IMAGE_BYTES_PER_LINE); + final int bitsPerPixel = image.getInt(IMAGE_BITS_PER_PIXEL); + final long redMask = image.getLong(IMAGE_RED_MASK); + final long greenMask = image.getLong(IMAGE_GREEN_MASK); + final long blueMask = image.getLong(IMAGE_BLUE_MASK); + + final Channel red = new Channel(redMask); + final Channel green = new Channel(greenMask); + final Channel blue = new Channel(blueMask); + + boolean changed = false; + final int copyWidth = Math.min(width, imageWidth); + final int copyHeight = Math.min(height, imageHeight); + for (int y = 0; y < copyHeight; y++) { + final int rowOffset = y * bytesPerLine; + final int destinationRow = y * width; + for (int x = 0; x < copyWidth; x++) { + final long pixelValue = readPixel(data, rowOffset, x, + bitsPerPixel, byteOrder); + final int argb = 0xFF000000 + | (red.extract(pixelValue) << 16) + | (green.extract(pixelValue) << 8) + | blue.extract(pixelValue); + final int index = destinationRow + x; + if (destination[index] != argb) { + destination[index] = argb; + changed = true; + } + } + } + return changed; + } + + private static long readPixel(final Pointer data, final int rowOffset, + final int x, final int bitsPerPixel, + final int byteOrder) { + if (bitsPerPixel == 32) { + final int value = data.getInt(rowOffset + (long) x * 4); + return Integer.toUnsignedLong( + byteOrder == LSB_FIRST ? value : Integer.reverseBytes(value)); + } + if (bitsPerPixel == 24) { + final long offset = rowOffset + (long) x * 3; + final int b0 = data.getByte(offset) & 0xFF; + final int b1 = data.getByte(offset + 1) & 0xFF; + final int b2 = data.getByte(offset + 2) & 0xFF; + return byteOrder == LSB_FIRST + ? b0 | (b1 << 8) | (b2 << 16) + : (b0 << 16) | (b1 << 8) | b2; + } + if (bitsPerPixel == 16) { + final short value = data.getShort(rowOffset + (long) x * 2); + return Short.toUnsignedLong( + byteOrder == LSB_FIRST ? value : Short.reverseBytes(value)); + } + throw new IllegalStateException( + "unsupported bits_per_pixel " + bitsPerPixel); + } + + /** + * Extracts one 8-bit color channel from a raw pixel using the server + * pixel mask (any position and width, scaled to 0..255). + */ + private static final class Channel { + private final long mask; + private final int shift; + private final long maxValue; + + Channel(final long mask) { + this.mask = mask; + shift = mask == 0 ? 0 : Long.numberOfTrailingZeros(mask); + maxValue = mask == 0 ? 1 : (mask >>> shift); + } + + int extract(final long pixelValue) { + if (mask == 0) + return 0; + return (int) (((pixelValue & mask) >>> shift) * 255 / maxValue); + } + } +} diff --git a/src/main/java/eu/svjatoslav/sixth/bridge/x11/XvfbServer.java b/src/main/java/eu/svjatoslav/sixth/bridge/x11/XvfbServer.java new file mode 100644 index 0000000..efe3ee7 --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/bridge/x11/XvfbServer.java @@ -0,0 +1,116 @@ +/* + * Sixth spatial computing environment. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + */ +package eu.svjatoslav.sixth.bridge.x11; + +import java.io.File; +import java.io.IOException; + +/** + * Lifecycle of a private {@code Xvfb} virtual X server that one captured + * GUI application renders into. + * + *

Each server gets its own display number (scanned upward from 90, + * skipping displays with an existing lock file or socket), so multiple + * captured apps never share a screen. Readiness is detected by polling for + * the unix socket and then proving the display with a real + * {@code XOpenDisplay} — just waiting for the socket is not enough.

+ */ +public final class XvfbServer implements AutoCloseable { + + private static final int FIRST_DISPLAY = 90; + private static final int LAST_DISPLAY = 200; + private static final long START_TIMEOUT_MS = 10_000; + + private final int displayNumber; + private final Process process; + + private XvfbServer(final int displayNumber, final Process process) { + this.displayNumber = displayNumber; + this.process = process; + } + + /** + * Starts an Xvfb server with a single screen of the given size and + * waits until the display actually accepts connections. + * + * @throws IOException if Xvfb fails to start within the timeout + */ + public static XvfbServer start(final int width, final int height) + throws IOException { + final int displayNumber = findFreeDisplay(); + final Process process = new ProcessBuilder("Xvfb", + ":" + displayNumber, + "-screen", "0", width + "x" + height + "x24", + "-nolisten", "tcp") + .redirectOutput(new File("/tmp/sixth-xvfb-" + + displayNumber + ".log")) + .redirectError(new File("/tmp/sixth-xvfb-" + + displayNumber + ".log")) + .start(); + + final XvfbServer server = new XvfbServer(displayNumber, process); + final long deadline = System.currentTimeMillis() + START_TIMEOUT_MS; + while (System.currentTimeMillis() < deadline) { + if (!process.isAlive()) + break; + if (new File(server.socketPath()).exists()) { + try { + final X11Native.Connection probe = + new X11Native.Connection(server.getDisplayName()); + probe.close(); + return server; + } catch (final Throwable ignored) { + // socket exists but server not accepting yet + } + } + try { + Thread.sleep(100); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + process.destroyForcibly(); + throw new IOException("Xvfb :" + displayNumber + + " did not become ready within " + START_TIMEOUT_MS + " ms"); + } + + private static int findFreeDisplay() throws IOException { + for (int number = FIRST_DISPLAY; number <= LAST_DISPLAY; number++) + if (!new File("/tmp/.X" + number + "-lock").exists() + && !new File(socketPath(number)).exists()) + return number; + throw new IOException("no free X display number between " + + FIRST_DISPLAY + " and " + LAST_DISPLAY); + } + + private static String socketPath(final int displayNumber) { + return "/tmp/.X11-unix/X" + displayNumber; + } + + private String socketPath() { + return socketPath(displayNumber); + } + + /** + * Display name for the {@code DISPLAY} environment variable + * (e.g. {@code ":93"}). + */ + public String getDisplayName() { + return ":" + displayNumber; + } + + @Override + public void close() { + process.destroy(); + try { + if (!process.waitFor(2, java.util.concurrent.TimeUnit.SECONDS)) + process.destroyForcibly(); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + process.destroyForcibly(); + } + } +} diff --git a/src/main/java/eu/svjatoslav/sixth/core/Main.java b/src/main/java/eu/svjatoslav/sixth/core/Main.java index 2e19b0c..b0d232e 100644 --- a/src/main/java/eu/svjatoslav/sixth/core/Main.java +++ b/src/main/java/eu/svjatoslav/sixth/core/Main.java @@ -4,6 +4,7 @@ */ package eu.svjatoslav.sixth.core; +import eu.svjatoslav.sixth.workspace.FirefoxPanel; import eu.svjatoslav.sixth.workspace.TerminalPanel; import eu.svjatoslav.sixth.workspace.Workspace; @@ -23,12 +24,39 @@ public class Main { private static final long SELFTEST_TIMEOUT_MS = 15_000; public static void main(final String[] args) throws Exception { + if (args.length > 0 && "--clicktest".equals(args[0])) + prepareClickTestPage(); + + if (args.length > 0 && "--keytest".equals(args[0])) + prepareKeyTestPage(); + + if (args.length > 0 && "--scrolltest".equals(args[0])) + prepareScrollTestPage(); + + if (args.length > 0 && "--hovertest".equals(args[0])) + prepareHoverTestPage(); + final Workspace workspace = new Workspace(); workspace.open(); if (args.length > 0 && "--selftest".equals(args[0])) System.exit(runSelfTest(workspace)); + if (args.length > 0 && "--guitest".equals(args[0])) + System.exit(runGuiTest(workspace)); + + if (args.length > 0 && "--clicktest".equals(args[0])) + System.exit(runClickTest(workspace)); + + if (args.length > 0 && "--keytest".equals(args[0])) + System.exit(runKeyTest(workspace)); + + if (args.length > 0 && "--scrolltest".equals(args[0])) + System.exit(runScrollTest(workspace)); + + if (args.length > 0 && "--hovertest".equals(args[0])) + System.exit(runHoverTest(workspace)); + if (args.length > 0 && "--headtest".equals(args[0])) System.exit(runHeadTest(workspace)); @@ -46,6 +74,26 @@ public class Main { private static int runSelfTest(final Workspace workspace) throws InterruptedException { final TerminalPanel terminal = workspace.getTerminalPanel(); + final var terminals = workspace.getTerminalPanels(); + final int expectedTerminalCount = Workspace.TERMINAL_GRID_COLUMNS + * Workspace.TERMINAL_GRID_ROWS; + if (terminals.size() != expectedTerminalCount) + return fail(terminal, "expected " + expectedTerminalCount + + " terminal panels, got " + terminals.size()); + final long sessionCount = terminals.stream() + .map(TerminalPanel::getSession) + .distinct() + .count(); + if (sessionCount != terminals.size()) + return fail(terminal, "terminal panels share PTY sessions: " + + sessionCount + " sessions for " + terminals.size() + + " panels"); + final long runningSessionCount = terminals.stream() + .filter(panel -> panel.getSession().isRunning()) + .count(); + if (runningSessionCount != terminals.size()) + return fail(terminal, "only " + runningSessionCount + " of " + + terminals.size() + " PTY sessions are running"); // wait for the real shell prompt (bashrc noise may contain a bare // "$", so match the user@host prompt text instead) @@ -176,6 +224,635 @@ public class Main { return sawMovement ? 0 : 1; } + /** + * Verifies the GUI-app bridge end to end without AWT events: waits + * for Firefox to paint into the panel texture, then checks the + * captured frame is a real image (not a flat fill). Exit 0 = pass. + */ + private static int runGuiTest(final Workspace workspace) + throws InterruptedException { + final var panel = workspace.getFirefoxPanel(); + final long deadline = System.currentTimeMillis() + 60_000; + while (System.currentTimeMillis() < deadline) { + if (panel.hasLiveFrame()) + break; + Thread.sleep(500); + } + if (!panel.hasLiveFrame()) { + System.out.println("GUITEST FAIL: no live frame within 60s" + + " (firefox running: " + + panel.getSession().isApplicationRunning() + ")"); + workspace.close(); + return 1; + } + + // a real browser screen has many distinct colors; a dead/black + // capture collapses to one or two + final int[] pixels = panel.getTexture().primaryBitmap.pixels; + final var distinctColors = new java.util.HashSet(); + long luminanceSum = 0; + int samples = 0; + for (int i = 0; i < pixels.length; i += 997) { + final int pixel = pixels[i]; + distinctColors.add(pixel); + luminanceSum += ((pixel >> 16) & 0xFF) + ((pixel >> 8) & 0xFF) + + (pixel & 0xFF); + samples++; + } + final double meanLuminance = (double) luminanceSum / samples / 3.0; + System.out.println("GUITEST stats: distinct sampled colors=" + + distinctColors.size() + " mean luminance=" + + String.format("%.1f", meanLuminance)); + if (distinctColors.size() < 16 || meanLuminance < 5) { + System.out.println("GUITEST FAIL: captured frame looks flat"); + workspace.close(); + return 1; + } + + System.out.println("GUITEST PASS — firefox is painting into the" + + " workspace panel"); + workspace.close(); + return 0; + } + + /** + * Verifies mouse click forwarding into the virtual display: loads a + * test page that paints the whole screen red on click, focuses the + * Firefox panel, sends a left click through the panel's mouse + * interaction path, and requires a large pixel change. Also verifies + * that a middle click releases focus and is NOT forwarded. Exit 0 = + * pass. + */ + private static int runClickTest(final Workspace workspace) + throws Exception { + final var panel = workspace.getFirefoxPanel(); + final long deadline = System.currentTimeMillis() + 60_000; + while (System.currentTimeMillis() < deadline) { + if (panel.hasLiveFrame()) + break; + Thread.sleep(500); + } + if (!panel.hasLiveFrame()) { + System.out.println("CLICKTEST FAIL: no live frame within 60s"); + workspace.close(); + return 1; + } + // let the page finish rendering after the first live frame + Thread.sleep(3000); + + final int[] pixels = panel.getTexture().primaryBitmap.pixels; + final int[] before = pixels.clone(); + + // click while unfocused: grabs focus only, page must not change + panel.mouseClicked( + eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_LEFT, + 640, 480); + Thread.sleep(2000); + if (countChangedPixels(before, pixels) > 1000) { + System.out.println("CLICKTEST FAIL: focus-grab click leaked" + + " into the page"); + workspace.close(); + return 1; + } + if (!panel.hasKeyboardFocus()) { + System.out.println("CLICKTEST FAIL: first click did not focus" + + " the panel"); + workspace.close(); + return 1; + } + + // left click while focused: forwarded, page turns red + System.out.println("CLICKTEST focus=" + panel.hasKeyboardFocus() + + " — sending forwarded click"); + panel.mouseClicked( + eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_LEFT, + 640, 480); + final long clickDeadline = System.currentTimeMillis() + 10_000; + int changed = 0; + while (System.currentTimeMillis() < clickDeadline) { + Thread.sleep(500); + changed = countChangedPixels(before, pixels); + if (changed > 100_000) + break; + } + if (changed <= 100_000) { + saveTexturePng(panel, "/tmp/clicktest-texture.png"); + System.out.println("CLICKTEST FAIL: forwarded click changed only " + + changed + " pixels (texture dump: " + + "/tmp/clicktest-texture.png)"); + workspace.close(); + return 1; + } + + // mouse back button: navigates back to the gradient page + final int[] redPage = pixels.clone(); + panel.mouseClicked( + eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_BACK, + 640, 480); + int backChanged = awaitPixelChange(pixels, redPage, 10_000); + if (backChanged <= 100_000) { + System.out.println("CLICKTEST FAIL: back button did not" + + " navigate back (" + backChanged + " pixels)"); + workspace.close(); + return 1; + } + + // mouse forward button: navigates forward to the red page + final int[] gradientPage = pixels.clone(); + panel.mouseClicked( + eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_FORWARD, + 640, 480); + int forwardChanged = awaitPixelChange(pixels, gradientPage, 10_000); + if (forwardChanged <= 100_000) { + System.out.println("CLICKTEST FAIL: forward button did not" + + " navigate forward (" + forwardChanged + " pixels)"); + workspace.close(); + return 1; + } + + // middle click: releases focus, not forwarded + final int[] red = pixels.clone(); + panel.mouseClicked( + eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_MIDDLE, + 640, 480); + Thread.sleep(2000); + if (panel.hasKeyboardFocus()) { + System.out.println("CLICKTEST FAIL: middle click did not" + + " release focus"); + workspace.close(); + return 1; + } + if (countChangedPixels(red, pixels) > 1000) { + System.out.println("CLICKTEST FAIL: middle click leaked into" + + " the page"); + workspace.close(); + return 1; + } + + System.out.println("CLICKTEST PASS — focus grab, forwarded click (" + + changed + " pixels changed), back/forward navigation," + + " middle-click release"); + workspace.close(); + return 0; + } + + /** + * Writes the hover-target page and points the Firefox panel at it. + * The page turns orange on a mousemove event in its top-left quadrant + * (clientX < 300 && clientY < 300). Must run BEFORE the + * workspace is constructed. + */ + private static void prepareHoverTestPage() throws Exception { + final java.io.File page = java.io.File.createTempFile( + "sixth-hovertest", ".html"); + page.deleteOnExit(); + java.nio.file.Files.writeString(page.toPath(), """ + HOVERTEST + +
+ + """); + System.setProperty("sixth.firefox.url", page.toURI().toString()); + } + + /** + * Verifies hover forwarding: while unfocused, hovering the panel must + * NOT move the pointer inside Firefox; once focused, hovering the + * top-left area must deliver a mousemove there (page turns orange). + */ + private static int runHoverTest(final Workspace workspace) + throws Exception { + final FirefoxPanel panel = workspace.getFirefoxPanel(); + if (panel == null) { + System.out.println("HOVERTEST FAIL: no firefox panel"); + workspace.close(); + return 1; + } + final long deadline = System.currentTimeMillis() + 60_000; + while (System.currentTimeMillis() < deadline) { + if (panel.hasLiveFrame()) + break; + Thread.sleep(500); + } + if (!panel.hasLiveFrame()) { + System.out.println("HOVERTEST FAIL: no live firefox capture"); + workspace.close(); + return 1; + } + Thread.sleep(2_000); + final int[] pixels = panel.getTexture().primaryBitmap.pixels; + + // unfocused hover must NOT reach the page + final int[] beforeHover = pixels.clone(); + panel.mouseHover(150, 200); + Thread.sleep(2_000); + if (countChangedPixels(beforeHover, pixels) > 100_000) { + System.out.println("HOVERTEST FAIL: unfocused hover leaked" + + " into the page"); + workspace.close(); + return 1; + } + + // focus (click at 640,480 — forwarded pointer lands outside the + // top-left quadrant, page stays unchanged) + panel.mouseClicked( + eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_LEFT, + 640, 480); + Thread.sleep(1_500); + if (!panel.hasKeyboardFocus()) { + System.out.println("HOVERTEST FAIL: focus was not acquired"); + workspace.close(); + return 1; + } + + // focused hover into the top-left quadrant must turn it orange + final int[] focused = pixels.clone(); + panel.mouseHover(150, 200); + final int changed = awaitPixelChange(pixels, focused, 10_000); + if (changed <= 100_000) { + System.out.println("HOVERTEST FAIL: focused hover changed only " + + changed + " pixels"); + workspace.close(); + return 1; + } + + System.out.println("HOVERTEST PASS — unfocused hover not forwarded," + + " focused hover moved the pointer (" + changed + + " pixels changed)"); + workspace.close(); + return 0; + } + + private static int awaitPixelChange(final int[] pixels, + final int[] reference, + final long timeoutMs) + throws InterruptedException { + final long deadline = System.currentTimeMillis() + timeoutMs; + int changed = 0; + while (System.currentTimeMillis() < deadline) { + Thread.sleep(500); + changed = countChangedPixels(reference, pixels); + if (changed > 100_000) + break; + } + return changed; + } + + /** + * Verifies keyboard forwarding into the virtual display: the test page + * has an input field that turns the whole screen green once it + * receives the text "hello". The panel is focused, the input is + * clicked (focus it in the page), then "hello" is typed as AWT key + * events through the panel's keyboard path. Also verifies that + * Shift+ESC releases focus. Exit 0 = pass. + */ + private static int runKeyTest(final Workspace workspace) + throws Exception { + final var panel = workspace.getFirefoxPanel(); + final long deadline = System.currentTimeMillis() + 60_000; + while (System.currentTimeMillis() < deadline) { + if (panel.hasLiveFrame()) + break; + Thread.sleep(500); + } + if (!panel.hasLiveFrame()) { + System.out.println("KEYTEST FAIL: no live frame within 60s"); + workspace.close(); + return 1; + } + Thread.sleep(3000); + + final int[] pixels = panel.getTexture().primaryBitmap.pixels; + + // focus the panel (first click grabs focus only) + panel.mouseClicked( + eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_LEFT, + 640, 480); + if (!panel.hasKeyboardFocus()) { + System.out.println("KEYTEST FAIL: first click did not focus" + + " the panel"); + workspace.close(); + return 1; + } + + // click the input field (page coords 600,280 + chrome offset + // lands it near texture 700,390) so typed text goes into it + panel.mouseClicked( + eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_LEFT, + 700, 390); + Thread.sleep(1500); + + // type "hello" as raw key events through the panel keyboard path + final int[] before = pixels.clone(); + final var viewPanel = workspace.getViewFrame().getViewPanel(); + for (final char c : "hello".toCharArray()) { + final int keyCode = java.awt.event.KeyEvent.getExtendedKeyCodeForChar(c); + panel.keyPressed(new java.awt.event.KeyEvent(viewPanel, + java.awt.event.KeyEvent.KEY_PRESSED, + System.currentTimeMillis(), 0, keyCode, c), viewPanel); + panel.keyReleased(new java.awt.event.KeyEvent(viewPanel, + java.awt.event.KeyEvent.KEY_RELEASED, + System.currentTimeMillis(), 0, keyCode, c), viewPanel); + Thread.sleep(150); + } + + // the page turns green once the input contains "hello" + final long typeDeadline = System.currentTimeMillis() + 15_000; + int changed = 0; + while (System.currentTimeMillis() < typeDeadline) { + Thread.sleep(500); + changed = countChangedPixels(before, pixels); + if (changed > 100_000) + break; + } + if (changed <= 100_000) { + saveTexturePng(panel, "/tmp/keytest-texture.png"); + System.out.println("KEYTEST FAIL: typing changed only " + + changed + " pixels (texture dump: " + + "/tmp/keytest-texture.png)"); + workspace.close(); + return 1; + } + + // Shift+ESC releases focus + final int shiftMods = java.awt.event.InputEvent.SHIFT_DOWN_MASK; + panel.keyPressed(new java.awt.event.KeyEvent(viewPanel, + java.awt.event.KeyEvent.KEY_PRESSED, + System.currentTimeMillis(), shiftMods, + java.awt.event.KeyEvent.VK_SHIFT, + java.awt.event.KeyEvent.CHAR_UNDEFINED), viewPanel); + panel.keyPressed(new java.awt.event.KeyEvent(viewPanel, + java.awt.event.KeyEvent.KEY_PRESSED, + System.currentTimeMillis(), shiftMods, + java.awt.event.KeyEvent.VK_ESCAPE, ''), viewPanel); + Thread.sleep(500); + if (panel.hasKeyboardFocus()) { + System.out.println("KEYTEST FAIL: Shift+ESC did not release" + + " focus"); + workspace.close(); + return 1; + } + + System.out.println("KEYTEST PASS — typed text reached the page (" + + changed + " pixels changed), Shift+ESC released focus"); + workspace.close(); + return 0; + } + + /** + * Verifies scroll wheel forwarding on both axes: + *
    + *
  • Firefox: the test page counts wheel events (deltaY and + * deltaX) and turns blue once it has seen 2 notches down and + * 2 notches left.
    • + *
  • Terminal: the wheel becomes arrow keys — wheel-up at a prompt + * with a drafted line recalls history, wheel-left moves the + * cursor inside the drafted line.
    • + *
+ * Exit 0 = pass. + */ + private static int runScrollTest(final Workspace workspace) + throws Exception { + final var panel = workspace.getFirefoxPanel(); + final long deadline = System.currentTimeMillis() + 60_000; + while (System.currentTimeMillis() < deadline) { + if (panel.hasLiveFrame()) + break; + Thread.sleep(500); + } + if (!panel.hasLiveFrame()) { + System.out.println("SCROLLTEST FAIL: no live frame within 60s"); + workspace.close(); + return 1; + } + Thread.sleep(3000); + + // --- firefox: both wheel axes reach the page --- + panel.mouseClicked( + eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_LEFT, + 640, 480); + if (!panel.hasKeyboardFocus()) { + System.out.println("SCROLLTEST FAIL: first click did not focus" + + " the panel"); + workspace.close(); + return 1; + } + + final int[] pixels = panel.getTexture().primaryBitmap.pixels; + final int[] before = pixels.clone(); + panel.mouseWheelMoved(2, 0); // two notches down + Thread.sleep(300); + panel.mouseWheelMoved(0, -2); // two notches left + final long scrollDeadline = System.currentTimeMillis() + 15_000; + int changed = 0; + while (System.currentTimeMillis() < scrollDeadline) { + Thread.sleep(500); + changed = countChangedPixels(before, pixels); + if (changed > 100_000) + break; + } + if (changed <= 100_000) { + saveTexturePng(panel, "/tmp/scrolltest-texture.png"); + System.out.println("SCROLLTEST FAIL: wheel changed only " + + changed + " pixels (texture dump: " + + "/tmp/scrolltest-texture.png)"); + workspace.close(); + return 1; + } + + // --- terminal: wheel becomes arrow keys --- + final TerminalPanel terminal = workspace.getTerminalPanel(); + if (!waitFor(terminal, "n0@tiny")) { + System.out.println("SCROLLTEST FAIL: no shell prompt appeared"); + workspace.close(); + return 1; + } + // controlled history: wheel-up is 3 arrow-ups per notch, so after + // A, B, C the recalled line is deterministically A + terminal.typeText("echo WHEEL_A\r"); + if (!waitFor(terminal, "WHEEL_A")) { + System.out.println("SCROLLTEST FAIL: base command A failed"); + workspace.close(); + return 1; + } + terminal.typeText("echo WHEEL_B\r"); + Thread.sleep(700); + terminal.typeText("echo WHEEL_C\r"); + Thread.sleep(700); + + // draft a partial command, then wheel up: bash history recall + // (3 up-arrows per notch) replaces the draft with WHEEL_A's line + terminal.typeText("echo DRAFT"); + Thread.sleep(500); + terminal.mouseWheelMoved(-1, 0); + Thread.sleep(1000); + final String screenAfterUp = terminal.getScreenText().trim(); + if (!screenAfterUp.endsWith("echo WHEEL_A")) { + System.out.println("SCROLLTEST FAIL: wheel-up did not recall" + + " history; screen:\n" + screenAfterUp); + workspace.close(); + return 1; + } + + // wheel left: cursor moves into the recalled line, typed text + // lands in the middle of it: "echo WHEEL_A" (12 chars), 6 + // cursor-lefts -> insert at position 6 -> "echo WQHEEL_A" + terminal.mouseWheelMoved(0, -2); // 6 cursor-lefts + Thread.sleep(500); + terminal.typeText("Q"); + Thread.sleep(500); + if (!terminal.getScreenText().trim().endsWith("echo WQHEEL_A")) { + System.out.println("SCROLLTEST FAIL: horizontal wheel did not" + + " move the cursor left; screen:\n" + + terminal.getScreenText()); + workspace.close(); + return 1; + } + + System.out.println("SCROLLTEST PASS — firefox wheel both axes (" + + changed + " pixels changed), terminal wheel-as-arrows"); + workspace.close(); + return 0; + } + + /** + * Writes the scroll-target page and points the Firefox panel at it. + * Must run BEFORE the workspace is constructed, because the panel + * reads the URL system property in its constructor. + */ + private static void prepareScrollTestPage() throws Exception { + final java.io.File page = java.io.File.createTempFile( + "sixth-scrolltest", ".html"); + page.deleteOnExit(); + java.nio.file.Files.writeString(page.toPath(), """ + SCROLL none + +
+
+ Scroll down twice and left twice.
+ + """); + System.setProperty("sixth.firefox.url", page.toURI().toString()); + } + + /** + * Writes the keyboard-target page and points the Firefox panel at it. + * Must run BEFORE the workspace is constructed, because the panel + * reads the URL system property in its constructor. + */ + private static void prepareKeyTestPage() throws Exception { + final java.io.File page = java.io.File.createTempFile( + "sixth-keytest", ".html"); + page.deleteOnExit(); + java.nio.file.Files.writeString(page.toPath(), """ + KEYTEST none + +
+ + + """); + System.setProperty("sixth.firefox.url", page.toURI().toString()); + } + + /** + * Writes the click-target page and points the Firefox panel at it. + * Must run BEFORE the workspace is constructed, because the panel + * reads the URL system property in its constructor. + */ + private static void prepareClickTestPage() throws Exception { + // two pages linked by real navigation, so the mouse back/forward + // buttons have history to walk + final java.io.File page2 = java.io.File.createTempFile( + "sixth-clicktest-2", ".html"); + page2.deleteOnExit(); + java.nio.file.Files.writeString(page2.toPath(), """ + WITNESS page2 + +

CLICKED

+ + """); + + final java.io.File page1 = java.io.File.createTempFile( + "sixth-clicktest-1", ".html"); + page1.deleteOnExit(); + java.nio.file.Files.writeString(page1.toPath(), """ + WITNESS none + +
+
+ Click anywhere to navigate to the red page.
+ + """.replace("PAGE2URL", page2.toURI().toString())); + System.setProperty("sixth.firefox.url", page1.toURI().toString()); + } + + private static void saveTexturePng(final eu.svjatoslav.sixth.workspace.FirefoxPanel panel, + final String path) throws Exception { + final int w = eu.svjatoslav.sixth.workspace.FirefoxPanel.CAPTURE_WIDTH; + final int h = eu.svjatoslav.sixth.workspace.FirefoxPanel.CAPTURE_HEIGHT; + final var image = new java.awt.image.BufferedImage(w, h, + java.awt.image.BufferedImage.TYPE_INT_ARGB); + image.setRGB(0, 0, w, h, panel.getTexture().primaryBitmap.pixels, 0, w); + javax.imageio.ImageIO.write(image, "png", new java.io.File(path)); + } + + private static int countChangedPixels(final int[] a, final int[] b) { + int changed = 0; + for (int i = 0; i < a.length; i++) + if (a[i] != b[i]) + changed++; + return changed; + } + private static boolean waitFor(final TerminalPanel terminal, final String needle) throws InterruptedException { diff --git a/src/main/java/eu/svjatoslav/sixth/workspace/FirefoxPanel.java b/src/main/java/eu/svjatoslav/sixth/workspace/FirefoxPanel.java new file mode 100644 index 0000000..a1bdd23 --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/workspace/FirefoxPanel.java @@ -0,0 +1,242 @@ +/* + * Sixth spatial computing environment. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + */ +package eu.svjatoslav.sixth.workspace; + +import eu.svjatoslav.sixth.bridge.x11.GuiAppSession; +import eu.svjatoslav.sixth.e3d.geometry.Point2D; +import eu.svjatoslav.sixth.e3d.gui.GuiComponent; +import eu.svjatoslav.sixth.e3d.gui.ViewPanel; +import eu.svjatoslav.sixth.e3d.math.Transform; +import eu.svjatoslav.sixth.e3d.renderer.raster.shapes.composite.TexturedRectangle; +import eu.svjatoslav.sixth.e3d.renderer.raster.texture.Texture; + +import java.io.IOException; + +/** + * A live Firefox browser panel in 3D space. + * + *

Firefox runs on a private Xvfb display (via {@link GuiAppSession}); + * a capture thread copies the virtual screen into the texture of a + * {@link TexturedRectangle}, so the browser window appears as an object + * in the world — including in stereo/VR rendering, like every other + * shape.

+ * + *

Interaction: the first click focuses the panel; while focused, + * left/right clicks are forwarded into the browser via the X TEST + * extension at the exact clicked position (the engine reports the + * perspective-correct texture coordinate of the hit), and all keyboard + * input is forwarded as X key events. Middle click or Shift+ESC releases + * focus and is never forwarded; plain ESC goes to the browser. The + * capture thread writes the texture's primary bitmap + * directly and requests a repaint — the same benign writer/renderer race + * the terminal panel already accepts.

+ */ +public class FirefoxPanel extends GuiComponent { + + /** + * Virtual screen (and texture) resolution: the browser window fills + * the whole Xvfb screen, so this is also the captured window size. + */ + public static final int CAPTURE_WIDTH = 1280; + public static final int CAPTURE_HEIGHT = 960; + + private final TexturedRectangle rectangle; + private final GuiAppSession session; + private volatile boolean liveFrameSeen; + private double lastHoverU = Double.NaN; + private double lastHoverV = Double.NaN; + private int lastForwardedHoverX = -1; + private int lastForwardedHoverY = -1; + + /** + * Creates the panel. Call {@link #start()} to launch the browser and + * begin streaming its screen into the texture. + * + * @param transform position in the world + * @param viewPanel the view panel this component belongs to + * @param sizeInWorldCoordinates panel size in world units; the browser + * capture is stretched to fill it + */ + public FirefoxPanel(final Transform transform, final ViewPanel viewPanel, + final Point2D sizeInWorldCoordinates) + throws IOException { + super(transform, viewPanel, sizeInWorldCoordinates.to3D()); + + rectangle = new TexturedRectangle(new Transform(), + (int) sizeInWorldCoordinates.x, + (int) sizeInWorldCoordinates.y, + CAPTURE_WIDTH, CAPTURE_HEIGHT, 1); + rectangle.setMouseInteractionController(this); + addShape(rectangle); + + session = GuiAppSession.firefox(CAPTURE_WIDTH, CAPTURE_HEIGHT, + System.getProperty("sixth.firefox.url", "about:home")); + } + + /** + * Launches Xvfb + Firefox and starts the screen capture. + */ + public void start() throws IOException { + session.start(rectangle.getTexture().primaryBitmap.pixels, + this::onFrameCaptured); + } + + private void onFrameCaptured() { + try { + // captured pixels are already in the primary bitmap; drop the + // stale mipmaps so they regenerate lazily from the new frame + rectangle.getTexture().resetResampledBitmapCache(); + if (!liveFrameSeen) + liveFrameSeen = looksLikeRealScreen( + rectangle.getTexture().primaryBitmap.pixels); + viewPanel.repaintDuringNextViewUpdate(); + } catch (final Throwable throwable) { + // renderer may briefly reallocate internals (e.g. on window + // resize); never let that kill the capture thread + } + } + + /** + * A real application screen has many distinct colors; an empty Xvfb + * root (opaque black) collapses to a single one. The alpha channel + * is ignored: the very first capture of a black screen differs from + * the zero-initialized buffer in alpha only, and must not count. + */ + static boolean looksLikeRealScreen(final int[] pixels) { + final var distinctColors = new java.util.HashSet(); + // dense enough to hit antialiased text and icons: a very coarse + // stride sees only the few flat background colors of a mostly + // empty page and reports a live browser screen as "flat" + for (int i = 0; i < pixels.length; i += 997) + distinctColors.add(pixels[i] & 0xFFFFFF); + return distinctColors.size() >= 16; + } + + /** + * Whether the browser has painted a real screen into the texture + * (many distinct colors) — used by the automated GUI test. A black + * empty Xvfb root does not count. + */ + public boolean hasLiveFrame() { + return liveFrameSeen; + } + + /** + * The texture the captured browser screen is written into. + */ + public Texture getTexture() { + return rectangle.getTexture(); + } + + public GuiAppSession getSession() { + return session; + } + + /** + * Focus and click-forwarding behavior: + *
    + *
  • click while unfocused: take keyboard focus (first click is a + * focus grab, not forwarded to the browser)
    • + *
  • left/right click while focused: forwarded into the browser at + * the clicked texture position (texture pixels map 1:1 onto the + * virtual screen, and the browser window fills it at +0+0)
    • + *
  • middle click: releases focus (workspace convention, like ESC) + * and is never forwarded to the browser
    • + *
+ */ + @Override + public boolean mouseClicked(final int button, final double textureU, + final double textureV) { + if (!hasKeyboardFocus() + || button == eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_MIDDLE + || Double.isNaN(textureU) || Double.isNaN(textureV)) + return super.mouseClicked(button); + + // Mouse back/forward buttons are X buttons 8/9. AWT reports them + // as 6/7 on Linux and 4/5 on other platforms — accept both. + final int xButton = switch (button) { + case 4, 6 -> 8; // back + case 5, 7 -> 9; // forward + default -> button; + }; + session.sendMouseClick((int) textureU, (int) textureV, xButton); + return true; + } + + @Override + public boolean mouseHover(final double textureU, final double textureV) { + lastHoverU = textureU; + lastHoverV = textureV; + // while focused the app follows the mouse cursor without clicking + if (hasKeyboardFocus() && !Double.isNaN(textureU)) { + final int x = (int) textureU; + final int y = (int) textureV; + if (x != lastForwardedHoverX || y != lastForwardedHoverY) { + lastForwardedHoverX = x; + lastForwardedHoverY = y; + session.sendMouseMove(x, y); + } + } + return false; // no repaint needed + } + + @Override + public boolean mouseWheelMoved(final int verticalUnits, + final int horizontalUnits) { + // scroll at the current pointer position so the element under the + // cursor scrolls, like on a real desktop + final int x = Double.isNaN(lastHoverU) ? CAPTURE_WIDTH / 2 + : (int) lastHoverU; + final int y = Double.isNaN(lastHoverV) ? CAPTURE_HEIGHT / 2 + : (int) lastHoverV; + session.sendScroll(verticalUnits, horizontalUnits, x, y); + return true; + } + + @Override + public boolean keyPressed(final java.awt.event.KeyEvent event, + final ViewPanel viewPanel) { + if (isShiftEscape(event)) + // focus-release shortcut, never forwarded to the browser + return super.keyPressed(event, viewPanel); + session.sendKeyEvent(event, true); + return true; + } + + @Override + public boolean keyReleased(final java.awt.event.KeyEvent event, + final ViewPanel viewPanel) { + if (isShiftEscape(event)) + // press side already consumed this shortcut; swallow release + return true; + session.sendKeyEvent(event, false); + return true; + } + + /** + * Shift+ESC releases focus (same convention as the terminal panels). + * Plain ESC is forwarded to the browser, which may need it itself. + */ + private static boolean isShiftEscape(final java.awt.event.KeyEvent event) { + return event.getKeyCode() == java.awt.event.KeyEvent.VK_ESCAPE + && event.isShiftDown(); + } + + @Override + public boolean focusLost(final ViewPanel viewPanel) { + // Shift+ESC pops focus while Shift is still held; the Shift press + // was forwarded but its release will go to the next focus owner, + // so release modifiers server-side to avoid a stuck Shift + session.releaseModifiers(); + return super.focusLost(viewPanel); + } + + /** + * Stops the capture thread, kills Firefox and its Xvfb server. + */ + public void stop() { + session.close(); + } +} diff --git a/src/main/java/eu/svjatoslav/sixth/workspace/TerminalPanel.java b/src/main/java/eu/svjatoslav/sixth/workspace/TerminalPanel.java index 239690b..f0af002 100644 --- a/src/main/java/eu/svjatoslav/sixth/workspace/TerminalPanel.java +++ b/src/main/java/eu/svjatoslav/sixth/workspace/TerminalPanel.java @@ -178,6 +178,22 @@ public class TerminalPanel extends GuiComponent { return true; } + @Override + public boolean mouseWheelMoved(final int verticalUnits, + final int horizontalUnits) { + // The emulator has no scrollback and no mouse reporting, so the + // wheel is translated to arrow keys — scrolls less/htop/mc, and + // walks the command history at a shell prompt. Three lines per + // notch, matching xterm. + for (int i = 0; i < Math.abs(verticalUnits) * 3; i++) + session.send(verticalUnits < 0 ? cursorKey('A', 'A') + : cursorKey('B', 'B')); + for (int i = 0; i < Math.abs(horizontalUnits) * 3; i++) + session.send(horizontalUnits < 0 ? cursorKey('D', 'D') + : cursorKey('C', 'C')); + return true; + } + /** * Whether the terminal is in application cursor keys mode (DECCKM). * Curses programs (mc, htop, vim) enable it; cursor keys must then be diff --git a/src/main/java/eu/svjatoslav/sixth/workspace/Workspace.java b/src/main/java/eu/svjatoslav/sixth/workspace/Workspace.java index d220892..6468596 100644 --- a/src/main/java/eu/svjatoslav/sixth/workspace/Workspace.java +++ b/src/main/java/eu/svjatoslav/sixth/workspace/Workspace.java @@ -18,6 +18,9 @@ import eu.svjatoslav.sixth.e3d.renderer.raster.shapes.composite.textcanvas.TextC import eu.svjatoslav.sixth.e3d.renderer.raster.shapes.composite.wireframe.Grid2D; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import static eu.svjatoslav.sixth.e3d.geometry.Point3D.point; import static eu.svjatoslav.sixth.e3d.renderer.raster.Color.hex; @@ -26,9 +29,11 @@ import static eu.svjatoslav.sixth.e3d.renderer.raster.Color.hex; * The Sixth workspace: a persistent 3D world where programs are placeable * objects. * - *

Current population: one text editor (engine component) and one live - * terminal running bash on a real PTY. Click a panel to focus it and type; - * ESC releases focus; fly with the engine's standard camera controls.

+ *

Current population: one text editor (engine component), a 4x4 + * grid of live terminals, each running its own bash on a real PTY, and a + * live Firefox browser (private Xvfb display captured into a texture). + * Click a panel to focus it and type; ESC or middle click releases focus; fly with the + * engine's standard camera controls.

*/ public class Workspace { @@ -37,9 +42,12 @@ public class Workspace { */ public static final int TERMINAL_COLUMNS = 100; public static final int TERMINAL_ROWS = 30; + public static final int TERMINAL_GRID_COLUMNS = 4; + public static final int TERMINAL_GRID_ROWS = 4; private final ViewFrame viewFrame; - private TerminalPanel terminalPanel; + private final List terminalPanels = new ArrayList<>(); + private FirefoxPanel firefoxPanel; public Workspace() { viewFrame = new ViewFrame("Sixth"); @@ -59,7 +67,8 @@ public class Workspace { addGrid(scene); addTextEditor(viewPanel, scene); - addTerminal(viewPanel, scene); + addTerminals(viewPanel, scene); + addFirefox(viewPanel, scene); viewPanel.repaintDuringNextViewUpdate(); } @@ -80,28 +89,61 @@ public class Workspace { new Point2D(400, 240), new LookAndFeel()); editor.setText("Sixth workspace\n\n" + "Click a panel to focus it.\n" - + "Type into it. ESC releases focus\n" - + "(in the terminal: Shift+ESC).\n\n" - + "The panel on the right is a real\n" - + "bash shell running on a PTY.\n" + + "Type into it. Middle click or ESC releases\n" + + "focus (in the terminal and browser: Shift+ESC).\n\n" + + "The 4x4 grid on the right contains\n" + + "independent bash shells.\n" + "Try: ls, mc, htop"); scene.addShape(editor); } - private void addTerminal(final ViewPanel viewPanel, + private void addTerminals(final ViewPanel viewPanel, final ShapeCollection scene) throws IOException { - final PtySession session = new PtySession(TERMINAL_COLUMNS, - TERMINAL_ROWS); - terminalPanel = new TerminalPanel( - new Transform(point(100, 0, 300)), viewPanel, - new Point2D(TERMINAL_COLUMNS * TextCanvas.FONT_CHAR_WIDTH, - TERMINAL_ROWS * TextCanvas.FONT_CHAR_HEIGHT), - session); - scene.addShape(terminalPanel); + for (int x = 0; x < TERMINAL_GRID_COLUMNS; x++) { + for (int y = 0; y < TERMINAL_GRID_ROWS; y++) { + final PtySession session = new PtySession(TERMINAL_COLUMNS, + TERMINAL_ROWS); + final TerminalPanel terminalPanel = new TerminalPanel( + new Transform(point(100 + (x * 900), + -100 - (y * 600), 300)), viewPanel, + new Point2D( + TERMINAL_COLUMNS * TextCanvas.FONT_CHAR_WIDTH, + TERMINAL_ROWS * TextCanvas.FONT_CHAR_HEIGHT), + session); + terminalPanels.add(terminalPanel); + scene.addShape(terminalPanel); + } + } } + /** + * Adds a live Firefox browser below the text editor: a private Xvfb + * display whose screen is captured into a textured rectangle. + */ + private void addFirefox(final ViewPanel viewPanel, + final ShapeCollection scene) throws IOException { + firefoxPanel = new FirefoxPanel(new Transform(point(-700, -450, 300)), + viewPanel, new Point2D(640, 480)); + firefoxPanel.start(); + scene.addShape(firefoxPanel); + } + + /** + * Returns the Firefox panel, used by the automated GUI test. + */ + public FirefoxPanel getFirefoxPanel() { + return firefoxPanel; + } + + /** + * Returns the first terminal, used by the automated terminal selftest. + */ public TerminalPanel getTerminalPanel() { - return terminalPanel; + return terminalPanels.get(0); + } + + public List getTerminalPanels() { + return Collections.unmodifiableList(terminalPanels); } public eu.svjatoslav.sixth.e3d.gui.headtrack.HeadTracker getHeadTracker() { @@ -113,10 +155,12 @@ public class Workspace { } /** - * Shuts down background processes (the shell). + * Shuts down all background shell processes. */ public void close() { - if (terminalPanel != null) + for (final TerminalPanel terminalPanel : terminalPanels) terminalPanel.getSession().stop(); + if (firefoxPanel != null) + firefoxPanel.stop(); } }