--- /dev/null
+* Put agents in 3D world
+
+- Agent has body that resembles human
+ - TODO: find good design for humanoid
+
+- Agent has ability to move around cyberspace
+
+- Agent has ability to see (project image from agent perspective)
+
+- Agent has ability to write and execute programs
+
+- Agent has ability to read and write documents in cyberspace
+
+- Agent can hear your voice (STT, whisper)
+
+- Agent can speak (TTS, Piper)
+
+- Local LLM as well as cloud based LLM can be used
+ - OpenRouter free LLMs
+ - Kimi K3
+
+
int posix_spawnattr_setflags(Pointer attributes, short flags);
+ int posix_spawn_file_actions_destroy(Pointer fileActions);
+
+ int posix_spawnattr_destroy(Pointer attributes);
+
int close(int fd);
int read(int fd, Pointer buffer, int count);
// the master. POSIX_SPAWN_SETSID makes the child a session leader;
// glibc applies it BEFORE file actions, so the addopen below (no
// O_NOCTTY) also acquires the slave as the controlling terminal.
- final Memory fileActions = new Memory(256);
- final Memory attributes = new Memory(256);
+ // glibc's posix_spawnattr_t is 336 bytes on amd64; allocate a
+ // padded block for both structs rather than relying on ABI sizes.
+ final Memory fileActions = new Memory(512);
+ final Memory attributes = new Memory(512);
CLib.INSTANCE.posix_spawn_file_actions_init(fileActions);
CLib.INSTANCE.posix_spawnattr_init(attributes);
CLib.INSTANCE.posix_spawnattr_setflags(attributes,
final Memory pidResult = new Memory(4);
final int rc = CLib.INSTANCE.posix_spawn(pidResult, "/bin/sh",
fileActions, attributes, shellArgv, envp);
+ CLib.INSTANCE.posix_spawn_file_actions_destroy(fileActions);
+ CLib.INSTANCE.posix_spawnattr_destroy(attributes);
if (rc != 0)
throw new IllegalStateException("posix_spawn failed: " + rc);
childPid = pidResult.getInt(0);
--- /dev/null
+/* Sixth spatial computing environment. Author: Svjatoslav Agejenko. This project is released under Creative Commons Zero (CC0) license. */
+package eu.svjatoslav.sixth.bridge.x11;
+
+import java.awt.event.KeyEvent;
+
+/**
+ * Translates AWT key events to X11 keysyms for injection into the virtual
+ * display via the X TEST extension.
+ *
+ * <p>Mapping is by {@code keyCode}, not {@code keyChar}: letters map to
+ * their unshifted keysyms and uppercase is produced by the real Shift key
+ * events, which are forwarded like any other key. This mirrors how a
+ * physical keyboard works and assumes the virtual server's keymap is a
+ * US-style layout (Xvfb default).</p>
+ *
+ * <p>Keys with no obvious mapping fall back to the Unicode keysym scheme
+ * ({@code 0x01000000 | codepoint}) when the event carries a character —
+ * this covers characters like ä and ö when they exist in the keymap.</p>
+ */
+final class AwtKeysyms {
+
+ // X11 keysym values (from keysymdef.h)
+ static final long XK_BACK_SPACE = 0xFF08;
+ static final long XK_TAB = 0xFF09;
+ static final long XK_RETURN = 0xFF0D;
+ static final long XK_ESCAPE = 0xFF1B;
+ static final long XK_HOME = 0xFF50;
+ static final long XK_LEFT = 0xFF51;
+ static final long XK_UP = 0xFF52;
+ static final long XK_RIGHT = 0xFF53;
+ static final long XK_DOWN = 0xFF54;
+ static final long XK_PAGE_UP = 0xFF55;
+ static final long XK_PAGE_DOWN = 0xFF56;
+ static final long XK_END = 0xFF57;
+ static final long XK_INSERT = 0xFF63;
+ static final long XK_F1 = 0xFFBE;
+ static final long XK_KP_MULTIPLY = 0xFFAA;
+ static final long XK_KP_ADD = 0xFFAB;
+ static final long XK_KP_SUBTRACT = 0xFFAD;
+ static final long XK_KP_DECIMAL = 0xFFAE;
+ static final long XK_KP_DIVIDE = 0xFFAF;
+ static final long XK_KP_0 = 0xFFB0;
+ static final long XK_SHIFT_L = 0xFFE1;
+ static final long XK_CONTROL_L = 0xFFE3;
+ static final long XK_CAPS_LOCK = 0xFFE5;
+ static final long XK_ALT_L = 0xFFE9;
+ static final long XK_DELETE = 0xFFFF;
+
+ private AwtKeysyms() {
+ }
+
+ /**
+ * Maps an AWT key event to an X keysym.
+ *
+ * @return the keysym, or {@code -1} when the key cannot be represented
+ */
+ static long keysymFor(final KeyEvent event) {
+ final int code = event.getKeyCode();
+
+ // letters: unshifted keysyms; shift state comes from real Shift
+ // key events
+ if (code >= KeyEvent.VK_A && code <= KeyEvent.VK_Z)
+ return 'a' + (code - KeyEvent.VK_A);
+ if (code >= KeyEvent.VK_0 && code <= KeyEvent.VK_9)
+ return '0' + (code - KeyEvent.VK_0);
+ if (code >= KeyEvent.VK_F1 && code <= KeyEvent.VK_F12)
+ return XK_F1 + (code - KeyEvent.VK_F1);
+ if (code >= KeyEvent.VK_NUMPAD0 && code <= KeyEvent.VK_NUMPAD9)
+ return XK_KP_0 + (code - KeyEvent.VK_NUMPAD0);
+
+ switch (code) {
+ case KeyEvent.VK_BACK_SPACE:
+ return XK_BACK_SPACE;
+ case KeyEvent.VK_TAB:
+ return XK_TAB;
+ case KeyEvent.VK_ENTER:
+ return XK_RETURN;
+ case KeyEvent.VK_ESCAPE:
+ return XK_ESCAPE;
+ case KeyEvent.VK_INSERT:
+ return XK_INSERT;
+ case KeyEvent.VK_DELETE:
+ return XK_DELETE;
+ case KeyEvent.VK_HOME:
+ return XK_HOME;
+ case KeyEvent.VK_END:
+ return XK_END;
+ case KeyEvent.VK_PAGE_UP:
+ return XK_PAGE_UP;
+ case KeyEvent.VK_PAGE_DOWN:
+ return XK_PAGE_DOWN;
+ case KeyEvent.VK_LEFT:
+ return XK_LEFT;
+ case KeyEvent.VK_UP:
+ return XK_UP;
+ case KeyEvent.VK_RIGHT:
+ return XK_RIGHT;
+ case KeyEvent.VK_DOWN:
+ return XK_DOWN;
+ case KeyEvent.VK_SHIFT:
+ return XK_SHIFT_L;
+ case KeyEvent.VK_CONTROL:
+ return XK_CONTROL_L;
+ case KeyEvent.VK_ALT:
+ return XK_ALT_L;
+ case KeyEvent.VK_CAPS_LOCK:
+ return XK_CAPS_LOCK;
+ case KeyEvent.VK_MULTIPLY:
+ return XK_KP_MULTIPLY;
+ case KeyEvent.VK_ADD:
+ return XK_KP_ADD;
+ case KeyEvent.VK_SUBTRACT:
+ return XK_KP_SUBTRACT;
+ case KeyEvent.VK_DECIMAL:
+ return XK_KP_DECIMAL;
+ case KeyEvent.VK_DIVIDE:
+ return XK_KP_DIVIDE;
+ case KeyEvent.VK_SPACE:
+ return ' ';
+ case KeyEvent.VK_COMMA:
+ return ',';
+ case KeyEvent.VK_PERIOD:
+ return '.';
+ case KeyEvent.VK_SLASH:
+ return '/';
+ case KeyEvent.VK_BACK_SLASH:
+ return '\\';
+ case KeyEvent.VK_SEMICOLON:
+ return ';';
+ case KeyEvent.VK_QUOTE:
+ return '\'';
+ case KeyEvent.VK_OPEN_BRACKET:
+ return '[';
+ case KeyEvent.VK_CLOSE_BRACKET:
+ return ']';
+ case KeyEvent.VK_BACK_QUOTE:
+ return '`';
+ case KeyEvent.VK_MINUS:
+ return '-';
+ case KeyEvent.VK_EQUALS:
+ return '=';
+ default:
+ return unicodeFallback(event.getKeyChar());
+ }
+ }
+
+ /**
+ * Unicode keysym fallback ({@code 0x01000000 | codepoint}) for
+ * characters outside the keyCode-based map, e.g. ä, ö, é.
+ */
+ private static long unicodeFallback(final char keyChar) {
+ if (keyChar == KeyEvent.CHAR_UNDEFINED || keyChar < 0x20)
+ return -1;
+ if (keyChar < 0x7F)
+ // printable ASCII not covered above
+ return keyChar;
+ return 0x01000000L | keyChar;
+ }
+}
--- /dev/null
+/*
+ * Sixth spatial computing environment. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ */
+package eu.svjatoslav.sixth.bridge.x11;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A GUI application running on a private Xvfb display, with its screen
+ * continuously captured into a caller-owned ARGB pixel buffer.
+ *
+ * <p>The session owns three things: the Xvfb server, the application
+ * process, and a capture thread that grabs the root window at a fixed
+ * rate. The capture thread writes directly into the destination array
+ * (typically a texture's primary bitmap pixels) and invokes the frame
+ * listener only when the image actually changed, so a static screen costs
+ * no repaints.</p>
+ */
+public final class GuiAppSession implements AutoCloseable {
+
+ /**
+ * Called on the capture thread after the destination buffer was
+ * updated with a changed frame.
+ */
+ public interface FrameListener {
+ void frameCaptured();
+ }
+
+ private static final long CAPTURE_INTERVAL_MS = 100;
+
+ private final int width;
+ private final int height;
+ private final List<String> command;
+ private final Map<String, String> extraEnvironment;
+ private final boolean privateDbusSession;
+ private final File logFile;
+
+ private XvfbServer server;
+ private Process application;
+ private X11Native.Connection connection;
+ private Thread captureThread;
+ private volatile boolean running;
+
+ public GuiAppSession(final int width, final int height,
+ final List<String> command,
+ final Map<String, String> extraEnvironment) {
+ this(width, height, command, extraEnvironment, false);
+ }
+
+ /**
+ * @param privateDbusSession wrap the app in {@code dbus-run-session}
+ * so it gets a private session bus. Needed
+ * because merely unsetting
+ * DBUS_SESSION_BUS_ADDRESS is not enough:
+ * GIO then falls back to the systemd user
+ * bus socket (/run/user/$UID/bus), where the
+ * app's desktop instance can still be found
+ * and command-line handling forwarded to it.
+ */
+ public GuiAppSession(final int width, final int height,
+ final List<String> command,
+ final Map<String, String> extraEnvironment,
+ final boolean privateDbusSession) {
+ this.width = width;
+ this.height = height;
+ this.command = command;
+ this.extraEnvironment = extraEnvironment;
+ this.privateDbusSession = privateDbusSession;
+ logFile = new File("/tmp/sixth-xapp.log");
+ }
+
+ /**
+ * Starts Xvfb, launches the application onto it, and begins capturing.
+ *
+ * @param destination ARGB pixel buffer of exactly
+ * {@code width * height} ints; every captured frame
+ * is written into this array
+ * @param listener notified (on the capture thread) whenever a frame
+ * changed the buffer
+ */
+ public void start(final int[] destination, final FrameListener listener)
+ throws IOException {
+ if (destination.length != width * height)
+ throw new IllegalArgumentException("destination buffer holds "
+ + destination.length + " pixels, expected "
+ + (width * height));
+
+ server = XvfbServer.start(width, height);
+ connection = new X11Native.Connection(server.getDisplayName());
+
+ final List<String> effectiveCommand = privateDbusSession
+ ? java.util.stream.Stream.concat(
+ java.util.stream.Stream.of("dbus-run-session", "--"),
+ command.stream()).toList()
+ : command;
+ final ProcessBuilder builder = new ProcessBuilder(effectiveCommand);
+ final Map<String, String> environment = builder.environment();
+ environment.put("DISPLAY", server.getDisplayName());
+ // Firefox/GTK apps locate an already-running instance through the
+ // SESSION D-Bus (GApplication), ignoring --no-remote and DISPLAY:
+ // the window then pops up on the user's desktop while our Xvfb
+ // screen stays black. Hide the session bus so the app is forced
+ // to be its own primary instance on the private display.
+ environment.remove("DBUS_SESSION_BUS_ADDRESS");
+ environment.put("MOZ_NO_REMOTE", "1");
+ // On Wayland desktops (GNOME/Mutter) Firefox ignores DISPLAY
+ // entirely and renders natively into the user's compositor —
+ // the window pops up on the real screen while the Xvfb display
+ // stays windowless. Force the X11 backend onto our display.
+ environment.remove("WAYLAND_DISPLAY");
+ environment.put("MOZ_ENABLE_WAYLAND", "0");
+ environment.putAll(extraEnvironment);
+ builder.redirectOutput(logFile);
+ builder.redirectError(logFile);
+ application = builder.start();
+
+ running = true;
+ captureThread = new Thread(
+ () -> captureLoop(destination, listener), "x11-capture");
+ captureThread.setDaemon(true);
+ captureThread.start();
+ }
+
+ private void captureLoop(final int[] destination,
+ final FrameListener listener) {
+ final boolean debug = Boolean.getBoolean("sixth.xcapture.debug");
+ int iterations = 0;
+ while (running) {
+ try {
+ final boolean changed = X11Native.captureRoot(connection,
+ width, height, destination);
+ if (debug && iterations++ % 10 == 0) {
+ final var distinct = new java.util.HashSet<Integer>();
+ for (int i = 0; i < destination.length; i += 97)
+ distinct.add(destination[i] & 0xFFFFFF);
+ System.out.println("XCAPTURE changed=" + changed
+ + " distinct=" + distinct.size()
+ + " appAlive=" + isApplicationRunning());
+ }
+ if (changed)
+ listener.frameCaptured();
+ } catch (final Throwable throwable) {
+ // capture failures (window closing, renderer racing the
+ // buffer) must never kill the capture thread
+ if (debug)
+ throwable.printStackTrace();
+ }
+ try {
+ Thread.sleep(CAPTURE_INTERVAL_MS);
+ } catch (final InterruptedException e) {
+ return;
+ }
+ }
+ }
+
+ /**
+ * Forwards a keyboard event (press or release) into the virtual
+ * display. Modifier keys are forwarded as ordinary keys, so the
+ * server-side modifier state tracks the user's real keyboard.
+ *
+ * @param event the AWT key event
+ * @param pressed {@code true} for key press, {@code false} for release
+ */
+ public void sendKeyEvent(final java.awt.event.KeyEvent event,
+ final boolean pressed) {
+ if (connection == null || !isApplicationRunning())
+ return;
+ final long keysym = AwtKeysyms.keysymFor(event);
+ if (keysym < 0)
+ return;
+ if (Boolean.getBoolean("sixth.xcapture.debug"))
+ System.out.println("XINPUT key keysym=0x"
+ + Long.toHexString(keysym) + (pressed ? " press" : " release")
+ + " on " + server.getDisplayName());
+ X11Native.sendKeyEvent(connection, keysym, pressed);
+ }
+
+ /**
+ * Releases Shift/Control/Alt server-side. Called when keyboard focus
+ * is lost mid-modifier (e.g. Shift+ESC pops focus while Shift is
+ * held), so the browser is not left with stuck modifiers.
+ */
+ public void releaseModifiers() {
+ if (connection == null || !isApplicationRunning())
+ return;
+ X11Native.sendKeyEvent(connection, AwtKeysyms.XK_SHIFT_L, false);
+ X11Native.sendKeyEvent(connection, AwtKeysyms.XK_CONTROL_L, false);
+ X11Native.sendKeyEvent(connection, AwtKeysyms.XK_ALT_L, false);
+ }
+
+ /**
+ * Forwards scroll wheel input into the virtual display at the given
+ * screen coordinates (typically the current pointer hover position on
+ * the panel).
+ *
+ * @param verticalUnits positive = scroll down, negative = scroll up
+ * @param horizontalUnits positive = scroll right, negative = left
+ */
+ public void sendScroll(final int verticalUnits, final int horizontalUnits,
+ final int x, final int y) {
+ if (connection == null || !isApplicationRunning())
+ return;
+ final int clampedX = Math.max(0, Math.min(x, width - 1));
+ final int clampedY = Math.max(0, Math.min(y, height - 1));
+ X11Native.sendScroll(connection, clampedX, clampedY, verticalUnits,
+ horizontalUnits);
+ }
+
+ /**
+ * Moves the pointer inside the virtual display without pressing
+ * buttons (hover forwarding).
+ */
+ public void sendMouseMove(final int x, final int y) {
+ if (connection == null || !isApplicationRunning())
+ return;
+ X11Native.sendMouseMove(connection,
+ Math.max(0, Math.min(x, width - 1)),
+ Math.max(0, Math.min(y, height - 1)));
+ }
+
+ public boolean isApplicationRunning() {
+ return application != null && application.isAlive();
+ }
+
+ /**
+ * Forwards a mouse click into the virtual display at screen
+ * coordinates (x, y). The captured texture IS the virtual screen, so
+ * texture pixels map 1:1 onto these coordinates. The click is
+ * synthesized with the X TEST extension, which applications treat as
+ * real device input.
+ *
+ * @param x screen X coordinate (clamped into the screen)
+ * @param y screen Y coordinate (clamped into the screen)
+ * @param button X button number: 1 = left, 3 = right
+ */
+ public void sendMouseClick(final int x, final int y, final int button) {
+ if (connection == null || !isApplicationRunning())
+ return;
+ final int clampedX = Math.max(0, Math.min(x, width - 1));
+ final int clampedY = Math.max(0, Math.min(y, height - 1));
+ if (Boolean.getBoolean("sixth.xcapture.debug"))
+ System.out.println("XINPUT click at " + clampedX + "," + clampedY
+ + " button " + button + " on "
+ + server.getDisplayName());
+ X11Native.sendMouseClick(connection, clampedX, clampedY, button);
+ }
+
+ @Override
+ public void close() {
+ running = false;
+ if (captureThread != null)
+ captureThread.interrupt();
+ if (application != null)
+ application.destroyForcibly();
+ if (connection != null)
+ connection.close();
+ if (server != null)
+ server.close();
+ }
+
+ /**
+ * Builds a session that runs Firefox on a fresh throwaway profile,
+ * sized to fill the whole virtual screen.
+ */
+ public static GuiAppSession firefox(final int width, final int height,
+ final String url)
+ throws IOException {
+ final File profile = Files.createTempDirectory(
+ "sixth-firefox-profile").toFile();
+
+ // keep first-run noise out of the captured screen
+ final String userJs = """
+ user_pref("browser.shell.checkDefaultBrowser", false);
+ user_pref("browser.aboutwelcome.enabled", false);
+ user_pref("datareporting.policy.dataSubmissionEnabled", false);
+ user_pref("datareporting.policy.firstRunURL", "");
+ user_pref("trailhead.firstrun.didSeeAboutWelcome", true);
+ // stop Debian's system-wide extensions (Web eID) from
+ // hijacking the first-run session with an active tab
+ user_pref("extensions.autoDisableScopes", 15);
+ user_pref("extensions.shownSelectionUI", true);
+ """;
+ Files.writeString(new File(profile, "user.js").toPath(), userJs);
+
+ return new GuiAppSession(width, height,
+ List.of("firefox", "--no-remote", "--new-instance",
+ "--profile", profile.getAbsolutePath(),
+ "--width", String.valueOf(width),
+ "--height", String.valueOf(height),
+ url),
+ Map.of(), true);
+ }
+}
--- /dev/null
+/*
+ * Sixth spatial computing environment. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ */
+package eu.svjatoslav.sixth.bridge.x11;
+
+import com.sun.jna.Function;
+import com.sun.jna.Library;
+import com.sun.jna.Native;
+import com.sun.jna.NativeLong;
+import com.sun.jna.Pointer;
+
+/**
+ * Minimal libX11 bindings for screen capture via JNA.
+ *
+ * <p>Only what {@link X11Capture} needs: open/close a display, resolve the
+ * root window, and {@code XGetImage} a rectangular area into ARGB pixels.
+ * The XImage struct is read through manual field offsets (amd64 layout)
+ * instead of a JNA {@code Structure} — the struct embeds function pointers
+ * and exact alignment is easier to get right explicitly.</p>
+ *
+ * <p>Linux-only by design; the workspace targets Linux.</p>
+ */
+final class X11Native {
+
+ /** ZPixmap format constant for XGetImage. */
+ static final int Z_PIXMAP = 2;
+
+ /** XImage field offsets (amd64 / LP64 layout). */
+ private static final int IMAGE_WIDTH = 0;
+ private static final int IMAGE_HEIGHT = 4;
+ private static final int IMAGE_DATA = 16;
+ private static final int IMAGE_BYTE_ORDER = 24;
+ private static final int IMAGE_DEPTH = 40;
+ private static final int IMAGE_BYTES_PER_LINE = 44;
+ private static final int IMAGE_BITS_PER_PIXEL = 48;
+ private static final int IMAGE_RED_MASK = 56;
+ private static final int IMAGE_GREEN_MASK = 64;
+ private static final int IMAGE_BLUE_MASK = 72;
+ private static final int IMAGE_DESTROY_FUNCTION = 96;
+
+ private static final int LSB_FIRST = 0;
+
+ private interface X11Lib extends Library {
+ X11Lib INSTANCE = Native.load("X11", X11Lib.class);
+
+ Pointer XOpenDisplay(String displayName);
+
+ int XCloseDisplay(Pointer display);
+
+ int XDefaultScreen(Pointer display);
+
+ NativeLong XRootWindow(Pointer display, int screenNumber);
+
+ Pointer XGetImage(Pointer display, NativeLong drawable, int x, int y,
+ int width, int height, NativeLong planeMask,
+ int format);
+
+ int XFlush(Pointer display);
+
+ /**
+ * Converts a keysym to a keycode in the server's current keymap.
+ * Returns 0 when the keysym is not present in the keymap.
+ */
+ int XKeysymToKeycode(Pointer display, NativeLong keysym);
+ }
+
+ /**
+ * libXtst (X TEST extension) bindings for synthesizing input. Fake
+ * input events are indistinguishable from real device events, unlike
+ * XSendEvent which many toolkits (GTK included) treat as untrusted.
+ */
+ private interface XTestLib extends Library {
+ XTestLib INSTANCE = Native.load("Xtst", XTestLib.class);
+
+ int XTestFakeMotionEvent(Pointer display, int screenNumber,
+ int x, int y, NativeLong delay);
+
+ int XTestFakeButtonEvent(Pointer display, int button,
+ boolean isPress, NativeLong delay);
+
+ /**
+ * Synthesizes a key press or release.
+ *
+ * @return nonzero on success
+ */
+ int XTestFakeKeyEvent(Pointer display, int keycode, boolean isPress,
+ NativeLong delay);
+ }
+
+ /**
+ * An open connection to an X server.
+ *
+ * <p>libX11 is not thread-safe unless XInitThreads was called; all
+ * requests on this connection are instead serialized through
+ * {@link #lock} (the capture loop and the input injector run on
+ * different threads).</p>
+ */
+ static final class Connection {
+ final Pointer display;
+ final NativeLong rootWindow;
+ final int screenNumber;
+ /** Serializes all X protocol traffic on this connection. */
+ final Object lock = new Object();
+
+ Connection(final String displayName) {
+ display = X11Lib.INSTANCE.XOpenDisplay(displayName);
+ if (display == null)
+ throw new IllegalStateException(
+ "cannot open X display " + displayName);
+ screenNumber = X11Lib.INSTANCE.XDefaultScreen(display);
+ rootWindow = X11Lib.INSTANCE.XRootWindow(display, screenNumber);
+ }
+
+ void close() {
+ synchronized (lock) {
+ if (display != null)
+ X11Lib.INSTANCE.XCloseDisplay(display);
+ }
+ }
+ }
+
+ /**
+ * Grabs the top-left rectangle of the root window and converts it to
+ * ARGB, honoring the server's pixel masks, bits-per-pixel and byte
+ * order. Writes directly into {@code destination} and reports whether
+ * any pixel changed.
+ *
+ * @return true when at least one written pixel differs from the
+ * previous buffer content
+ */
+ static boolean captureRoot(final Connection connection,
+ final int width, final int height,
+ final int[] destination) {
+ synchronized (connection.lock) {
+ final Pointer image = X11Lib.INSTANCE.XGetImage(connection.display,
+ connection.rootWindow, 0, 0, width, height,
+ new NativeLong(-1L), Z_PIXMAP);
+ if (image == null)
+ return false;
+ try {
+ return convert(image, width, height, destination);
+ } finally {
+ // XDestroyImage is a macro calling image->f.destroy_image
+ final Pointer destroyFunction = image.getPointer(
+ IMAGE_DESTROY_FUNCTION);
+ Function.getFunction(destroyFunction).invokeInt(
+ new Object[]{image});
+ }
+ }
+ }
+
+ /**
+ * Sends a single key press or release, identified by X keysym, to the
+ * virtual display. Modifier state is NOT synthesized here — real
+ * modifier key events (Shift/Ctrl/Alt) are forwarded like any other
+ * key, so the server-side modifier state tracks the user's keyboard.
+ *
+ * @param keysym the X keysym (see {@link AwtKeysyms})
+ * @param press {@code true} for press, {@code false} for release
+ */
+ static void sendKeyEvent(final Connection connection, final long keysym,
+ final boolean press) {
+ synchronized (connection.lock) {
+ final int keycode = X11Lib.INSTANCE.XKeysymToKeycode(
+ connection.display, new NativeLong(keysym));
+ if (keycode == 0)
+ // keysym not present in the server keymap; nothing to send
+ return;
+ XTestLib.INSTANCE.XTestFakeKeyEvent(connection.display, keycode,
+ press, new NativeLong(0));
+ X11Lib.INSTANCE.XFlush(connection.display);
+ }
+ }
+
+ /**
+ * Sends scroll wheel input at the given root-window coordinates: moves
+ * the pointer there, then presses/releases the X scroll buttons
+ * (4 = up, 5 = down, 6 = left, 7 = right), one click per notch.
+ *
+ * @param verticalUnits positive = scroll down, negative = scroll up
+ * @param horizontalUnits positive = scroll right, negative = left
+ */
+ static void sendScroll(final Connection connection, final int x,
+ final int y, final int verticalUnits,
+ final int horizontalUnits) {
+ synchronized (connection.lock) {
+ XTestLib.INSTANCE.XTestFakeMotionEvent(connection.display,
+ connection.screenNumber, x, y, new NativeLong(0));
+ final int verticalButton = verticalUnits < 0 ? 4 : 5;
+ for (int i = 0; i < Math.abs(verticalUnits); i++) {
+ XTestLib.INSTANCE.XTestFakeButtonEvent(connection.display,
+ verticalButton, true, new NativeLong(0));
+ XTestLib.INSTANCE.XTestFakeButtonEvent(connection.display,
+ verticalButton, false, new NativeLong(0));
+ }
+ final int horizontalButton = horizontalUnits < 0 ? 6 : 7;
+ for (int i = 0; i < Math.abs(horizontalUnits); i++) {
+ XTestLib.INSTANCE.XTestFakeButtonEvent(connection.display,
+ horizontalButton, true, new NativeLong(0));
+ XTestLib.INSTANCE.XTestFakeButtonEvent(connection.display,
+ horizontalButton, false, new NativeLong(0));
+ }
+ X11Lib.INSTANCE.XFlush(connection.display);
+ }
+ }
+
+ /**
+ * Moves the pointer to the given root-window coordinates without
+ * pressing any button (hover forwarding).
+ */
+ static void sendMouseMove(final Connection conn, final int x,
+ final int y) {
+ synchronized (conn.lock) {
+ XTestLib.INSTANCE.XTestFakeMotionEvent(conn.display,
+ conn.screenNumber, x, y, new NativeLong(0));
+ X11Lib.INSTANCE.XFlush(conn.display);
+ }
+ }
+
+ /**
+ * Synthesizes a full mouse click (move pointer, press, release) at the
+ * given root-window coordinates. The pointer is moved first because a
+ * button press acts at the current pointer location. The button is
+ * held briefly because Firefox does not synthesize a DOM click from a
+ * zero-duration press+release.
+ *
+ * @param x root-window X coordinate
+ * @param y root-window Y coordinate
+ * @param button X button number (1 = left, 3 = right)
+ */
+ static void sendMouseClick(final Connection connection, final int x,
+ final int y, final int button) {
+ synchronized (connection.lock) {
+ XTestLib.INSTANCE.XTestFakeMotionEvent(connection.display,
+ connection.screenNumber, x, y, new NativeLong(0));
+ XTestLib.INSTANCE.XTestFakeButtonEvent(connection.display, button,
+ true, new NativeLong(0));
+ X11Lib.INSTANCE.XFlush(connection.display);
+ // Firefox (EventStateManager) does not synthesize a DOM click
+ // from a zero-duration press+release; hold the button briefly
+ try {
+ Thread.sleep(80);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ XTestLib.INSTANCE.XTestFakeButtonEvent(connection.display, button,
+ false, new NativeLong(0));
+ X11Lib.INSTANCE.XFlush(connection.display);
+ }
+ }
+
+ private static boolean convert(final Pointer image, final int width,
+ final int height, final int[] destination) {
+ final int imageWidth = image.getInt(IMAGE_WIDTH);
+ final int imageHeight = image.getInt(IMAGE_HEIGHT);
+ final Pointer data = image.getPointer(IMAGE_DATA);
+ final int byteOrder = image.getInt(IMAGE_BYTE_ORDER);
+ final int bytesPerLine = image.getInt(IMAGE_BYTES_PER_LINE);
+ final int bitsPerPixel = image.getInt(IMAGE_BITS_PER_PIXEL);
+ final long redMask = image.getLong(IMAGE_RED_MASK);
+ final long greenMask = image.getLong(IMAGE_GREEN_MASK);
+ final long blueMask = image.getLong(IMAGE_BLUE_MASK);
+
+ final Channel red = new Channel(redMask);
+ final Channel green = new Channel(greenMask);
+ final Channel blue = new Channel(blueMask);
+
+ boolean changed = false;
+ final int copyWidth = Math.min(width, imageWidth);
+ final int copyHeight = Math.min(height, imageHeight);
+ for (int y = 0; y < copyHeight; y++) {
+ final int rowOffset = y * bytesPerLine;
+ final int destinationRow = y * width;
+ for (int x = 0; x < copyWidth; x++) {
+ final long pixelValue = readPixel(data, rowOffset, x,
+ bitsPerPixel, byteOrder);
+ final int argb = 0xFF000000
+ | (red.extract(pixelValue) << 16)
+ | (green.extract(pixelValue) << 8)
+ | blue.extract(pixelValue);
+ final int index = destinationRow + x;
+ if (destination[index] != argb) {
+ destination[index] = argb;
+ changed = true;
+ }
+ }
+ }
+ return changed;
+ }
+
+ private static long readPixel(final Pointer data, final int rowOffset,
+ final int x, final int bitsPerPixel,
+ final int byteOrder) {
+ if (bitsPerPixel == 32) {
+ final int value = data.getInt(rowOffset + (long) x * 4);
+ return Integer.toUnsignedLong(
+ byteOrder == LSB_FIRST ? value : Integer.reverseBytes(value));
+ }
+ if (bitsPerPixel == 24) {
+ final long offset = rowOffset + (long) x * 3;
+ final int b0 = data.getByte(offset) & 0xFF;
+ final int b1 = data.getByte(offset + 1) & 0xFF;
+ final int b2 = data.getByte(offset + 2) & 0xFF;
+ return byteOrder == LSB_FIRST
+ ? b0 | (b1 << 8) | (b2 << 16)
+ : (b0 << 16) | (b1 << 8) | b2;
+ }
+ if (bitsPerPixel == 16) {
+ final short value = data.getShort(rowOffset + (long) x * 2);
+ return Short.toUnsignedLong(
+ byteOrder == LSB_FIRST ? value : Short.reverseBytes(value));
+ }
+ throw new IllegalStateException(
+ "unsupported bits_per_pixel " + bitsPerPixel);
+ }
+
+ /**
+ * Extracts one 8-bit color channel from a raw pixel using the server
+ * pixel mask (any position and width, scaled to 0..255).
+ */
+ private static final class Channel {
+ private final long mask;
+ private final int shift;
+ private final long maxValue;
+
+ Channel(final long mask) {
+ this.mask = mask;
+ shift = mask == 0 ? 0 : Long.numberOfTrailingZeros(mask);
+ maxValue = mask == 0 ? 1 : (mask >>> shift);
+ }
+
+ int extract(final long pixelValue) {
+ if (mask == 0)
+ return 0;
+ return (int) (((pixelValue & mask) >>> shift) * 255 / maxValue);
+ }
+ }
+}
--- /dev/null
+/*
+ * Sixth spatial computing environment. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ */
+package eu.svjatoslav.sixth.bridge.x11;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Lifecycle of a private {@code Xvfb} virtual X server that one captured
+ * GUI application renders into.
+ *
+ * <p>Each server gets its own display number (scanned upward from 90,
+ * skipping displays with an existing lock file or socket), so multiple
+ * captured apps never share a screen. Readiness is detected by polling for
+ * the unix socket and then proving the display with a real
+ * {@code XOpenDisplay} — just waiting for the socket is not enough.</p>
+ */
+public final class XvfbServer implements AutoCloseable {
+
+ private static final int FIRST_DISPLAY = 90;
+ private static final int LAST_DISPLAY = 200;
+ private static final long START_TIMEOUT_MS = 10_000;
+
+ private final int displayNumber;
+ private final Process process;
+
+ private XvfbServer(final int displayNumber, final Process process) {
+ this.displayNumber = displayNumber;
+ this.process = process;
+ }
+
+ /**
+ * Starts an Xvfb server with a single screen of the given size and
+ * waits until the display actually accepts connections.
+ *
+ * @throws IOException if Xvfb fails to start within the timeout
+ */
+ public static XvfbServer start(final int width, final int height)
+ throws IOException {
+ final int displayNumber = findFreeDisplay();
+ final Process process = new ProcessBuilder("Xvfb",
+ ":" + displayNumber,
+ "-screen", "0", width + "x" + height + "x24",
+ "-nolisten", "tcp")
+ .redirectOutput(new File("/tmp/sixth-xvfb-"
+ + displayNumber + ".log"))
+ .redirectError(new File("/tmp/sixth-xvfb-"
+ + displayNumber + ".log"))
+ .start();
+
+ final XvfbServer server = new XvfbServer(displayNumber, process);
+ final long deadline = System.currentTimeMillis() + START_TIMEOUT_MS;
+ while (System.currentTimeMillis() < deadline) {
+ if (!process.isAlive())
+ break;
+ if (new File(server.socketPath()).exists()) {
+ try {
+ final X11Native.Connection probe =
+ new X11Native.Connection(server.getDisplayName());
+ probe.close();
+ return server;
+ } catch (final Throwable ignored) {
+ // socket exists but server not accepting yet
+ }
+ }
+ try {
+ Thread.sleep(100);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ process.destroyForcibly();
+ throw new IOException("Xvfb :" + displayNumber
+ + " did not become ready within " + START_TIMEOUT_MS + " ms");
+ }
+
+ private static int findFreeDisplay() throws IOException {
+ for (int number = FIRST_DISPLAY; number <= LAST_DISPLAY; number++)
+ if (!new File("/tmp/.X" + number + "-lock").exists()
+ && !new File(socketPath(number)).exists())
+ return number;
+ throw new IOException("no free X display number between "
+ + FIRST_DISPLAY + " and " + LAST_DISPLAY);
+ }
+
+ private static String socketPath(final int displayNumber) {
+ return "/tmp/.X11-unix/X" + displayNumber;
+ }
+
+ private String socketPath() {
+ return socketPath(displayNumber);
+ }
+
+ /**
+ * Display name for the {@code DISPLAY} environment variable
+ * (e.g. {@code ":93"}).
+ */
+ public String getDisplayName() {
+ return ":" + displayNumber;
+ }
+
+ @Override
+ public void close() {
+ process.destroy();
+ try {
+ if (!process.waitFor(2, java.util.concurrent.TimeUnit.SECONDS))
+ process.destroyForcibly();
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ process.destroyForcibly();
+ }
+ }
+}
*/
package eu.svjatoslav.sixth.core;
+import eu.svjatoslav.sixth.workspace.FirefoxPanel;
import eu.svjatoslav.sixth.workspace.TerminalPanel;
import eu.svjatoslav.sixth.workspace.Workspace;
private static final long SELFTEST_TIMEOUT_MS = 15_000;
public static void main(final String[] args) throws Exception {
+ if (args.length > 0 && "--clicktest".equals(args[0]))
+ prepareClickTestPage();
+
+ if (args.length > 0 && "--keytest".equals(args[0]))
+ prepareKeyTestPage();
+
+ if (args.length > 0 && "--scrolltest".equals(args[0]))
+ prepareScrollTestPage();
+
+ if (args.length > 0 && "--hovertest".equals(args[0]))
+ prepareHoverTestPage();
+
final Workspace workspace = new Workspace();
workspace.open();
if (args.length > 0 && "--selftest".equals(args[0]))
System.exit(runSelfTest(workspace));
+ if (args.length > 0 && "--guitest".equals(args[0]))
+ System.exit(runGuiTest(workspace));
+
+ if (args.length > 0 && "--clicktest".equals(args[0]))
+ System.exit(runClickTest(workspace));
+
+ if (args.length > 0 && "--keytest".equals(args[0]))
+ System.exit(runKeyTest(workspace));
+
+ if (args.length > 0 && "--scrolltest".equals(args[0]))
+ System.exit(runScrollTest(workspace));
+
+ if (args.length > 0 && "--hovertest".equals(args[0]))
+ System.exit(runHoverTest(workspace));
+
if (args.length > 0 && "--headtest".equals(args[0]))
System.exit(runHeadTest(workspace));
private static int runSelfTest(final Workspace workspace)
throws InterruptedException {
final TerminalPanel terminal = workspace.getTerminalPanel();
+ final var terminals = workspace.getTerminalPanels();
+ final int expectedTerminalCount = Workspace.TERMINAL_GRID_COLUMNS
+ * Workspace.TERMINAL_GRID_ROWS;
+ if (terminals.size() != expectedTerminalCount)
+ return fail(terminal, "expected " + expectedTerminalCount
+ + " terminal panels, got " + terminals.size());
+ final long sessionCount = terminals.stream()
+ .map(TerminalPanel::getSession)
+ .distinct()
+ .count();
+ if (sessionCount != terminals.size())
+ return fail(terminal, "terminal panels share PTY sessions: "
+ + sessionCount + " sessions for " + terminals.size()
+ + " panels");
+ final long runningSessionCount = terminals.stream()
+ .filter(panel -> panel.getSession().isRunning())
+ .count();
+ if (runningSessionCount != terminals.size())
+ return fail(terminal, "only " + runningSessionCount + " of "
+ + terminals.size() + " PTY sessions are running");
// wait for the real shell prompt (bashrc noise may contain a bare
// "$", so match the user@host prompt text instead)
return sawMovement ? 0 : 1;
}
+ /**
+ * Verifies the GUI-app bridge end to end without AWT events: waits
+ * for Firefox to paint into the panel texture, then checks the
+ * captured frame is a real image (not a flat fill). Exit 0 = pass.
+ */
+ private static int runGuiTest(final Workspace workspace)
+ throws InterruptedException {
+ final var panel = workspace.getFirefoxPanel();
+ final long deadline = System.currentTimeMillis() + 60_000;
+ while (System.currentTimeMillis() < deadline) {
+ if (panel.hasLiveFrame())
+ break;
+ Thread.sleep(500);
+ }
+ if (!panel.hasLiveFrame()) {
+ System.out.println("GUITEST FAIL: no live frame within 60s"
+ + " (firefox running: "
+ + panel.getSession().isApplicationRunning() + ")");
+ workspace.close();
+ return 1;
+ }
+
+ // a real browser screen has many distinct colors; a dead/black
+ // capture collapses to one or two
+ final int[] pixels = panel.getTexture().primaryBitmap.pixels;
+ final var distinctColors = new java.util.HashSet<Integer>();
+ long luminanceSum = 0;
+ int samples = 0;
+ for (int i = 0; i < pixels.length; i += 997) {
+ final int pixel = pixels[i];
+ distinctColors.add(pixel);
+ luminanceSum += ((pixel >> 16) & 0xFF) + ((pixel >> 8) & 0xFF)
+ + (pixel & 0xFF);
+ samples++;
+ }
+ final double meanLuminance = (double) luminanceSum / samples / 3.0;
+ System.out.println("GUITEST stats: distinct sampled colors="
+ + distinctColors.size() + " mean luminance="
+ + String.format("%.1f", meanLuminance));
+ if (distinctColors.size() < 16 || meanLuminance < 5) {
+ System.out.println("GUITEST FAIL: captured frame looks flat");
+ workspace.close();
+ return 1;
+ }
+
+ System.out.println("GUITEST PASS — firefox is painting into the"
+ + " workspace panel");
+ workspace.close();
+ return 0;
+ }
+
+ /**
+ * Verifies mouse click forwarding into the virtual display: loads a
+ * test page that paints the whole screen red on click, focuses the
+ * Firefox panel, sends a left click through the panel's mouse
+ * interaction path, and requires a large pixel change. Also verifies
+ * that a middle click releases focus and is NOT forwarded. Exit 0 =
+ * pass.
+ */
+ private static int runClickTest(final Workspace workspace)
+ throws Exception {
+ final var panel = workspace.getFirefoxPanel();
+ final long deadline = System.currentTimeMillis() + 60_000;
+ while (System.currentTimeMillis() < deadline) {
+ if (panel.hasLiveFrame())
+ break;
+ Thread.sleep(500);
+ }
+ if (!panel.hasLiveFrame()) {
+ System.out.println("CLICKTEST FAIL: no live frame within 60s");
+ workspace.close();
+ return 1;
+ }
+ // let the page finish rendering after the first live frame
+ Thread.sleep(3000);
+
+ final int[] pixels = panel.getTexture().primaryBitmap.pixels;
+ final int[] before = pixels.clone();
+
+ // click while unfocused: grabs focus only, page must not change
+ panel.mouseClicked(
+ eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_LEFT,
+ 640, 480);
+ Thread.sleep(2000);
+ if (countChangedPixels(before, pixels) > 1000) {
+ System.out.println("CLICKTEST FAIL: focus-grab click leaked"
+ + " into the page");
+ workspace.close();
+ return 1;
+ }
+ if (!panel.hasKeyboardFocus()) {
+ System.out.println("CLICKTEST FAIL: first click did not focus"
+ + " the panel");
+ workspace.close();
+ return 1;
+ }
+
+ // left click while focused: forwarded, page turns red
+ System.out.println("CLICKTEST focus=" + panel.hasKeyboardFocus()
+ + " — sending forwarded click");
+ panel.mouseClicked(
+ eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_LEFT,
+ 640, 480);
+ final long clickDeadline = System.currentTimeMillis() + 10_000;
+ int changed = 0;
+ while (System.currentTimeMillis() < clickDeadline) {
+ Thread.sleep(500);
+ changed = countChangedPixels(before, pixels);
+ if (changed > 100_000)
+ break;
+ }
+ if (changed <= 100_000) {
+ saveTexturePng(panel, "/tmp/clicktest-texture.png");
+ System.out.println("CLICKTEST FAIL: forwarded click changed only "
+ + changed + " pixels (texture dump: "
+ + "/tmp/clicktest-texture.png)");
+ workspace.close();
+ return 1;
+ }
+
+ // mouse back button: navigates back to the gradient page
+ final int[] redPage = pixels.clone();
+ panel.mouseClicked(
+ eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_BACK,
+ 640, 480);
+ int backChanged = awaitPixelChange(pixels, redPage, 10_000);
+ if (backChanged <= 100_000) {
+ System.out.println("CLICKTEST FAIL: back button did not"
+ + " navigate back (" + backChanged + " pixels)");
+ workspace.close();
+ return 1;
+ }
+
+ // mouse forward button: navigates forward to the red page
+ final int[] gradientPage = pixels.clone();
+ panel.mouseClicked(
+ eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_FORWARD,
+ 640, 480);
+ int forwardChanged = awaitPixelChange(pixels, gradientPage, 10_000);
+ if (forwardChanged <= 100_000) {
+ System.out.println("CLICKTEST FAIL: forward button did not"
+ + " navigate forward (" + forwardChanged + " pixels)");
+ workspace.close();
+ return 1;
+ }
+
+ // middle click: releases focus, not forwarded
+ final int[] red = pixels.clone();
+ panel.mouseClicked(
+ eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_MIDDLE,
+ 640, 480);
+ Thread.sleep(2000);
+ if (panel.hasKeyboardFocus()) {
+ System.out.println("CLICKTEST FAIL: middle click did not"
+ + " release focus");
+ workspace.close();
+ return 1;
+ }
+ if (countChangedPixels(red, pixels) > 1000) {
+ System.out.println("CLICKTEST FAIL: middle click leaked into"
+ + " the page");
+ workspace.close();
+ return 1;
+ }
+
+ System.out.println("CLICKTEST PASS — focus grab, forwarded click ("
+ + changed + " pixels changed), back/forward navigation,"
+ + " middle-click release");
+ workspace.close();
+ return 0;
+ }
+
+ /**
+ * Writes the hover-target page and points the Firefox panel at it.
+ * The page turns orange on a mousemove event in its top-left quadrant
+ * (clientX < 300 && clientY < 300). Must run BEFORE the
+ * workspace is constructed.
+ */
+ private static void prepareHoverTestPage() throws Exception {
+ final java.io.File page = java.io.File.createTempFile(
+ "sixth-hovertest", ".html");
+ page.deleteOnExit();
+ java.nio.file.Files.writeString(page.toPath(), """
+ <!doctype html><html><head><title>HOVERTEST</title></head>
+ <body style="margin:0">
+ <div style="height:960px;background:linear-gradient(
+ to right,#e6194b,#f58231,#ffe119,#bfef45,#3cb44b,
+ #42d4f4,#4363d8,#911eb4,#f032e6)"></div>
+ <script>
+ window.addEventListener('mousemove', e => {
+ document.title = 'HOVER ' + e.clientX + ','
+ + e.clientY;
+ if (e.clientX < 300 && e.clientY < 300) {
+ document.body.style.background = '#f80';
+ document.body.innerHTML =
+ '<h1 style="font-size:100px">HOVERED</h1>';
+ }
+ }, {capture: true, passive: true});
+ </script></body></html>
+ """);
+ System.setProperty("sixth.firefox.url", page.toURI().toString());
+ }
+
+ /**
+ * Verifies hover forwarding: while unfocused, hovering the panel must
+ * NOT move the pointer inside Firefox; once focused, hovering the
+ * top-left area must deliver a mousemove there (page turns orange).
+ */
+ private static int runHoverTest(final Workspace workspace)
+ throws Exception {
+ final FirefoxPanel panel = workspace.getFirefoxPanel();
+ if (panel == null) {
+ System.out.println("HOVERTEST FAIL: no firefox panel");
+ workspace.close();
+ return 1;
+ }
+ final long deadline = System.currentTimeMillis() + 60_000;
+ while (System.currentTimeMillis() < deadline) {
+ if (panel.hasLiveFrame())
+ break;
+ Thread.sleep(500);
+ }
+ if (!panel.hasLiveFrame()) {
+ System.out.println("HOVERTEST FAIL: no live firefox capture");
+ workspace.close();
+ return 1;
+ }
+ Thread.sleep(2_000);
+ final int[] pixels = panel.getTexture().primaryBitmap.pixels;
+
+ // unfocused hover must NOT reach the page
+ final int[] beforeHover = pixels.clone();
+ panel.mouseHover(150, 200);
+ Thread.sleep(2_000);
+ if (countChangedPixels(beforeHover, pixels) > 100_000) {
+ System.out.println("HOVERTEST FAIL: unfocused hover leaked"
+ + " into the page");
+ workspace.close();
+ return 1;
+ }
+
+ // focus (click at 640,480 — forwarded pointer lands outside the
+ // top-left quadrant, page stays unchanged)
+ panel.mouseClicked(
+ eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_LEFT,
+ 640, 480);
+ Thread.sleep(1_500);
+ if (!panel.hasKeyboardFocus()) {
+ System.out.println("HOVERTEST FAIL: focus was not acquired");
+ workspace.close();
+ return 1;
+ }
+
+ // focused hover into the top-left quadrant must turn it orange
+ final int[] focused = pixels.clone();
+ panel.mouseHover(150, 200);
+ final int changed = awaitPixelChange(pixels, focused, 10_000);
+ if (changed <= 100_000) {
+ System.out.println("HOVERTEST FAIL: focused hover changed only "
+ + changed + " pixels");
+ workspace.close();
+ return 1;
+ }
+
+ System.out.println("HOVERTEST PASS — unfocused hover not forwarded,"
+ + " focused hover moved the pointer (" + changed
+ + " pixels changed)");
+ workspace.close();
+ return 0;
+ }
+
+ private static int awaitPixelChange(final int[] pixels,
+ final int[] reference,
+ final long timeoutMs)
+ throws InterruptedException {
+ final long deadline = System.currentTimeMillis() + timeoutMs;
+ int changed = 0;
+ while (System.currentTimeMillis() < deadline) {
+ Thread.sleep(500);
+ changed = countChangedPixels(reference, pixels);
+ if (changed > 100_000)
+ break;
+ }
+ return changed;
+ }
+
+ /**
+ * Verifies keyboard forwarding into the virtual display: the test page
+ * has an input field that turns the whole screen green once it
+ * receives the text "hello". The panel is focused, the input is
+ * clicked (focus it in the page), then "hello" is typed as AWT key
+ * events through the panel's keyboard path. Also verifies that
+ * Shift+ESC releases focus. Exit 0 = pass.
+ */
+ private static int runKeyTest(final Workspace workspace)
+ throws Exception {
+ final var panel = workspace.getFirefoxPanel();
+ final long deadline = System.currentTimeMillis() + 60_000;
+ while (System.currentTimeMillis() < deadline) {
+ if (panel.hasLiveFrame())
+ break;
+ Thread.sleep(500);
+ }
+ if (!panel.hasLiveFrame()) {
+ System.out.println("KEYTEST FAIL: no live frame within 60s");
+ workspace.close();
+ return 1;
+ }
+ Thread.sleep(3000);
+
+ final int[] pixels = panel.getTexture().primaryBitmap.pixels;
+
+ // focus the panel (first click grabs focus only)
+ panel.mouseClicked(
+ eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_LEFT,
+ 640, 480);
+ if (!panel.hasKeyboardFocus()) {
+ System.out.println("KEYTEST FAIL: first click did not focus"
+ + " the panel");
+ workspace.close();
+ return 1;
+ }
+
+ // click the input field (page coords 600,280 + chrome offset
+ // lands it near texture 700,390) so typed text goes into it
+ panel.mouseClicked(
+ eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_LEFT,
+ 700, 390);
+ Thread.sleep(1500);
+
+ // type "hello" as raw key events through the panel keyboard path
+ final int[] before = pixels.clone();
+ final var viewPanel = workspace.getViewFrame().getViewPanel();
+ for (final char c : "hello".toCharArray()) {
+ final int keyCode = java.awt.event.KeyEvent.getExtendedKeyCodeForChar(c);
+ panel.keyPressed(new java.awt.event.KeyEvent(viewPanel,
+ java.awt.event.KeyEvent.KEY_PRESSED,
+ System.currentTimeMillis(), 0, keyCode, c), viewPanel);
+ panel.keyReleased(new java.awt.event.KeyEvent(viewPanel,
+ java.awt.event.KeyEvent.KEY_RELEASED,
+ System.currentTimeMillis(), 0, keyCode, c), viewPanel);
+ Thread.sleep(150);
+ }
+
+ // the page turns green once the input contains "hello"
+ final long typeDeadline = System.currentTimeMillis() + 15_000;
+ int changed = 0;
+ while (System.currentTimeMillis() < typeDeadline) {
+ Thread.sleep(500);
+ changed = countChangedPixels(before, pixels);
+ if (changed > 100_000)
+ break;
+ }
+ if (changed <= 100_000) {
+ saveTexturePng(panel, "/tmp/keytest-texture.png");
+ System.out.println("KEYTEST FAIL: typing changed only "
+ + changed + " pixels (texture dump: "
+ + "/tmp/keytest-texture.png)");
+ workspace.close();
+ return 1;
+ }
+
+ // Shift+ESC releases focus
+ final int shiftMods = java.awt.event.InputEvent.SHIFT_DOWN_MASK;
+ panel.keyPressed(new java.awt.event.KeyEvent(viewPanel,
+ java.awt.event.KeyEvent.KEY_PRESSED,
+ System.currentTimeMillis(), shiftMods,
+ java.awt.event.KeyEvent.VK_SHIFT,
+ java.awt.event.KeyEvent.CHAR_UNDEFINED), viewPanel);
+ panel.keyPressed(new java.awt.event.KeyEvent(viewPanel,
+ java.awt.event.KeyEvent.KEY_PRESSED,
+ System.currentTimeMillis(), shiftMods,
+ java.awt.event.KeyEvent.VK_ESCAPE, '\e'), viewPanel);
+ Thread.sleep(500);
+ if (panel.hasKeyboardFocus()) {
+ System.out.println("KEYTEST FAIL: Shift+ESC did not release"
+ + " focus");
+ workspace.close();
+ return 1;
+ }
+
+ System.out.println("KEYTEST PASS — typed text reached the page ("
+ + changed + " pixels changed), Shift+ESC released focus");
+ workspace.close();
+ return 0;
+ }
+
+ /**
+ * Verifies scroll wheel forwarding on both axes:
+ * <ul>
+ * <li>Firefox: the test page counts wheel events (deltaY and
+ * deltaX) and turns blue once it has seen 2 notches down and
+ * 2 notches left.</li>
+ * <li>Terminal: the wheel becomes arrow keys — wheel-up at a prompt
+ * with a drafted line recalls history, wheel-left moves the
+ * cursor inside the drafted line.</li>
+ * </ul>
+ * Exit 0 = pass.
+ */
+ private static int runScrollTest(final Workspace workspace)
+ throws Exception {
+ final var panel = workspace.getFirefoxPanel();
+ final long deadline = System.currentTimeMillis() + 60_000;
+ while (System.currentTimeMillis() < deadline) {
+ if (panel.hasLiveFrame())
+ break;
+ Thread.sleep(500);
+ }
+ if (!panel.hasLiveFrame()) {
+ System.out.println("SCROLLTEST FAIL: no live frame within 60s");
+ workspace.close();
+ return 1;
+ }
+ Thread.sleep(3000);
+
+ // --- firefox: both wheel axes reach the page ---
+ panel.mouseClicked(
+ eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_LEFT,
+ 640, 480);
+ if (!panel.hasKeyboardFocus()) {
+ System.out.println("SCROLLTEST FAIL: first click did not focus"
+ + " the panel");
+ workspace.close();
+ return 1;
+ }
+
+ final int[] pixels = panel.getTexture().primaryBitmap.pixels;
+ final int[] before = pixels.clone();
+ panel.mouseWheelMoved(2, 0); // two notches down
+ Thread.sleep(300);
+ panel.mouseWheelMoved(0, -2); // two notches left
+ final long scrollDeadline = System.currentTimeMillis() + 15_000;
+ int changed = 0;
+ while (System.currentTimeMillis() < scrollDeadline) {
+ Thread.sleep(500);
+ changed = countChangedPixels(before, pixels);
+ if (changed > 100_000)
+ break;
+ }
+ if (changed <= 100_000) {
+ saveTexturePng(panel, "/tmp/scrolltest-texture.png");
+ System.out.println("SCROLLTEST FAIL: wheel changed only "
+ + changed + " pixels (texture dump: "
+ + "/tmp/scrolltest-texture.png)");
+ workspace.close();
+ return 1;
+ }
+
+ // --- terminal: wheel becomes arrow keys ---
+ final TerminalPanel terminal = workspace.getTerminalPanel();
+ if (!waitFor(terminal, "n0@tiny")) {
+ System.out.println("SCROLLTEST FAIL: no shell prompt appeared");
+ workspace.close();
+ return 1;
+ }
+ // controlled history: wheel-up is 3 arrow-ups per notch, so after
+ // A, B, C the recalled line is deterministically A
+ terminal.typeText("echo WHEEL_A\r");
+ if (!waitFor(terminal, "WHEEL_A")) {
+ System.out.println("SCROLLTEST FAIL: base command A failed");
+ workspace.close();
+ return 1;
+ }
+ terminal.typeText("echo WHEEL_B\r");
+ Thread.sleep(700);
+ terminal.typeText("echo WHEEL_C\r");
+ Thread.sleep(700);
+
+ // draft a partial command, then wheel up: bash history recall
+ // (3 up-arrows per notch) replaces the draft with WHEEL_A's line
+ terminal.typeText("echo DRAFT");
+ Thread.sleep(500);
+ terminal.mouseWheelMoved(-1, 0);
+ Thread.sleep(1000);
+ final String screenAfterUp = terminal.getScreenText().trim();
+ if (!screenAfterUp.endsWith("echo WHEEL_A")) {
+ System.out.println("SCROLLTEST FAIL: wheel-up did not recall"
+ + " history; screen:\n" + screenAfterUp);
+ workspace.close();
+ return 1;
+ }
+
+ // wheel left: cursor moves into the recalled line, typed text
+ // lands in the middle of it: "echo WHEEL_A" (12 chars), 6
+ // cursor-lefts -> insert at position 6 -> "echo WQHEEL_A"
+ terminal.mouseWheelMoved(0, -2); // 6 cursor-lefts
+ Thread.sleep(500);
+ terminal.typeText("Q");
+ Thread.sleep(500);
+ if (!terminal.getScreenText().trim().endsWith("echo WQHEEL_A")) {
+ System.out.println("SCROLLTEST FAIL: horizontal wheel did not"
+ + " move the cursor left; screen:\n"
+ + terminal.getScreenText());
+ workspace.close();
+ return 1;
+ }
+
+ System.out.println("SCROLLTEST PASS — firefox wheel both axes ("
+ + changed + " pixels changed), terminal wheel-as-arrows");
+ workspace.close();
+ return 0;
+ }
+
+ /**
+ * Writes the scroll-target page and points the Firefox panel at it.
+ * Must run BEFORE the workspace is constructed, because the panel
+ * reads the URL system property in its constructor.
+ */
+ private static void prepareScrollTestPage() throws Exception {
+ final java.io.File page = java.io.File.createTempFile(
+ "sixth-scrolltest", ".html");
+ page.deleteOnExit();
+ java.nio.file.Files.writeString(page.toPath(), """
+ <!doctype html><html><head><title>SCROLL none</title></head>
+ <body style="margin:0">
+ <div style="height:200px;background:linear-gradient(
+ to right,#e6194b,#f58231,#ffe119,#bfef45,#3cb44b,
+ #42d4f4,#4363d8,#911eb4,#f032e6)"></div>
+ <div style="padding:20px;font-size:24px">
+ Scroll down twice and left twice.</div>
+ <script>
+ let v = 0, h = 0;
+ window.addEventListener('wheel', e => {
+ v += Math.sign(e.deltaY);
+ h += Math.sign(e.deltaX);
+ document.title = 'SCROLL v=' + v + ' h=' + h;
+ if (v >= 2 && h <= -2) {
+ document.body.style.background = '#00f';
+ document.body.innerHTML =
+ '<h1 style="font-size:100px">SCROLLED</h1>';
+ }
+ }, {capture: true, passive: true});
+ </script></body></html>
+ """);
+ System.setProperty("sixth.firefox.url", page.toURI().toString());
+ }
+
+ /**
+ * Writes the keyboard-target page and points the Firefox panel at it.
+ * Must run BEFORE the workspace is constructed, because the panel
+ * reads the URL system property in its constructor.
+ */
+ private static void prepareKeyTestPage() throws Exception {
+ final java.io.File page = java.io.File.createTempFile(
+ "sixth-keytest", ".html");
+ page.deleteOnExit();
+ java.nio.file.Files.writeString(page.toPath(), """
+ <!doctype html><html><head><title>KEYTEST none</title></head>
+ <body style="margin:0">
+ <div style="height:200px;background:linear-gradient(
+ to right,#e6194b,#f58231,#ffe119,#bfef45,#3cb44b,
+ #42d4f4,#4363d8,#911eb4,#f032e6)"></div>
+ <input id="i" style="position:absolute;left:600px;top:280px;
+ width:200px;height:40px;font-size:30px">
+ <script>
+ const i = document.getElementById('i');
+ i.oninput = () => {
+ document.title = 'KEYTEST ' + i.value;
+ if (i.value === 'hello') {
+ document.body.style.background = '#0f0';
+ document.body.innerHTML =
+ '<h1 style="font-size:100px">TYPED-OK</h1>';
+ }
+ };
+ </script></body></html>
+ """);
+ System.setProperty("sixth.firefox.url", page.toURI().toString());
+ }
+
+ /**
+ * Writes the click-target page and points the Firefox panel at it.
+ * Must run BEFORE the workspace is constructed, because the panel
+ * reads the URL system property in its constructor.
+ */
+ private static void prepareClickTestPage() throws Exception {
+ // two pages linked by real navigation, so the mouse back/forward
+ // buttons have history to walk
+ final java.io.File page2 = java.io.File.createTempFile(
+ "sixth-clicktest-2", ".html");
+ page2.deleteOnExit();
+ java.nio.file.Files.writeString(page2.toPath(), """
+ <!doctype html><html><head><title>WITNESS page2</title></head>
+ <body style="margin:0;background:#f00">
+ <h1 style="font-size:100px">CLICKED</h1>
+ </body></html>
+ """);
+
+ final java.io.File page1 = java.io.File.createTempFile(
+ "sixth-clicktest-1", ".html");
+ page1.deleteOnExit();
+ java.nio.file.Files.writeString(page1.toPath(), """
+ <!doctype html><html><head><title>WITNESS none</title></head>
+ <body style="margin:0">
+ <div style="height:200px;background:linear-gradient(
+ to right,#e6194b,#f58231,#ffe119,#bfef45,#3cb44b,
+ #42d4f4,#4363d8,#911eb4,#f032e6)"></div>
+ <div style="padding:20px;font-size:24px">
+ Click anywhere to navigate to the red page.</div>
+ <script>
+ window.onmousedown = e => document.title =
+ 'WITNESS down ' + e.clientX + ',' + e.clientY;
+ window.addEventListener('click', e => {
+ document.title = 'WITNESS winclick ' + e.clientX
+ + ',' + e.clientY;
+ location.href = 'PAGE2URL';
+ }, true);
+ </script></body></html>
+ """.replace("PAGE2URL", page2.toURI().toString()));
+ System.setProperty("sixth.firefox.url", page1.toURI().toString());
+ }
+
+ private static void saveTexturePng(final eu.svjatoslav.sixth.workspace.FirefoxPanel panel,
+ final String path) throws Exception {
+ final int w = eu.svjatoslav.sixth.workspace.FirefoxPanel.CAPTURE_WIDTH;
+ final int h = eu.svjatoslav.sixth.workspace.FirefoxPanel.CAPTURE_HEIGHT;
+ final var image = new java.awt.image.BufferedImage(w, h,
+ java.awt.image.BufferedImage.TYPE_INT_ARGB);
+ image.setRGB(0, 0, w, h, panel.getTexture().primaryBitmap.pixels, 0, w);
+ javax.imageio.ImageIO.write(image, "png", new java.io.File(path));
+ }
+
+ private static int countChangedPixels(final int[] a, final int[] b) {
+ int changed = 0;
+ for (int i = 0; i < a.length; i++)
+ if (a[i] != b[i])
+ changed++;
+ return changed;
+ }
+
private static boolean waitFor(final TerminalPanel terminal,
final String needle)
throws InterruptedException {
--- /dev/null
+/*
+ * Sixth spatial computing environment. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ */
+package eu.svjatoslav.sixth.workspace;
+
+import eu.svjatoslav.sixth.bridge.x11.GuiAppSession;
+import eu.svjatoslav.sixth.e3d.geometry.Point2D;
+import eu.svjatoslav.sixth.e3d.gui.GuiComponent;
+import eu.svjatoslav.sixth.e3d.gui.ViewPanel;
+import eu.svjatoslav.sixth.e3d.math.Transform;
+import eu.svjatoslav.sixth.e3d.renderer.raster.shapes.composite.TexturedRectangle;
+import eu.svjatoslav.sixth.e3d.renderer.raster.texture.Texture;
+
+import java.io.IOException;
+
+/**
+ * A live Firefox browser panel in 3D space.
+ *
+ * <p>Firefox runs on a private Xvfb display (via {@link GuiAppSession});
+ * a capture thread copies the virtual screen into the texture of a
+ * {@link TexturedRectangle}, so the browser window appears as an object
+ * in the world — including in stereo/VR rendering, like every other
+ * shape.</p>
+ *
+ * <p>Interaction: the first click focuses the panel; while focused,
+ * left/right clicks are forwarded into the browser via the X TEST
+ * extension at the exact clicked position (the engine reports the
+ * perspective-correct texture coordinate of the hit), and all keyboard
+ * input is forwarded as X key events. Middle click or Shift+ESC releases
+ * focus and is never forwarded; plain ESC goes to the browser. The
+ * capture thread writes the texture's primary bitmap
+ * directly and requests a repaint — the same benign writer/renderer race
+ * the terminal panel already accepts.</p>
+ */
+public class FirefoxPanel extends GuiComponent {
+
+ /**
+ * Virtual screen (and texture) resolution: the browser window fills
+ * the whole Xvfb screen, so this is also the captured window size.
+ */
+ public static final int CAPTURE_WIDTH = 1280;
+ public static final int CAPTURE_HEIGHT = 960;
+
+ private final TexturedRectangle rectangle;
+ private final GuiAppSession session;
+ private volatile boolean liveFrameSeen;
+ private double lastHoverU = Double.NaN;
+ private double lastHoverV = Double.NaN;
+ private int lastForwardedHoverX = -1;
+ private int lastForwardedHoverY = -1;
+
+ /**
+ * Creates the panel. Call {@link #start()} to launch the browser and
+ * begin streaming its screen into the texture.
+ *
+ * @param transform position in the world
+ * @param viewPanel the view panel this component belongs to
+ * @param sizeInWorldCoordinates panel size in world units; the browser
+ * capture is stretched to fill it
+ */
+ public FirefoxPanel(final Transform transform, final ViewPanel viewPanel,
+ final Point2D sizeInWorldCoordinates)
+ throws IOException {
+ super(transform, viewPanel, sizeInWorldCoordinates.to3D());
+
+ rectangle = new TexturedRectangle(new Transform(),
+ (int) sizeInWorldCoordinates.x,
+ (int) sizeInWorldCoordinates.y,
+ CAPTURE_WIDTH, CAPTURE_HEIGHT, 1);
+ rectangle.setMouseInteractionController(this);
+ addShape(rectangle);
+
+ session = GuiAppSession.firefox(CAPTURE_WIDTH, CAPTURE_HEIGHT,
+ System.getProperty("sixth.firefox.url", "about:home"));
+ }
+
+ /**
+ * Launches Xvfb + Firefox and starts the screen capture.
+ */
+ public void start() throws IOException {
+ session.start(rectangle.getTexture().primaryBitmap.pixels,
+ this::onFrameCaptured);
+ }
+
+ private void onFrameCaptured() {
+ try {
+ // captured pixels are already in the primary bitmap; drop the
+ // stale mipmaps so they regenerate lazily from the new frame
+ rectangle.getTexture().resetResampledBitmapCache();
+ if (!liveFrameSeen)
+ liveFrameSeen = looksLikeRealScreen(
+ rectangle.getTexture().primaryBitmap.pixels);
+ viewPanel.repaintDuringNextViewUpdate();
+ } catch (final Throwable throwable) {
+ // renderer may briefly reallocate internals (e.g. on window
+ // resize); never let that kill the capture thread
+ }
+ }
+
+ /**
+ * A real application screen has many distinct colors; an empty Xvfb
+ * root (opaque black) collapses to a single one. The alpha channel
+ * is ignored: the very first capture of a black screen differs from
+ * the zero-initialized buffer in alpha only, and must not count.
+ */
+ static boolean looksLikeRealScreen(final int[] pixels) {
+ final var distinctColors = new java.util.HashSet<Integer>();
+ // dense enough to hit antialiased text and icons: a very coarse
+ // stride sees only the few flat background colors of a mostly
+ // empty page and reports a live browser screen as "flat"
+ for (int i = 0; i < pixels.length; i += 997)
+ distinctColors.add(pixels[i] & 0xFFFFFF);
+ return distinctColors.size() >= 16;
+ }
+
+ /**
+ * Whether the browser has painted a real screen into the texture
+ * (many distinct colors) — used by the automated GUI test. A black
+ * empty Xvfb root does not count.
+ */
+ public boolean hasLiveFrame() {
+ return liveFrameSeen;
+ }
+
+ /**
+ * The texture the captured browser screen is written into.
+ */
+ public Texture getTexture() {
+ return rectangle.getTexture();
+ }
+
+ public GuiAppSession getSession() {
+ return session;
+ }
+
+ /**
+ * Focus and click-forwarding behavior:
+ * <ul>
+ * <li>click while unfocused: take keyboard focus (first click is a
+ * focus grab, not forwarded to the browser)</li>
+ * <li>left/right click while focused: forwarded into the browser at
+ * the clicked texture position (texture pixels map 1:1 onto the
+ * virtual screen, and the browser window fills it at +0+0)</li>
+ * <li>middle click: releases focus (workspace convention, like ESC)
+ * and is never forwarded to the browser</li>
+ * </ul>
+ */
+ @Override
+ public boolean mouseClicked(final int button, final double textureU,
+ final double textureV) {
+ if (!hasKeyboardFocus()
+ || button == eu.svjatoslav.sixth.e3d.gui.humaninput.MouseEvent.BUTTON_MIDDLE
+ || Double.isNaN(textureU) || Double.isNaN(textureV))
+ return super.mouseClicked(button);
+
+ // Mouse back/forward buttons are X buttons 8/9. AWT reports them
+ // as 6/7 on Linux and 4/5 on other platforms — accept both.
+ final int xButton = switch (button) {
+ case 4, 6 -> 8; // back
+ case 5, 7 -> 9; // forward
+ default -> button;
+ };
+ session.sendMouseClick((int) textureU, (int) textureV, xButton);
+ return true;
+ }
+
+ @Override
+ public boolean mouseHover(final double textureU, final double textureV) {
+ lastHoverU = textureU;
+ lastHoverV = textureV;
+ // while focused the app follows the mouse cursor without clicking
+ if (hasKeyboardFocus() && !Double.isNaN(textureU)) {
+ final int x = (int) textureU;
+ final int y = (int) textureV;
+ if (x != lastForwardedHoverX || y != lastForwardedHoverY) {
+ lastForwardedHoverX = x;
+ lastForwardedHoverY = y;
+ session.sendMouseMove(x, y);
+ }
+ }
+ return false; // no repaint needed
+ }
+
+ @Override
+ public boolean mouseWheelMoved(final int verticalUnits,
+ final int horizontalUnits) {
+ // scroll at the current pointer position so the element under the
+ // cursor scrolls, like on a real desktop
+ final int x = Double.isNaN(lastHoverU) ? CAPTURE_WIDTH / 2
+ : (int) lastHoverU;
+ final int y = Double.isNaN(lastHoverV) ? CAPTURE_HEIGHT / 2
+ : (int) lastHoverV;
+ session.sendScroll(verticalUnits, horizontalUnits, x, y);
+ return true;
+ }
+
+ @Override
+ public boolean keyPressed(final java.awt.event.KeyEvent event,
+ final ViewPanel viewPanel) {
+ if (isShiftEscape(event))
+ // focus-release shortcut, never forwarded to the browser
+ return super.keyPressed(event, viewPanel);
+ session.sendKeyEvent(event, true);
+ return true;
+ }
+
+ @Override
+ public boolean keyReleased(final java.awt.event.KeyEvent event,
+ final ViewPanel viewPanel) {
+ if (isShiftEscape(event))
+ // press side already consumed this shortcut; swallow release
+ return true;
+ session.sendKeyEvent(event, false);
+ return true;
+ }
+
+ /**
+ * Shift+ESC releases focus (same convention as the terminal panels).
+ * Plain ESC is forwarded to the browser, which may need it itself.
+ */
+ private static boolean isShiftEscape(final java.awt.event.KeyEvent event) {
+ return event.getKeyCode() == java.awt.event.KeyEvent.VK_ESCAPE
+ && event.isShiftDown();
+ }
+
+ @Override
+ public boolean focusLost(final ViewPanel viewPanel) {
+ // Shift+ESC pops focus while Shift is still held; the Shift press
+ // was forwarded but its release will go to the next focus owner,
+ // so release modifiers server-side to avoid a stuck Shift
+ session.releaseModifiers();
+ return super.focusLost(viewPanel);
+ }
+
+ /**
+ * Stops the capture thread, kills Firefox and its Xvfb server.
+ */
+ public void stop() {
+ session.close();
+ }
+}
return true;
}
+ @Override
+ public boolean mouseWheelMoved(final int verticalUnits,
+ final int horizontalUnits) {
+ // The emulator has no scrollback and no mouse reporting, so the
+ // wheel is translated to arrow keys — scrolls less/htop/mc, and
+ // walks the command history at a shell prompt. Three lines per
+ // notch, matching xterm.
+ for (int i = 0; i < Math.abs(verticalUnits) * 3; i++)
+ session.send(verticalUnits < 0 ? cursorKey('A', 'A')
+ : cursorKey('B', 'B'));
+ for (int i = 0; i < Math.abs(horizontalUnits) * 3; i++)
+ session.send(horizontalUnits < 0 ? cursorKey('D', 'D')
+ : cursorKey('C', 'C'));
+ return true;
+ }
+
/**
* Whether the terminal is in application cursor keys mode (DECCKM).
* Curses programs (mc, htop, vim) enable it; cursor keys must then be
import eu.svjatoslav.sixth.e3d.renderer.raster.shapes.composite.wireframe.Grid2D;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import static eu.svjatoslav.sixth.e3d.geometry.Point3D.point;
import static eu.svjatoslav.sixth.e3d.renderer.raster.Color.hex;
* The Sixth workspace: a persistent 3D world where programs are placeable
* objects.
*
- * <p>Current population: one text editor (engine component) and one live
- * terminal running bash on a real PTY. Click a panel to focus it and type;
- * ESC releases focus; fly with the engine's standard camera controls.</p>
+ * <p>Current population: one text editor (engine component), a 4x4
+ * grid of live terminals, each running its own bash on a real PTY, and a
+ * live Firefox browser (private Xvfb display captured into a texture).
+ * Click a panel to focus it and type; ESC or middle click releases focus; fly with the
+ * engine's standard camera controls.</p>
*/
public class Workspace {
*/
public static final int TERMINAL_COLUMNS = 100;
public static final int TERMINAL_ROWS = 30;
+ public static final int TERMINAL_GRID_COLUMNS = 4;
+ public static final int TERMINAL_GRID_ROWS = 4;
private final ViewFrame viewFrame;
- private TerminalPanel terminalPanel;
+ private final List<TerminalPanel> terminalPanels = new ArrayList<>();
+ private FirefoxPanel firefoxPanel;
public Workspace() {
viewFrame = new ViewFrame("Sixth");
addGrid(scene);
addTextEditor(viewPanel, scene);
- addTerminal(viewPanel, scene);
+ addTerminals(viewPanel, scene);
+ addFirefox(viewPanel, scene);
viewPanel.repaintDuringNextViewUpdate();
}
new Point2D(400, 240), new LookAndFeel());
editor.setText("Sixth workspace\n\n"
+ "Click a panel to focus it.\n"
- + "Type into it. ESC releases focus\n"
- + "(in the terminal: Shift+ESC).\n\n"
- + "The panel on the right is a real\n"
- + "bash shell running on a PTY.\n"
+ + "Type into it. Middle click or ESC releases\n"
+ + "focus (in the terminal and browser: Shift+ESC).\n\n"
+ + "The 4x4 grid on the right contains\n"
+ + "independent bash shells.\n"
+ "Try: ls, mc, htop");
scene.addShape(editor);
}
- private void addTerminal(final ViewPanel viewPanel,
+ private void addTerminals(final ViewPanel viewPanel,
final ShapeCollection scene) throws IOException {
- final PtySession session = new PtySession(TERMINAL_COLUMNS,
- TERMINAL_ROWS);
- terminalPanel = new TerminalPanel(
- new Transform(point(100, 0, 300)), viewPanel,
- new Point2D(TERMINAL_COLUMNS * TextCanvas.FONT_CHAR_WIDTH,
- TERMINAL_ROWS * TextCanvas.FONT_CHAR_HEIGHT),
- session);
- scene.addShape(terminalPanel);
+ for (int x = 0; x < TERMINAL_GRID_COLUMNS; x++) {
+ for (int y = 0; y < TERMINAL_GRID_ROWS; y++) {
+ final PtySession session = new PtySession(TERMINAL_COLUMNS,
+ TERMINAL_ROWS);
+ final TerminalPanel terminalPanel = new TerminalPanel(
+ new Transform(point(100 + (x * 900),
+ -100 - (y * 600), 300)), viewPanel,
+ new Point2D(
+ TERMINAL_COLUMNS * TextCanvas.FONT_CHAR_WIDTH,
+ TERMINAL_ROWS * TextCanvas.FONT_CHAR_HEIGHT),
+ session);
+ terminalPanels.add(terminalPanel);
+ scene.addShape(terminalPanel);
+ }
+ }
}
+ /**
+ * Adds a live Firefox browser below the text editor: a private Xvfb
+ * display whose screen is captured into a textured rectangle.
+ */
+ private void addFirefox(final ViewPanel viewPanel,
+ final ShapeCollection scene) throws IOException {
+ firefoxPanel = new FirefoxPanel(new Transform(point(-700, -450, 300)),
+ viewPanel, new Point2D(640, 480));
+ firefoxPanel.start();
+ scene.addShape(firefoxPanel);
+ }
+
+ /**
+ * Returns the Firefox panel, used by the automated GUI test.
+ */
+ public FirefoxPanel getFirefoxPanel() {
+ return firefoxPanel;
+ }
+
+ /**
+ * Returns the first terminal, used by the automated terminal selftest.
+ */
public TerminalPanel getTerminalPanel() {
- return terminalPanel;
+ return terminalPanels.get(0);
+ }
+
+ public List<TerminalPanel> getTerminalPanels() {
+ return Collections.unmodifiableList(terminalPanels);
}
public eu.svjatoslav.sixth.e3d.gui.headtrack.HeadTracker getHeadTracker() {
}
/**
- * Shuts down background processes (the shell).
+ * Shuts down all background shell processes.
*/
public void close() {
- if (terminalPanel != null)
+ for (final TerminalPanel terminalPanel : terminalPanels)
terminalPanel.getSession().stop();
+ if (firefoxPanel != null)
+ firefoxPanel.stop();
}
}
projects / sixth.git / commitdiff