From 7314f1185992cec62bcde6905e227051d060f1f4 Mon Sep 17 00:00:00 2001 From: Svjatoslav Agejenko Date: Sun, 4 Oct 2026 22:36:02 +0300 Subject: [PATCH] feat(tools): add incremental org-mode HTML exporter Shared tool for the sibling hobby projects (qbasicapps, crtbasic, physical, alyverkko-cli), whose "Update web site" scripts used to re-export every page in one Emacs batch process per file. - Re-exports only changed pages: per-page sha256 fingerprint covers the .org file plus its first-level #+INCLUDE dependencies (so editing an included .bas/SVG re-exports the including page) - Runs all dirty exports in a single Emacs batch process with per-file error isolation (condition-case); --sequential for debugging, --force to rebuild everything - Sentinel files (--sentinel, e.g. style.css) invalidate all pages - State kept in a per-repo .org-export-state cache (gitignored) Also correct the documented dependency version to 2.0.0. --- AGENTS.md | 6 + Documentation/index.org | 2 +- Tools/Org export incremental | 334 +++++++++++++++++++++++++++++++++++ 3 files changed, 341 insertions(+), 1 deletion(-) create mode 100755 Tools/Org export incremental diff --git a/AGENTS.md b/AGENTS.md index 29c64ef..a217e2a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,3 +76,9 @@ Uses JUnit 4 with standard `@Test` annotations. mvn test # Run all tests mvn test -Dtest=String2Test # Run specific test class ``` + +## Tools + +| Tool | Purpose | +|------|---------| +| `Tools/Org export incremental` | Shared incremental org-mode → HTML exporter used by sibling hobby-project repos (qbasicapps, crtbasic, physical, alyverkko-cli). Hashes each .org file plus its `#+INCLUDE` dependencies into a per-repo `.org-export-state` (gitignored) and re-exports only changed pages in one Emacs batch process. Supports `--force`, `--sentinel`, `--exclude`, `--sequential`, `--dry-run`. | diff --git a/Documentation/index.org b/Documentation/index.org index 03fbcc3..0d7a711 100644 --- a/Documentation/index.org +++ b/Documentation/index.org @@ -36,7 +36,7 @@ Add the *svjatoslavcommons* dependency to your Maven project: eu.svjatoslav svjatoslavcommons - 1.9 + 2.0.0 #+END_SRC diff --git a/Tools/Org export incremental b/Tools/Org export incremental new file mode 100755 index 0000000..35738c2 --- /dev/null +++ b/Tools/Org export incremental @@ -0,0 +1,334 @@ +#!/bin/bash +# ============================================================================= +# Org export incremental — incremental org-mode → HTML export +# ============================================================================= +# +# Shared tool (lives in svjatoslav_commons/Tools/) used by every hobby +# project's "Tools/Update web site" wrapper. Solves: "every publish +# re-exports every org file even when only one changed." +# +# HOW IT WORKS +# 1. Finds all .org files under --root (find -name --pattern). +# 2. For each, computes a fingerprint = sha256 over the .org file PLUS +# every existing first-level `#+INCLUDE: "file"` target (relative to +# the .org file's directory). This catches edits to included sources +# (.bas files, SVG diagrams) that a plain org-file hash would miss. +# 3. Compares fingerprints against a state file (default: +# /.org-export-state). A file is re-exported only when its +# fingerprint changed, its .html output is missing, or a --sentinel +# file (e.g. style.css) changed (which invalidates ALL pages). +# 4. All dirty files are exported in ONE Emacs batch process (per-file +# errors isolated via condition-case) instead of one Emacs startup +# per file. --sequential falls back to the old one-process-per-file +# behavior for debugging. +# +# USAGE +# Org export incremental --root DIR [options] +# --root DIR search root for org files (repeatable; required) +# --pattern GLOB find -name pattern (default: *.org) +# --maxdepth N find -maxdepth (default: none) +# --exclude NAME skip these basenames (repeatable) +# --state FILE state file (default: /.org-export-state) +# --sentinel FILE global sentinel; if changed, re-export everything +# (repeatable; e.g. style.css) +# --force ignore the cache, re-export everything +# --sequential one Emacs process per file (old, slow behavior) +# --dry-run print what would be exported, export nothing +# +# EXIT CODE: 0 when everything exported/skipped cleanly, 1 when any +# export failed. State is updated only for successfully exported files, +# so a failed file is retried on the next run. +# ============================================================================= +set -u + +ROOTS=() +PATTERN="*.org" +MAXDEPTH="" +EXCLUDES=() +SENTINELS=() +STATE_FILE="" +FORCE=0 +SEQUENTIAL=0 +DRYRUN=0 + +while [ $# -gt 0 ]; do + case "$1" in + --root) ROOTS+=("$2"); shift 2 ;; + --pattern) PATTERN="$2"; shift 2 ;; + --maxdepth) MAXDEPTH="$2"; shift 2 ;; + --exclude) EXCLUDES+=("$2"); shift 2 ;; + --state) STATE_FILE="$2"; shift 2 ;; + --sentinel) SENTINELS+=("$2"); shift 2 ;; + --force) FORCE=1; shift ;; + --sequential) SEQUENTIAL=1; shift ;; + --dry-run) DRYRUN=1; shift ;; + *) echo "❌ Unknown argument: $1" >&2; exit 2 ;; + esac +done + +if [ ${#ROOTS[@]} -eq 0 ]; then + echo "❌ At least one --root is required" >&2 + exit 2 +fi + +[ -z "$STATE_FILE" ] && STATE_FILE="${ROOTS[0]%/}/.org-export-state" + +# --- Collect org files -------------------------------------------------------- + +FIND_ARGS=() +[ -n "$MAXDEPTH" ] && FIND_ARGS+=(-maxdepth "$MAXDEPTH") + +mapfile -t ORG_FILES < <( + find "${ROOTS[@]}" "${FIND_ARGS[@]}" -type f -name "$PATTERN" | sort +) + +# Apply excludes (basename match) +if [ ${#EXCLUDES[@]} -gt 0 ]; then + FILTERED=() + for f in "${ORG_FILES[@]}"; do + base=$(basename "$f") + skip=0 + for ex in "${EXCLUDES[@]}"; do + [ "$base" = "$ex" ] && skip=1 && break + done + [ "$skip" -eq 0 ] && FILTERED+=("$f") + done + ORG_FILES=("${FILTERED[@]}") +fi + +if [ ${#ORG_FILES[@]} -eq 0 ]; then + echo "❌ No org files found under: ${ROOTS[*]}" + exit 0 +fi + +echo "🔍 Found ${#ORG_FILES[@]} org file(s) under: ${ROOTS[*]}" + +# --- Hashing helpers ---------------------------------------------------------- + +# file_sha — sha256 of file, empty when missing +file_sha() { + [ -f "$1" ] && sha256sum "$1" | awk '{print $1}' +} + +# fingerprint +# sha256 over the org file plus every existing first-level #+INCLUDE +# target. Missing include targets contribute a marker so that creating +# the file later changes the fingerprint. +fingerprint() { + local org="$1" + local dir + dir=$(dirname "$org") + { + file_sha "$org" + # Extract first-level include paths: #+INCLUDE: "path" [args...] + grep -oE '^#\+INCLUDE:[[:space:]]+"[^"]+"' "$org" 2>/dev/null \ + | sed -E 's/^#\+INCLUDE:[[:space:]]+"([^"]+)"/\1/' \ + | while IFS= read -r inc; do + if [ -f "$dir/$inc" ]; then + file_sha "$dir/$inc" + else + echo "MISSING:$inc" + fi + done + } | sha256sum | awk '{print $1}' +} + +# --- State file (TAB-separated, version 2) ------------------------------------ + +declare -A STATE_HASHES=() +STATE_SENTINEL="" + +load_state() { + [ -f "$STATE_FILE" ] || return 1 + local kind hash path + while IFS=$'\t' read -r kind hash path; do + case "$kind" in + file) STATE_HASHES["$path"]="$hash" ;; + sentinel) STATE_SENTINEL="$hash" ;; + esac + done < "$STATE_FILE" + return 0 +} + +write_state() { + local tmp="${STATE_FILE}.tmp.$$" + { + echo -e "# Org export incremental state v2 — do not edit by hand" + for path in "${!STATE_HASHES[@]}"; do + echo -e "file\t${STATE_HASHES[$path]}\t$path" + done + echo -e "sentinel\t${CURRENT_SENTINEL}\t-" + } > "$tmp" + mv "$tmp" "$STATE_FILE" +} + +# Combined sentinel hash over all declared sentinel files (missing sentinels +# contribute a marker so adding style.css later invalidates the cache). +compute_sentinel() { + { + for s in "${SENTINELS[@]}"; do + if [ -f "$s" ]; then + file_sha "$s" + else + echo "ABSENT:$s" + fi + done + } | sha256sum | awk '{print $1}' +} + +# --- Decide what needs exporting ---------------------------------------------- + +LOADED=0 +if [ "$FORCE" -eq 0 ] && load_state; then + LOADED=1 + echo "📋 Loaded state from $STATE_FILE (${#STATE_HASHES[@]} entries)" +else + [ "$FORCE" -eq 1 ] && echo "📋 --force: ignoring cache" \ + || echo "📋 No state file — first run exports everything" +fi + +CURRENT_SENTINEL="$(compute_sentinel)" +SENTINEL_CHANGED=0 +if [ ${#SENTINELS[@]} -gt 0 ] && [ "$LOADED" -eq 1 ] \ + && [ -n "$STATE_SENTINEL" ] && [ "$STATE_SENTINEL" != "$CURRENT_SENTINEL" ]; then + echo "🎨 Sentinel changed (${SENTINELS[*]}) — invalidating all pages" + SENTINEL_CHANGED=1 +fi + +DIRTY=() +declare -A DIRTY_HASH=() +SKIPPED=0 + +for org in "${ORG_FILES[@]}"; do + fp=$(fingerprint "$org") + html="${org%.org}.html" + if [ "$FORCE" -eq 0 ] && [ "$SENTINEL_CHANGED" -eq 0 ] \ + && [ "${STATE_HASHES[$org]:-}" = "$fp" ] && [ -f "$html" ]; then + echo "↻ Skipping (unchanged): $org" + ((SKIPPED++)) + continue + fi + DIRTY+=("$org") + DIRTY_HASH["$org"]="$fp" +done + +# --- Export ------------------------------------------------------------------- + +FAILED=0 +declare -A OK_FILES=() + +export_sequential() { + for org in "${DIRTY[@]}"; do + local dir base + dir=$(dirname "$org"); base=$(basename "$org" .org) + rm -f "$dir/$base.html" + echo "Exporting: $org" + if ( cd "$dir" && emacs --batch -l ~/.emacs --visit="$base.org" \ + --funcall=org-html-export-to-html --kill ); then + echo "✓ $org" + OK_FILES["$org"]=1 + else + echo "✗ FAILED: $org" + FAILED=$((FAILED + 1)) + fi + done +} + +export_batch() { + [ ${#DIRTY[@]} -eq 0 ] && return 0 + # Build the elisp list as (absolute-path . relative-tag) cons pairs: + # find-file needs the absolute path (sets default-directory right), + # while OK markers must carry the relative path used as state key. + local elisp_list="" org abs rel + for org in "${DIRTY[@]}"; do + rel="$org" + abs=$(cd "$(dirname "$org")" && pwd)/$(basename "$org") + case "$abs" in + *\"*|*\\*) echo "❌ Path unsafe for batch elisp: $abs" >&2; return 1 ;; + esac + # NOTE: (cons "a" "b") evaluates to the pair; a quoted-looking + # ("a" . "b") inside (list ...) would be EVALUATED as a function + # call and crash with wrong-type-argument. + elisp_list="$elisp_list (cons \"$abs\" \"$rel\")" + rm -f "${org%.org}.html" + echo "Exporting: $org" + done + + local eval_form + eval_form="(dolist (pair (list$elisp_list)) + (let ((f (car pair)) (tag (cdr pair))) + (condition-case err + (progn (find-file f) + (org-html-export-to-html) + (princ (format \"ORGEXPORT-OK %s\n\" tag))) + (error (princ (format \"ORGEXPORT-FAIL %s :: %s\n\" tag (error-message-string err))))))) +(kill-emacs)" + + local out rc + out=$(emacs --batch -l ~/.emacs --eval "$eval_form" 2>&1) + rc=$? + + # Parse per-file markers; update state only for OK files. + local line f + while IFS= read -r line; do + case "$line" in + ORGEXPORT-OK\ *) + f="${line#ORGEXPORT-OK }" + OK_FILES["$f"]=1 + echo "✓ $f" + ;; + ORGEXPORT-FAIL\ *) + f="${line#ORGEXPORT-FAIL }" + echo "✗ FAILED: $f" + FAILED=$((FAILED + 1)) + ;; + esac + done <<< "$out" + + # Emacs dying without markers (rc!=0 and no OK lines) = batch failure. + if [ "$rc" -ne 0 ] && [ ${#OK_FILES[@]} -eq 0 ]; then + echo "$out" | tail -20 + echo "❌ Batch Emacs failed (rc=$rc). Retry with --sequential for detail." + FAILED=$((FAILED + ${#DIRTY[@]})) + fi + return 0 +} + +if [ ${#DIRTY[@]} -gt 0 ]; then + if [ "$DRYRUN" -eq 1 ]; then + echo "🏜 Dry run — would export ${#DIRTY[@]} file(s)" + else + if [ "$SEQUENTIAL" -eq 1 ]; then + export_sequential + else + export_batch + fi + fi +fi + +# --- Persist state (only current files; only successfully exported/skipped) --- + +if [ "$DRYRUN" -eq 0 ]; then + declare -A FINAL=() + for org in "${ORG_FILES[@]}"; do + if [ -n "${OK_FILES[$org]:-}" ]; then + FINAL["$org"]="${DIRTY_HASH[$org]}" + elif [ -n "${STATE_HASHES[$org]:-}" ] && [ -f "${org%.org}.html" ]; then + # skipped or previously-known file: carry over old fingerprint + # unless it was dirty and failed (then drop → retry next run) + was_dirty=0 + for d in "${DIRTY[@]}"; do [ "$d" = "$org" ] && was_dirty=1 && break; done + [ "$was_dirty" -eq 0 ] && FINAL["$org"]="${STATE_HASHES[$org]}" + fi + done + STATE_HASHES=() + for k in "${!FINAL[@]}"; do STATE_HASHES["$k"]="${FINAL[$k]}"; done + write_state + echo "💾 Wrote state to $STATE_FILE (${#STATE_HASHES[@]} entries)" +fi + +echo "=======================================" +echo "📊 SUMMARY: exported $(( ${#DIRTY[@]} - FAILED )) · skipped $SKIPPED · failed $FAILED · total ${#ORG_FILES[@]}" + +[ "$FAILED" -gt 0 ] && exit 1 +exit 0 -- 2.20.1