ci(visual): add visual-regression test pipeline + Xwayland scenario

.gitattributes

@@ -0,0 +1,5 @@
+*.ppm binary
+*.png binary
+*.jpg binary
+*.bin binary
+*.img binary

.github/workflows/ci.yml

@@ -414,6 +414,53 @@ jobs:
             container_image: base
             limit_to_owner: ""
             main_pr_only: false
+          # Visual regression matrix: re-enable per arch once at least one
+          # scenario has that arch listed in its `arches` file and a verified
+          # golden is committed. Until then run_all.sh exits 1 (no scenarios
+          # ran), so the matrix would be uniformly red. The DRM card0 PR (#506)
+          # is the current prerequisite for weston_shm_baseline / xwayland_xeyes.
+          # - name: Visual starry riscv64
+          #   use_container: true
+          #   runs_on: '["ubuntu-latest"]'
+          #   command: |
+          #     apt-get update && apt-get install -y --no-install-recommends e2tools
+          #     cargo xtask starry build --arch riscv64
+          #     cargo xtask starry rootfs --arch riscv64
+          #     python3 scripts/visual-test/prepare_rootfs_extras.py --arch riscv64 --scenario xwayland_xeyes
+          #     bash scripts/visual-test/run_all.sh --arch riscv64
+          #   cache_key: test-visual-riscv64
+          #   container_image: base
+          #   apk_region: us
+          #   limit_to_owner: ""
+          #   main_pr_only: false
+          # - name: Visual starry aarch64
+          #   use_container: true
+          #   runs_on: '["ubuntu-latest"]'
+          #   command: |
+          #     apt-get update && apt-get install -y --no-install-recommends e2tools
+          #     cargo xtask starry build --arch aarch64
+          #     cargo xtask starry rootfs --arch aarch64
+          #     python3 scripts/visual-test/prepare_rootfs_extras.py --arch aarch64 --scenario xwayland_xeyes
+          #     bash scripts/visual-test/run_all.sh --arch aarch64
+          #   cache_key: test-visual-aarch64
+          #   container_image: base
+          #   apk_region: us
+          #   limit_to_owner: ""
+          #   main_pr_only: false
+          # - name: Visual starry x86_64
+          #   use_container: true
+          #   runs_on: '["ubuntu-latest"]'
+          #   command: |
+          #     apt-get update && apt-get install -y --no-install-recommends e2tools
+          #     cargo xtask starry build --arch x86_64
+          #     cargo xtask starry rootfs --arch x86_64
+          #     python3 scripts/visual-test/prepare_rootfs_extras.py --arch x86_64 --scenario xwayland_xeyes
+          #     bash scripts/visual-test/run_all.sh --arch x86_64
+          #   cache_key: test-visual-x86_64
+          #   container_image: base
+          #   apk_region: us
+          #   limit_to_owner: ""
+          #   main_pr_only: false
           # Stress tests: uncomment to enable (runs only on PRs targeting main).
           # - name: Stress starry aarch64
           #   use_container: true

os/StarryOS/starryos/src/init.sh

@@ -29,6 +29,21 @@ for i in 0 1 2 3 4 5 6 7; do
     : > "/run/udev/data/c13:$((64 + i))" 2>/dev/null
 done
 
+# Visual-CI hook: when run_scenario.sh injects /test_runner.sh into the
+# rootfs, fire it asynchronously before dropping to the login shell.
+# Absence of /test_runner.sh in normal/interactive boots leaves this a
+# true no-op, so this hook is harmless on user images.
+#
+# setsid detaches from the controlling tty so weston's children don't
+# get SIGHUP when init re-execs the login shell; /dev/console captures
+# the runner's progress prints into the serial log used by the harness
+# to assert that the scenario actually launched.
+if [ -x /test_runner.sh ]; then
+    echo "[init] /test_runner.sh detected, launching visual scenario"
+    setsid /test_runner.sh </dev/null >/dev/console 2>&1 &
+    echo "[init] /test_runner.sh started pid=$!"
+fi
+
 cd "$HOME" || cd /
 cat > /tmp/starry-shrc <<'EOF'
 export PS1='${USER}@${HOSTNAME}:${PWD} # '

scripts/visual-test/perceptual_diff.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""Perceptual PPM diff with a tolerance band.
+
+Counts pixels whose RGB channels differ by more than `--delta` from the
+golden. If that count exceeds `--max-changed-pct` of the total, exits
+non-zero. On failure, emits a per-channel histogram + bounding box of
+the changed region to help pinpoint what moved.
+
+Tolerance defaults are chosen to absorb the jitter we actually see:
+anti-aliasing on text (usually <2% of pixels), clock-time redraws
+(a few hundred pixels per minute boundary), compositor cursor-hide
+fade, and TCG's slightly non-deterministic frame boundaries. If a
+test frames things in ways that need tighter or looser bounds, pass
+custom thresholds on the invocation.
+
+Input files are P6 (binary) PPM written by rfb_capture.py. PNG isn't
+directly supported to keep this script dependency-free (core Python
+only); the visual-test CI job converts goldens back-and-forth via
+ImageMagick in the orchestrator.
+"""
+import argparse
+import struct
+import sys
+
+
+def read_ppm(path: str):
+    with open(path, "rb") as f:
+        magic = f.readline().strip()
+        if magic != b"P6":
+            raise RuntimeError(f"{path}: expected P6 PPM, got {magic!r}")
+        # Skip comment lines, tolerate the `width height` + `maxval` pair
+        # with or without interleaved whitespace/comments.
+        while True:
+            line = f.readline()
+            if not line.startswith(b"#"):
+                break
+        w, h = map(int, line.split())
+        maxval = int(f.readline().strip())
+        if maxval != 255:
+            raise RuntimeError(f"{path}: only maxval=255 supported, got {maxval}")
+        data = f.read(w * h * 3)
+        if len(data) != w * h * 3:
+            raise RuntimeError(
+                f"{path}: truncated pixel data ({len(data)} < {w*h*3})"
+            )
+    return w, h, data
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("golden")
+    p.add_argument("actual")
+    p.add_argument(
+        "--delta", type=int, default=8,
+        help="per-channel allowed drift before a pixel is 'changed'",
+    )
+    p.add_argument(
+        "--max-changed-pct", type=float, default=1.5,
+        help="threshold — if more than this %% of pixels differ beyond "
+             "`--delta`, the test fails",
+    )
+    args = p.parse_args()
+
+    gw, gh, gdata = read_ppm(args.golden)
+    aw, ah, adata = read_ppm(args.actual)
+    if (gw, gh) != (aw, ah):
+        print(
+            f"FAIL geometry mismatch: golden {gw}x{gh} vs actual {aw}x{ah}",
+            file=sys.stderr,
+        )
+        return 1
+
+    total = gw * gh
+    changed = 0
+    # Bounding box of changed region (for debugging regressions).
+    min_x, min_y, max_x, max_y = gw, gh, -1, -1
+    # Channel-mean delta on changed pixels — distinguishes "color shift"
+    # from "everything moved a few pixels".
+    sum_dr = sum_dg = sum_db = 0
+    for i in range(total):
+        off = i * 3
+        gr, gg, gb = gdata[off], gdata[off + 1], gdata[off + 2]
+        ar, ag, ab = adata[off], adata[off + 1], adata[off + 2]
+        dr, dg, db = abs(gr - ar), abs(gg - ag), abs(gb - ab)
+        if dr > args.delta or dg > args.delta or db > args.delta:
+            changed += 1
+            y, x = divmod(i, gw)
+            if x < min_x: min_x = x
+            if y < min_y: min_y = y
+            if x > max_x: max_x = x
+            if y > max_y: max_y = y
+            sum_dr += dr
+            sum_dg += dg
+            sum_db += db
+
+    pct = 100.0 * changed / total
+    if changed > 0:
+        avg_dr = sum_dr / changed
+        avg_dg = sum_dg / changed
+        avg_db = sum_db / changed
+        bbox = f"[{min_x},{min_y})→[{max_x+1},{max_y+1})"
+    else:
+        avg_dr = avg_dg = avg_db = 0
+        bbox = "(none)"
+
+    status = "PASS" if pct <= args.max_changed_pct else "FAIL"
+    print(
+        f"{status} changed={changed}/{total} ({pct:.3f}%) "
+        f"threshold={args.max_changed_pct}% bbox={bbox} "
+        f"avg_delta=(R:{avg_dr:.1f},G:{avg_dg:.1f},B:{avg_db:.1f})"
+    )
+    return 0 if status == "PASS" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/visual-test/prepare_rootfs_extras.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""Materialize visual-test rootfs_extras from Alpine package manifests."""
+
+from __future__ import annotations
+
+import argparse
+import io
+import os
+import re
+import shutil
+import tarfile
+import tempfile
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+
+
+ALPINE_BRANCH = os.environ.get("VISUAL_ALPINE_BRANCH", "edge")
+ALPINE_MIRROR = os.environ.get("VISUAL_ALPINE_MIRROR", "https://dl-cdn.alpinelinux.org/alpine")
+REPOS = ("main", "community")
+
+
+@dataclass(frozen=True)
+class Package:
+    name: str
+    version: str
+    repo: str
+    depends: tuple[str, ...]
+    provides: tuple[str, ...]
+
+    @property
+    def filename(self) -> str:
+        return f"{self.name}-{self.version}.apk"
+
+
+def fetch_bytes(url: str) -> bytes:
+    with urllib.request.urlopen(url) as response:
+        return response.read()
+
+
+def parse_manifest(path: Path) -> list[str]:
+    packages: list[str] = []
+    for raw_line in path.read_text(encoding="utf-8").splitlines():
+        line = raw_line.split("#", 1)[0].strip()
+        if line:
+            packages.append(line)
+    if not packages:
+        raise SystemExit(f"{path} does not list any packages")
+    return packages
+
+
+def normalize_dependency(dep: str) -> str:
+    dep = dep.strip()
+    if dep.startswith("!"):
+        return ""
+    return re.split(r"[<>=~]", dep, maxsplit=1)[0]
+
+
+def parse_apkindex(data: bytes, repo: str) -> list[Package]:
+    with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive:
+        index_member = archive.extractfile("APKINDEX")
+        if index_member is None:
+            raise SystemExit(f"APKINDEX missing from {repo}")
+        text = index_member.read().decode("utf-8")
+
+    packages: list[Package] = []
+    for record in text.strip().split("\n\n"):
+        fields: dict[str, list[str]] = {}
+        for line in record.splitlines():
+            if len(line) >= 2 and line[1] == ":":
+                fields.setdefault(line[0], []).append(line[2:])
+        names = fields.get("P", [])
+        versions = fields.get("V", [])
+        if not names or not versions:
+            continue
+        depends = tuple(
+            dep for value in fields.get("D", []) for dep in value.split() if normalize_dependency(dep)
+        )
+        provides = tuple(
+            normalize_dependency(provide)
+            for value in fields.get("p", [])
+            for provide in value.split()
+            if normalize_dependency(provide)
+        )
+        packages.append(Package(names[0], versions[0], repo, depends, provides))
+    return packages
+
+
+def load_indexes(arch: str) -> tuple[dict[str, Package], dict[str, Package]]:
+    by_name: dict[str, Package] = {}
+    by_provide: dict[str, Package] = {}
+    for repo in REPOS:
+        url = f"{ALPINE_MIRROR}/{ALPINE_BRANCH}/{repo}/{arch}/APKINDEX.tar.gz"
+        print(f"[visual] fetching {url}")
+        for package in parse_apkindex(fetch_bytes(url), repo):
+            by_name.setdefault(package.name, package)
+            for provide in package.provides:
+                by_provide.setdefault(provide, package)
+    return by_name, by_provide
+
+
+def resolve_packages(roots: list[str], by_name: dict[str, Package], by_provide: dict[str, Package]) -> list[Package]:
+    resolved: dict[str, Package] = {}
+    visiting = list(roots)
+    while visiting:
+        dep = normalize_dependency(visiting.pop())
+        if not dep:
+            continue
+        package = by_name.get(dep) or by_provide.get(dep)
+        if package is None:
+            raise SystemExit(f"could not resolve Alpine dependency {dep!r}")
+        if package.name in resolved:
+            continue
+        resolved[package.name] = package
+        visiting.extend(package.depends)
+    return sorted(resolved.values(), key=lambda package: package.name)
+
+
+def safe_extract_apk(apk_path: Path, dest: Path) -> None:
+    with tarfile.open(apk_path, mode="r:gz") as archive:
+        for member in archive.getmembers():
+            name = member.name
+            if name.startswith(".") or name in {"dev", "proc", "sys"}:
+                continue
+            target = (dest / name).resolve()
+            if not str(target).startswith(str(dest.resolve()) + os.sep):
+                raise SystemExit(f"refusing unsafe apk member path {name!r}")
+            archive.extract(member, dest)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--arch", required=True, help="Alpine architecture, for example riscv64")
+    parser.add_argument("--scenario", required=True, help="visual scenario name")
+    parser.add_argument("--repo-root", type=Path, default=None)
+    args = parser.parse_args()
+
+    repo_root = args.repo_root or Path(__file__).resolve().parents[2]
+    scenario_dir = repo_root / "test-suit" / "starryos" / "visual" / args.scenario
+    manifest = scenario_dir / "rootfs_extras.packages"
+    output = scenario_dir / "rootfs_extras"
+
+    roots = parse_manifest(manifest)
+    by_name, by_provide = load_indexes(args.arch)
+    packages = resolve_packages(roots, by_name, by_provide)
+
+    with tempfile.TemporaryDirectory(prefix="visual-rootfs-extras-") as temp_name:
+        temp = Path(temp_name)
+        staging = temp / "rootfs_extras"
+        staging.mkdir()
+        for package in packages:
+            url = f"{ALPINE_MIRROR}/{ALPINE_BRANCH}/{package.repo}/{args.arch}/{package.filename}"
+            apk_path = temp / package.filename
+            print(f"[visual] fetching {package.name} ({package.repo})")
+            apk_path.write_bytes(fetch_bytes(url))
+            safe_extract_apk(apk_path, staging)
+
+        if output.exists():
+            shutil.rmtree(output)
+        shutil.copytree(staging, output, symlinks=True)
+
+    print(f"[visual] wrote {output} with {len(packages)} packages")
+
+
+if __name__ == "__main__":
+    main()