From 5ec29d3209c8d07893f39e3162130abc291c0352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E6=99=A8=20=28Leo=20Cheng=29?= Date: Mon, 6 Jul 2026 04:12:50 +0800 Subject: [PATCH] test(starry): add monitoring-stack carpet (apps/starry/monitor) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit on-target carpet for the Prometheus stack (prometheus/promtool/node_exporter), Grafana and glances (cli/headless/tui-pyte/client-server/web) across the four single-core qemu arches; run_monitor.py gates on MONITOR_OK=N/N + TEST PASSED. Signed-off-by: 林晨 (Leo Cheng) Signed-off-by: 林晨 --- apps/starry/monitor/APPLY.md | 69 +++ apps/starry/monitor/README.md | 88 ++++ apps/starry/monitor/SOURCES.md | 70 +++ apps/starry/monitor/assets/MANIFEST.md | 80 +++ .../monitor/assets/build-loong-binaries.sh | 92 ++++ apps/starry/monitor/assets/prometheus.yml | 13 + .../build-aarch64-unknown-none-softfloat.toml | 13 + ...ld-loongarch64-unknown-none-softfloat.toml | 18 + .../build-riscv64gc-unknown-none-elf.toml | 14 + .../monitor/build-x86_64-unknown-none.toml | 9 + apps/starry/monitor/prebuild.sh | 344 ++++++++++++ apps/starry/monitor/programs/pty_tui_drive.py | 202 +++++++ apps/starry/monitor/programs/pyte_assert.py | 99 ++++ apps/starry/monitor/programs/run-monitor.sh | 32 ++ .../starry/monitor/python/GlancesCliCarpet.py | 128 +++++ apps/starry/monitor/python/GlancesCsCarpet.py | 120 +++++ .../monitor/python/GlancesHeadlessCarpet.py | 186 +++++++ .../starry/monitor/python/GlancesTuiCarpet.py | 242 +++++++++ .../starry/monitor/python/GlancesWebCarpet.py | 120 +++++ apps/starry/monitor/python/GrafanaCarpet.py | 282 ++++++++++ .../monitor/python/NodeExporterCarpet.py | 360 +++++++++++++ .../starry/monitor/python/PrometheusCarpet.py | 495 ++++++++++++++++++ apps/starry/monitor/python/run_monitor.py | 63 +++ apps/starry/monitor/qemu-aarch64.toml | 16 + apps/starry/monitor/qemu-loongarch64.toml | 19 + apps/starry/monitor/qemu-riscv64.toml | 15 + apps/starry/monitor/qemu-x86_64.toml | 18 + 27 files changed, 3207 insertions(+) create mode 100644 apps/starry/monitor/APPLY.md create mode 100644 apps/starry/monitor/README.md create mode 100644 apps/starry/monitor/SOURCES.md create mode 100644 apps/starry/monitor/assets/MANIFEST.md create mode 100644 apps/starry/monitor/assets/build-loong-binaries.sh create mode 100644 apps/starry/monitor/assets/prometheus.yml create mode 100644 apps/starry/monitor/build-aarch64-unknown-none-softfloat.toml create mode 100644 apps/starry/monitor/build-loongarch64-unknown-none-softfloat.toml create mode 100644 apps/starry/monitor/build-riscv64gc-unknown-none-elf.toml create mode 100644 apps/starry/monitor/build-x86_64-unknown-none.toml create mode 100644 apps/starry/monitor/prebuild.sh create mode 100644 apps/starry/monitor/programs/pty_tui_drive.py create mode 100644 apps/starry/monitor/programs/pyte_assert.py create mode 100644 apps/starry/monitor/programs/run-monitor.sh create mode 100644 apps/starry/monitor/python/GlancesCliCarpet.py create mode 100644 apps/starry/monitor/python/GlancesCsCarpet.py create mode 100644 apps/starry/monitor/python/GlancesHeadlessCarpet.py create mode 100644 apps/starry/monitor/python/GlancesTuiCarpet.py create mode 100644 apps/starry/monitor/python/GlancesWebCarpet.py create mode 100644 apps/starry/monitor/python/GrafanaCarpet.py create mode 100644 apps/starry/monitor/python/NodeExporterCarpet.py create mode 100644 apps/starry/monitor/python/PrometheusCarpet.py create mode 100644 apps/starry/monitor/python/run_monitor.py create mode 100644 apps/starry/monitor/qemu-aarch64.toml create mode 100644 apps/starry/monitor/qemu-loongarch64.toml create mode 100644 apps/starry/monitor/qemu-riscv64.toml create mode 100644 apps/starry/monitor/qemu-x86_64.toml diff --git a/apps/starry/monitor/APPLY.md b/apps/starry/monitor/APPLY.md new file mode 100644 index 0000000000..fa9aabc53a --- /dev/null +++ b/apps/starry/monitor/APPLY.md @@ -0,0 +1,69 @@ +# APPLY - 把 monitor app 落入 StarryOS / tgoskits 测试树 + +## 目录布局（apps/starry/monitor/） + +``` +apps/starry/monitor/ +├── prebuild.sh # 构建期 provision（下载 prometheus/node_exporter + apk add glances 闭包 + 解包 pyte/uvicorn wheel + 注入 carpet） +├── build-x86_64-unknown-none.toml # 四架构内核 build feature +├── build-aarch64-unknown-none-softfloat.toml +├── build-riscv64gc-unknown-none-elf.toml +├── build-loongarch64-unknown-none-softfloat.toml # LoongArch 必须 dynamic 平台（ax-driver/serial，不 opt-out） +├── qemu-x86_64.toml # 四架构 qemu 启动 + success/fail regex + timeout +├── qemu-aarch64.toml +├── qemu-riscv64.toml +├── qemu-loongarch64.toml # uefi=false / to_bin=true（dynamic 平台 raw boot） +├── assets/ +│ ├── prometheus.yml # scrape 配置（node 作业 -> loopback :9100） +│ ├── build-loong-binaries.sh # loong64 prometheus/node_exporter/grafana 的 Go 交叉编译 recipe +│ └── MANIFEST.md # 逐档 URL + sha256（prometheus/node_exporter/grafana/wheel） +├── programs/ +│ ├── run-monitor.sh # on-target 启动器（装到 /usr/bin/run-monitor.sh） +│ ├── pty_tui_drive.py # PTY 驱动器（SS3 应用键模式） +│ └── pyte_assert.py # pyte 屏幕重建 + 三重不变量断言 +└── python/ + ├── run_monitor.py # 编排器 + 唯一门控锚点（MONITOR_OK=8/8 + TEST PASSED） + ├── PrometheusCarpet.py + ├── GrafanaCarpet.py # headless grafana server + HTTP/API 断言 + ├── GlancesCliCarpet.py + ├── GlancesHeadlessCarpet.py + ├── GlancesTuiCarpet.py # pyte 真实断言的 curses TUI carpet（核心难点） + ├── GlancesCsCarpet.py + └── GlancesWebCarpet.py +``` + +## 运行 + +```bash +source <仓库根>/.starry-env.sh # qemu-10 +cargo xtask starry app qemu -t monitor --arch x86_64 +cargo xtask starry app qemu -t monitor --arch aarch64 +cargo xtask starry app qemu -t monitor --arch riscv64 +cargo xtask starry app qemu -t monitor --arch loongarch64 +``` + +判据：xtask `rc=0` + 日志 `SUCCESS PATTERN MATCHED` + `^MONITOR_OK=8/8` + `TEST PASSED`。 + +## prebuild 可调环境变量（均有安全默认） + +- `MONITOR_APK_BRANCH`（默认 `v3.23`）、`ALPINE_CDN`、`MONITOR_APK_CACHE`（离线加速，miss 走网络非早退）。 +- `MONITOR_ROOTFS_SIZE`（默认 `6G`；grow-only，overlay 经 debugfs 注入不 resize，欠大会静默截断大 .so/大二进制/grafana public）。 +- `MONITOR_BINS_DIR`（可选）：预置可复现二进制目录，离线用；**loong64** 若已按 `assets/build-loong-binaries.sh` + 预编译成 `$MONITOR_BINS_DIR/{prometheus//prometheus-3.11.3.linux-loong64.tar.gz, node_exporter//node_exporter, + grafana//grafana-13.0.1.linux-loong64.tar.gz}`，prebuild 直接取用，免去构建期交叉编译。可再放 + `$MONITOR_BINS_DIR/grafana/grafana-13.0.1.db`（架构无关预迁移 SQLite 播种）。未预置时 prebuild 会尝试现场 Go + 交叉编译（**仅需 Go≥1.25，无需 Node**：prometheus 不带内嵌 web UI = 无 npm/yarn 前端构建，grafana v13 后端亦无需前端构建）。 +- `MONITOR_GRAFANA_PREMIGRATE`（默认 `1`）：构建期尽力预迁移 grafana.db（native/qemu-user，有界，失败即回退 on-target 迁移）。 +- `MONITOR_TUI_SETTLE` / `MONITOR_TUI_DWELL`（默认 6 / 2 秒）：TUI soak 首帧稳定 + 每步停留时长。慢架构可上调。 +- `MONITOR_TUI_COLS` / `MONITOR_TUI_ROWS`（默认 200 / 50）：pty 尺寸。 +- `MONITOR_GRAFANA_HEALTH_TRIES`（默认 `1200`）：grafana /api/health 轮询上限（1s 一次；给 TCG 首启迁移留足预算）。 + +## 关键约定 + +- **四架构均运行，无 SKIP**：prometheus/node_exporter/grafana 的 amd64/arm64/riscv64 走官方 release，loong64 从源码 + Go 交叉编译（grafana 仅后端 + 嫁接官方前端）；glances 四架构 apk 齐全。 +- **LoongArch 必须 dynamic 平台**：`build-loongarch64*.toml` 保留 `ax-driver/serial` 且不 opt-out 动态平台， + `qemu-loongarch64.toml` 用 `uefi=false / to_bin=true`（静态平台缺串口 TTY 绑定 → 早退）。 +- **rootfs grow-only**：`prebuild.sh` 只增不减，never 吞掉 resize2fs 失败。 +- **门控锚点唯一**：`MONITOR_OK=8/8` + `TEST PASSED` 只在 `run_monitor.py` 里输出（`run-monitor.sh` 从不输出）， + 故 success_regex 不会被命令回显自匹配。 diff --git a/apps/starry/monitor/README.md b/apps/starry/monitor/README.md new file mode 100644 index 0000000000..aed2020da0 --- /dev/null +++ b/apps/starry/monitor/README.md @@ -0,0 +1,88 @@ +# monitor - Prometheus 监控栈 + Grafana + glances 系统监控器（StarryOS 四架构地毯测试） + +本 app 在 StarryOS 四架构（x86_64 / aarch64 / riscv64 / loongarch64）单核 qemu 上，对三大监控软件做工业级、端到端、逐断言的地毯测试（非 “进程能起来” 的冒烟）： + +- **Prometheus**（CNCF 监控系统：拉取式指标抓取 + TSDB + PromQL 引擎；`prometheus` 3.11.3 + `promtool`，CGO-free 纯静态 Go 二进制）＋ **node_exporter** 1.11.1（最简 exporter，作为真实抓取目标）。 +- **Grafana**（观测/可视化 web 应用，13.0.1；单个 CGO-free 纯静态 Go 二进制 + 内嵌前端 SPA + 内嵌 SQLite） - headless 起 server + HTTP 断言（无浏览器、无 TUI）。 +- **glances**（基于 psutil 的 Python 系统监控器，4.4.1）的全部五种运行形态：CLI / headless / **TUI（pyte 真实断言）** / client-server / web。 + +单次启动运行 8 个子 carpet，全过才由 `run_monitor.py` 输出唯一门控锚点 `MONITOR_OK=8/8` + `TEST PASSED`。 + +## 测试维度 + +### Prometheus（`PrometheusCarpet.py`） +1. **版本红线**：`prometheus --version` 与 `promtool --version` 必须精确报 `3.11.3`；`node_exporter --version` 报 `1.11.1`。版本不符即判测例无效。 +2. **`--help` 全树**：`prometheus --help` / `--help-long` 的旗标树 + `promtool --help` 子命令树（check / query / debug / test / tsdb / push，及 check 的 config/rules/metrics）逐项断言存在。 +3. **promtool 校验**：`promtool check config /etc/prometheus.yml` 返回 SUCCESS；`promtool query instant` 作为 PromQL 客户端向在跑的 server 发查询。 +4. **promtool tsdb 功能腿**：`tsdb create-blocks-from openmetrics`（从 OpenMetrics 造块）→ `tsdb list`（打印块 ULID 验创建成功）→ `tsdb analyze`（读块统计）。全在 `/root`（ext4 磁盘）而非 `/tmp`：TSDB 与临时目录落磁盘既是生产正确做法，也规避 starry tmpfs 的 mmap read-back 返零页问题（analyze 经 mmap 读块 meta/index 会得 `\x00`；`tsdb list` 走 `read()` 不受影响）。此 tmpfs mmap 缺陷单列内核跟进。 +5. **ready**：headless 拉起 prometheus server 于 loopback `:9090`，断言日志 `Server is ready to receive web requests.` 且 `/-/ready` 返回 `Ready`（HTTP 服务起来 + TSDB 打开）。 +6. **PromQL 引擎**：`/api/v1/query?query=vector(42)` 返回 `status:success` + 值 `42`。 +7. **scrape + 端到端集成测**：前台拉起 node_exporter 于 `:9100/metrics`（启 `--collector.uname/cpu/meminfo/loadavg/netdev/diskstats/filesystem`，覆盖 starry 现渲染的全部 procfs 源）作为真实抓取目标，prometheus 配 `prometheus.yml` scrape job `node`。先断言 `/api/v1/query?query=up{job="node"}` 返回 `1`（真抓到 + 入库），再经 **180s soak**（2s 抓取间隔多轮）后断言 `node_cpu_seconds_total`、`node_memory_MemTotal_bytes` 与 `node_network_receive_bytes_total`（netdev collector 产）均已以数值样本入 TSDB、且 `query_range` 在时窗内返回多个数据点。这是 scrape→store→query 全链路的工业级端到端验证：node_exporter 真读 starry `/proc/stat`·`/proc/meminfo`·`/proc/loadavg`·`/proc/net/dev`·`/proc/diskstats`·`/proc/mounts` 暴露指标，prometheus 真抓真存真查。 + +验证内核面：Go runtime（M:N 调度 / GC / futex 停泊 / getrandom）、loopback TCP/HTTP 双向、磁盘（ext4）TSDB 块写读 + mmap（analyze）、定时器 / WAL、procfs（`/proc/stat`·`/proc/meminfo`·`/proc/loadavg`·`/proc/net/dev`·`/proc/diskstats`·`/proc/mounts` 喂 node_exporter collector）。 + +### Grafana（`GrafanaCarpet.py`） +Grafana 是单个纯静态 Go 二进制（`grafana server` 子命令），serve 内嵌前端 SPA + REST API 于 loopback `:3000`，后端为内嵌 SQLite - 故像 glances web / prometheus 一样 **headless 起 server + HTTP 断言**，不测 TUI。 +1. **版本红线**：`grafana --version` 必须精确报 `13.0.1`。 +2. **启动**：`grafana server --homepath --config ` 起在 loopback，断言启动日志出现 `HTTP Server Listen` 与 `migrations completed`（server 起 + SQLite 迁移完成）。 +3. **health**：`GET /api/health` 返回 `200` + JSON `{"database":"ok","version":"13.0.1"}`（SQLite 后端就绪）。 +4. **frontend**：`GET /login` 返回 `200`，serve 内嵌 Grafana SPA（`Grafana` + `grafanaBootData`）。 +5. **API/静态**：`GET /`（根）到达 app（200/30x → /login），`GET /robots.txt` 从内嵌 `public/` 静态 serve `200`。 + +确定性：固定 homepath/config + 每次运行独立临时 data 目录（从构建期预迁移的 `grafana.db` 播种，跳过 709 个首启迁移），无外部 datasource，无网。验证内核面：Go runtime、loopback HTTP、内嵌 SQLite 存储（ext4/fsync 路径）。 + +### glances（5 个 carpet） +- **CLI**（`GlancesCliCarpet.py`）：`--version` 版本红线（glances 4.4.1 / API 4 / PsUtil 版本行）＋ `--help` 全选项树逐项断言（每个文档化选项、五种运行模式 `-c/-s/-w/--browser/--stdout`、`--sort-processes` 全部取值、默认端口 61209）＋ `--modules-list` ＋ `--print-completion`。 +- **headless**（`GlancesHeadlessCarpet.py`）：`--stdout cpu,mem,load` 单帧快照，断言 psutil 真读到 `/proc` 的数（`mem.total > 1 MiB`、cpu 规范字段、load `min1`）＋ `--stdout-json` 机器可读路径。 +- **TUI（pyte 真实断言，核心难点）**（`GlancesTuiCarpet.py`）：见下节。 +- **client-server**（`GlancesCsCarpet.py`）：`glances -s`（XML-RPC server）＋ `glances -c`（client）经 loopback 拉取快照，断言 client 从 server 拿到合理 `mem.total`（c/s 往返真通）。 +- **web**（`GlancesWebCarpet.py`）：`glances -w`（FastAPI + uvicorn）起 ASGI server，断言 REST API `/api/4/status`、`/api/4/all`（完整 JSON 快照）、`/api/4/mem/total`、`/api/4/cpu` 的 HTTP + JSON 往返。 + +## TUI pyte 方法学（`GlancesTuiCarpet.py` + `programs/pty_tui_drive.py` + `programs/pyte_assert.py`） + +glances 是全屏 curses 程序。本 carpet 用**离线终端仿真（pyte）**对其做真实断言，而不是 “进程起来了” 的冒烟： + +1. **真 PTY 驱动**：`pty.fork` 在真伪终端里跑 glances（走 initscr → smkx → 备用屏的生产渲染路径），用 `TIOCSWINSZ` 设窗口尺寸，持续抽取它写回的原始 ANSI 字节流。 +2. **SS3 方向键（关键事实）**：glances 调 `keypad(True)` → 进入应用光标键模式（DECCKM），它期望的方向键是 **SS3 形式 `ESC O A/B/C/D`**，不是 CSI 形式 `ESC [ A`。若发 CSI，ncurses 把落单的 ESC 当 “退出” → glances 立刻退出。因此驱动器发 SS3 方向键；PgUp/PgDn/功能键用各自的 terminfo 序列。**“glances 熬过整个方向键 soak 没退出” 本身就是对 StarryOS pty 正确投递 SS3 的硬断言**。 +3. **交互中显示**：发排序热键 `m` / `c` / `a`，断言屏上 “Threads sorted …” 状态行相应变为 memory / CPU / automatic（交互真落地且屏幕连贯重绘）。 +4. **pyte 三重断言**（对捕获流做离线重建）： + - **golden 静态骨架（core 硬断言）**：进程表表头（CPU% / MEM% / PID / USER / TIME+ / Command）＋核心区块 TASKS / MEM / LOAD 确实被画出。这些读的是 StarryOS 稳定提供的核心 /proc（`/proc/[pid]/stat`、`/proc/stat`、`/proc/meminfo`、`/proc/loadavg` - headless carpet 已在 on-target 证明）。 + - **左侧栏 procfs 区块（硬断言）**：NETWORK（`/proc/net/dev`）/ DISK I/O（`/proc/diskstats`）/ FILE SYS（`/proc/mounts`+statfs）。StarryOS 现真实渲染这三个 `/proc` 源（procfs 已补齐真数据），glances 仅当插件有非空 stats 时才画出该区块，故重建屏幕里出现这三个区块标签本身就是插件读到真数据的证据：NETWORK 带回环接口、DISK I/O 带根 virtio-blk 盘 `vda`、FILE SYS 带 ext4 根挂载。三区块作**硬门控**断言（非 best-effort），并额外断言区块内出现具体数据 token（`vda` 在 DISK I/O 与 FILE SYS 根行 `/ (vda)`；`eth0` 在 NETWORK）。headless carpet 对同样三区块做数值级硬断言（`lo` 累计字节 >0 / `vda` 累计 read_bytes >0 / 根 fs size >0）。 + - **跨帧稳定性**：稳定锚点用**始终渲染的 core 进程表表头**，每帧落在**同一行同一列** - 无闪烁 / 无残影 / 无错位刷新（左侧栏行的数值随刷新变化，故不作定位锚点）。 + - **差分渲染不变量**：把字节流按任意分块边界喂 pyte，得到的屏幕矩阵与整块一次性喂**逐格完全一致** - 任何转义序列被切断 / 丢失 / 损坏都会被抓出。 +5. **大区域重绘**：`-2`（左侧栏）/ `-3`（quicklook）开关来回，断言 **core 骨架**重绘回来、无残影。 +6. **内容驱动捕获（跨 arch 时序鲁棒）**：每次交互（启动 / 发 `m`/`c`/`a`/方向键 / 开关）后，驱动器持续把 PTY 输出增量喂给一块 live pyte 屏，**轮询等到预期内容真渲染出来再捕获断言帧**（启动等进程表头 `CPU%`/`MEM%`/`PID`；排序热键等排序指示行真出现 `by memory`/`by CPU`/`automatically`），设慷慨的 per-arch 上限（settle 90s、每步 45s，`MONITOR_TUI_SETTLE_WAIT`/`MONITOR_TUI_STEP_WAIT` 可调）+ 每步最小停留（保持真 soak）。快 arch（x86）立即满足即捕获，慢 arch（aa/rv/loong TCG）会等到渲染完成 - **捕获时机由内容驱动而非固定延时**，故同一脚本在四架构都稳；到上限仍无预期内容则**据实失败**（捕获帧缺内容，下游断言真报错，绝不静默放过）。 + +> **glances CLI 版本感知说明**：`--print-completion`（shell 补全）由可选依赖 `shtab` 提供；交付版 Alpine glances 4.4.1-r1 的 apk 闭包**不含 shtab**，故该选项本就不提供 - carpet 对它做**能力探测**（存在则测其产出补全脚本，缺失则据实记为 documented SKIP），不假设 host 版本。其余全部 `--help` 选项按**交付版 4.4.1-r1 实测选项全集**逐项硬断言。 + +## 四架构来源（provenance，据实注明） + +| 组件 | 版本 | x86_64 | aarch64 | riscv64 | loongarch64 | +|------|------|--------|---------|---------|-------------| +| prometheus + promtool | 3.11.3 | 官方预编译 | 官方预编译 | **官方预编译**（上游发 riscv64）| **Go 交叉编译**（上游无 loong64，从 v3.11.3 tag，**无内嵌 web UI**：API 全功能 scrape/PromQL/alert 都在，仅无内置 /graph 网页 - PROM carpet 只测 API，网页本由 grafana 提供，故不影响）| +| node_exporter | 1.11.1 | 官方预编译 | 官方预编译 | 官方预编译 | **Go 交叉编译**（从 v1.11.1 tag，无前端）| +| grafana | 13.0.1 | 官方预编译 | 官方预编译 | **官方预编译**（上游发 riscv64）| **Go 交叉编译后端** + 嫁接官方前端（上游无 loong64；v13 后端不嵌前端，只交叉编译 `bin/grafana`，无需 Node）| +| glances + psutil + FastAPI/uvicorn 栈 | 4.4.1 | apk | apk | apk | apk | + +- prometheus / node_exporter / grafana 为 CGO-free 纯静态 Go 二进制，在 musl/StarryOS 直接跑，无需 libc；四架构均运行，无任何 SKIP。amd64/arm64/riscv64 走官方 release（prometheus/node_exporter 的 URL + sha256 与上游 `sha256sums.txt` 逐字一致；grafana 与 dl.grafana.com `*.tar.gz.sha256` 一致），loong64 从源码 Go 交叉编译（`assets/build-loong-binaries.sh`）。逐档 URL/sha256 见 `assets/MANIFEST.md`。 +- glances 及其闭包（psutil native、FastAPI/Starlette/pydantic、jinja2）由 `apk add` 解析当前版本 + 目标架构 musl `.so` 全闭包，四架构均有；Alpine v3.23 未收录的 `pyte` / `uvicorn` 为纯 python（noarch）wheel，按钉死 URL + sha256 取回校验后解包进 site-packages。 +- **无任何二进制入 git**；全部构建期抓取 + 校验（见 `prebuild.sh`）。 + +## 运行 + +```bash +# 1) 构建 overlay（下载 prometheus/node_exporter/grafana + apk add glances 闭包 + 解包 pyte/uvicorn wheel + grafana.db 预迁移 + 注入 carpet） +# 由 app 运行框架自动调用 prebuild.sh；四架构分别构建。 +# 2) 运行（单核） +source <仓库根>/.starry-env.sh # 使用 qemu-10 +cargo xtask starry app qemu -t monitor --arch +# 成功判据：rc=0 + 日志 SUCCESS PATTERN MATCHED + ^MONITOR_OK=8/8 + TEST PASSED +``` + +## 确定性与说明 + +- **网络/容器探测插件**：TUI/headless/web/cs carpet 均禁用会在离线 SLIRP guest 上阻塞首帧的环境探测插件（`ip` 公网 IP 查询、`cloud` 169.254 元数据、`containers`/`docker` socket、`ports`/`folders`/`connections`/`sensors` 扫描）。核心仪表盘（cpu/mem/load/quicklook/uptime/processlist）照常渲染，正是 golden 所硬断言的对象。golden 全为结构性（标签/列位），从不钉死宿主相关的数值。 +- **左侧栏 procfs 区块（硬断言，据实真渲染）**：TUI/headless carpet 的 NETWORK/DISK I/O/FILE SYS 区块读 `/proc/net/dev`·`/proc/diskstats`·`/proc/mounts`+statfs。StarryOS 现真实渲染这三个源（`/proc/net/dev` 真 RX/TX 计数、`/proc/diskstats` 根盘 `vda` 真请求/扇区计数、`/proc/mounts` 真挂载表 + `statfs` 真容量），故三区块**硬门控**断言其真渲染 + 携带真数据：TUI 断言三区块标签 + `vda`/`eth0` 数据 token 出现，headless 断言 `lo` 累计字节 >0、`vda` 累计 read_bytes >0、根 fs size >0；node_exporter 同时启 netdev/diskstats/filesystem collector 并断言 `node_network_receive_bytes_total{lo}>0`、`node_disk_reads_completed_total{vda}>0`、`node_filesystem_size_bytes{"/"}>0`。core 区块（进程表 + MEM/LOAD/TASKS）一并硬断言。仍是长 soak + 三态 + 四不变量，非降级 smoke。 +- **loopback**：grafana / glances web（uvicorn/ASGI）/ client-server（XML-RPC）均在 guest 内 loopback 上 bind/listen/accept，压 StarryOS 网络栈。 +- **grafana SQLite 迁移**：grafana 首启要跑 709 个 SQLite 迁移（重压 ext4/fsync）。prebuild **构建期尽力预迁移**架构无关的 `grafana.db` 播种，on-target 跳过这 709 个迁移；carpet 用 WAL 模式 + 对前端端点做退避重试化解启动期 `SQLITE_BUSY` 争用。若预迁移未成，on-target 现场迁移（更慢但仍能过）。 +- **超时**：本 app 单次启动跑 prometheus 冷启动 + scrape + grafana server（SQLite 迁移/启动）+ glances 五形态（含 TUI soak + web + c/s），故 `timeout` 取较大值（慢架构 TCG 下 Go/SQLite 冷启动需足够长，非真 hang）。 diff --git a/apps/starry/monitor/SOURCES.md b/apps/starry/monitor/SOURCES.md new file mode 100644 index 0000000000..842cb8f818 --- /dev/null +++ b/apps/starry/monitor/SOURCES.md @@ -0,0 +1,70 @@ +# monitor - 软件来源与可复现性（provenance） + +适配目标：StarryOS × 四架构（x86_64 / aarch64 / riscv64 / loongarch64）。所有二进制/wheel/apk 均在 +`prebuild.sh` 构建期抓取 + 校验，**无任何二进制入 git**。逐档 URL + sha256 见 `assets/MANIFEST.md`。 + +## 1. Prometheus 3.11.3 + promtool（github.com/prometheus/prometheus, v3.11.3 release） + +- 资产：`prometheus-3.11.3.linux-.tar.gz`，内含 `prometheus`（server）+ `promtool`，均为 + CGO-free 纯静态 Go 二进制（`file` 报 statically linked，`netgo,builtinassets` → web UI 已嵌），在 + musl/StarryOS 直接跑，无需 libc/ld-musl 接线。 +- 官方 linux 架构含 amd64/arm64/**riscv64**（有 riscv64，**无 loong64**）。 +- **x86_64 / aarch64 / riscv64**：官方预编译。sha256 与上游 `sha256sums.txt` 逐字一致（2026-07-02 + 对 live release 复核）。 +- **loongarch64**：上游不发 loong64 → 从 `v3.11.3` tag **Go 交叉编译**（`assets/build-loong-binaries.sh prometheus`）： + `CGO_ENABLED=0 GOOS=linux GOARCH=loong64 go build -trimpath -tags netgo + -ldflags "-X .../version.Version=3.11.3 …" ./cmd/{prometheus,promtool}`（需 Go≥1.25，无需 Node）。 + **无内嵌 web UI**：不带 `builtinassets` tag，故**不做** Node≥22 的 npm/yarn react-scripts 前端构建（该步重、脆、 + arch 无关，且 Node<22 会失败 `assets_embed.go: undefined EmbedFS`）。产物 API **全功能**（scrape / TSDB / + PromQL / alerting 全在），仅缺内置 `/graph` HTML 页 - 而本 app 的 PROM carpet 只测 API/CLI（`--version` / + `--help` / promtool / `/-/ready` / PromQL / scrape），且仪表盘网页由 grafana 提供，故不受影响。产物 `file` + 报 `LoongArch ELF, statically linked`，内嵌版本串 `3.11.3`。这是从官方源码交叉编译的产物，不是 SKIP。 + +## 2. node_exporter 1.11.1（github.com/prometheus/node_exporter, v1.11.1 release） + +- 作用：prometheus 的真实抓取目标（暴露 `:9100/metrics`），使 scrape → ingest → query 主链真正跑通 + （`up{job="node"}==1`），而非只测合成 `vector(42)`。 +- 单个纯 Go 静态二进制。官方 linux 架构含 amd64/arm64/**riscv64**（无 loong64）。 +- x86_64 / aarch64 / riscv64：官方预编译（TARBALL sha256 与上游 `sha256sums.txt` 一致）。 +- loongarch64：从 `v1.11.1` tag Go 交叉编译（一条命令，无前端；`assets/build-loong-binaries.sh node_exporter`）。 + +## 3. Grafana 13.0.1 OSS（dl.grafana.com） + +- 资产：`grafana-13.0.1.linux-.tar.gz`，内含单个 CGO-free 纯静态 Go 二进制 `bin/grafana` + （`grafana server` / `grafana cli` 为子命令）+ `public/`（前端 SPA）+ `conf/`（defaults.ini）；内嵌 + SQLite 存储。在 musl/StarryOS 直接跑，无需 libc。 +- 官方 linux 架构含 amd64/arm64/**riscv64**（有 riscv64，**无 loong64**）。 +- **x86_64 / aarch64 / riscv64**：官方预编译（tarball sha256 与 dl.grafana.com `*.tar.gz.sha256` 一致， + 2026-07-03 对 live 复核）。 +- **loongarch64**：上游不发 → grafana v13 后端**不嵌**前端（`embed.go` 仅嵌 cue.mod schema），故只需 + **Go 交叉编译后端** `bin/grafana`（无需 Node/yarn 建前端），再嫁接官方 riscv64 tar 的 `public/`/`conf/` + （架构无关）。见 `assets/build-loong-binaries.sh grafana`。这是源码交叉编译的产物，不是 SKIP。 +- prebuild 剥离 `public/**/*.map`（~290MB 浏览器调试件，server 端从不读），并**构建期尽力预迁移**架构无关的 + `grafana.db`（跳过 709 个首启 SQLite 迁移；失败则 on-target 现场迁移，carpet 两种情况都能过）。 + +## 4. glances 4.4.1 + 依赖闭包（Alpine v3.23, musl, 四架构） + +- `apk add` 解析当前版本 + 目标架构 musl `.so` 全闭包（无钉死漂移 URL、无 cache-miss 早退）： + - `glances 4.4.1-r1`（community）+ `py3-psutil 7.1.3-r0`（main，native `_psutil_linux.abi3.so` 每 + 架构各自 musl so）。 + - web 形态闭包：`py3-fastapi` / `py3-starlette` / `py3-pydantic`（+ native `pydantic-core`）/ + `py3-anyio` / `py3-sniffio` / `py3-h11` / `py3-click` / `py3-jinja2`。 + - TUI：`py3-wcwidth`（pyte 唯一依赖）+ Alpine 自带 ncursesw/terminfo。 + - 以上四架构在 Alpine v3.23 均存在（已核对 APKINDEX）。 +- `glances --version` 同时覆盖 glances（红线钉 4.4.1）+ psutil（版本柔性）两个版本号，API version 钉 4。 + +## 5. 纯 python wheel（Alpine v3.23 未收录，noarch） + +- `pyte 0.8.2`、`uvicorn 0.34.0`：`py3-none-any` 纯 python wheel，按钉死 URL + sha256 取回校验后解包进 + site-packages。 + - pyte 运行时依赖仅 wcwidth（Alpine `py3-wcwidth`）。 + - uvicorn 运行时依赖 `click>=7` + `h11>=0.8`（Alpine `py3-click` / `py3-h11` 满足）；`standard` 额外项 + （httptools/uvloop/websockets）为可选且不使用 - glances 以默认 asyncio loop + h11 跑 uvicorn。 + +## 6. 依赖内核面 + +- prometheus/node_exporter/grafana：Go runtime（goroutine 调度 / GC / futex / getrandom / SIGURG 抢占 / + epoll netpoll）、loopback TCP/HTTP、mmap-backed TSDB 写盘（prometheus）、内嵌 SQLite 存储 + 迁移 + （grafana，压 ext4/fsync 路径）、定时器/WAL。 +- glances：psutil 重度读 `/proc/stat`、`/proc/meminfo`、`/proc//*`；TUI 走 pty/tty（SS3 应用键模式、 + TIOCSWINSZ、备用屏）；web/cs 压 loopback socket（bind/listen/accept）。 diff --git a/apps/starry/monitor/assets/MANIFEST.md b/apps/starry/monitor/assets/MANIFEST.md new file mode 100644 index 0000000000..84665eccbb --- /dev/null +++ b/apps/starry/monitor/assets/MANIFEST.md @@ -0,0 +1,80 @@ +# monitor app -- pinned binary/wheel provenance (URLs + sha256) + +All artifacts are fetched at build time by `prebuild.sh`; NOTHING binary is committed. amd64/arm64/ +riscv64 prometheus + node_exporter sha256 are byte-for-byte identical to the upstream release +`sha256sums.txt` (re-verified against the live release on 2026-07-02). loong64 has no upstream +prebuilt -> Go-cross-compiled from the pinned source tag (`assets/build-loong-binaries.sh`), which +is non-deterministic; the recorded sha is of the reference reproducible artifact. + +## prometheus 3.11.3 (github.com/prometheus/prometheus) +Release asset: `prometheus-3.11.3.linux-.tar.gz` (contains `prometheus` + `promtool`, +static CGO-free Go, netgo,builtinassets => embedded UI). Upstream linux arches include amd64/arm64/ +riscv64 (has riscv64, NO loong64). + + URL base: https://github.com/prometheus/prometheus/releases/download/v3.11.3/prometheus-3.11.3.linux-{amd64,arm64,riscv64}.tar.gz + +| arch | goarch | sha256 (tarball) | source | +|-------------|---------|--------------------------------------------------------------------|-------------------| +| x86_64 | amd64 | 9479af67673316278958cda1f39b88a09f8921084e039c65acca060d0447bb38 | official prebuilt | +| aarch64 | arm64 | d2ec0a96259afde955ad1560ced303cef99cac4dac676bd4dd7614d76adb708a | official prebuilt | +| riscv64 | riscv64 | bd6978937d64f4afa82919e0c4b3b83ace50808b953ab6174e480ca7dda2ba9a | official prebuilt | +| loongarch64 | loong64 | ca42f362268d3e404deeec608aa23faadc06f1c0ad9f0036c8650b6a12a69d73 | Go cross-compiled (go1.26.3, tag v3.11.3, **no embedded web UI**; non-deterministic) | + +loong64 prometheus/promtool are built WITHOUT the embedded `/graph` web UI (no `builtinassets` tag -> +no Node/npm frontend build; just `go build ./cmd/{prometheus,promtool}`, Go>=1.25). The binary keeps +full API/server functionality (scrape / TSDB / PromQL / alerting); only the built-in HTML dashboard is +absent, which the PROM carpet does not test (it tests the API/CLI) and which grafana provides in this +stack. Verified: `file` -> `LoongArch ELF, statically linked`; under qemu-loongarch64: `prometheus +--version` -> `prometheus, version 3.11.3 ... platform: linux/loong64`, promtool likewise 3.11.3. + +## node_exporter 1.11.1 (github.com/prometheus/node_exporter) +Release asset: `node_exporter-1.11.1.linux-.tar.gz` (single static Go binary). Upstream +linux arches include amd64/arm64/riscv64 (has riscv64, NO loong64). + + URL base: https://github.com/prometheus/node_exporter/releases/download/v1.11.1/node_exporter-1.11.1.linux-{amd64,arm64,riscv64}.tar.gz + +TARBALL sha256 (upstream sha256sums.txt): + +| arch | goarch | sha256 (tarball) | binary sha256 (extracted) | source | +|-------------|---------|--------------------------------------------------------------------|------------------------------------------------------------------|-------------------| +| x86_64 | amd64 | 9f5ea48e5bc7b656f8a91a32e7d7deb89f70f73dabd0d974418aca15f37d6810 | 3a01a3cc7f69798698fbb31b24e5ee279dd2c39727be2a5a65071536fc16b455 | official prebuilt | +| aarch64 | arm64 | ba1886efbd76cb96b0087c695ea8d1b9cb6e8aa946c996d744e9ee16c8e3591a | c92bd1e9eeb4061f1bdbf60a7b41d446220a4a44d87fb77ae889f034cb8cf3bc | official prebuilt | +| riscv64 | riscv64 | 8d73447c47488a94f7eba467838c815ea7dceb449c75b1b8e91fa6dc3e0e364e | 3932dd6b4456eda301f3198fe0c22860b66afbe5ffa8214a32df532e490e5d21 | official prebuilt | +| loongarch64 | loong64 | (Go cross-compiled; no tarball) | 76c56223816d403761564b581e8961d59360e0d50f7343d50a60767630306eba | Go cross-compiled (tag v1.11.1) | + +## grafana 13.0.1 OSS (dl.grafana.com) +Release asset: `grafana-13.0.1.linux-.tar.gz` (single static CGO-free Go `bin/grafana` + +`public/` frontend SPA + `conf/`; embedded SQLite store). Upstream linux arches include amd64/arm64/ +riscv64 (has riscv64, NO loong64). Tarball sha256 == dl.grafana.com `*.tar.gz.sha256` (re-verified live 2026-07-03). + + URL base: https://dl.grafana.com/oss/release/grafana-13.0.1.linux-{amd64,arm64,riscv64}.tar.gz + +| arch | goarch | sha256 (tarball) | source | +|-------------|---------|--------------------------------------------------------------------|-------------------| +| x86_64 | amd64 | 187ddc4badb69aecb7cd3fae2884add7ed21adde7124a6f8093b7b4033d722f2 | official prebuilt | +| aarch64 | arm64 | 553d5ee3fb1600c83ef2fbf336579ed6cc64fffc328843ea7d662f85b876c261 | official prebuilt | +| riscv64 | riscv64 | 233ac9bf87390f203e45a1beb47630b28d3eb0c0dce3bfc5838e0e1603eb2cee | official prebuilt | +| loongarch64 | loong64 | a3f14993f0fce3419ded61f41f8ab0695181a822501a783168e37993bcac660f | Go cross-compiled BACKEND (tag v13.0.1) + official riscv64 tar public/ grafted; non-deterministic | + +grafana v13's backend does NOT embed the frontend, so loong64 needs only the Go backend cross-compiled +(no Node/yarn build) with the official arch-independent frontend grafted -- `assets/build-loong-binaries.sh grafana`. +prebuild strips `public/**/*.map` (~290MB browser debug artifacts, never read server-side) and +best-effort pre-migrates the arch-independent `grafana.db` (skips the 709 first-run SQLite migrations). + +## glances 4.4.1 stack (Alpine v3.23, musl, all 4 arches -- resolved live by `apk add`) +No pinned URLs: `apk add` resolves the CURRENT version + full musl .so closure per arch (no drift). + + glances 4.4.1-r1 (community) + py3-psutil 7.1.3-r0 (main, native _psutil_linux.abi3.so per arch) + py3-fastapi 0.121.2 / py3-starlette 0.47.2 / py3-pydantic 2.12.3 (+pydantic-core native) / + py3-anyio 4.11.0 / py3-sniffio 1.3.1 / py3-h11 0.16.0 / py3-click 8.1.8 / py3-wcwidth 0.2.13 / + py3-jinja2 -- all present for x86_64/aarch64/riscv64/loongarch64 in Alpine v3.23. + +## vendored pure-python wheels (py3-none-any; NOT in Alpine v3.23) +| package | version | sha256 | url | +|----------------|---------|------------------------------------------------------------------|-----| +| pyte | 0.8.2 | 85db42a35798a5aafa96ac4d8da78b090b2c933248819157fc0e6f78876a0135 | https://files.pythonhosted.org/packages/59/d0/bb522283b90853afbf506cd5b71c650cf708829914efd0003d615cf426cd/pyte-0.8.2-py3-none-any.whl | +| uvicorn | 0.34.0 | 023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4 | https://files.pythonhosted.org/packages/61/14/33a3a1352cfa71812a3a21e8c9bfb83f60b0011f5e36f2b1399d51928209/uvicorn-0.34.0-py3-none-any.whl | + +pyte's only runtime dep is wcwidth (Alpine py3-wcwidth). uvicorn's runtime deps click>=7 + h11>=0.8 +are satisfied by Alpine py3-click / py3-h11 (the "standard" extras httptools/uvloop/websockets are +optional and unused -- glances runs uvicorn with the default asyncio loop + h11). diff --git a/apps/starry/monitor/assets/build-loong-binaries.sh b/apps/starry/monitor/assets/build-loong-binaries.sh new file mode 100644 index 0000000000..eedff95fd4 --- /dev/null +++ b/apps/starry/monitor/assets/build-loong-binaries.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# build-loong-binaries.sh -- reproducible LoongArch64 cross-compile of prometheus 3.11.3 + promtool +# and node_exporter 1.11.1 from their pinned upstream source tags. Upstream ships NO loong64 +# prebuilt release, and StarryOS runs real loong64 -- so we cross-compile (Go makes this simple), +# NEVER skip the arch. Output tarball/binary land in $MONITOR_LOONG_OUT (default: current dir). +# +# Usage: build-loong-binaries.sh prometheus # -> prometheus-3.11.3.linux-loong64.tar.gz +# build-loong-binaries.sh node_exporter # -> node_exporter (static loong64 ELF) +# +# Requirements: a Go toolchain >= 1.25 (prometheus/node_exporter go.mod require go 1.25). node_exporter +# is a one-command cross-compile. prometheus additionally needs its web UI embedded (built once with +# Node >= 22, arch-independent) so the binary carries `builtinassets`; if the prebuilt UI embed is +# absent this script builds it. Both are CGO_ENABLED=0 pure-static Go -> no libc needed on StarryOS. +set -euo pipefail +what="${1:?usage: build-loong-binaries.sh prometheus|node_exporter|grafana}" +OUT="${MONITOR_LOONG_OUT:-$(pwd)}"; mkdir -p "$OUT" +PROM_VER="3.11.3"; NE_VER="1.11.1"; GRAFANA_VER="13.0.1" +WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT +command -v go >/dev/null 2>&1 || { echo "build-loong: need a Go toolchain (>=1.25)"; exit 3; } +export GOTOOLCHAIN="${GOTOOLCHAIN:-auto}" GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=loong64 +UTS="$(date -u +%Y%m%dT%H:%M:%SZ)" + +case "$what" in + node_exporter) + echo "=== cross-compile node_exporter $NE_VER (loong64) ===" + git clone --depth 1 --branch "v$NE_VER" https://github.com/prometheus/node_exporter "$WORK/ne" + cd "$WORK/ne"; rev="$(git rev-parse HEAD)" + go build -trimpath \ + -ldflags "-X github.com/prometheus/common/version.Version=$NE_VER \ + -X github.com/prometheus/common/version.Revision=$rev \ + -X github.com/prometheus/common/version.Branch=HEAD \ + -X github.com/prometheus/common/version.BuildUser=rcore-os-tgoskits-crosscompile \ + -X github.com/prometheus/common/version.BuildDate=$UTS" \ + -o "$OUT/node_exporter" . + file "$OUT/node_exporter" | sed 's/,.*statically/ [static]/' + echo "=== node_exporter loong64 -> $OUT/node_exporter ===" + ;; + prometheus) + echo "=== cross-compile prometheus $PROM_VER + promtool (loong64, NO embedded web UI) ===" + # The embedded /graph web UI is deliberately NOT built. Building it needs a Node>=22 + npm/yarn + # react-scripts frontend build (the `builtinassets` tag); that step is heavy, fragile, and + # arch-irrelevant, and on Node<22 it fails (assets_embed.go: undefined EmbedFS). The monitor PROM + # carpet exercises ONLY the API/CLI surface (--version / --help / promtool / /-/ready / PromQL / + # scrape) -- NOT the web UI -- and the dashboard UI is provided by grafana in this stack. So build + # the two commands WITHOUT builtinassets: `go build ./cmd/{prometheus,promtool}` yields a fully + # API-functional prometheus (scrape / TSDB / PromQL / alerting all present; only the built-in + # /graph HTML page is absent). Still a from-source cross-compile of the official v3.11.3 tag -- NOT + # a SKIP. (Requires Go>=1.25 per prometheus go.mod; no Node needed.) + git clone --depth 1 --branch "v$PROM_VER" https://github.com/prometheus/prometheus "$WORK/p" + cd "$WORK/p"; rev="$(git rev-parse HEAD)" + local_ld="-X github.com/prometheus/common/version.Version=$PROM_VER \ + -X github.com/prometheus/common/version.Revision=$rev \ + -X github.com/prometheus/common/version.Branch=HEAD \ + -X github.com/prometheus/common/version.BuildUser=rcore-os-tgoskits-crosscompile \ + -X github.com/prometheus/common/version.BuildDate=$UTS" + mkdir -p "$WORK/stage/prometheus-$PROM_VER.linux-loong64" + for cmd in prometheus promtool; do + CGO_ENABLED=0 go build -trimpath -tags netgo -ldflags "$local_ld" \ + -o "$WORK/stage/prometheus-$PROM_VER.linux-loong64/$cmd" "./cmd/$cmd" + done + cp -f LICENSE NOTICE documentation/examples/prometheus.yml \ + "$WORK/stage/prometheus-$PROM_VER.linux-loong64/" 2>/dev/null || true + ( cd "$WORK/stage" && tar czf "$OUT/prometheus-$PROM_VER.linux-loong64.tar.gz" "prometheus-$PROM_VER.linux-loong64" ) + file "$WORK/stage/prometheus-$PROM_VER.linux-loong64/prometheus" | sed 's/,.*/ .../' + echo "=== prometheus loong64 (no web UI) -> $OUT/prometheus-$PROM_VER.linux-loong64.tar.gz ===" + ;; + grafana) + echo "=== cross-compile grafana $GRAFANA_VER backend (loong64) + graft official frontend ===" + # grafana v13 backend is a single bin/grafana that does NOT embed the frontend (embed.go only + # embeds cue.mod schema); the frontend SPA ships separately in the release tar's public/. So we + # cross-compile ONLY the Go backend and graft the OFFICIAL riscv64 tar's public/+conf/ (both + # arch-independent) onto it -- no Node/yarn frontend build needed for loong64. + git clone --depth 1 --branch "v$GRAFANA_VER" https://github.com/grafana/grafana "$WORK/g" + cd "$WORK/g"; rev="$(git rev-parse HEAD)" + # No GOSUMDB=off here: grafana v13's go.mod pins `toolchain go1.25.9`, and GOTOOLCHAIN=auto must + # be able to fetch+verify that toolchain (prometheus/node_exporter build without GOSUMDB=off too). + go build -buildvcs=false -trimpath \ + -ldflags "-X main.version=$GRAFANA_VER -X main.commit=$rev -X main.buildBranch=HEAD -X main.buildstamp=$(date -u +%s)" \ + -o "$WORK/bin/grafana" ./pkg/cmd/grafana + file "$WORK/bin/grafana" | sed 's/,.*/ .../' + # graft onto the official riscv64 release tar (public/ + conf/ are arch-independent). + local_rv="$WORK/grafana-$GRAFANA_VER.linux-riscv64.tar.gz" + curl -fL --retry 3 -o "$local_rv" "https://dl.grafana.com/oss/release/grafana-$GRAFANA_VER.linux-riscv64.tar.gz" + tar xzf "$local_rv" -C "$WORK" + cp -f "$WORK/bin/grafana" "$WORK/grafana-$GRAFANA_VER/bin/grafana" + # Keep the archive's top dir as `grafana-$GRAFANA_VER` (matching the official tars + what + # prebuild.sh extracts, topdir="grafana-$GRAFANA_VER"); only the output filename carries the arch. + ( cd "$WORK" && tar czf "$OUT/grafana-$GRAFANA_VER.linux-loong64.tar.gz" "grafana-$GRAFANA_VER" ) + echo "=== grafana loong64 -> $OUT/grafana-$GRAFANA_VER.linux-loong64.tar.gz ===" + ;; + *) echo "unknown target: $what"; exit 2 ;; +esac diff --git a/apps/starry/monitor/assets/prometheus.yml b/apps/starry/monitor/assets/prometheus.yml new file mode 100644 index 0000000000..6a88d856e7 --- /dev/null +++ b/apps/starry/monitor/assets/prometheus.yml @@ -0,0 +1,13 @@ +# Prometheus scrape config for the StarryOS `monitor` app carpet. +# A single `node` job points at the in-guest node_exporter on loopback :9100 so the case exercises +# the CORE scrape -> ingest -> query pipeline (up{job="node"} == 1), not just a synthetic scalar. +# scrape_interval 2s => the first scrape lands fast under TCG. +global: + scrape_interval: 15s + evaluation_interval: 15s +scrape_configs: + - job_name: node + scrape_interval: 2s + scrape_timeout: 2s + static_configs: + - targets: ['127.0.0.1:9100'] diff --git a/apps/starry/monitor/build-aarch64-unknown-none-softfloat.toml b/apps/starry/monitor/build-aarch64-unknown-none-softfloat.toml new file mode 100644 index 0000000000..17af0fa9fd --- /dev/null +++ b/apps/starry/monitor/build-aarch64-unknown-none-softfloat.toml @@ -0,0 +1,13 @@ +target = "aarch64-unknown-none-softfloat" +log = "Warn" +features = [ + "ax-feat/display", + "ax-feat/rtc", + "ax-driver/virtio-blk", + "ax-driver/virtio-net", + "ax-driver/virtio-gpu", + "ax-driver/virtio-input", + "ax-driver/virtio-socket", + "starry-kernel/input", + "starry-kernel/vsock", +] diff --git a/apps/starry/monitor/build-loongarch64-unknown-none-softfloat.toml b/apps/starry/monitor/build-loongarch64-unknown-none-softfloat.toml new file mode 100644 index 0000000000..3ad44f3fa1 --- /dev/null +++ b/apps/starry/monitor/build-loongarch64-unknown-none-softfloat.toml @@ -0,0 +1,18 @@ +# LoongArch64 MUST use the dynamic platform (axplat-dyn): keep ax-driver/serial + do NOT opt out of +# dynamic platform mode. The retired static LoongArch path lacked the serial-console binding +# ("/dev/console has no serial TTY binding" -> early exit). qemu-loongarch64.toml boots the raw +# binary (uefi=false / to_bin=true). +target = "loongarch64-unknown-none-softfloat" +log = "Warn" +features = [ + "ax-feat/display", + "ax-feat/rtc", + "ax-driver/serial", + "ax-driver/virtio-blk", + "ax-driver/virtio-net", + "ax-driver/virtio-gpu", + "ax-driver/virtio-input", + "ax-driver/virtio-socket", + "starry-kernel/input", + "starry-kernel/vsock", +] diff --git a/apps/starry/monitor/build-riscv64gc-unknown-none-elf.toml b/apps/starry/monitor/build-riscv64gc-unknown-none-elf.toml new file mode 100644 index 0000000000..b29f512a4f --- /dev/null +++ b/apps/starry/monitor/build-riscv64gc-unknown-none-elf.toml @@ -0,0 +1,14 @@ +target = "riscv64gc-unknown-none-elf" +log = "Warn" +features = [ + "ax-feat/display", + "ax-feat/rtc", + "ax-driver/serial", + "ax-driver/virtio-blk", + "ax-driver/virtio-net", + "ax-driver/virtio-gpu", + "ax-driver/virtio-input", + "ax-driver/virtio-socket", + "starry-kernel/input", + "starry-kernel/vsock", +] diff --git a/apps/starry/monitor/build-x86_64-unknown-none.toml b/apps/starry/monitor/build-x86_64-unknown-none.toml new file mode 100644 index 0000000000..a8c92b85d8 --- /dev/null +++ b/apps/starry/monitor/build-x86_64-unknown-none.toml @@ -0,0 +1,9 @@ +target = "x86_64-unknown-none" +log = "Warn" +features = [ + "ax-driver/virtio-blk", + "ax-driver/virtio-net", + "ax-driver/virtio-gpu", + "ax-driver/virtio-input", + "ax-driver/virtio-socket", +] diff --git a/apps/starry/monitor/prebuild.sh b/apps/starry/monitor/prebuild.sh new file mode 100644 index 0000000000..a9437be049 --- /dev/null +++ b/apps/starry/monitor/prebuild.sh @@ -0,0 +1,344 @@ +#!/usr/bin/env bash +# prebuild.sh -- provision the StarryOS `monitor` app overlay, reproducibly: +# * the Prometheus monitoring stack -- prometheus 3.11.3 + promtool (fully-static CGO-free Go +# binaries) + node_exporter 1.11.1 (the simplest exporter, a static Go binary), fetched at +# build time from the official GitHub releases (per-arch tarball, sha256-verified). loong64 +# upstream ships NO prebuilt, so it is Go-cross-compiled from the pinned source tag (or taken +# from a maintainer-staged reproducible artifact) -- NEVER a SKIP. See assets/MANIFEST.md. +# * the glances system monitor -- `apk add glances` (Alpine musl, all 4 arches) plus its FastAPI/ +# Starlette/pydantic/uvicorn web closure, resolved INTO the staging tree by qemu-user-static so +# apk pulls the CURRENT version + the full musl .so closure for the TARGET arch (no drifting +# hardcoded apk URLs, no cache-miss exit). pyte + uvicorn are not in Alpine v3.23, so their +# pinned pure-python (noarch) wheels are fetched by URL + sha256 and unpacked into site-packages. +# * the on-target carpet suite (prometheus + glances CLI/headless/TUI-pyte/client-server/web) and +# the pyte harness (pty_tui_drive.py + pyte_assert.py) staged under /root/monitor. +# +# NOTHING binary is committed to the source tree; every binary/wheel/apk is fetched at build time. +# Env from the app runner: STARRY_ARCH, STARRY_ROOTFS, STARRY_STAGING_ROOT, STARRY_OVERLAY_DIR, +# STARRY_APP_DIR. App-specific overrides documented inline below. +set -euo pipefail + +app_dir="${STARRY_APP_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +arch="${STARRY_ARCH:?prebuild: STARRY_ARCH required}" +base_rootfs="${STARRY_ROOTFS:?prebuild: STARRY_ROOTFS required}" +staging_root="${STARRY_STAGING_ROOT:?prebuild: STARRY_STAGING_ROOT required}" +overlay_dir="${STARRY_OVERLAY_DIR:?prebuild: STARRY_OVERLAY_DIR required}" + +APK_BRANCH="${MONITOR_APK_BRANCH:-v3.23}" +ALPINE_CDN="${ALPINE_CDN:-https://dl-cdn.alpinelinux.org/alpine}" +ROOTFS_SIZE="${MONITOR_ROOTFS_SIZE:-6G}" # grafana public/ + Go binaries + glances closure are large +APK_CACHE="${MONITOR_APK_CACHE:-}" +# Optional: a maintainer-staged directory of reproducible binaries (used offline and, crucially, +# for the loong64 prometheus/node_exporter/grafana that upstream does not prebuild). Layout: +# $MONITOR_BINS_DIR/prometheus//prometheus-3.11.3.linux-.tar.gz +# $MONITOR_BINS_DIR/node_exporter//node_exporter +# $MONITOR_BINS_DIR/grafana//grafana-13.0.1.linux-.tar.gz +# $MONITOR_BINS_DIR/grafana/grafana-13.0.1.db (optional pre-migrated, arch-independent seed) +BINS_DIR="${MONITOR_BINS_DIR:-}" +GRAFANA_PREMIGRATE="${MONITOR_GRAFANA_PREMIGRATE:-1}" # best-effort build-time SQLite migration (skips 709 on-target) + +PROM_VER="3.11.3" +NE_VER="1.11.1" +GRAFANA_VER="13.0.1" +PYTE_WHL_URL="https://files.pythonhosted.org/packages/59/d0/bb522283b90853afbf506cd5b71c650cf708829914efd0003d615cf426cd/pyte-0.8.2-py3-none-any.whl" +PYTE_WHL_SHA="85db42a35798a5aafa96ac4d8da78b090b2c933248819157fc0e6f78876a0135" +UVICORN_WHL_URL="https://files.pythonhosted.org/packages/61/14/33a3a1352cfa71812a3a21e8c9bfb83f60b0011f5e36f2b1399d51928209/uvicorn-0.34.0-py3-none-any.whl" +UVICORN_WHL_SHA="023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4" + +case "$arch" in + aarch64) qemu_runner="qemu-aarch64-static"; goarch="arm64" ;; + riscv64) qemu_runner="qemu-riscv64-static"; goarch="riscv64" ;; + x86_64) qemu_runner="qemu-x86_64-static"; goarch="amd64" ;; + loongarch64) qemu_runner="qemu-loongarch64-static"; goarch="loong64" ;; + *) echo "prebuild: unsupported arch: $arch" >&2; exit 1 ;; +esac + +# Official release sha256 per goarch (amd64/arm64/riscv64 == upstream sha256sums.txt, verified +# byte-for-byte). loong64 has no official release -> resolved via cross-compile / staged artifact. +prom_sha() { case "$1" in + amd64) echo 9479af67673316278958cda1f39b88a09f8921084e039c65acca060d0447bb38 ;; + arm64) echo d2ec0a96259afde955ad1560ced303cef99cac4dac676bd4dd7614d76adb708a ;; + riscv64) echo bd6978937d64f4afa82919e0c4b3b83ace50808b953ab6174e480ca7dda2ba9a ;; + *) echo "" ;; +esac ; } +ne_sha() { case "$1" in # node_exporter release TARBALL sha256 (upstream sha256sums.txt) + amd64) echo 9f5ea48e5bc7b656f8a91a32e7d7deb89f70f73dabd0d974418aca15f37d6810 ;; + arm64) echo ba1886efbd76cb96b0087c695ea8d1b9cb6e8aa946c996d744e9ee16c8e3591a ;; + riscv64) echo 8d73447c47488a94f7eba467838c815ea7dceb449c75b1b8e91fa6dc3e0e364e ;; + *) echo "" ;; +esac ; } +grafana_sha() { case "$1" in # grafana OSS TARBALL sha256 (dl.grafana.com *.sha256; amd64/arm64/riscv64 official) + amd64) echo 187ddc4badb69aecb7cd3fae2884add7ed21adde7124a6f8093b7b4033d722f2 ;; + arm64) echo 553d5ee3fb1600c83ef2fbf336579ed6cc64fffc328843ea7d662f85b876c261 ;; + riscv64) echo 233ac9bf87390f203e45a1beb47630b28d3eb0c0dce3bfc5838e0e1603eb2cee ;; + *) echo "" ;; # loong64: self-cross-compiled backend + official frontend (non-deterministic); staged +esac ; } + +ensure_host_tools() { + local missing=() + for t in debugfs resize2fs e2fsck truncate readelf curl tar sha256sum unzip; do + command -v "$t" >/dev/null 2>&1 || missing+=("$t") + done + command -v "$qemu_runner" >/dev/null 2>&1 || missing+=("$qemu_runner") + if [[ ${#missing[@]} -gt 0 ]]; then + echo "prebuild: missing host tools: ${missing[*]}" >&2 + command -v apt-get >/dev/null 2>&1 && { + apt-get update && apt-get install -y --no-install-recommends \ + e2fsprogs coreutils binutils curl tar coreutils unzip qemu-user-static || true + } + fi +} + +grow_rootfs() { + [[ -f "$base_rootfs" ]] || { echo "prebuild: rootfs image missing: $base_rootfs" >&2; exit 2; } + local before; before=$(stat -c %s "$base_rootfs") + local target; target=$(numfmt --from=iec "${ROOTFS_SIZE/G/G}" 2>/dev/null || echo $((4*1024*1024*1024))) + echo "prebuild: rootfs $base_rootfs is $((before/1024/1024)) MiB; growing to $ROOTFS_SIZE (grow-only)" + if [[ "$before" -lt "$target" ]]; then + truncate -s "$ROOTFS_SIZE" "$base_rootfs" + e2fsck -f -y "$base_rootfs" >/dev/null 2>&1 || true + resize2fs "$base_rootfs" >/dev/null 2>&1 || { echo "prebuild: resize2fs failed" >&2; exit 2; } + fi + echo "prebuild: rootfs sized to $(( $(stat -c %s "$base_rootfs")/1024/1024 )) MiB" +} + +extract_base_rootfs() { + rm -rf "$staging_root"; mkdir -p "$staging_root" + debugfs -R "rdump / $staging_root" "$base_rootfs" >/dev/null 2>&1 + [[ -x "$staging_root/sbin/apk" ]] || { echo "prebuild: base rootfs has no apk" >&2; exit 2; } +} + +normalize_symlinks() { + local link tgt rel + while IFS= read -r link; do + tgt="$(readlink "$link")"; [[ "$tgt" == /* ]] || continue + rel="$(realpath -m --relative-to="$(dirname "$link")" "$staging_root$tgt")" + ln -sf "$rel" "$link" + done < <(find "$staging_root/lib" "$staging_root/usr/lib" -type l 2>/dev/null) +} + +install_glances_closure() { + normalize_symlinks + [[ -f /etc/resolv.conf ]] && cp -f /etc/resolv.conf "$staging_root/etc/resolv.conf" || true + printf '%s/%s/main\n%s/%s/community\n' \ + "$ALPINE_CDN" "$APK_BRANCH" "$ALPINE_CDN" "$APK_BRANCH" \ + > "$staging_root/etc/apk/repositories" + local cache_args=(); [[ -n "$APK_CACHE" ]] && { mkdir -p "$APK_CACHE"; cache_args=(--cache-dir "$APK_CACHE"); } + # glances + psutil (core) + the FastAPI/Starlette/pydantic/uvicorn web-server closure that + # `glances -w` needs (py3-uvicorn is not in Alpine -> vendored as a wheel below) + py3-wcwidth + # (pyte's only dep) + jinja2 (web templates). apk resolves the full musl .so closure per arch. + local pkgs="glances python3 py3-psutil py3-fastapi py3-starlette py3-pydantic py3-anyio py3-sniffio py3-h11 py3-click py3-wcwidth py3-jinja2" + echo "prebuild: apk add ($APK_BRANCH) via $qemu_runner: $pkgs" + QEMU_LD_PREFIX="$staging_root" LD_LIBRARY_PATH="$staging_root/lib:$staging_root/usr/lib" \ + "$qemu_runner" -L "$staging_root" "$staging_root/sbin/apk" --root "$staging_root" \ + --repositories-file "$staging_root/etc/apk/repositories" --keys-dir "$staging_root/etc/apk/keys" \ + "${cache_args[@]}" --update-cache --no-progress --no-scripts add $pkgs + [[ -e "$staging_root/usr/bin/glances" ]] || { echo "prebuild: glances not provisioned" >&2; exit 3; } + local pyver; pyver="$(ls -d "$staging_root"/usr/lib/python3.* 2>/dev/null | grep -oE 'python3\.[0-9]+' | head -1)" + case "$pyver" in python3.1[2-9]|python3.2[0-9]) echo "prebuild: provisioned glances + $pyver" ;; + *) echo "prebuild: need CPython>=3.12, got '$pyver'" >&2; exit 3 ;; esac +} + +vendor_wheel() { # $1=url $2=sha $3=name -- fetch pinned noarch wheel, verify, unzip into site-packages + local url="$1" sha="$2" name="$3" + local pyver; pyver="$(ls -d "$staging_root"/usr/lib/python3.* 2>/dev/null | grep -oE 'python3\.[0-9]+' | head -1)" + local sp="$staging_root/usr/lib/$pyver/site-packages" + mkdir -p "$sp"; local whl="$staging_root/tmp/$name.whl"; mkdir -p "$staging_root/tmp" + echo "prebuild: fetch wheel $name" + curl -fL --retry 3 -o "$whl" "$url" + echo "$sha $whl" | sha256sum -c - || { echo "prebuild: $name wheel sha256 MISMATCH" >&2; exit 4; } + unzip -oq "$whl" -d "$sp"; rm -f "$whl" + echo "prebuild: vendored $name into $sp" +} + +install_prometheus_stack() { + local prom_bin_dir="$staging_root/usr/local/bin" ne_bin_dir="$staging_root/usr/bin" + mkdir -p "$prom_bin_dir" "$ne_bin_dir" + local topdir="prometheus-$PROM_VER.linux-$goarch" + local tb="$staging_root/tmp/prom.tgz" ne_tb="$staging_root/tmp/ne.tgz" + mkdir -p "$staging_root/tmp" + + # --- prometheus + promtool --- + if [[ "$goarch" == "loong64" ]]; then + # upstream ships no loong64: use the maintainer-staged reproducible artifact (built from the + # pinned source tag per assets/build-loong-binaries.sh) -- NEVER a SKIP. + local staged="${BINS_DIR:+$BINS_DIR/prometheus/$arch/prometheus-$PROM_VER.linux-loong64.tar.gz}" + if [[ -n "$staged" && -f "$staged" ]]; then + echo "prebuild: loong64 prometheus from staged artifact $staged"; cp -f "$staged" "$tb" + else + echo "prebuild: loong64 prometheus not staged -> cross-compiling from source (assets/build-loong-binaries.sh)" + MONITOR_LOONG_OUT="$staging_root/tmp" bash "$app_dir/assets/build-loong-binaries.sh" prometheus \ + || { echo "prebuild: loong64 prometheus cross-compile FAILED (set MONITOR_BINS_DIR to a staged artifact)" >&2; exit 5; } + cp -f "$staging_root/tmp/prometheus-$PROM_VER.linux-loong64.tar.gz" "$tb" + fi + else + local url="https://github.com/prometheus/prometheus/releases/download/v$PROM_VER/$topdir.tar.gz" + local staged="${BINS_DIR:+$BINS_DIR/prometheus/$arch/$topdir.tar.gz}" + if [[ -n "$staged" && -f "$staged" ]]; then cp -f "$staged" "$tb"; echo "prebuild: prometheus from staged $staged" + else echo "prebuild: download prometheus $url"; curl -fL --retry 3 -o "$tb" "$url"; fi + echo "$(prom_sha "$goarch") $tb" | sha256sum -c - || { echo "prebuild: prometheus sha256 MISMATCH ($goarch)" >&2; exit 5; } + fi + tar xzf "$tb" -C "$staging_root/tmp" "$topdir/prometheus" "$topdir/promtool" + mv "$staging_root/tmp/$topdir/prometheus" "$prom_bin_dir/prometheus" + mv "$staging_root/tmp/$topdir/promtool" "$prom_bin_dir/promtool" + chmod 0755 "$prom_bin_dir/prometheus" "$prom_bin_dir/promtool"; rm -rf "$staging_root/tmp/$topdir" "$tb" + + # --- node_exporter (pre-extracted static binary) --- + if [[ "$goarch" == "loong64" ]]; then + local staged="${BINS_DIR:+$BINS_DIR/node_exporter/$arch/node_exporter}" + if [[ -n "$staged" && -f "$staged" ]]; then cp -f "$staged" "$ne_bin_dir/node_exporter"; echo "prebuild: loong64 node_exporter from staged $staged" + else + echo "prebuild: loong64 node_exporter not staged -> cross-compiling (assets/build-loong-binaries.sh)" + MONITOR_LOONG_OUT="$staging_root/tmp" bash "$app_dir/assets/build-loong-binaries.sh" node_exporter \ + || { echo "prebuild: loong64 node_exporter cross-compile FAILED (set MONITOR_BINS_DIR)" >&2; exit 5; } + cp -f "$staging_root/tmp/node_exporter" "$ne_bin_dir/node_exporter" + fi + else + local staged="${BINS_DIR:+$BINS_DIR/node_exporter/$arch/node_exporter}" + if [[ -n "$staged" && -f "$staged" ]]; then cp -f "$staged" "$ne_bin_dir/node_exporter"; echo "prebuild: node_exporter from staged $staged" + else + local url="https://github.com/prometheus/node_exporter/releases/download/v$NE_VER/node_exporter-$NE_VER.linux-$goarch.tar.gz" + echo "prebuild: download node_exporter $url"; curl -fL --retry 3 -o "$ne_tb" "$url" + echo "$(ne_sha "$goarch") $ne_tb" | sha256sum -c - || { echo "prebuild: node_exporter sha256 MISMATCH ($goarch)" >&2; exit 5; } + tar xzf "$ne_tb" -C "$staging_root/tmp" "node_exporter-$NE_VER.linux-$goarch/node_exporter" + mv "$staging_root/tmp/node_exporter-$NE_VER.linux-$goarch/node_exporter" "$ne_bin_dir/node_exporter" + rm -rf "$staging_root/tmp/node_exporter-$NE_VER.linux-$goarch" "$ne_tb" + fi + fi + chmod 0755 "$ne_bin_dir/node_exporter" + # scrape config: a `node` job pointing at the in-guest node_exporter on loopback :9100. + install -Dm0644 "$app_dir/assets/prometheus.yml" "$staging_root/etc/prometheus.yml" + echo "prebuild: prometheus+promtool+node_exporter staged ($goarch); version red-line $PROM_VER / $NE_VER" +} + +install_grafana() { + # grafana OSS 13.0.1: single fully-static CGO-free Go binary (`grafana server` subcommand) + the + # frontend SPA in public/ + conf/. amd64/arm64/riscv64 are official dl.grafana.com releases; + # loong64 upstream ships none -> self-cross-compiled backend + official frontend (staged/recipe). + local gdir="$staging_root/opt/grafana" topdir="grafana-$GRAFANA_VER" tb="$staging_root/tmp/grafana.tgz" + mkdir -p "$gdir" "$staging_root/tmp" + if [[ "$goarch" == "loong64" ]]; then + local staged="${BINS_DIR:+$BINS_DIR/grafana/$arch/grafana-$GRAFANA_VER.linux-loong64.tar.gz}" + if [[ -n "$staged" && -f "$staged" ]]; then echo "prebuild: loong64 grafana from staged $staged"; cp -f "$staged" "$tb" + else + echo "prebuild: loong64 grafana not staged -> cross-compiling backend + grafting official frontend (assets/build-loong-binaries.sh)" + MONITOR_LOONG_OUT="$staging_root/tmp" bash "$app_dir/assets/build-loong-binaries.sh" grafana \ + || { echo "prebuild: loong64 grafana build FAILED (set MONITOR_BINS_DIR to a staged artifact)" >&2; exit 6; } + cp -f "$staging_root/tmp/grafana-$GRAFANA_VER.linux-loong64.tar.gz" "$tb" + fi + else + local url="https://dl.grafana.com/oss/release/grafana-$GRAFANA_VER.linux-$goarch.tar.gz" + local staged="${BINS_DIR:+$BINS_DIR/grafana/$arch/grafana-$GRAFANA_VER.linux-$goarch.tar.gz}" + if [[ -n "$staged" && -f "$staged" ]]; then cp -f "$staged" "$tb"; echo "prebuild: grafana from staged $staged" + else echo "prebuild: download grafana $url"; curl -fL --retry 3 -o "$tb" "$url"; fi + echo "$(grafana_sha "$goarch") $tb" | sha256sum -c - || { echo "prebuild: grafana sha256 MISMATCH ($goarch)" >&2; exit 6; } + fi + tar xzf "$tb" -C "$staging_root/tmp" "$topdir" + # bin/ + public/ (frontend SPA) + conf/ (defaults.ini); drop *.map browser debug artifacts + # (~290MB, never read server-side) and docs/ to keep the overlay lean. + cp -a "$staging_root/tmp/$topdir/bin" "$gdir/" + cp -a "$staging_root/tmp/$topdir/public" "$gdir/" + cp -a "$staging_root/tmp/$topdir/conf" "$gdir/" + find "$gdir/public" -name '*.map' -delete 2>/dev/null || true + chmod 0755 "$gdir/bin/grafana" + mkdir -p "$gdir/data" "$gdir/logs" "$gdir/plugins" \ + "$gdir/provisioning/datasources" "$gdir/provisioning/dashboards" \ + "$gdir/provisioning/plugins" "$gdir/provisioning/alerting" \ + "$gdir/provisioning/access-control" "$gdir/provisioning/notifiers" + rm -rf "$staging_root/tmp/$topdir" "$tb" + + # --- best-effort BUILD-TIME pre-migration of the arch-independent grafana.db (skips the 709 + # first-run SQLite migrations on-target). Prefer a staged db, else run the target grafana + # (native for x86; qemu-user otherwise), bounded; on any failure ship un-seeded (the carpet + # migrates on the spot). grafana.db is arch-independent, so a db built for ANY arch is valid. + local seeded=0 + local staged_db="${BINS_DIR:+$BINS_DIR/grafana/grafana-$GRAFANA_VER.db}" + if [[ -n "$staged_db" && -f "$staged_db" ]]; then + cp -f "$staged_db" "$gdir/data/grafana.db"; seeded=1; echo "prebuild: grafana pre-migrated db from staged $staged_db" + elif [[ "$GRAFANA_PREMIGRATE" == 1 ]]; then + premigrate_grafana "$gdir" && seeded=1 || echo "prebuild: grafana pre-migration skipped (on-target will migrate)" + fi + echo "prebuild: grafana $GRAFANA_VER staged ($goarch); pre-migrated=$seeded" +} + +# run grafana once so the SQLite store migrates, then stop; copy the migrated db as the seed. +premigrate_grafana() { + # loongarch64: `grafana server` under qemu-loongarch64-static never reaches "HTTP Server Listen" + # (unlike the amd64 native / aarch64 / riscv64 qemu-user paths), so the build-time seed is skipped + # and the carpet migrates on-target within its 1200s TCG budget (a stricter, real ext4/fsync test). + [[ "$goarch" == "loong64" ]] && { echo "prebuild: grafana pre-migration skipped for loong64 (qemu-user grafana server does not listen; on-target migrates)"; return 1; } + local gdir="$1" port=13300 work="$staging_root/tmp/gfmig" + rm -rf "$work"; mkdir -p "$work/data" "$work/logs" "$work/plugins" \ + "$work/provisioning/datasources" "$work/provisioning/dashboards" "$work/provisioning/plugins" \ + "$work/provisioning/alerting" "$work/provisioning/access-control" "$work/provisioning/notifiers" + cat > "$work/grafana.ini" <"$work/mig.log" 2>&1 & + local pid=$! up=0 i=0 + while [[ $i -lt 300 ]]; do # bounded: native/qemu-user grafana that listens comes up well under this + kill -0 "$pid" 2>/dev/null || break + if curl -fs --max-time 3 --noproxy '*' "http://127.0.0.1:$port/api/health" >/dev/null 2>&1; then up=1; break; fi + i=$((i+1)); sleep 1 + done + kill "$pid" 2>/dev/null; sleep 1; kill -9 "$pid" 2>/dev/null || true + if [[ $up == 1 && -f "$work/data/grafana.db" ]]; then + cp -f "$work/data/grafana.db" "$gdir/data/grafana.db"; rm -rf "$work"; return 0 + fi + rm -rf "$work"; return 1 +} + +stage_overlay() { + mkdir -p "$overlay_dir/usr" + # wholesale /usr closure: glances + python3 + psutil + fastapi/starlette/pydantic + pyte/uvicorn + # + prometheus/promtool (/usr/local/bin) + node_exporter (/usr/bin) + musl .so closure + terminfo. + cp -a "$staging_root/usr/bin" "$overlay_dir/usr/" + cp -a "$staging_root/usr/lib" "$overlay_dir/usr/" + [[ -d "$staging_root/usr/local/bin" ]] && { mkdir -p "$overlay_dir/usr/local"; cp -a "$staging_root/usr/local/bin" "$overlay_dir/usr/local/"; } + [[ -d "$staging_root/usr/share/terminfo" ]] && { mkdir -p "$overlay_dir/usr/share"; cp -a "$staging_root/usr/share/terminfo" "$overlay_dir/usr/share/"; } + [[ -d "$staging_root/etc/terminfo" ]] && { mkdir -p "$overlay_dir/etc"; cp -a "$staging_root/etc/terminfo" "$overlay_dir/etc/"; } + install -Dm0644 "$staging_root/etc/prometheus.yml" "$overlay_dir/etc/prometheus.yml" + ln -sf python3 "$overlay_dir/usr/bin/python" 2>/dev/null || true + # grafana homepath (bin/ + public/ + conf/ + pre-migrated data/) under /opt/grafana. + [[ -d "$staging_root/opt/grafana" ]] && { mkdir -p "$overlay_dir/opt"; cp -a "$staging_root/opt/grafana" "$overlay_dir/opt/"; } + + # carpet suite + pyte harness under /root/monitor; launcher on PATH. + mkdir -p "$overlay_dir/root/monitor/python" "$overlay_dir/root/monitor/programs" + local n=0 + for f in "$app_dir"/python/*.py; do install -Dm0644 "$f" "$overlay_dir/root/monitor/python/$(basename "$f")"; n=$((n+1)); done + install -Dm0644 "$app_dir/programs/pty_tui_drive.py" "$overlay_dir/root/monitor/programs/pty_tui_drive.py" + install -Dm0644 "$app_dir/programs/pyte_assert.py" "$overlay_dir/root/monitor/programs/pyte_assert.py" + install -Dm0755 "$app_dir/programs/run-monitor.sh" "$overlay_dir/usr/bin/run-monitor.sh" + local sz; sz=$(du -sm "$overlay_dir" 2>/dev/null | cut -f1) + echo "prebuild: staged $n python carpet(s) + pyte harness; monitor overlay ready for $arch -- ${sz}M" +} + +ensure_host_tools +grow_rootfs +extract_base_rootfs +install_glances_closure +vendor_wheel "$PYTE_WHL_URL" "$PYTE_WHL_SHA" "pyte-0.8.2" +vendor_wheel "$UVICORN_WHL_URL" "$UVICORN_WHL_SHA" "uvicorn-0.34.0" +install_prometheus_stack +install_grafana +stage_overlay diff --git a/apps/starry/monitor/programs/pty_tui_drive.py b/apps/starry/monitor/programs/pty_tui_drive.py new file mode 100644 index 0000000000..d272dc24c1 --- /dev/null +++ b/apps/starry/monitor/programs/pty_tui_drive.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# pty_tui_drive.py -- headless PTY interaction driver for a real curses/TUI program. +# +# Runs inside a REAL pseudo-terminal (pty.fork), so the program takes its production +# terminal render path (initscr / smkx / alternate screen), sets the window size on the master +# with TIOCSWINSZ, then plays a scripted key sequence with a settle dwell after each key while +# continuously draining the raw ANSI byte stream the program writes back. Returns the full raw +# capture plus a checkpoint offset after every scripted step, and whether the program stayed +# alive across the whole soak. +# +# ============================================================================================ +# CRITICAL, EMPIRICALLY-VERIFIED FACT -- application-keypad / DECCKM arrow keys +# ============================================================================================ +# A curses app that calls keypad(True) (glances, top, ...) emits smkx, putting the terminal +# into APPLICATION cursor-key mode (DECCKM). In that mode the arrow keys the app expects are the +# SS3 forms ESC O A / ESC O B / ESC O C / ESC O D (up/down/right/left) -- NOT the CSI forms +# ESC [ A ... . If a driver sends the CSI arrows, ncurses cannot match them against its +# application-mode terminfo (kcuu1=\EOA), times out on the lone ESC, and returns a bare ESC to the +# app -- glances treats a bare ESC as "quit" and exits. So this driver sends SS3 arrows. +# PgUp/PgDn/Home/End/function keys use their real terminfo sequences (still CSI/tilde forms, which +# do not change under DECCKM). This was proven on StarryOS: SS3 -> program survives + keeps +# redrawing; CSI -> program quits immediately. StarryOS's tty/pty layer delivers the bytes and +# honours TIOCSWINSZ correctly, so the same SS3 sequences that work on Linux work on StarryOS. +# ============================================================================================ +import os, sys, pty, time, select, signal, fcntl, termios, struct + +# SS3 (application cursor-key mode / DECCKM) arrow sequences -- the ONLY correct arrows for a +# keypad(True) curses app. Do NOT substitute the CSI forms (ESC [ A ...). +SS3_UP = b"\x1bOA" +SS3_DOWN = b"\x1bOB" +SS3_RIGHT = b"\x1bOC" +SS3_LEFT = b"\x1bOD" +# These do not change between normal/application cursor mode. +PGDN = b"\x1b[6~" +PGUP = b"\x1b[5~" +HOME = b"\x1b[H" +END = b"\x1b[F" + + +# Optional live-screen emulation so the driver can wait UNTIL the TUI has rendered expected content +# (content-driven capture) instead of a fixed delay -- essential for slow (TCG) arches where glances +# renders much later than on x86. Falls back to fixed dwells if pyte is unavailable. +try: + import pyte as _pyte +except Exception: + _pyte = None + + +def drive(cmd, script, cols=200, rows=50, env_extra=None, quit_key=b"q", tail_dwell=2.0): + """Run `cmd` (list) in a pty and play `script`. + + Each script step is (label, key_bytes, spec) where spec is EITHER: + * a number -> fixed dwell in seconds (legacy, time-driven), OR + * a dict {"wait_for": [tokens], "max_wait": s, "min_dwell": s} -> CONTENT-DRIVEN: after the key + is sent, keep pumping + feeding a live pyte screen until every token in `wait_for` is present + on that screen (AND at least min_dwell has elapsed), then capture; if `max_wait` passes first, + capture anyway and flag the step `met=False` (a REAL,据实 failure surfaces downstream because + the captured frame lacks the expected content -- never a silent pass). This makes capture + timing track each arch's actual render speed (x86 satisfies instantly; aa/rv/loong wait). + + Returns dict: raw(bytes), checkpoints(list[(label, offset, met)]), dead(str|None), + survived(bool), sent(list[label]). + """ + pid, fd = pty.fork() + if pid == 0: + env = os.environ + env["TERM"] = "xterm-256color" + env["COLUMNS"], env["LINES"] = str(cols), str(rows) + if env_extra: + for k, v in env_extra.items(): + env[k] = v + try: + os.execvp(cmd[0], cmd) + except Exception: + os._exit(127) + # parent: fix the master window size so the child renders to `rows`x`cols`. + try: + fcntl.ioctl(fd, termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0)) + except Exception: + pass + + raw = bytearray() + dead = [None] + checkpoints = [] + sent = [] + # live incremental screen: fed the SAME bytes as raw, so raw[:offset] full-fed == live screen + # (pyte is deterministic; the differential-render invariant asserts this equivalence separately). + live_screen = _pyte.Screen(cols, rows) if _pyte else None + live_stream = _pyte.ByteStream(live_screen) if _pyte else None + + def _read_step(timeout): + r, _, _ = select.select([fd], [], [], timeout) + if fd in r: + try: + data = os.read(fd, 65536) + except OSError: + dead[0] = "eof" + return + if not data: + dead[0] = "eof" + return + raw.extend(data) + if live_stream is not None: + try: + live_stream.feed(data) + except Exception: + pass + try: + wpid, st = os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + dead[0] = "reaped" + return + if wpid == pid: + dead[0] = "exit:%d" % (os.WEXITSTATUS(st) if os.WIFEXITED(st) else -os.WTERMSIG(st)) + + def _screen_has(tokens): + if live_screen is None or not tokens: + return True + disp = live_screen.display + return all(any(t in line for line in disp) for t in tokens) + + def pump_fixed(seconds): + end = time.time() + seconds + while time.time() < end and not dead[0]: + _read_step(0.2) + + def pump_until(tokens, max_wait, min_dwell): + start = time.time() + end = start + max_wait + met = False + while time.time() < end and not dead[0]: + _read_step(0.2) + if (time.time() - start) >= min_dwell and _screen_has(tokens): + met = True + break + # content satisfied before min_dwell? keep pumping to min_dwell so the frame fully settles. + while (time.time() - start) < min_dwell and not dead[0]: + _read_step(0.2) + return met + + for label, key, spec in script: + if dead[0]: + break + if key: + try: + os.write(fd, key) + except OSError as e: + dead[0] = "write:%s" % e + break + sent.append(label) + if isinstance(spec, dict): + met = pump_until(spec.get("wait_for", []), + float(spec.get("max_wait", 30.0)), + float(spec.get("min_dwell", 1.0))) + else: + pump_fixed(float(spec)) + met = None + checkpoints.append((label, len(raw), met)) + + # ask the program to quit cleanly (single-key 'q' for glances/top). + if not dead[0]: + try: + os.write(fd, quit_key) + except OSError: + pass + pump_fixed(tail_dwell) + + # a clean quit (exit:0) or "still alive when we stopped reading" both count as survived; only a + # NON-zero exit or a mid-soak death (before we sent quit) is a failure. + survived = True + if dead[0] and dead[0].startswith("exit:") and dead[0] != "exit:0": + survived = False + if dead[0] in ("eof", "reaped") and not sent: + survived = False + + try: + os.close(fd) + except OSError: + pass + try: + os.waitpid(pid, 0) + except Exception: + pass + + return { + "raw": bytes(raw), + "checkpoints": checkpoints, + "dead": dead[0], + "survived": survived, + "sent": sent, + } + + +if __name__ == "__main__": + # Standalone smoke: pty_tui_drive.py + signal.signal(signal.SIGALRM, lambda *_: os._exit(0)) + signal.alarm(int(float(sys.argv[1])) + 40 if len(sys.argv) > 1 else 60) + dur = float(sys.argv[1]) if len(sys.argv) > 1 else 12 + cmd = sys.argv[2:] or ["glances"] + scr = [("settle", b"", dur)] + res = drive(cmd, scr) + print("survived=%s dead=%s rawlen=%d" % (res["survived"], res["dead"], len(res["raw"]))) diff --git a/apps/starry/monitor/programs/pyte_assert.py b/apps/starry/monitor/programs/pyte_assert.py new file mode 100644 index 0000000000..26c92d466e --- /dev/null +++ b/apps/starry/monitor/programs/pyte_assert.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# pyte_assert.py -- offline terminal-emulation assertions over a captured raw ANSI stream. +# +# Feeds a captured raw byte stream to pyte (a pure-python VT/ANSI terminal emulator) to reconstruct +# the exact on-screen cell matrix the TUI painted, then asserts against it. Three families of check: +# +# 1. golden-frame content -- required "static chrome" tokens (section labels, table headers) are +# present in the reconstructed screen (proves the TUI actually rendered its dashboard, not a +# blank/garbled screen). +# 2. cross-frame stability -- each anchor token lands at the SAME (row, col) at every checkpoint +# across the whole soak: no flicker (chrome never disappears), no residue (no stale duplicate), +# no misaligned refresh (columns never shift). This is the differential-across-time invariant. +# 3. differential-render invariant -- feeding the byte stream in arbitrary split chunks yields a +# cell-for-cell IDENTICAL screen to feeding it whole. A mismatch means an escape sequence was +# split/dropped/corrupted (capture or tty delivery bug). This is the differential-render +# (incremental == full-redraw) invariant. +import pyte + + +def full_screen(raw, cols, rows): + s = pyte.Screen(cols, rows) + st = pyte.ByteStream(s) + st.feed(raw) + return s + + +def incremental_screen(raw, split_offsets, cols, rows): + """Feed `raw` into ONE persistent Screen, split at each offset in split_offsets (arbitrary + boundaries that mimic os.read() chunking / packet arrival).""" + s = pyte.Screen(cols, rows) + st = pyte.ByteStream(s) + prev = 0 + for off in sorted(set(o for o in split_offsets if 0 < o < len(raw))): + st.feed(raw[prev:off]) + prev = off + st.feed(raw[prev:]) + return s + + +def display_lines(screen): + return list(screen.display) + + +def find_token(disp, token): + for r, line in enumerate(disp): + c = line.find(token) + if c >= 0: + return (r, c) + return (-1, -1) + + +def tokens_present(disp, tokens): + """Return {token: (row,col)} for tokens found, and a list of missing tokens.""" + found = {} + missing = [] + for t in tokens: + rc = find_token(disp, t) + if rc[0] >= 0: + found[t] = rc + else: + missing.append(t) + return found, missing + + +def screens_equal(a, b): + """Cell-for-cell compare via the rendered display lines. Returns (bool, first_diff_row, detail).""" + da, db = list(a.display), list(b.display) + if len(da) != len(db): + return (False, -1, "row count %d != %d" % (len(da), len(db))) + for r in range(len(da)): + if da[r] != db[r]: + return (False, r, "row %d: %r != %r" % (r, da[r][:60], db[r][:60])) + return (True, -1, "") + + +def stability_violations(frames, anchors): + """frames = list of (label, disp). anchors = list of tokens whose (row,col) must be identical + across every frame in which the TUI is fully painted. Returns list of violation strings. + + An anchor is required to be present in EVERY frame (absence = flicker/erased chrome) and at the + SAME coordinate (movement = misaligned refresh / residue).""" + violations = [] + for tok in anchors: + coords = [] + for label, disp in frames: + rc = find_token(disp, tok) + coords.append((label, rc)) + present = [rc for _, rc in coords if rc[0] >= 0] + if len(present) != len(coords): + missing_at = [lbl for lbl, rc in coords if rc[0] < 0] + violations.append("anchor %r missing at frames %s (flicker/erased chrome)" % (tok, missing_at)) + continue + first = present[0] + for label, rc in coords: + if rc != first: + violations.append("anchor %r moved: %s@%s != baseline %s (misaligned refresh)" + % (tok, label, rc, first)) + break + return violations diff --git a/apps/starry/monitor/programs/run-monitor.sh b/apps/starry/monitor/programs/run-monitor.sh new file mode 100644 index 0000000000..9106d84ad6 --- /dev/null +++ b/apps/starry/monitor/programs/run-monitor.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# run-monitor.sh -- on-target launcher for the StarryOS `monitor` app carpet. +# +# Sets the musl dynamic-loader search path (so the loader finds the glances CPython .so closure +# injected under /lib + /usr/lib), puts the app's python/ + programs/ on PYTHONPATH, forces a +# deterministic environment, then hands off to python3 run_monitor.py which runs the prometheus + +# glances carpets and emits the PASS/FAIL gate. The `TEST PASSED` / `MONITOR_OK=N/N` anchor lives +# ONLY in run_monitor.py, never in this wrapper, so the success regex cannot self-match the launch +# command echoed on the serial console. +# +# The musl loader reads only /etc/ld-musl-.path; writing all four names is harmless and +# keeps the launcher arch-agnostic. +for a in x86_64 aarch64 riscv64 loongarch64; do + printf '/lib\n/usr/lib\n' > "/etc/ld-musl-$a.path" +done + +export PATH=/usr/local/bin:/usr/bin:/bin:/sbin:/usr/sbin +export HOME=/root +export PYTHONDONTWRITEBYTECODE=1 +export PYTHONUNBUFFERED=1 +export GLANCES_BIN=glances +# prometheus stack binaries (injected by prebuild.sh) +export PROMETHEUS_BIN=/usr/local/bin/prometheus +export PROMTOOL_BIN=/usr/local/bin/promtool +export NODE_EXPORTER_BIN=/usr/bin/node_exporter +export PROMETHEUS_CFG=/etc/prometheus.yml +# grafana (injected by prebuild.sh) +export GRAFANA_BIN=/opt/grafana/bin/grafana +export GRAFANA_HOMEPATH=/opt/grafana + +cd /root/monitor/python || exit 1 +exec python3 run_monitor.py diff --git a/apps/starry/monitor/python/GlancesCliCarpet.py b/apps/starry/monitor/python/GlancesCliCarpet.py new file mode 100644 index 0000000000..4e38aa502c --- /dev/null +++ b/apps/starry/monitor/python/GlancesCliCarpet.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# GlancesCliCarpet.py -- glances CLI surface carpet: version RED-LINE + exhaustive `--help` option +# tree + module list + shell-completion. Every documented option/mode is asserted present against +# the real `glances --help` output (ground truth = the --help tree of glances 4.4.1), not a subset. +# +# Emits `GCLI_RESULT ok= fail=` and, only when F==0, `GCLI_DONE`. +import os, re, subprocess, sys + +GLANCES = os.environ.get("GLANCES_BIN", "glances") +PY = sys.executable +_ok = 0 +_fail = 0 + + +def check(cond, label): + global _ok, _fail + if cond: + _ok += 1 + else: + _fail += 1 + print(" FAIL %s" % label) + + +def run(args, timeout=60): + r = subprocess.run([GLANCES] + args, capture_output=True, text=True, timeout=timeout) + return r.returncode, (r.stdout or "") + (r.stderr or "") + + +def main(): + print("=== GlancesCliCarpet: version red-line + exhaustive --help tree ===") + + # --- VERSION RED-LINE: glances pinned to 4.4.1; API version 4; psutil present (version-flexible + # because the musl apk and a host glibc psutil differ). Wrong glances version = invalid test. + rc, out = run(["--version"]) + check(rc == 0, "`glances --version` exits 0") + m = re.search(r"Glances version:\s+([0-9]+\.[0-9]+\.[0-9]+)", out) + check(bool(m), "`--version` prints a Glances version line") + check(bool(m) and m.group(1) == "4.4.1", "Glances version RED-LINE == 4.4.1 (got %s)" + % (m.group(1) if m else None)) + check(re.search(r"Glances API version:\s+4", out) is not None, "Glances API version == 4") + check(re.search(r"PsUtil version:\s+[0-9]+\.[0-9]+", out) is not None, "PsUtil version line present") + # -V short form + rc2, out2 = run(["-V"]) + check(rc2 == 0 and "Glances version:" in out2, "`glances -V` short form works") + + # --- EXHAUSTIVE --help OPTION TREE: assert every option of the DELIVERED build (Alpine glances + # 4.4.1-r1) is listed. Ground truth = the actual `glances --help` of the musl apk (captured + # under qemu-user-static from the delivery closure), NOT a host pip build. `--print-completion` + # is deliberately NOT here: it is shtab-gated and Alpine glances 4.4.1-r1 ships without shtab, + # so the option is legitimately absent -> handled as a capability probe below, never a red-line. + rc, help_out = run(["--help"]) + check(rc == 0, "`glances --help` exits 0") + REQUIRED_OPTS = [ + # short forms + "-h", "-V", "-d", "-C", "-P", "-0", "-1", "-2", "-3", "-4", "-5", "-6", + "-c", "-s", "-p", "-B", "-u", "-t", "-w", "-q", "-f", "-b", + # long forms (the full delivered 4.4.1-r1 set) + "--help", "--version", "--debug", "--config", "--plugins", "--modules-list", "--module-list", + "--disable-plugin", "--disable-plugins", "--disable", "--enable-plugin", "--enable-plugins", + "--enable", "--disable-process", "--disable-webui", "--light", "--enable-light", + "--disable-irix", "--percpu", "--per-cpu", "--disable-left-sidebar", "--disable-quicklook", + "--full-quicklook", "--disable-top", "--meangpu", "--disable-history", "--disable-bold", + "--disable-bg", "--enable-irq", "--enable-process-extended", "--disable-separator", + "--disable-cursor", "--sort-processes", "--programs", "--program", "--export", + "--export-csv-file", "--export-csv-overwrite", "--export-json-file", "--export-graph-path", + "--export-process-filter", "--client", "--server", "--browser", "--disable-autodiscover", + "--port", "--bind", "--username", "--password", "--snmp-community", "--snmp-port", + "--snmp-version", "--snmp-user", "--snmp-auth", "--snmp-force", "--time", "--webserver", + "--cached-time", "--stop-after", "--open-web-browser", "--quiet", "--process-filter", + "--process-short-name", "--process-long-name", "--stdout", "--stdout-json", "--stdout-csv", + "--issue", "--trace-malloc", "--memory-leak", "--api-doc", "--api-restful-doc", + "--hide-kernel-threads", "--byte", "--diskio-show-ramfs", "--diskio-iops", "--diskio-latency", + "--fahrenheit", "--fs-free-space", "--sparkline", "--disable-unicode", "--hide-public-info", + "--disable-check-update", "--strftime", "--fetch", "--fetch-template", "--stdout-fetch", + "--stdout-fetch-template", + ] + missing = [o for o in REQUIRED_OPTS if o not in help_out] + check(not missing, "`--help` lists all %d delivered-build options (missing: %s)" + % (len(REQUIRED_OPTS), missing)) + + # the FIVE run modes glances documents (standalone is the default; the other four are flags). + for mode_flag, name in [("--client", "client"), ("--server", "server"), + ("--webserver", "web server"), ("--browser", "browser"), + ("--stdout", "stdout")]: + check(mode_flag in help_out, "run-mode flag %s (%s) documented in --help" % (mode_flag, name)) + + # --sort-processes choices are the canonical psutil sort keys. + for choice in ["cpu_percent", "memory_percent", "username", "cpu_times", "io_counters", "name"]: + check(choice in help_out, "--sort-processes choice %s documented" % choice) + + # default client/server port red-line. + check("61209" in help_out, "default client/server port 61209 documented") + + # --- --modules-list: enumerates plugins + exporters (rc 0, mentions core plugins). + rc, out = run(["--modules-list"]) + check(rc == 0, "`glances --modules-list` exits 0") + check(all(p in out for p in ("cpu", "mem", "load", "network")), + "--modules-list enumerates core plugins (cpu/mem/load/network)") + + # --- shell completion is CAPABILITY-AWARE. glances only offers `--print-completion` when the + # optional `shtab` dependency is importable; the delivery build (Alpine glances 4.4.1-r1) + # ships WITHOUT shtab (it is not in the apk closure), so the option is legitimately absent. + # If a build DOES offer it, assert the generator emits a script (gate on CONTENT, not rc: + # shtab exits non-zero on some builds even after printing the script). If absent -> a + # documented capability SKIP, never a failure (据实按交付版本, not the host build). + if "--print-completion" in help_out: + rc, out = run(["--print-completion", "bash"]) + check(len(out) > 20 and ("shtab" in out or "_glances" in out or "complete" in out.lower()), + "`--print-completion bash` emits a completion script (shtab present in this build)") + else: + print(" SKIP --print-completion -- shtab not bundled in this glances build (Alpine 4.4.1-r1); " + "the option is not offered (capability absent by design, not a failure)") + + print("GCLI_RESULT ok=%d fail=%d" % (_ok, _fail)) + if _fail == 0: + print("GCLI_DONE") + return 0 + return 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as e: + import traceback + traceback.print_exc() + print("GCLI_RESULT ok=%d fail=%d" % (_ok, _fail + 1)) + sys.exit(1) diff --git a/apps/starry/monitor/python/GlancesCsCarpet.py b/apps/starry/monitor/python/GlancesCsCarpet.py new file mode 100644 index 0000000000..e4a99d66ad --- /dev/null +++ b/apps/starry/monitor/python/GlancesCsCarpet.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# GlancesCsCarpet.py -- glances CLIENT/SERVER mode. Starts a glances XML-RPC server (`glances -s`) +# on loopback, then runs a glances client (`glances -c`) that connects to it and pulls a one-shot +# snapshot over the wire. Asserts the client received a plausible mem.total FROM the server -> +# proves the XML-RPC server (bind/listen/accept) + client (connect) round-trip on StarryOS loopback. +# +# Emits `GCS_RESULT ok= fail=` and, only when F==0, `GCS_DONE`. +import os, re, signal, subprocess, sys, time + +GLANCES = os.environ.get("GLANCES_BIN", "glances") +DISABLE = "ip,cloud,containers,docker,folders,ports,smart,wifi,gpu,connections,sensors" +PORT = int(os.environ.get("MONITOR_CS_PORT", "61234")) +HOST = "127.0.0.1" +_ok = 0 +_fail = 0 + + +def check(cond, label): + global _ok, _fail + if cond: + _ok += 1 + else: + _fail += 1 + print(" FAIL %s" % label) + + +def main(): + print("=== GlancesCsCarpet: glances -s (XML-RPC server) + glances -c (client) on :%d ===" % PORT) + slog = open("/tmp/glances-server.log", "w") + server = subprocess.Popen( + [GLANCES, "-s", "-p", str(PORT), "-B", HOST, "--disable-autodiscover", "-t", "2", + "--disable-check-update", "--disable-plugin", DISABLE], + stdout=slog, stderr=subprocess.STDOUT) + client = None + try: + # wait for the server to announce it is listening. + up = False + for _ in range(90): + if server.poll() is not None: + break + try: + txt = open("/tmp/glances-server.log").read() + except Exception: + txt = "" + if "XML-RPC server is running" in txt or ("running on %s:%d" % (HOST, PORT)) in txt: + up = True + break + time.sleep(1) + check(up, "glances -s XML-RPC server came up (announced listening)") + + # run the client bounded: it fetches from the server and prints mem.total/cpu.total. In + # client stdout mode --stop-after is not always honoured, so bound it with a timeout and + # read whatever it printed to the log. + clog_path = "/tmp/glances-client.log" + clog = open(clog_path, "w") + client = subprocess.Popen( + [GLANCES, "-c", HOST, "-p", str(PORT), "--stdout", "mem.total,cpu.total", + "--stop-after", "1", "-t", "2", "--disable-check-update"], + stdout=clog, stderr=subprocess.STDOUT) + got = None + deadline = time.time() + 40 + while time.time() < deadline: + try: + txt = open(clog_path).read() + except Exception: + txt = "" + m = re.search(r"^mem\.total:\s*([0-9]+)\s*$", txt, re.M) + if m: + got = int(m.group(1)) + break + if client.poll() is not None: + # process finished; re-read once more + try: + txt = open(clog_path).read() + except Exception: + pass + m = re.search(r"^mem\.total:\s*([0-9]+)\s*$", txt, re.M) + if m: + got = int(m.group(1)) + break + time.sleep(1) + clog.close() + check(got is not None, "client received a mem.total line from the server") + check(got is not None and got > (1 << 20), + "client mem.total > 1 MiB over the wire (got %s) -- c/s round-trip real" % got) + c = None + try: + c = re.search(r"^cpu\.total:\s*([0-9.]+)\s*$", open(clog_path).read(), re.M) + except Exception: + pass + check(c is not None, "client also received cpu.total from the server") + finally: + for p in (client, server): + if p is None: + continue + try: + p.send_signal(signal.SIGTERM) + p.wait(timeout=6) + except Exception: + try: + p.kill() + except Exception: + pass + slog.close() + + print("GCS_RESULT ok=%d fail=%d" % (_ok, _fail)) + if _fail == 0: + print("GCS_DONE") + return 0 + return 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as e: + import traceback + traceback.print_exc() + print("GCS_RESULT ok=%d fail=%d" % (_ok, _fail + 1)) + sys.exit(1) diff --git a/apps/starry/monitor/python/GlancesHeadlessCarpet.py b/apps/starry/monitor/python/GlancesHeadlessCarpet.py new file mode 100644 index 0000000000..7345ae1a6b --- /dev/null +++ b/apps/starry/monitor/python/GlancesHeadlessCarpet.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# GlancesHeadlessCarpet.py -- glances one-shot NON-curses stdout path. Exercises the standalone +# GlancesStdout interface (serve_n(1): one refresh, no curses) and asserts the psutil-read numbers +# are real (mem.total a plausible RAM size, cpu has the canonical fields, load present), plus the +# machine-readable --stdout-json path. This proves glances reads StarryOS /proc correctly, not just +# "it didn't crash". It also asserts the psutil plugins backed by the newer procfs sources -- +# network (/proc/net/dev), diskio (/proc/diskstats) and fs (/proc/mounts + statfs) -- return real, +# non-empty stats: the loopback interface with a byte counter, the root virtio-blk disk "vda", and +# the ext4 root filesystem with a non-zero size. These are the same three left-sidebar sections the +# curses TUI carpet renders. +# +# Emits `GHDL_RESULT ok= fail=` and, only when F==0, `GHDL_DONE`. +import ast, json, os, re, socket, subprocess, sys + +GLANCES = os.environ.get("GLANCES_BIN", "glances") +DISABLE = "ip,cloud,containers,docker,folders,ports,smart,wifi,gpu,connections,sensors" +_ok = 0 +_fail = 0 + + +def warm_loopback(nbytes=131072): + """Push real traffic through the loopback interface so /proc/net/dev's `lo` + byte counters are non-zero when the snapshot is taken. + + The kernel counts loopback bytes only when packets actually flow (the router + increments lo rx/tx on the loopback egress fast path); an idle boot leaves + them at 0. This carpet runs before any server carpet binds loopback, so + without self-generated traffic `lo` would legitimately read 0. Generating a + deterministic loopback transfer here makes the `lo bytes > 0` assertion a + self-contained proof that the counters are live -- the network analogue of + the diskio (boot reads) and fs (statfs) proofs below.""" + rx = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + rx.bind(("127.0.0.1", 0)) + rx.settimeout(0.2) + tx = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + addr = rx.getsockname() + payload = b"x" * 1024 + sent = 0 + while sent < nbytes: + tx.sendto(payload, addr) + sent += len(payload) + try: + rx.recvfrom(2048) + except socket.timeout: + pass + tx.close() + rx.close() + + +def check(cond, label): + global _ok, _fail + if cond: + _ok += 1 + else: + _fail += 1 + print(" FAIL %s" % label) + + +def run(extra, timeout=90): + args = [GLANCES] + extra + ["--time", "1", "--stop-after", "1", + "--disable-check-update", "--disable-plugin", DISABLE] + r = subprocess.run(args, capture_output=True, text=True, timeout=timeout) + return r.returncode, (r.stdout or ""), (r.stderr or "") + + +def main(): + print("=== GlancesHeadlessCarpet: one-shot --stdout snapshot (psutil /proc read) ===") + + # 1. cpu,mem,load whole-plugin snapshot -> three plugin dict lines. + rc, out, err = run(["--stdout", "cpu,mem,load"]) + check(rc == 0, "`--stdout cpu,mem,load` exits 0") + print(out.strip()[:400]) + cpu_line = next((l for l in out.splitlines() if l.startswith("cpu:")), None) + mem_line = next((l for l in out.splitlines() if l.startswith("mem:")), None) + load_line = next((l for l in out.splitlines() if l.startswith("load:")), None) + check(cpu_line is not None, "cpu plugin line printed") + check(mem_line is not None, "mem plugin line printed") + check(load_line is not None, "load plugin line printed") + + # parse the python-dict literals glances prints and assert field shape + plausibility. + def parse(line): + try: + return ast.literal_eval(line.split(":", 1)[1].strip()) + except Exception: + return None + + cpu = parse(cpu_line) if cpu_line else None + mem = parse(mem_line) if mem_line else None + load = parse(load_line) if load_line else None + check(isinstance(cpu, dict) and all(k in cpu for k in ("total", "user", "system", "idle")), + "cpu dict has canonical fields total/user/system/idle") + check(isinstance(cpu, dict) and "cpucore" in cpu and int(cpu["cpucore"]) >= 1, + "cpu.cpucore >= 1 (real core count)") + check(isinstance(mem, dict) and all(k in mem for k in ("total", "available", "percent", "used")), + "mem dict has canonical fields total/available/percent/used") + check(isinstance(mem, dict) and int(mem.get("total", 0)) > (1 << 20), + "mem.total > 1 MiB (psutil really read /proc/meminfo MemTotal, not a 0 fallback)") + check(isinstance(load, dict) and "min1" in load and "cpucore" in load, + "load dict has min1 + cpucore") + + # 2. attribute form: a clean machine-checkable mem.total + cpu.total. + rc, out, err = run(["--stdout", "mem.total,cpu.total"]) + check(rc == 0, "`--stdout mem.total,cpu.total` exits 0") + m = re.search(r"^mem\.total:\s*([0-9]+)\s*$", out, re.M) + check(bool(m) and int(m.group(1)) > (1 << 20), "mem.total attribute is an integer > 1 MiB") + c = re.search(r"^cpu\.total:\s*([0-9.]+)\s*$", out, re.M) + check(bool(c) and 0.0 <= float(c.group(1)) <= 100.0, "cpu.total attribute is a 0..100 percent") + + # 3. JSON stdout path (machine-consumable) -> valid JSON, plugin nested under its name: + # `--stdout-json mem` -> {"mem": {"total": ..., ...}}. + rc, out, err = run(["--stdout-json", "mem"]) + check(rc == 0, "`--stdout-json mem` exits 0") + try: + blob = out[out.index("{"):out.rindex("}") + 1] + obj = json.loads(blob) + memobj = obj.get("mem", obj) if isinstance(obj, dict) else {} + check(isinstance(memobj, dict) and int(memobj.get("total", 0)) > (1 << 20), + "--stdout-json mem is valid JSON with mem.total > 1 MiB") + except Exception as e: + check(False, "--stdout-json mem parses as JSON (%r)" % e) + + # 4. procfs-backed plugins: network (/proc/net/dev), diskio (/proc/diskstats), + # fs (/proc/mounts + statfs). Each `--stdout ` prints `: [ {..}, .. ]` -- one + # dict per interface / disk / mount. Assert the lists are non-empty AND carry the specific + # StarryOS entities with real values (not an empty list that a missing /proc source would give). + # Drive deterministic loopback traffic first so `lo` has live byte counters (see warm_loopback). + warm_loopback() + rc, out, err = run(["--stdout", "network,diskio,fs"]) + check(rc == 0, "`--stdout network,diskio,fs` exits 0") + + def parse_list(prefix): + line = next((l for l in out.splitlines() if l.startswith(prefix + ":")), None) + if line is None: + return None + try: + v = ast.literal_eval(line.split(":", 1)[1].strip()) + return v if isinstance(v, list) else None + except Exception: + return None + + # network: /proc/net/dev -> at least the loopback interface, with a cumulative byte gauge. + net = parse_list("network") + check(isinstance(net, list) and len(net) > 0, "network plugin returns a non-empty interface list") + ifaces = {d.get("interface_name") for d in net} if net else set() + print(" network interfaces: %s" % sorted(x for x in ifaces if x)) + lo = next((d for d in (net or []) if d.get("interface_name") == "lo"), None) + check(lo is not None, "network plugin exposes the 'lo' loopback interface (from /proc/net/dev)") + check(lo is not None and int(lo.get("bytes_all_gauge", 0)) > 0, + "lo has a non-zero cumulative byte counter (bytes_all_gauge > 0): %r" + % (lo.get("bytes_all_gauge") if lo else None)) + + # diskio: /proc/diskstats -> the root virtio-blk disk 'vda' with cumulative read counters. + dio = parse_list("diskio") + check(isinstance(dio, list) and len(dio) > 0, "diskio plugin returns a non-empty disk list") + disks = {d.get("disk_name") for d in dio} if dio else set() + print(" diskio disks: %s" % sorted(x for x in disks if x)) + vda = next((d for d in (dio or []) if d.get("disk_name") == "vda"), None) + check(vda is not None, "diskio plugin exposes the 'vda' root disk (from /proc/diskstats)") + check(vda is not None and int(vda.get("read_bytes", 0)) > 0, + "vda has non-zero cumulative read_bytes (boot reads): %r" % (vda.get("read_bytes") if vda else None)) + + # fs: /proc/mounts + statfs -> the ext4 root filesystem with a real total size. + fs = parse_list("fs") + check(isinstance(fs, list) and len(fs) > 0, "fs plugin returns a non-empty filesystem list") + mounts = {d.get("mnt_point") for d in fs} if fs else set() + print(" fs mount points: %s" % sorted(x for x in mounts if x)) + root = next((d for d in (fs or []) if d.get("mnt_point") == "/"), None) + check(root is not None, "fs plugin exposes the '/' root mount (from /proc/mounts)") + check(root is not None and int(root.get("size", 0)) > 0, + "root filesystem has a non-zero total size from statfs: %r" % (root.get("size") if root else None)) + + print("GHDL_RESULT ok=%d fail=%d" % (_ok, _fail)) + if _fail == 0: + print("GHDL_DONE") + return 0 + return 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as e: + import traceback + traceback.print_exc() + print("GHDL_RESULT ok=%d fail=%d" % (_ok, _fail + 1)) + sys.exit(1) diff --git a/apps/starry/monitor/python/GlancesTuiCarpet.py b/apps/starry/monitor/python/GlancesTuiCarpet.py new file mode 100644 index 0000000000..096fb55c88 --- /dev/null +++ b/apps/starry/monitor/python/GlancesTuiCarpet.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +# GlancesTuiCarpet.py -- industrial, pyte-verified assertion carpet for the glances curses TUI on +# StarryOS (4 arches). This is the hard/novel dimension: a real curses full-screen monitor driven +# through a live pseudo-terminal over a minutes-long soak, its raw ANSI output reconstructed by an +# offline terminal emulator (pyte) and asserted cell-by-cell. +# +# WHAT IS EXERCISED (NOT a "process starts" smoke): +# * glances is launched inside a REAL pty (pty.fork -> initscr -> smkx alternate screen) with the +# window size set via TIOCSWINSZ, so it takes its production terminal render path on StarryOS. +# * A scripted key soak drives display + interaction + display-during-interaction: +# - process sort hotkeys m / c / a (sort by memory / cpu / automatic) -- and we assert the +# on-screen "Threads sorted ..." status line CHANGES accordingly (interaction really landed +# and the screen repainted coherently). +# - SS3 application-mode arrow keys (ESC O A/B/C/D) for process-cursor navigation. glances +# does keypad(True) => DECCKM => it expects SS3 arrows; a CSI arrow would be mis-decoded to +# a bare ESC and glances would QUIT. So "glances SURVIVED the whole arrow soak" is itself a +# hard assertion that StarryOS's pty delivered SS3 correctly (verified fact). +# - large-region redraw toggles (-2 left sidebar off/on, -3 quicklook off/on) to force full +# dirty-block repaints, then assert the chrome comes back with no residue. +# * The captured raw stream is fed to pyte and asserted three ways (see pyte_assert.py): +# 1. golden chrome present (section labels + process-table header actually painted) +# 2. cross-frame stability (every anchor token at the SAME row/col across all soak frames: +# no flicker / no residue / no misaligned refresh) +# 3. differential-render invariant (stream fed in arbitrary split chunks == fed whole, +# cell-for-cell -- no split/dropped escape sequence) +# +# Deterministic: fixed 200x50 terminal; environment-probe / network / container plugins that would +# block the first render on an offline SLIRP guest (ip public-IP lookup, cloud 169.254 metadata, +# containers/docker socket, ports/folders/connections scans) are disabled. Goldens are structural +# (labels/columns), never pinned to host-specific values. +# +# HARD-asserted sections (StarryOS renders every /proc source these read): +# * CORE dashboard -- the process table (/proc/[pid]/stat + /proc/stat) and the top gauges MEM +# (/proc/meminfo), LOAD (/proc/loadavg), TASKS (process count). +# * left-sidebar procfs sections -- NETWORK (/proc/net/dev), DISK I/O (/proc/diskstats), +# FILE SYS (/proc/mounts + statfs). glances paints a section ONLY when its plugin has non-empty +# stats, so the section label appearing in the reconstructed screen is itself proof the psutil +# plugin read live data: NETWORK carries the loopback interface, DISK I/O carries the root +# virtio-blk disk "vda", and FILE SYS carries the ext4 root mount. These are asserted as HARD +# gates (not best-effort) -- the same three sections the headless carpet proves numerically. +# The pyte-soak stability anchors use the always-present CORE process-table header (the sidebar +# rows churn as stats update, so they are not used as positional anchors). +# +# Emits `GTUI_RESULT ok= fail=` and, only when F==0, `GTUI_DONE`. +import os, sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +# programs/ (pty_tui_drive.py, pyte_assert.py) is a sibling of python/ in the app tree; on-target +# both are staged under the same run dir. Support both layouts. +for cand in (os.path.join(HERE, "..", "programs"), HERE, "/root/monitor/programs", "/root/monitor"): + if os.path.isfile(os.path.join(cand, "pty_tui_drive.py")): + sys.path.insert(0, cand) + break +import pty_tui_drive as ptd +import pyte_assert as pa + +GLANCES = os.environ.get("GLANCES_BIN", "glances") +COLS = int(os.environ.get("MONITOR_TUI_COLS", "200")) +ROWS = int(os.environ.get("MONITOR_TUI_ROWS", "50")) +# min dwell per step (keeps a real soak even on fast arches); big per-step max-wait budget so slow +# TCG arches (aa/rv/loong) can finish rendering before we capture (content-driven, not time-driven). +SETTLE = float(os.environ.get("MONITOR_TUI_SETTLE", "4")) # min dwell for the first-render settle +DWELL = float(os.environ.get("MONITOR_TUI_DWELL", "2")) # min dwell per interaction +SETTLE_WAIT = float(os.environ.get("MONITOR_TUI_SETTLE_WAIT", "90")) # max wait for the FIRST full render +STEP_WAIT = float(os.environ.get("MONITOR_TUI_STEP_WAIT", "45")) # max wait per interaction render + +# Plugins that do network / socket / external probing and would block the FIRST render on an +# offline guest (proven on host: sensors blocked; on a SLIRP guest ip/cloud/containers block). +# Disabling them is legitimate -- we test the TUI render+interaction, not those probes. +DISABLE = "ip,cloud,containers,docker,folders,ports,smart,wifi,gpu,connections,sensors" + +CMD = [GLANCES, "--time", "1", "--disable-check-update", "--disable-plugin", DISABLE] + +# CONTENT-DRIVEN capture: each step waits UNTIL the expected content is rendered (or a generous +# per-arch timeout) before capturing, so the same script is timing-robust on x86 AND slow TCG arches. +# The process-table header is the readiness signal for the layout; the sort steps wait for the sort +# status line to actually show the new sort keyword. Phase A = constant layout (stability + sort +# interaction + SS3 arrows). Phase B = large-region redraw toggles. +HDR = ["CPU%", "MEM%", "PID"] # process-table header = "the dashboard has rendered" signal +def _w(tokens, mx, mn): + return {"wait_for": tokens, "max_wait": mx, "min_dwell": mn} +SCRIPT = [ + ("A_settle", b"", _w(HDR, SETTLE_WAIT, SETTLE)), + ("A_sort_mem", b"m", _w(["by memory"], STEP_WAIT, DWELL)), + ("A_sort_cpu", b"c", _w(["by CPU"], STEP_WAIT, DWELL)), + ("A_sort_auto",b"a", _w(["automatically"], STEP_WAIT, DWELL)), + ("A_obs", b"", _w(HDR, STEP_WAIT, DWELL)), + ("A_down", ptd.SS3_DOWN, _w(HDR, STEP_WAIT, DWELL)), + ("A_down2", ptd.SS3_DOWN, _w(HDR, STEP_WAIT, DWELL)), + ("A_up", ptd.SS3_UP, _w(HDR, STEP_WAIT, DWELL)), + ("A_right", ptd.SS3_RIGHT, _w(HDR, STEP_WAIT, DWELL)), + ("A_left", ptd.SS3_LEFT, _w(HDR, STEP_WAIT, DWELL)), + ("B_sidebar_off", b"2", _w(HDR, STEP_WAIT, DWELL)), + ("B_sidebar_on", b"2", _w(HDR, STEP_WAIT, DWELL)), + ("B_quicklook_off", b"3", _w(HDR, STEP_WAIT, DWELL)), + ("B_quicklook_on", b"3", _w(HDR, STEP_WAIT, DWELL)), + ("B_final", b"", _w(HDR, STEP_WAIT, DWELL)), +] + +_ok = 0 +_fail = 0 + + +def check(cond, label): + global _ok, _fail + if cond: + _ok += 1 + else: + _fail += 1 + print(" FAIL %s" % label) + + +def sorted_line(disp): + for line in disp: + j = line.find("sorted") + if j >= 0: + return line[max(0, j - 8):j + 45].strip() + return "" + + +def main(): + print("=== GlancesTuiCarpet: pyte-verified curses TUI soak (%dx%d, content-driven capture) ===" + % (COLS, ROWS)) + res = ptd.drive(CMD, SCRIPT, cols=COLS, rows=ROWS, quit_key=b"q") + raw = res["raw"] + # per-step wait outcome: which steps rendered their expected content in time (met=True) vs timed + # out (met=False -> a real failure will surface below since the captured frame lacks the content). + timed_out = [lbl for lbl, _off, met in res["checkpoints"] if met is False] + print(" driver: survived=%s dead=%s rawbytes=%d sent=%d checkpoints=%d timed_out=%s" + % (res["survived"], res["dead"], len(raw), len(res["sent"]), len(res["checkpoints"]), timed_out)) + + # 1. SS3 survival -- glances never quit across the arrow + hotkey soak (the DECCKM/SS3 fact). + check(res["survived"], "glances SURVIVED the full SS3-arrow + hotkey soak (no bare-ESC quit)") + check(res["dead"] != "exit:1" and (res["dead"] is None or not res["dead"].startswith("exit:") or res["dead"] == "exit:0"), + "glances did not crash mid-soak (dead=%s)" % res["dead"]) + + # 2. it actually painted a full-screen dashboard (not blank / not a traceback). + check(len(raw) > 4000, "TUI painted a substantial ANSI stream (>4000 bytes, got %d)" % len(raw)) + + # 2b. the FIRST full render happened within the (generous, per-arch) settle budget -- if the + # process table never rendered in time, that is a REAL failure (据实), not a silent pass. + settle_met = next((met for lbl, _o, met in res["checkpoints"] if lbl == "A_settle"), None) + check(settle_met is not False, "process table rendered within the settle budget (%.0fs); timed_out=%s" + % (SETTLE_WAIT, timed_out)) + + # reconstruct per-checkpoint frames. Content assertions use the LAST PRE-QUIT checkpoint frame + # (B_final): after 'q', curses endwin() exits the alternate screen and the primary buffer is + # blank, so full_screen(raw) is not the live dashboard. The differential-render invariant below + # deliberately still runs over the WHOLE raw (byte-stream integrity, teardown included). + frames = [] + for label, off, _met in res["checkpoints"]: + frames.append((label, list(pa.full_screen(raw[:off], COLS, ROWS).display))) + final_disp = frames[-1][1] if frames else list(pa.full_screen(raw, COLS, ROWS).display) + + # 3. golden chrome present in the final painted frame (structural labels, never host values). + # HARD: the CORE dashboard that renders from the /proc StarryOS reliably provides -- the process + # table (/proc/[pid]/stat + /proc/stat) and the top gauges MEM (/proc/meminfo), LOAD + # (/proc/loadavg), TASKS (process count). These are the same core sources the headless carpet + # already proves on-target. + header_tokens = ["CPU%", "MEM%", "PID", "USER", "TIME+", "Command"] + hf, hmiss = pa.tokens_present(final_disp, header_tokens) + check(not hmiss, "process-table header tokens rendered %s (missing=%s)" % (header_tokens, hmiss)) + core_sections = ["TASKS", "LOAD", "MEM"] + cf, cmiss = pa.tokens_present(final_disp, core_sections) + check(not cmiss, "core dashboard sections rendered %s (missing=%s)" % (core_sections, cmiss)) + + # 3b. HARD: the left-sidebar procfs sections. glances paints a section ONLY when its plugin has + # non-empty stats (network/diskio/fs early-return with no stats), so a rendered section label + # is proof the psutil plugin read live data from StarryOS /proc: + # NETWORK <- psutil.net_io_counters (/proc/net/dev) + # DISK I/O <- psutil.disk_io_counters (/proc/diskstats) + # FILE SYS <- psutil.disk_partitions (/proc/mounts) + statfs + for opt in ["NETWORK", "DISK I/O", "FILE SYS"]: + print(" section %-9s rendered=%s" % (opt, pa.find_token(final_disp, opt)[0] >= 0)) + sidebar_sections = ["NETWORK", "DISK I/O", "FILE SYS"] + sf, smiss = pa.tokens_present(final_disp, sidebar_sections) + check(not smiss, "left-sidebar procfs sections rendered %s (missing=%s)" % (sidebar_sections, smiss)) + # concrete per-source data tokens inside those sections (glances only prints them with real stats): + # 'vda' -> the root virtio-blk disk name, shown by BOTH DISK I/O and the FILE SYS root row + # "/ (vda)"; its presence proves /proc/diskstats + /proc/mounts data reached the render. + # 'eth0' -> the ethernet interface row in NETWORK, proving /proc/net/dev data reached the render. + data_tokens = ["vda", "eth0"] + df, dmiss = pa.tokens_present(final_disp, data_tokens) + check(not dmiss, "sidebar sections carry live procfs data %s (missing=%s)" % (data_tokens, dmiss)) + + # 4. display-during-interaction: the sort hotkeys changed the on-screen sort status line. + ck = {lbl: dsp for lbl, dsp in frames} + if "A_sort_mem" in ck: + line = sorted_line(ck["A_sort_mem"]) + check("memory" in line.lower(), "after 'm' the sort line reads memory: %r" % line) + else: + check(False, "A_sort_mem checkpoint captured") + if "A_sort_cpu" in ck: + line = sorted_line(ck["A_sort_cpu"]) + check("cpu" in line.lower(), "after 'c' the sort line reads CPU: %r" % line) + else: + check(False, "A_sort_cpu checkpoint captured") + if "A_sort_auto" in ck: + line = sorted_line(ck["A_sort_auto"]) + check("automatically" in line.lower(), "after 'a' the sort line reads automatically: %r" % line) + else: + check(False, "A_sort_auto checkpoint captured") + + # 5. cross-frame stability over the CONSTANT-layout Phase A frames: chrome anchors never flicker, + # move, or leave residue (no misaligned refresh) as the dynamic stats churn + cursor moves. + # Anchors are the CORE process-table header tokens -- always rendered (the process list is + # core) and positionally stable within a run -- so the invariant holds on a minimal guest too + # (network/diskio anchors would be unstable there and are deliberately NOT used). + phaseA = [(lbl, dsp) for lbl, dsp in frames if lbl.startswith("A_")] + anchors = ["CPU%", "MEM%", "PID", "USER", "Command"] + viol = pa.stability_violations(phaseA, anchors) + check(not viol, "Phase-A chrome stable across %d frames (no flicker/residue/misalign): %s" + % (len(phaseA), viol[:3])) + + # 5b. after the Phase-B large-region toggles-and-back (-2 sidebar, -3 quicklook), the CORE chrome + # REDREW with no residue (assert the process-table header + core sections came back). + bf, bmiss = pa.tokens_present(final_disp, ["PID", "Command", "MEM", "LOAD"]) + check(not bmiss, "core chrome redrew after -2/-3 toggles (no residue); missing=%s" % bmiss) + + # 6. differential-render invariant: feeding the stream split at every checkpoint boundary yields + # a cell-for-cell IDENTICAL screen to feeding it whole (no split/dropped escape sequence). + splits = [off for _, off, _ in res["checkpoints"]] + inc = pa.incremental_screen(raw, splits, COLS, ROWS) + whole = pa.full_screen(raw, COLS, ROWS) + eq, diffrow, detail = pa.screens_equal(inc, whole) + check(eq, "differential-render invariant: chunked==whole (first diff %s: %s)" % (diffrow, detail)) + + print("GTUI_RESULT ok=%d fail=%d" % (_ok, _fail)) + if _fail == 0: + print("GTUI_DONE") + return 0 + # on failure, dump the final frame for diagnosis + print("--- final frame (nonempty rows) ---") + for i, l in enumerate(final_disp): + t = l.rstrip() + if t: + print("%2d|%s" % (i, t[:150])) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/apps/starry/monitor/python/GlancesWebCarpet.py b/apps/starry/monitor/python/GlancesWebCarpet.py new file mode 100644 index 0000000000..5b74c4c82a --- /dev/null +++ b/apps/starry/monitor/python/GlancesWebCarpet.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# GlancesWebCarpet.py -- glances WEB server mode (`glances -w`): a FastAPI app served by uvicorn on +# loopback. Drives the REAL HTTP REST API (Starlette/uvicorn ASGI + the Go-free python net stack on +# StarryOS loopback) and asserts JSON round-trips: /api/4/status, /api/4/all (full snapshot), +# /api/4/mem/total (a plausible RAM size), /api/4/cpu (canonical fields). +# +# Emits `GWEB_RESULT ok= fail=` and, only when F==0, `GWEB_DONE`. +import json, os, signal, subprocess, sys, time, urllib.request, urllib.error + +GLANCES = os.environ.get("GLANCES_BIN", "glances") +DISABLE = "ip,cloud,containers,docker,folders,ports,smart,wifi,gpu,connections,sensors" +PORT = int(os.environ.get("MONITOR_WEB_PORT", "62080")) +HOST = "127.0.0.1" +_ok = 0 +_fail = 0 + +# ignore any inherited HTTP(S)_PROXY env: loopback must be reached directly. +_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def check(cond, label): + global _ok, _fail + if cond: + _ok += 1 + else: + _fail += 1 + print(" FAIL %s" % label) + + +def get(path, timeout=8): + req = urllib.request.Request("http://%s:%d%s" % (HOST, PORT, path)) + try: + with _OPENER.open(req, timeout=timeout) as r: + return r.getcode(), r.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + except Exception: + return 0, b"" + + +def main(): + print("=== GlancesWebCarpet: glances -w (FastAPI/uvicorn) REST API on :%d ===" % PORT) + logf = open("/tmp/glances-web.log", "w") + proc = subprocess.Popen( + [GLANCES, "-w", "-p", str(PORT), "--disable-webui", "-t", "2", + "--disable-check-update", "--disable-plugin", DISABLE], + stdout=logf, stderr=subprocess.STDOUT) + try: + # wait for uvicorn to serve /api/4/status (Go-free ASGI cold start can lag under TCG). + up = False + for _ in range(120): + if proc.poll() is not None: + break + code, _b = get("/api/4/status", timeout=4) + if code == 200: + up = True + break + time.sleep(1) + check(up, "glances -w uvicorn served /api/4/status == 200") + if not up: + logf.flush() + try: + print("--- glances-web.log tail ---\n" + open("/tmp/glances-web.log").read()[-1200:]) + except Exception: + pass + else: + # /api/4/all -- the full stats snapshot as one JSON document. + code, body = get("/api/4/all", timeout=12) + check(code == 200 and len(body) > 500, "/api/4/all == 200 with a substantial JSON body (%d B)" % len(body)) + try: + allobj = json.loads(body) + check(isinstance(allobj, dict) and "cpu" in allobj and "mem" in allobj, + "/api/4/all JSON has cpu + mem sections") + except Exception as e: + check(False, "/api/4/all parses as JSON (%r)" % e) + + # /api/4/mem/total -- plausible RAM size via the REST path. + code, body = get("/api/4/mem/total", timeout=8) + check(code == 200, "/api/4/mem/total == 200") + try: + obj = json.loads(body) + check(int(obj.get("total", 0)) > (1 << 20), "/api/4/mem/total total > 1 MiB (got %s)" % obj) + except Exception as e: + check(False, "/api/4/mem/total parses as JSON (%r)" % e) + + # /api/4/cpu -- canonical cpu fields via REST. + code, body = get("/api/4/cpu", timeout=8) + check(code == 200, "/api/4/cpu == 200") + try: + obj = json.loads(body) + check(isinstance(obj, dict) and "total" in obj and "system" in obj, + "/api/4/cpu JSON has total + system") + except Exception as e: + check(False, "/api/4/cpu parses as JSON (%r)" % e) + finally: + try: + proc.send_signal(signal.SIGTERM) + proc.wait(timeout=8) + except Exception: + try: + proc.kill() + except Exception: + pass + logf.close() + + print("GWEB_RESULT ok=%d fail=%d" % (_ok, _fail)) + if _fail == 0: + print("GWEB_DONE") + return 0 + return 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as e: + import traceback + traceback.print_exc() + print("GWEB_RESULT ok=%d fail=%d" % (_ok, _fail + 1)) + sys.exit(1) diff --git a/apps/starry/monitor/python/GrafanaCarpet.py b/apps/starry/monitor/python/GrafanaCarpet.py new file mode 100644 index 0000000000..3a14cea594 --- /dev/null +++ b/apps/starry/monitor/python/GrafanaCarpet.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# GrafanaCarpet.py -- headless carpet for Grafana (the monitoring/observability web app). Grafana is +# a single fully-static CGO-free Go binary (`grafana server` subcommand) that serves its embedded +# frontend SPA + REST API on loopback :3000 backed by an embedded SQLite store -- so, like glances +# -w and prometheus, it is exercised HEADLESS via HTTP assertions (no browser, no TUI). +# +# DIMENSIONS (gated PASS/TOTAL -- no silent pass): +# VER -- `grafana --version` reports EXACTLY 13.0.1 (red-line). +# LISTEN -- `grafana server` reaches "HTTP Server Listen" and answers /api/health (server up + +# SQLite migrations done). Uses a fresh writable data dir seeded from the prebuild's +# pre-migrated arch-independent grafana.db when present (skips the 709 first-run +# migrations); otherwise grafana migrates on the spot (exercises the StarryOS ext4/fsync +# path -- legitimate, just slower). +# HEALTH -- GET /api/health -> 200 + JSON {"database":"ok","version":"13.0.1"} (SQLite backend up). +# FRONTEND -- GET /login -> 200 serving the embedded Grafana SPA (title "Grafana" + bootdata). +# API -- GET / (root) reaches the app (200/30x to /login), and /robots.txt is served (static +# asset path from the embedded public/ tree). +# +# Exercises the Go runtime, loopback HTTP, and the embedded SQLite store on the ext4 image. +# Emits `GRAF_RESULT ok= fail=` and, only when F==0, `GRAF_DONE`. +import json, os, re, shutil, signal, subprocess, sys, tempfile, time, urllib.request, urllib.error + +GRAFANA = os.environ.get("GRAFANA_BIN", "/opt/grafana/bin/grafana") +HOMEPATH = os.environ.get("GRAFANA_HOMEPATH", "/opt/grafana") +PORT = int(os.environ.get("MONITOR_GRAFANA_PORT", "3000")) +HOST = "127.0.0.1" +VER = "13.0.1" +TRIES = int(os.environ.get("MONITOR_GRAFANA_HEALTH_TRIES", "1200")) # 1s each; big budget for TCG migration +_ok = 0 +_fail = 0 +_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def check(cond, label): + global _ok, _fail + if cond: + _ok += 1 + else: + _fail += 1 + print(" FAIL %s" % label) + + +def http(path, timeout=8): + req = urllib.request.Request("http://%s:%d%s" % (HOST, PORT, path)) + try: + with _OPENER.open(req, timeout=timeout) as r: + return r.getcode(), r.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + except Exception: + return 0, b"" + + +def run(args, timeout=90): + try: + r = subprocess.run(args, capture_output=True, text=True, timeout=timeout) + return r.returncode, (r.stdout or "") + (r.stderr or "") + except subprocess.TimeoutExpired: + return 124, "" + + +def carpet_help(path, needles, timeout=90): + # `grafana --help` -- urfave/cli prints this node's help and exits 0. Pure + # introspection: no config, no db, no network, no server spun up. Proves the command/subcommand + # actually EXISTS + invokable, then asserts its documented flag/subcommand surface. The + # --help/flag surface is arch-independent, so the x86 ground truth holds on every target arch + # under musl (the overlay binary is the target-arch build of the identical CLI tree). + label = "grafana " + " ".join(path) if path else "grafana" + rc, out = run([GRAFANA] + list(path) + ["--help"], timeout=timeout) + check(rc == 0, "`%s --help` exits 0" % label) + for n in needles: + check(n in out, "`%s --help` documents %r" % (label, n)) + + +def main(): + print("=== GrafanaCarpet: headless grafana %s server + HTTP/API assertions on :%d ===" % (VER, PORT)) + check(os.path.exists(GRAFANA), "grafana binary present: %s" % GRAFANA) + if not os.path.exists(GRAFANA): + print("GRAF_RESULT ok=%d fail=%d" % (_ok, _fail)) + return 1 + + # VER red-line. + r = subprocess.run([GRAFANA, "--version"], capture_output=True, text=True, timeout=60) + out = (r.stdout or "") + (r.stderr or "") + check(r.returncode == 0 and re.search(r"grafana version %s\b" % re.escape(VER), out) is not None, + "grafana --version == %s (red-line): %r" % (VER, out.strip()[:60])) + # short-form version flag (the top-level `-v` mirrors `--version`). + rc, vout = run([GRAFANA, "-v"]) + check(rc == 0 and ("grafana version %s" % VER) in vout, "grafana -v == grafana version %s" % VER) + + # ---- CLI HELP TREE (grafana 13.0.1) ---------------------------------------------------------- + # Carpet-level coverage of the ENTIRE grafana CLI surface: every command x subcommand x + # documented flag, ground-truthed against the binary's OWN `--help` at each node (depth reaches 4 + # at `grafana cli admin secrets-migration re-encrypt`). Only flags/subcommands that real --help + # actually prints are asserted -- never invented. All nodes are pure `--help` introspection that + # exits 0 without a config/db/network/server, so the whole tree is deterministic + bounded. + + # top level: the two real commands (cli, server) + built-in help + the global options. No + # standalone grafana-cli/grafana-server binaries exist; the legacy names are only deprecated + # bash wrappers that exec `grafana cli|server`, so `grafana cli`/`grafana server` IS that surface. + carpet_help([], ["cli", "server", "help, h", "--help, -h", "--version, -v", + "Grafana server and command line interface"]) + + # `grafana cli`: the plugin flags (--pluginsDir/--repo/--pluginUrl/--insecure/--debug) and their + # env-var hints live on THIS level, not on the individual plugins/admin leaves. + carpet_help(["cli"], ["plugins", "admin", "help, h", "--config", "--homepath", + "--configOverrides", "--version, -v", "--debug", "--pluginsDir", "--repo", + "--pluginUrl", "--insecure", "--help, -h", + "$GF_PLUGIN_DIR", "$GF_PLUGIN_REPO", "$GF_PLUGIN_URL"]) + + # `grafana cli plugins`: the alias renderings (update, upgrade / update-all, upgrade-all / + # uninstall, remove) are part of the real --help and are asserted verbatim. + carpet_help(["cli", "plugins"], + ["install", "list-remote", "list-versions", "update, upgrade", + "update-all, upgrade-all", "ls", "uninstall, remove", "help, h", "--help, -h"]) + # plugins leaves: install/list-remote/list-versions/update/update-all/ls/uninstall each reach out + # to grafana.com/api/plugins (or a plugin-zip URL) to do real work -- a live network path that is + # out of scope for a headless on-target carpet. We assert each leaf's --help (proving the + # subcommand exists via its NAME line + its --help,-h flag surface); the real args are positional + # (e.g. `install `), so --help,-h is the entire documented flag set. + for leaf in ("install", "list-remote", "list-versions", "update", "update-all", "ls", + "uninstall"): + carpet_help(["cli", "plugins", leaf], ["cli plugins " + leaf, "--help, -h"]) + + # `grafana cli admin`: reset-admin-password / data-migration / secrets-migration / + # secrets-consolidation / flush-rbac-seed-assignment each spin up the FULL sqlstore and mutate a + # migrated grafana.db. On-target under TCG that path is unbounded (the 700+ migration chain) and + # mutates DB state, so we assert their --help surface here; the live migrate+serve DB path is + # exercised instead by the headless `grafana server` run below. + carpet_help(["cli", "admin"], + ["reset-admin-password", "data-migration", "secrets-migration", + "secrets-consolidation", "flush-rbac-seed-assignment", "help, h", "--help, -h"]) + carpet_help(["cli", "admin", "reset-admin-password"], + ["--password-from-stdin", "--user-id", "--help, -h"]) + carpet_help(["cli", "admin", "data-migration"], + ["encrypt-datasource-passwords", "help, h", "--help, -h"]) + carpet_help(["cli", "admin", "data-migration", "encrypt-datasource-passwords"], + ["data-migration encrypt-datasource-passwords", "--help, -h"]) + carpet_help(["cli", "admin", "secrets-migration"], + ["re-encrypt", "rollback", "re-encrypt-data-keys", "help, h", "--help, -h"]) + carpet_help(["cli", "admin", "secrets-migration", "re-encrypt"], + ["secrets-migration re-encrypt", "--help, -h"]) + carpet_help(["cli", "admin", "secrets-migration", "rollback"], + ["secrets-migration rollback", "--help, -h"]) + carpet_help(["cli", "admin", "secrets-migration", "re-encrypt-data-keys"], + ["secrets-migration re-encrypt-data-keys", "--help, -h"]) + carpet_help(["cli", "admin", "secrets-consolidation"], + ["consolidate", "help, h", "--help, -h"]) + # the one admin leaf with a rich flag set. + carpet_help(["cli", "admin", "secrets-consolidation", "consolidate"], + ["--config", "--homepath", "--configOverrides", "--chunk-size", "--threads", + "--benchmark", "--cpuprofile", "--memprofile", "--cpu-profile-rate", "--help, -h"]) + carpet_help(["cli", "admin", "flush-rbac-seed-assignment"], + ["flush-rbac-seed-assignment", "--help, -h"]) + + # `grafana server` full flag set (the actual server run below drives the functional path). Its + # `target` subcommand mirrors the identical flag set for targeted-service selection. + server_flags = ["--config", "--homepath", "--pidfile", "--packaging", "--configOverrides", + "--version, -v", "--vv", "--profile", "--profile-addr", "--profile-port", + "--profile-block-rate", "--profile-mutex-rate", "--tracing", "--tracing-file", + "--help, -h"] + carpet_help(["server"], ["target"] + server_flags) + carpet_help(["server", "target"], server_flags) + + # fresh, writable, isolated data dir seeded from the pre-migrated db when present. + workdir = tempfile.mkdtemp(prefix="grafana-run-") + data = os.path.join(workdir, "data") + logs = os.path.join(workdir, "logs") + plugins = os.path.join(workdir, "plugins") + prov = os.path.join(workdir, "provisioning") + for d in (data, logs, plugins, + os.path.join(prov, "datasources"), os.path.join(prov, "dashboards"), + os.path.join(prov, "plugins"), os.path.join(prov, "alerting"), + os.path.join(prov, "access-control"), os.path.join(prov, "notifiers")): + os.makedirs(d, exist_ok=True) + seed = os.path.join(HOMEPATH, "data", "grafana.db") + if os.path.isfile(seed): + shutil.copy2(seed, os.path.join(data, "grafana.db")) + print(" seeded pre-migrated grafana.db (migrations will be skipped)") + else: + print(" no pre-migrated grafana.db -> grafana will migrate on the spot (slower)") + + ini = os.path.join(workdir, "grafana.ini") + with open(ini, "w") as f: + f.write( + "app_mode = production\n" + "[paths]\ndata = %s\nlogs = %s\nplugins = %s\nprovisioning = %s\n" + "[server]\nhttp_addr = %s\nhttp_port = %d\n" + # wal = true: grafana 13's unified-storage apiserver and the sqlstore both touch + # grafana.db during startup; WAL (multi-reader + single-writer) cuts the transient + # SQLITE_BUSY contention so the frontend endpoints come up cleanly. level=info so the + # "HTTP Server Listen" / "migrations completed" log lines are emitted for assertion. + "[database]\ntype = sqlite3\nwal = true\n" + "[analytics]\nreporting_enabled = false\ncheck_for_updates = false\ncheck_for_plugin_updates = false\n" + "[log]\nmode = console\nlevel = info\n" + % (data, logs, plugins, prov, HOST, PORT)) + + logpath = os.path.join(workdir, "server.log") + logf = open(logpath, "w") + proc = subprocess.Popen([GRAFANA, "server", "--homepath", HOMEPATH, "--config", ini], + stdout=logf, stderr=subprocess.STDOUT) + try: + up = False + for _ in range(TRIES): + if proc.poll() is not None: + break + code, _b = http("/api/health", timeout=4) + if code == 200: + up = True + break + time.sleep(1) + logf.flush() + try: + log = open(logpath).read() + except Exception: + log = "" + check(up, "grafana server reached /api/health == 200") + check("HTTP Server Listen" in log, "server log emitted 'HTTP Server Listen'") + check("migrations completed" in log, "server log emitted 'migrations completed' (SQLite store migrated)") + if not up: + print("--- server.log tail ---\n" + log[-1500:]) + else: + # HEALTH JSON: database ok + version red-line. + code, body = http("/api/health", timeout=8) + try: + h = json.loads(body) + check(code == 200 and h.get("database") == "ok", "/api/health database == ok: %s" % h) + check(h.get("version") == VER, "/api/health version == %s (red-line): %s" % (VER, h.get("version"))) + except Exception as e: + check(False, "/api/health parses as JSON (%r): %r" % (e, body[:80])) + + # FRONTEND: embedded SPA served on /login. The unified-storage apiserver keeps grafana.db + # transiently busy right after /api/health flips green, so retry a few times until the + # SPA route settles (this is grafana startup churn, not a real failure). + code, txt = 0, "" + for _ in range(30): + code, body = http("/login", timeout=10) + txt = body.decode("utf-8", "replace") + if code == 200: + break + time.sleep(2) + check(code == 200, "/login == 200 (embedded frontend served)") + check("Grafana" in txt and ("grafanaBootData" in txt or "Grafana" in txt), + "/login serves the embedded Grafana SPA (title/bootdata)") + + # API/static: root reaches the app; robots.txt served from embedded public/. + code, _b = http("/", timeout=8) + check(code in (200, 301, 302), "GET / reaches the app (code %s)" % code) + rc = 0 + for _ in range(15): + rc, _b = http("/robots.txt", timeout=8) + if rc == 200: + break + time.sleep(2) + check(rc == 200, "/robots.txt static asset served (code %s)" % rc) + finally: + try: + proc.send_signal(signal.SIGTERM) + proc.wait(timeout=15) + except Exception: + try: + proc.kill() + except Exception: + pass + logf.close() + shutil.rmtree(workdir, ignore_errors=True) + + print("GRAF_RESULT ok=%d fail=%d" % (_ok, _fail)) + if _fail == 0: + print("GRAF_DONE") + return 0 + return 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as e: + import traceback + traceback.print_exc() + print("GRAF_RESULT ok=%d fail=%d" % (_ok, _fail + 1)) + sys.exit(1) diff --git a/apps/starry/monitor/python/NodeExporterCarpet.py b/apps/starry/monitor/python/NodeExporterCarpet.py new file mode 100644 index 0000000000..773afe20d6 --- /dev/null +++ b/apps/starry/monitor/python/NodeExporterCarpet.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +# NodeExporterCarpet.py -- industrial CLI + live carpet for node_exporter 1.11.1 on StarryOS. +# node_exporter is a fully-static, CGO-disabled Go binary (the same target-arch build the Prometheus +# carpet scrapes at /usr/bin/node_exporter). It has NO subcommands: its whole surface is one flat +# kingpin flag tree (`node_exporter []`) plus a Prometheus /metrics HTTP endpoint. This carpet +# grounds every assertion in the tool's OWN `--help` output (exit 0) -- the exact, complete flag +# surface, which is arch-independent, so the x86 ground truth applies to every arch the overlay ships. +# +# DIMENSIONS (all real, gated PASS/TOTAL -- no silent pass): +# VER -- `--version` first line is EXACTLY `node_exporter, version 1.11.1 ...` (red-line). +# HELP -- `--help` exits 0 and documents the general/web/path/log flags with their EXACT +# printed defaults (proves we read the real help, not an invented surface). +# COLLECTOR -- `--help` enumerates ALL 77 `--[no-]collector.` enable toggles (48 enabled + +# 29 disabled by default); assert a broad representative subset AND exact set-equality +# to the documented 77, plus a spread of per-collector configuration flags + defaults. +# REJECT -- bad enum values and invented flags are REJECTED with rc!=0 (the parser validates its +# own surface -- this is the guard that keeps every asserted flag a REAL one). +# LIVE -- start the exporter headless on loopback with a curated collector set whose data comes +# from the /proc files StarryOS renders: cpu/stat (/proc/stat), meminfo (/proc/meminfo), +# loadavg (/proc/loadavg), netdev (/proc/net/dev), diskstats (/proc/diskstats), +# filesystem (/proc/mounts + statfs) plus the pure-syscall collectors uname/time. GET the +# metrics path and assert the documented node_* families are exposed AND carry real, +# non-zero values: node_cpu_seconds_total, node_memory_*, node_load1, +# node_network_receive_bytes_total{device="lo"} (loopback scrape traffic), +# node_disk_reads_completed_total{device="vda"} (root virtio-blk), and +# node_filesystem_size_bytes{mountpoint="/"} (ext4 root via statfs). A custom +# --web.telemetry-path is honored (moved off /metrics) and the landing page is served; +# then stop it. +# +# The vast majority of collectors (nvme/zfs/systemd/perf/ntp/supervisord/...) drive hardware, daemons +# or subsystems that do not exist on-target, so their flag surface is asserted from `--help` only -- +# the help tree is the arch/kernel-independent ground truth. The live run enables exactly the /proc- +# and syscall-backed collectors that StarryOS genuinely serves; vmstat is intentionally left out +# because StarryOS renders no /proc/vmstat, so that collector would have no data to export. +# +# Exercises the Go runtime (M:N scheduler, GC, futex, getrandom), the loopback net stack +# (bind/listen/accept), block-device I/O accounting and the procfs reads on the StarryOS image. Emits +# `NE_RESULT ok= fail=` and, only when F==0, `NE_DONE`. +import os, re, signal, subprocess, sys, time, urllib.request, urllib.error + +NODE_EXP = os.environ.get("NODE_EXPORTER_BIN", "/usr/bin/node_exporter") +PORT = int(os.environ.get("MONITOR_NODE_EXPORTER_PORT", "19100")) +HOST = "127.0.0.1" +TELEMETRY = "/nodemetrics" # deliberately NOT /metrics, to prove --web.telemetry-path is honored. +VER = "1.11.1" +_ok = 0 +_fail = 0 +_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({})) + +# The complete, documented set of `--[no-]collector.` enable toggles (node_exporter 1.11.1). +COLLECTORS_ENABLED = [ + "arp", "bcache", "bcachefs", "bonding", "btrfs", "conntrack", "cpu", "cpufreq", "diskstats", + "dmi", "edac", "entropy", "fibrechannel", "filefd", "filesystem", "hwmon", "infiniband", "ipvs", + "kernel_hung", "loadavg", "mdadm", "meminfo", "netclass", "netdev", "netstat", "nfs", "nfsd", + "nvme", "os", "powersupplyclass", "pressure", "rapl", "schedstat", "selinux", "sockstat", + "softnet", "stat", "tapestats", "textfile", "thermal_zone", "time", "timex", "udp_queues", + "uname", "vmstat", "watchdog", "xfs", "zfs", +] +COLLECTORS_DISABLED = [ + "buddyinfo", "cgroups", "cpu_vulnerabilities", "drbd", "drm", "ethtool", "interrupts", "ksmd", + "lnstat", "logind", "meminfo_numa", "mountstats", "network_route", "ntp", "pcidevice", "perf", + "processes", "qdisc", "runit", "slabinfo", "softirqs", "supervisord", "swap", "sysctl", + "systemd", "tcpstat", "wifi", "xfrm", "zoneinfo", +] +COLLECTORS_ALL = set(COLLECTORS_ENABLED) | set(COLLECTORS_DISABLED) + +# A broad representative slice (enabled + disabled families) checked one-by-one for readable evidence. +COLLECTOR_SAMPLE = [ + "arp", "cpu", "meminfo", "loadavg", "diskstats", "filesystem", "netdev", "netstat", "stat", + "uname", "os", "time", "hwmon", "thermal_zone", "nvme", "zfs", "xfs", "pressure", + "buddyinfo", "cgroups", "ethtool", "interrupts", "ntp", "perf", "processes", "systemd", + "tcpstat", "wifi", "sysctl", "slabinfo", +] + +# General / web / path / log flags with their EXACT printed default (substring must appear verbatim). +FLAGS_WITH_DEFAULT = [ + '--web.listen-address=:9100', + '--web.telemetry-path="/metrics"', + '--web.config.file=""', + '--web.max-requests=40', + '--runtime.gomaxprocs=1', + '--log.level=info', + '--log.format=logfmt', + '--path.procfs="/proc"', + '--path.sysfs="/sys"', + '--path.rootfs="/"', + '--path.udev.data="/run/udev/data"', +] + +# Boolean/global flags rendered as `--[no-]NAME`; the bare NAME must appear. +FLAGS_BOOL = [ + "web.disable-exporter-metrics", "web.systemd-socket", "collector.disable-defaults", + "version", "help", +] + +# Per-collector configuration flags with their EXACT documented default (proves real help reading). +CONFIG_WITH_DEFAULT = [ + '--collector.diskstats.device-exclude="^(z?ram|loop|fd|(h|s|v|xv)d[a-z]|nvme', + '--collector.filesystem.mount-points-exclude="^/(dev|proc|run/credentials', + '--collector.filesystem.fs-types-exclude="^(autofs|binfmt_misc|bpf|cgroup2?', + '--collector.netstat.fields="^(.*_(InErrors|InErrs)', + '--collector.ntp.server="127.0.0.1"', + '--collector.ntp.server-port=123', + '--collector.ntp.protocol-version=4', + '--collector.ipvs.backend-labels="local_address,local_port,remote_address', + '--collector.supervisord.url="http://localhost:9001/RPC2"', + '--collector.vmstat.fields="^(oom_kill|pgpg|pswp', + '--collector.runit.servicedir="/etc/service"', + '--collector.slabinfo.slabs-include=".*"', + '--collector.slabinfo.slabs-exclude=""', + '--collector.ethtool.metrics-include=".*"', + '--collector.netclass.ignored-devices="^$"', + '--collector.powersupply.ignored-supplies="^$"', + '--collector.tapestats.ignored-devices="^$"', + '--collector.systemd.unit-include=".+"', + '--collector.perf.cpus=""', + '--collector.qdisc.fixtures=""', + '--collector.wifi.fixtures=""', +] + +# Per-collector configuration flags asserted by NAME (no default, or a value/regexp placeholder). +CONFIG_NAMES = [ + "--collector.arp.device-include", "--collector.arp.device-exclude", + "--collector.cpu.info.flags-include", "--collector.cpu.info.bugs-include", + "--collector.netdev.device-include", "--collector.netdev.device-exclude", + "--collector.hwmon.chip-include", "--collector.hwmon.sensor-include", + "--collector.textfile.directory", "--collector.sysctl.include", + "--collector.sysctl.include-info", "--collector.pcidevice.idsfile", + "--collector.qdisc.device-include", +] + +# Boolean per-collector config toggles rendered as `--[no-]collector.X.Y`; the bare tail must appear. +CONFIG_BOOL = [ + "collector.arp.netlink", "collector.bcache.priorityStats", "collector.cpu.guest", + "collector.cpu.info", "collector.interrupts.include-zeros", "collector.netclass.netlink", + "collector.netclass_rtnl.with-stats", "collector.netdev.address-info", "collector.netdev.netlink", + "collector.netdev.label-ifalias", "collector.pcidevice.names", "collector.rapl.enable-zone-label", + "collector.stat.softirq", "collector.systemd.enable-task-metrics", + "collector.systemd.enable-restarts-metrics", "collector.ntp.server-is-local", +] + +# node_* metric families that the LIVE curated collector set exposes on-target. The first group is +# backed by /proc/stat, /proc/meminfo, /proc/loadavg or pure syscalls; the second group is backed by +# the /proc/net/dev, /proc/diskstats and /proc/mounts (+statfs) sources. +LIVE_NODE_METRICS = [ + "node_cpu_seconds_total", "node_memory_MemTotal_bytes", "node_memory_MemFree_bytes", + "node_load1", "node_boot_time_seconds", "node_context_switches_total", "node_intr_total", + "node_time_seconds", "node_uname_info", "node_scrape_collector_success", + "node_network_receive_bytes_total", "node_network_transmit_bytes_total", + "node_network_receive_packets_total", "node_disk_reads_completed_total", + "node_disk_read_bytes_total", "node_disk_writes_completed_total", + "node_filesystem_size_bytes", "node_filesystem_avail_bytes", "node_filesystem_files", +] + + +def check(cond, label): + global _ok, _fail + if cond: + _ok += 1 + else: + _fail += 1 + print(" FAIL %s" % label) + + +def run(args, timeout=60): + try: + r = subprocess.run(args, capture_output=True, text=True, timeout=timeout) + return r.returncode, (r.stdout or "") + (r.stderr or "") + except subprocess.TimeoutExpired: + return 124, "" + + +def http_get(path, timeout=8): + try: + with _OPENER.open("http://%s:%d%s" % (HOST, PORT, path), timeout=timeout) as r: + return r.getcode(), r.read().decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, e.read().decode("utf-8", "replace") + except Exception: + return 0, "" + + +def metric_value(body, name, label_substr=None): + """First sample value of Prometheus metric `name` whose line contains `label_substr`.""" + for line in body.splitlines(): + if not (line.startswith(name + "{") or line.startswith(name + " ")): + continue + if label_substr is not None and label_substr not in line: + continue + try: + return float(line.rsplit(None, 1)[1]) + except (ValueError, IndexError): + continue + return None + + +def main(): + print("=== NodeExporterCarpet: node_exporter %s CLI + live /metrics on %s:%d ===" % (VER, HOST, PORT)) + check(os.path.exists(NODE_EXP), "binary present: %s" % NODE_EXP) + if not os.path.exists(NODE_EXP): + print("NE_RESULT ok=%d fail=%d" % (_ok, _fail)) + print("--- binary missing; cannot run ---") + return 1 + + # --- VER red-line: first line of --version --- + rc, out = run([NODE_EXP, "--version"]) + first = out.splitlines()[0] if out.splitlines() else "" + check(rc == 0 and re.match(r"^node_exporter, version %s " % re.escape(VER), first) is not None, + "node_exporter --version first line == 'node_exporter, version %s ...': %r" % (VER, first[:64])) + + # --- HELP tree (exit 0, usage line, every general flag + exact default) --- + rc, help_txt = run([NODE_EXP, "--help"]) + check(rc == 0, "node_exporter --help exits 0") + check("usage: node_exporter []" in help_txt, "--help prints 'usage: node_exporter []'") + for s in FLAGS_WITH_DEFAULT: + check(s in help_txt, "--help documents %s" % s) + for name in FLAGS_BOOL: + check(("--[no-]%s" % name) in help_txt, "--help documents --[no-]%s" % name) + + # --- COLLECTOR enumeration: representative subset, exact 77-set equality, default classification --- + for name in COLLECTOR_SAMPLE: + check(("--[no-]collector.%s " % name) in help_txt, "--help enumerates --[no-]collector.%s" % name) + # Parse the plain enable toggles: `--[no-]collector.` where has NO trailing dot + # (config flags like --[no-]collector.arp.netlink are excluded by the whitespace lookahead). + parsed = set(re.findall(r"--\[no-\]collector\.([a-z0-9_]+)(?=\s)", help_txt)) + check(len(parsed) == 77, "--help enumerates exactly 77 collector enable toggles (got %d)" % len(parsed)) + check(parsed == COLLECTORS_ALL, + "parsed collector toggles == documented 77 (missing=%s extra=%s)" + % (sorted(COLLECTORS_ALL - parsed), sorted(parsed - COLLECTORS_ALL))) + # A few exact default-classification lines (single-line in help; wrapped ones are avoided). + check("Enable the cpu collector (default: enabled)." in help_txt, "cpu collector default: enabled") + check("Enable the ipvs collector (default: enabled)." in help_txt, "ipvs collector default: enabled") + check("Enable the ntp collector (default: disabled)." in help_txt, "ntp collector default: disabled") + check("Enable the swap collector (default: disabled)." in help_txt, "swap collector default: disabled") + + # --- per-collector configuration flags (defaults + names + bool toggles) --- + for s in CONFIG_WITH_DEFAULT: + check(s in help_txt, "--help documents %s" % s) + for s in CONFIG_NAMES: + check(s in help_txt, "--help documents %s" % s) + for name in CONFIG_BOOL: + check(("--[no-]%s" % name) in help_txt, "--help documents --[no-]%s" % name) + + # --- REJECT: bad enum values / invented flags are rejected (guards against fake-flag assertions) --- + rc, out = run([NODE_EXP, "--log.level=BOGUS"]) + check(rc != 0 and "unrecognized log level" in out, "--log.level=BOGUS rejected (rc!=0 + message)") + rc, out = run([NODE_EXP, "--log.format=BOGUS"]) + check(rc != 0 and "unrecognized log format" in out, "--log.format=BOGUS rejected (rc!=0 + message)") + rc, out = run([NODE_EXP, "--collector.bogus_nonexistent"]) + check(rc != 0 and "unknown long flag" in out, "invented --collector flag rejected (rc!=0 + 'unknown long flag')") + rc, out = run([NODE_EXP, "--this.flag.does.not.exist"]) + check(rc != 0 and "unknown long flag" in out, "invented global flag rejected (rc!=0 + 'unknown long flag')") + + # --- LIVE: headless exporter with /proc- and syscall-backed collectors only --- + # These are the collectors whose data StarryOS genuinely provides: cpu/stat/meminfo/loadavg read + # /proc files the kernel renders; netdev reads /proc/net/dev (procfs mode -- the netlink backend + # is disabled because StarryOS's rtnetlink does not carry per-link rtnl_link_stats64); diskstats + # reads /proc/diskstats; filesystem reads /proc/self/mountinfo and statfs()es each mount; + # uname/time are pure syscalls. Custom telemetry path + accepted log/path/limit flags (server + # coming up == those flags accepted). + log = open("/tmp/node_exporter_carpet.log", "w") + proc = subprocess.Popen( + [NODE_EXP, + "--web.listen-address=%s:%d" % (HOST, PORT), + "--web.telemetry-path=%s" % TELEMETRY, + "--collector.disable-defaults", + "--collector.cpu", "--collector.stat", "--collector.meminfo", + "--collector.loadavg", "--collector.uname", "--collector.time", + "--collector.netdev", "--no-collector.netdev.netlink", + "--collector.diskstats", "--collector.filesystem", + "--log.level=info", "--log.format=logfmt", + "--web.max-requests=40", "--path.procfs=/proc"], + stdout=log, stderr=subprocess.STDOUT) + try: + up = False + body = "" + for _ in range(90): + if proc.poll() is not None: + break + code, body = http_get(TELEMETRY, timeout=4) + if code == 200 and len(body) > 0: + up = True + break + time.sleep(1) + check(up, "exporter serving %s on %s:%d (custom --web.telemetry-path honored)" % (TELEMETRY, HOST, PORT)) + if not up: + log.flush() + try: + print("--- node_exporter.log tail ---\n" + open("/tmp/node_exporter_carpet.log").read()[-1200:]) + except Exception: + pass + + # /metrics must now be 404 -- proves the telemetry path really moved off the default. + code404, _ = http_get("/metrics", timeout=6) + check(code404 == 404, "default /metrics -> 404 after telemetry-path override (got %s)" % code404) + + # landing page served on / with the exporter's own title. + codel, landing = http_get("/", timeout=6) + check(codel == 200 and "Node Exporter" in landing, "GET / -> 200 landing page with 'Node Exporter' (got %s)" % codel) + + # build_info exposes the pinned version as a label (version collector, always registered). + check(re.search(r'node_exporter_build_info\{[^}]*version="%s"[^}]*\}\s+1' % re.escape(VER), body) is not None, + "node_exporter_build_info exposes version=%s" % VER) + + # the documented node_* families the curated collectors produce. + for m in LIVE_NODE_METRICS: + check(re.search(r"^%s(\{|\s)" % re.escape(m), body, re.M) is not None, "/metrics exposes %s" % m) + + # --- procfs-backed collectors carry REAL, non-zero data (not just an exposed-but-empty family) --- + # netdev: the scrape itself flows over loopback, so lo has received bytes by collection time. + lo_rx = metric_value(body, "node_network_receive_bytes_total", 'device="lo"') + check(lo_rx is not None and lo_rx > 0, + 'node_network_receive_bytes_total{device="lo"} > 0 from /proc/net/dev (got %r)' % lo_rx) + # diskstats: the root virtio-blk device served boot reads, so vda read counters are non-zero. + vda_reads = metric_value(body, "node_disk_reads_completed_total", 'device="vda"') + check(vda_reads is not None and vda_reads > 0, + 'node_disk_reads_completed_total{device="vda"} > 0 from /proc/diskstats (got %r)' % vda_reads) + vda_rbytes = metric_value(body, "node_disk_read_bytes_total", 'device="vda"') + check(vda_rbytes is not None and vda_rbytes > 0, + 'node_disk_read_bytes_total{device="vda"} > 0 from /proc/diskstats (got %r)' % vda_rbytes) + # filesystem: statfs() on the ext4 root returns a real total size. + root_size = metric_value(body, "node_filesystem_size_bytes", 'mountpoint="/"') + check(root_size is not None and root_size > 0, + 'node_filesystem_size_bytes{mountpoint="/"} > 0 from /proc/mounts + statfs (got %r)' % root_size) + root_avail = metric_value(body, "node_filesystem_avail_bytes", 'mountpoint="/"') + check(root_avail is not None and root_avail >= 0, + 'node_filesystem_avail_bytes{mountpoint="/"} present from statfs (got %r)' % root_avail) + + # Go-runtime exporter metric (present unless --web.disable-exporter-metrics; we do NOT set it). + check(re.search(r"^go_goroutines(\{|\s)", body, re.M) is not None, "/metrics exposes go_goroutines") + + # Prometheus exposition format sanity: node_ HELP + TYPE header lines present. + check("# HELP node_" in body, "/metrics has '# HELP node_' exposition headers") + check("# TYPE node_" in body, "/metrics has '# TYPE node_' exposition headers") + finally: + try: + proc.send_signal(signal.SIGTERM) + proc.wait(timeout=8) + except Exception: + try: + proc.kill() + except Exception: + pass + log.close() + + print("NE_RESULT ok=%d fail=%d" % (_ok, _fail)) + if _fail == 0: + print("NE_DONE") + return 0 + return 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as e: + import traceback + traceback.print_exc() + print("NE_RESULT ok=%d fail=%d" % (_ok, _fail + 1)) + sys.exit(1) diff --git a/apps/starry/monitor/python/PrometheusCarpet.py b/apps/starry/monitor/python/PrometheusCarpet.py new file mode 100644 index 0000000000..9660a1a90d --- /dev/null +++ b/apps/starry/monitor/python/PrometheusCarpet.py @@ -0,0 +1,495 @@ +#!/usr/bin/env python3 +# PrometheusCarpet.py -- industrial end-to-end carpet for the Prometheus monitoring stack on +# StarryOS. prometheus + promtool are fully-static, CGO-disabled Go binaries (netgo,builtinassets => +# embedded web UI, no libc/ld-musl wiring). node_exporter 1.11.1 (the simplest exporter, also a +# static Go binary) is run as a REAL scrape target so the CORE scrape->ingest->query pipeline is +# exercised, not just a synthetic scalar. +# +# DIMENSIONS (all real, gated PASS/TOTAL -- no silent pass): +# VER -- prometheus --version AND promtool --version report EXACTLY 3.11.3 (red-line; +# a wrong version = invalid test); node_exporter --version == 1.11.1. +# HELP -- CARPET-LEVEL CLI surface. Ground truth = the tools' OWN --help. Every prometheus +# server flag is asserted in --help-long; the promtool root --help lists every one of +# its 27 leaf commands (exact usage signature) plus the persistent global flags; and +# every leaf's own ` --help` is spawned to assert its usage line, the four +# persistent global flags, and its documented group-persistent + leaf-specific flag +# surface. No flag is invented -- each substring is one the real --help prints. +# FUNC -- deterministic, server-FREE promtool behaviors: check config (good+bad), check rules +# (good+bad), check metrics lint (good+bad, over stdin), check web-config (good+bad), +# promql format + label-matchers set/delete (the --experimental PromQL rewriters), +# test rules (a rule unit-test driving the PromQL engine), and the TSDB import path +# (create-blocks-from openmetrics -> tsdb list -> tsdb analyze, which exercises block +# writing + WAL replay on the ext4 image). +# READY -- headless prometheus server on loopback :9090 logs "Server is ready to receive web +# requests." AND /-/ready answers Ready (HTTP serving + TSDB open). +# QUERY -- /api/v1/query?query=vector(42) -> status success + value "42" (PromQL engine), and +# promtool query instant/labels/series driven as a live PromQL client. +# SCRAPE -- /api/v1/query?query=up{job="node"} -> value "1" (node_exporter really scraped + +# ingested through loopback :9090 -> :9100). +# +# Live-server-only promtool leaves (check healthy/ready, query range/analyze, debug pprof/metrics/all, +# push metrics, tsdb bench write, tsdb create-blocks-from rules) are covered at the --help/usage +# level: driving them needs a second running server, a remote-write receiver, the multi-MB +# 20kseries.json fixture, or histogram-typed data that the on-target scrape doesn't produce, so their +# existence + flag surface is the in-scope assertion (the --help surface is arch-independent). +# +# Exercises the Go runtime (M:N scheduler, GC, futex parking, getrandom), the TSDB head/WAL on the +# ext4 image, and the loopback net stack (bind/listen/accept + the scrape client). Emits +# `PROM_RESULT ok= fail=` and, only when F==0, `PROM_DONE`. +import os, re, shutil, signal, subprocess, sys, tempfile, time, urllib.request, urllib.error + +PROM = os.environ.get("PROMETHEUS_BIN", "/usr/local/bin/prometheus") +PROMTOOL = os.environ.get("PROMTOOL_BIN", "/usr/local/bin/promtool") +NODE_EXP = os.environ.get("NODE_EXPORTER_BIN", "/usr/bin/node_exporter") +CFG = os.environ.get("PROMETHEUS_CFG", "/etc/prometheus.yml") +TSDB = os.environ.get("PROMETHEUS_TSDB", "/root/prom") +EP = os.environ.get("PROMETHEUS_EP", "127.0.0.1:9090") +NE_EP = os.environ.get("NODE_EXPORTER_EP", "127.0.0.1:9100") +VER = "3.11.3" +NE_VER = "1.11.1" +_ok = 0 +_fail = 0 +_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +# Persistent global flags kingpin merges into EVERY promtool command (root -> group -> leaf); the +# ground truth lists them as present on every leaf, so the per-leaf loop asserts all four persist. +GLOBALS = ["-h, --[no-]help", "--[no-]version", "--[no-]experimental", "--enable-feature="] + +# Every prometheus server flag, exactly as --help-long renders it (bare name substring so a value +# flag `--name=DEFAULT` and a boolean `--[no-]name` both match the same token). +SERVER_FLAGS = [ + "config.file", "config.auto-reload-interval", "web.listen-address", + "auto-gomaxprocs", "auto-gomemlimit", "auto-gomemlimit.ratio", "web.config.file", + "web.read-timeout", "web.max-connections", "web.max-notifications-subscribers", + "web.external-url", "web.route-prefix", "web.user-assets", "web.enable-lifecycle", + "web.enable-admin-api", "web.enable-remote-write-receiver", + "web.remote-write-receiver.accepted-protobuf-messages", "web.enable-otlp-receiver", + "web.console.templates", "web.console.libraries", "web.page-title", "web.cors.origin", + "storage.tsdb.path", "storage.tsdb.retention.time", "storage.tsdb.retention.size", + "storage.tsdb.no-lockfile", "storage.tsdb.head-chunks-write-queue-size", + "storage.tsdb.delay-compact-file.path", "storage.agent.path", "storage.agent.wal-compression", + "storage.agent.retention.min-time", "storage.agent.retention.max-time", + "storage.agent.no-lockfile", "storage.remote.flush-deadline", "storage.remote.read-sample-limit", + "storage.remote.read-concurrent-limit", "storage.remote.read-max-bytes-in-frame", + "rules.alert.for-outage-tolerance", "rules.alert.for-grace-period", "rules.alert.resend-delay", + "rules.max-concurrent-evals", "alertmanager.notification-queue-capacity", + "alertmanager.notification-batch-size", "alertmanager.drain-notification-queue-on-shutdown", + "query.lookback-delta", "query.timeout", "query.max-concurrency", "query.max-samples", + "enable-feature", "agent", "log.level", "log.format", +] +# A sample of the server's --enable-feature valid options (rendered inside the flag description). +SERVER_FEATURES = ["exemplar-storage", "memory-snapshot-on-shutdown", "promql-experimental-functions", + "otlp-deltatocumulative", "concurrent-rule-eval"] + +# The full promtool leaf tree. Each entry: (argv, exact usage signature, leaf-specific+group flags). +# The signature is asserted BOTH in the root --help command listing AND as "usage: promtool " +# in the leaf's own --help; the flag tokens are asserted in the leaf's --help alongside the GLOBALS. +LEAVES = [ + (["check", "service-discovery"], "check service-discovery [] ", + ["--query.lookback-delta", "--timeout=", "", ""]), + (["check", "config"], "check config [] ...", + ["--query.lookback-delta", "--[no-]syntax-only", "--lint=", "--[no-]lint-fatal", + "--[no-]ignore-unknown-fields", "--[no-]agent", ""]), + (["check", "web-config"], "check web-config ...", + ["--query.lookback-delta", ""]), + (["check", "healthy"], "check healthy []", + ["--query.lookback-delta", "--http.config.file", "--url=http://localhost:9090"]), + (["check", "ready"], "check ready []", + ["--query.lookback-delta", "--http.config.file", "--url=http://localhost:9090"]), + (["check", "rules"], "check rules [] [...]", + ["--query.lookback-delta", "--lint=", "--[no-]lint-fatal", "--[no-]ignore-unknown-fields", + ""]), + (["check", "metrics"], "check metrics []", + ["--query.lookback-delta", "--[no-]extended", "--lint="]), + (["query", "instant"], "query instant [] ", + ["--format=promql", "--http.config.file", "--time=", "", ""]), + (["query", "range"], "query range [] ", + ["--format=promql", "--http.config.file", "--header=", "--start=", "--end=", "--step=", + "", ""]), + (["query", "series"], "query series --match=MATCH [] ", + ["--format=promql", "--http.config.file", "--match=", "--start=", "--end=", ""]), + (["query", "labels"], "query labels [] ", + ["--format=promql", "--http.config.file", "--start=", "--end=", "--match=", "", + ""]), + (["query", "analyze"], "query analyze --server=SERVER --type=TYPE --match=MATCH []", + ["--format=promql", "--http.config.file", "--server=", "--type=", "--duration=", "--time=", + "--match="]), + (["debug", "pprof"], "debug pprof ", [""]), + (["debug", "metrics"], "debug metrics ", [""]), + (["debug", "all"], "debug all ", [""]), + (["push", "metrics"], "push metrics [] [...]", + ["--http.config.file", "--label=", "--timeout=", "--header=", "--protobuf_message=", + "", ""]), + (["test", "rules"], "test rules [] ...", + ["--junit=", "--run=", "--[no-]debug", "--[no-]diff", "--[no-]ignore-unknown-fields", + ""]), + (["tsdb", "bench", "write"], "tsdb bench write [] []", + ["--out=", "--metrics=", "--scrapes="]), + (["tsdb", "analyze"], "tsdb analyze [] [] []", + ["--limit=", "--[no-]extended", "--match=", "", ""]), + (["tsdb", "list"], "tsdb list [] []", + ["--[no-]human-readable", ""]), + (["tsdb", "dump"], "tsdb dump [] []", + ["--sandbox-dir-root=", "--min-time=", "--max-time=", "--match=", "--format=", ""]), + (["tsdb", "dump-openmetrics"], "tsdb dump-openmetrics [] []", + ["--sandbox-dir-root=", "--min-time=", "--max-time=", "--match=", ""]), + (["tsdb", "create-blocks-from", "openmetrics"], + "tsdb create-blocks-from openmetrics [] []", + ["--[no-]human-readable", "--[no-]quiet", "--label=", "", ""]), + (["tsdb", "create-blocks-from", "rules"], + "tsdb create-blocks-from rules --start=START [] ...", + ["--[no-]human-readable", "--[no-]quiet", "--http.config.file", "--url=", "--start=", "--end=", + "--output-dir=", "--eval-interval=", ""]), + (["promql", "format"], "promql format ", [""]), + (["promql", "label-matchers", "set"], "promql label-matchers set [] ", + ["--type=", "", "", ""]), + (["promql", "label-matchers", "delete"], "promql label-matchers delete ", + ["", ""]), +] + + +def check(cond, label): + global _ok, _fail + if cond: + _ok += 1 + else: + _fail += 1 + print(" FAIL %s" % label) + + +def run(args, timeout=90, input=None, cwd=None): + r = subprocess.run(args, capture_output=True, text=True, timeout=timeout, input=input, cwd=cwd) + return r.returncode, (r.stdout or "") + (r.stderr or "") + + +def http_get(path, timeout=8): + try: + with _OPENER.open("http://%s%s" % (EP, path), timeout=timeout) as r: + return r.getcode(), r.read().decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, e.read().decode("utf-8", "replace") + except Exception: + return 0, "" + + +def help_tree(): + # prometheus (kingpin) prints a curated set under --help and the EXHAUSTIVE flag tree under + # --help-long; assert both work and check every documented server flag against --help-long. + rc0, _ = run([PROM, "--help"]) + check(rc0 == 0, "prometheus --help exits 0") + rc, ph = run([PROM, "--help-long"]) + check(rc == 0, "prometheus --help-long exits 0") + for flag in SERVER_FLAGS: + check(flag in ph, "prometheus --help-long documents %s" % flag) + for feat in SERVER_FEATURES: + check(feat in ph, "prometheus --enable-feature lists %s" % feat) + + # promtool root --help: the seven command groups + `help`, the persistent globals, and the exact + # usage signature of every one of the 27 leaves (proves the whole command tree from the top). + rc, th = run([PROMTOOL, "--help"]) + check(rc == 0, "promtool --help exits 0") + for grp in ["check", "query", "debug", "test", "push", "tsdb", "promql"]: + check(("\n%s " % grp) in th or ("\n%s\n" % grp) in th or (" %s " % grp) in th, + "promtool --help lists group %s" % grp) + check("help [...]" in th, "promtool --help lists the help command") + for g in GLOBALS: + check(g in th, "promtool --help documents global %s" % g) + for argv, sig, _flags in LEAVES: + check(sig in th, "promtool --help lists leaf: %s" % sig) + + # every leaf's own --help: usage line + persistent globals + its full documented flag surface. + for argv, sig, flags in LEAVES: + name = " ".join(argv) + rc, h = run([PROMTOOL] + argv + ["--help"]) + check(rc == 0, "promtool %s --help exits 0" % name) + check(("usage: promtool " + sig) in h, "promtool %s --help usage line" % name) + for g in GLOBALS: + check(g in h, "promtool %s --help documents global %s" % (name, g)) + for tok in flags: + check(tok in h, "promtool %s --help documents %s" % (name, tok)) + + +def functional_cli(work): + # ---- check config: the real target config validates; a bad duration is rejected non-zero. ---- + rc, out = run([PROMTOOL, "check", "config", CFG]) + check(rc == 0 and "SUCCESS" in out, "promtool check config %s -> SUCCESS" % CFG) + badcfg = os.path.join(work, "bad.yml") + with open(badcfg, "w") as f: + f.write("global:\n scrape_interval: notaduration\n") + rc, out = run([PROMTOOL, "check", "config", badcfg]) + check(rc != 0 and "FAILED" in out, "promtool check config (bad duration) -> non-zero + FAILED") + + # ---- check rules: a well-formed group validates; an unbalanced expr is rejected non-zero. ---- + goodrules = os.path.join(work, "rules.yml") + with open(goodrules, "w") as f: + f.write("groups:\n - name: example\n rules:\n" + " - record: job:up:sum\n expr: sum(up) by (job)\n" + " - alert: InstanceDown\n expr: up == 0\n for: 5m\n") + rc, out = run([PROMTOOL, "check", "rules", goodrules]) + check(rc == 0 and "SUCCESS" in out, "promtool check rules (valid group) -> SUCCESS") + badrules = os.path.join(work, "bad-rules.yml") + with open(badrules, "w") as f: + f.write("groups:\n - name: bad\n rules:\n" + " - record: job:up:sum\n expr: sum(up) by (job\n") + rc, out = run([PROMTOOL, "check", "rules", badrules]) + check(rc != 0 and "FAILED" in out, "promtool check rules (unclosed paren) -> non-zero + FAILED") + + # ---- check metrics: reads a metrics exposition over stdin; a counter without _total lints. ---- + rc, out = run([PROMTOOL, "check", "metrics"], + input="# HELP a_total h\n# TYPE a_total counter\na_total 1\n") + check(rc == 0, "promtool check metrics (well-named) -> exit 0 over stdin") + rc, out = run([PROMTOOL, "check", "metrics"], + input="# HELP my_metric h\n# TYPE my_metric counter\nmy_metric 1\n") + check(rc != 0 and "_total" in out, "promtool check metrics (counter without _total) -> lint error") + + # ---- check web-config: a basic_auth users file validates; a missing cert is rejected. ---- + goodweb = os.path.join(work, "web.yml") + with open(goodweb, "w") as f: + f.write("basic_auth_users:\n admin: $2y$10$" + "a" * 53 + "\n") + rc, out = run([PROMTOOL, "check", "web-config", goodweb]) + check(rc == 0 and "SUCCESS" in out, "promtool check web-config (basic_auth) -> SUCCESS") + badweb = os.path.join(work, "webbad.yml") + with open(badweb, "w") as f: + f.write("tls_server_config:\n cert_file: /nonexistent/x.crt\n key_file: /nonexistent/x.key\n") + rc, out = run([PROMTOOL, "check", "web-config", badweb]) + check(rc != 0 and "FAILED" in out, "promtool check web-config (missing cert) -> non-zero + FAILED") + + # ---- promql (the --experimental PromQL rewriters; pure functions, no server). ---- + rc, out = run([PROMTOOL, "--experimental", "promql", "format", 'up{job="x"}+1']) + check(rc == 0 and 'up{job="x"} + 1' in out, "promtool promql format pretty-prints the query") + rc, out = run([PROMTOOL, "--experimental", "promql", "format", "up{{"]) + check(rc != 0 and "parse error" in out, "promtool promql format (bad query) -> parse error") + rc, out = run([PROMTOOL, "--experimental", "promql", "label-matchers", "set", "up", "foo", "bar"]) + check(rc == 0 and 'foo="bar"' in out, "promtool promql label-matchers set adds foo=\"bar\"") + rc, out = run([PROMTOOL, "--experimental", "promql", "label-matchers", "delete", + 'up{foo="bar"}', "foo"]) + check(rc == 0 and "up" in out and "foo" not in out, + "promtool promql label-matchers delete removes foo") + + # ---- test rules: a rule unit-test drives the PromQL engine over synthetic input_series. ---- + with open(os.path.join(work, "alert_rules.yml"), "w") as f: + f.write("groups:\n - name: example\n rules:\n" + " - alert: InstanceDown\n expr: up == 0\n for: 1m\n" + " labels:\n severity: page\n") + with open(os.path.join(work, "unittest.yml"), "w") as f: + f.write( + "rule_files:\n - alert_rules.yml\nevaluation_interval: 1m\ntests:\n" + " - interval: 1m\n input_series:\n" + " - series: 'up{job=\"prometheus\", instance=\"localhost:9090\"}'\n" + " values: '0 0 0 0 0'\n" + " alert_rule_test:\n" + " - eval_time: 3m\n alertname: InstanceDown\n exp_alerts:\n" + " - exp_labels:\n severity: page\n" + " job: prometheus\n instance: localhost:9090\n") + rc, out = run([PROMTOOL, "test", "rules", "unittest.yml"], cwd=work) + check(rc == 0 and "SUCCESS" in out, "promtool test rules (alert unit-test) -> SUCCESS") + + # ---- TSDB import path: OpenMetrics -> blocks, then list + analyze the produced block. This is + # the server-free way to exercise block writing + WAL replay + index build on the ext4 image + # (tsdb dump/dump-openmetrics need a data dir WITH a live WAL, which only the running server + # produces and tears down, so those two stay --help-level above). + om = os.path.join(work, "om.txt") + with open(om, "w") as f: + f.write("# HELP test_metric A test metric\n# TYPE test_metric gauge\n" + 'test_metric{foo="bar"} 1 1600000000\n' + 'test_metric{foo="bar"} 2 1600000060\n' + 'test_metric{foo="bar"} 3 1600000120\n# EOF\n') + blocks = os.path.join(work, "blocks") + rc, out = run([PROMTOOL, "tsdb", "create-blocks-from", "openmetrics", om, blocks], timeout=180) + # create-blocks-from does not echo the block table to stdout in 3.11.3; the block IS created on + # disk. Verify creation via the block dir; the following `tsdb list` confirms the ULID is readable. + check(rc == 0 and os.path.isdir(blocks) and len(os.listdir(blocks)) >= 1, + "promtool tsdb create-blocks-from openmetrics -> creates a block (rc=0 + block dir)") + rc, out = run([PROMTOOL, "tsdb", "list", blocks], timeout=60) + check(rc == 0 and "BLOCK ULID" in out, "promtool tsdb list -> lists the produced block") + rc, out = run([PROMTOOL, "tsdb", "analyze", blocks], timeout=120) + check(rc == 0 and "Total Series" in out, "promtool tsdb analyze -> Total Series report") + + +def main(): + print("=== PrometheusCarpet: prometheus %s + node_exporter %s end-to-end ===" % (VER, NE_VER)) + for b in (PROM, PROMTOOL, NODE_EXP): + check(os.path.exists(b), "binary present: %s" % b) + if _fail: + print("PROM_RESULT ok=%d fail=%d" % (_ok, _fail)) + print("--- binaries missing; cannot run ---") + return 1 + + # --- VER red-line --- + rc, out = run([PROM, "--version"]) + check(rc == 0 and re.search(r"^prometheus, version %s " % re.escape(VER), out, re.M) is not None, + "prometheus --version == %s (red-line)" % VER) + rc, out = run([PROMTOOL, "--version"]) + check(rc == 0 and re.search(r"version %s" % re.escape(VER), out) is not None, + "promtool --version == %s (red-line)" % VER) + rc, out = run([NODE_EXP, "--version"]) + check(rc == 0 and NE_VER in out, "node_exporter --version == %s" % NE_VER) + + # --- HELP tree (carpet-level: every command x subcommand x flag) --- + help_tree() + + # --- FUNC: deterministic, server-free promtool behaviors --- + work = tempfile.mkdtemp(prefix="promtool-func-", dir="/root") + try: + functional_cli(work) + finally: + shutil.rmtree(work, ignore_errors=True) + + # --- launch node_exporter (the scrape target) --- + ne_log = open("/tmp/node_exporter.log", "w") + ne = subprocess.Popen( + [NODE_EXP, "--web.listen-address=%s" % NE_EP, + "--collector.disable-defaults", + "--collector.uname", "--collector.cpu", "--collector.meminfo", "--collector.loadavg", + "--collector.netdev", "--no-collector.netdev.netlink", + "--collector.diskstats", "--collector.filesystem"], + stdout=ne_log, stderr=subprocess.STDOUT) + ne_up = False + for _ in range(40): + if ne.poll() is not None: + break + try: + with _OPENER.open("http://%s/metrics" % NE_EP, timeout=3) as r: + if r.getcode() == 200 and len(r.read()) > 0: + ne_up = True + break + except Exception: + pass + time.sleep(1) + check(ne_up, "node_exporter serving /metrics on %s" % NE_EP) + + # --- launch prometheus headless --- + if os.path.isdir(TSDB): + shutil.rmtree(TSDB, ignore_errors=True) + os.makedirs(TSDB, exist_ok=True) + p_log = "/tmp/prometheus.log" + plog = open(p_log, "w") + prom = subprocess.Popen( + [PROM, "--config.file=%s" % CFG, "--web.listen-address=%s" % EP, + "--storage.tsdb.path=%s" % TSDB, "--web.enable-lifecycle"], + stdout=plog, stderr=subprocess.STDOUT) + try: + # READY + ready = False + for _ in range(300): + if prom.poll() is not None: + break + try: + log = open(p_log).read() + except Exception: + log = "" + if "Server is ready to receive web requests" in log: + code, body = http_get("/-/ready", timeout=4) + if code == 200 and "Ready" in body: + ready = True + break + time.sleep(1) + check(ready, "log 'Server is ready' + /-/ready answers Ready") + if not ready: + try: + print("--- prometheus.log tail ---\n" + open(p_log).read()[-1500:]) + except Exception: + pass + + # QUERY: PromQL engine round-trips vector(42) + query_ok = False + if ready: + code, body = http_get("/api/v1/query?query=vector(42)", timeout=6) + query_ok = code == 200 and '"status":"success"' in body and '"42"' in body + check(query_ok, "/api/v1/query vector(42) -> success + value 42 (PromQL engine)") + + # PROMTOOL as a PromQL client against the live server + if ready: + rc, out = run([PROMTOOL, "query", "instant", "http://%s" % EP, "vector(42)"], timeout=30) + check(rc == 0 and "42" in out, "promtool query instant vector(42) -> 42 (PromQL client)") + else: + check(False, "promtool query instant (server not ready)") + + # SCRAPE: prometheus scraped node_exporter -> up{job=node} == 1 + scrape_ok = False + if ready and ne_up: + for _ in range(240): + code, body = http_get( + '/api/v1/query?query=up%7Bjob%3D%22node%22%7D', timeout=8) + if code == 200 and '"status":"success"' in body and '"1"]' in body: + scrape_ok = True + break + time.sleep(1) + check(scrape_ok, "up{job=node} == 1 (node_exporter scraped + ingested through :9090->:9100)") + + # promtool query labels/series as a live PromQL client, against the ingested `up` series. + labels_ok = series_ok = False + if scrape_ok: + rc, out = run([PROMTOOL, "query", "labels", "http://%s" % EP, "__name__"], timeout=30) + labels_ok = rc == 0 and "up" in out + rc, out = run([PROMTOOL, "query", "series", "--match=up", "http://%s" % EP], timeout=30) + series_ok = rc == 0 and 'job="node"' in out + check(labels_ok, "promtool query labels __name__ -> lists 'up' (live PromQL client)") + check(series_ok, "promtool query series --match=up -> job=node (live PromQL client)") + + # ===== INTEGRATION SOAK: scrape -> store -> query end-to-end ===== + # Let prometheus scrape node_exporter across a multi-minute soak (2s interval), then verify + # node_exporter's OWN metrics landed in the TSDB and a range query shows many samples across + # the whole window (proves multi-round scrape + persistence, not a single lucky point). + soak_ok = cpu_ok = mem_ok = range_ok = False + SOAK = int(os.environ.get("PROM_SOAK_SECS", "180")) + if scrape_ok: + t0 = int(time.time()) + time.sleep(SOAK) + t1 = int(time.time()) + soak_ok = True + # node_cpu_seconds_total: node_exporter counter (from /proc/stat) must be in the TSDB. + code, body = http_get("/api/v1/query?query=node_cpu_seconds_total", timeout=8) + cpu_ok = (code == 200 and '"status":"success"' in body + and "node_cpu_seconds_total" in body + and re.search(r'"value":\[[0-9.]+,"[0-9]', body) is not None) + check(cpu_ok, "node_cpu_seconds_total ingested with numeric samples (node_exporter -> TSDB)") + # node_memory_MemTotal_bytes: node_exporter gauge (from /proc/meminfo). + code, body = http_get("/api/v1/query?query=node_memory_MemTotal_bytes", timeout=8) + mem_ok = (code == 200 and '"status":"success"' in body + and re.search(r'"value":\[[0-9.]+,"[0-9]', body) is not None) + check(mem_ok, "node_memory_MemTotal_bytes ingested with numeric value (node_exporter -> TSDB)") + # node_network_receive_bytes_total: procfs-backed counter (from /proc/net/dev) -- proves + # the netdev collector's data flowed scrape->store->query, not just cpu/meminfo. + code, body = http_get('/api/v1/query?query=node_network_receive_bytes_total', timeout=8) + net_ok = (code == 200 and '"status":"success"' in body + and re.search(r'"value":\[[0-9.]+,"[0-9]', body) is not None) + check(net_ok, "node_network_receive_bytes_total ingested (netdev collector -> TSDB)") + # query_range over the soak window: many up{job=node} samples proves multi-round scraping. + q = ("/api/v1/query_range?query=up%%7Bjob%%3D%%22node%%22%%7D" + "&start=%d&end=%d&step=15") % (t0, t1) + code, body = http_get(q, timeout=12) + npts = (body.count("],[") + 1) if '"values":[' in body else 0 + range_ok = code == 200 and '"status":"success"' in body and npts >= 5 + check(range_ok, "query_range up{job=node} over %ds -> %d samples (multi-round scrape+store)" + % (SOAK, npts)) + check(soak_ok, "prometheus soaked node_exporter scrapes for %ds (scrape->store)" % SOAK) + finally: + for p in (prom, ne): + try: + p.send_signal(signal.SIGTERM) + p.wait(timeout=6) + except Exception: + try: + p.kill() + except Exception: + pass + plog.close() + ne_log.close() + + print("PROM_RESULT ok=%d fail=%d" % (_ok, _fail)) + if _fail == 0: + print("PROM_DONE") + return 0 + return 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as e: + import traceback + traceback.print_exc() + print("PROM_RESULT ok=%d fail=%d" % (_ok, _fail + 1)) + sys.exit(1) diff --git a/apps/starry/monitor/python/run_monitor.py b/apps/starry/monitor/python/run_monitor.py new file mode 100644 index 0000000000..498ab5084e --- /dev/null +++ b/apps/starry/monitor/python/run_monitor.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# run_monitor.py -- on-target gate for the StarryOS `monitor` app: the Prometheus monitoring stack +# (prometheus + promtool + node_exporter) and the glances system monitor (CLI / headless / TUI / +# client-server / web). +# +# Runs each carpet as its OWN subprocess; a carpet counts as PASS only when its DONE marker is +# present, it exited 0, AND no ` FAIL ` line appears in its output. Prints `MONITOR_OK=

/` and +# `TEST PASSED` only when EVERY carpet passes, else `TEST FAILED`. The PASS/FAIL anchor lives ONLY +# here so the harness success_regex can never self-match a launch command echoed on the console. +import os, subprocess, sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +PROGRAMS = os.path.normpath(os.path.join(HERE, "..", "programs")) +os.chdir(HERE) + +PY = sys.executable +env0 = dict(os.environ) +# make pty_tui_drive / pyte_assert importable by the TUI carpet subprocess. +env0["PYTHONPATH"] = os.pathsep.join( + [p for p in (PROGRAMS, HERE, env0.get("PYTHONPATH", "")) if p]) +env0.setdefault("PYTHONDONTWRITEBYTECODE", "1") +env0.setdefault("PYTHONUNBUFFERED", "1") +env0.setdefault("GLANCES_BIN", "glances") + +# (name, module, DONE marker). Fast/deterministic first; the Go stack and the TUI soak last. +CARPETS = [ + ("glances-cli", "GlancesCliCarpet.py", "GCLI_DONE"), + ("glances-headless", "GlancesHeadlessCarpet.py", "GHDL_DONE"), + ("node_exporter", "NodeExporterCarpet.py", "NE_DONE"), + ("grafana", "GrafanaCarpet.py", "GRAF_DONE"), + ("glances-cs", "GlancesCsCarpet.py", "GCS_DONE"), + ("glances-web", "GlancesWebCarpet.py", "GWEB_DONE"), + ("prometheus", "PrometheusCarpet.py", "PROM_DONE"), + ("glances-tui", "GlancesTuiCarpet.py", "GTUI_DONE"), +] + +print("=== python3 %s ===" % sys.version.split()[0]) +print("=== monitor: prometheus + node_exporter + grafana + glances (cli|headless|tui|client-server|web) carpets ===") + +passed = 0 +for name, fn, marker in CARPETS: + print("\n----- %s (%s) -----" % (name, fn)) + r = subprocess.run([PY, os.path.join(HERE, fn)], capture_output=True, text=True, env=env0) + out = (r.stdout or "") + (r.stderr or "") + result = [ln for ln in out.splitlines() if "_RESULT ok=" in ln] + ok = marker in out and r.returncode == 0 and " FAIL " not in out + # echo the carpet's own lines so the console log shows the evidence. + for ln in out.splitlines(): + print(" | " + ln) + if ok: + print(" OK %s (%s)" % (name, result[0].strip() if result else "")) + passed += 1 + else: + print(" FAIL %s (marker %s, rc=%s)" % (name, marker, r.returncode)) + +total = len(CARPETS) +print("\nMONITOR_RESULT PASS=%d TOTAL=%d" % (passed, total)) +print("MONITOR_OK=%d/%d" % (passed, total)) +if passed == total and total > 0: + print("TEST PASSED") + sys.exit(0) +print("TEST FAILED") +sys.exit(1) diff --git a/apps/starry/monitor/qemu-aarch64.toml b/apps/starry/monitor/qemu-aarch64.toml new file mode 100644 index 0000000000..17de3ed098 --- /dev/null +++ b/apps/starry/monitor/qemu-aarch64.toml @@ -0,0 +1,16 @@ +# monitor aarch64 -- Prometheus stack + glances carpet. +# -cpu max exposes the full AArch64 feature set (LSE atomics etc.); the arm64 Go runtime cpu probe +# reads EL0 MRS ID_AA64* (kernel-emulated). uefi=false / to_bin=true is the aarch64 raw-binary boot. +args = [ + "-nographic", "-cpu", "max", "-m", "2048M", "-smp", "1", + "-device", "virtio-blk-pci,drive=disk0", + "-drive", "id=disk0,if=none,format=raw,file=${workspace}/tmp/axbuild/rootfs/rootfs-aarch64-alpine.img", + "-device", "virtio-net-pci,netdev=net0", "-netdev", "user,id=net0", +] +uefi = false +to_bin = true +shell_prefix = "root@starry:" +shell_init_cmd = "sh /usr/bin/run-monitor.sh" +success_regex = ['(?m)^MONITOR_OK=8/8', '(?m)^TEST PASSED\s*$'] +fail_regex = ['(?i)\bpanic(?:ked)?\b', '(?m)^TEST FAILED\s*$'] +timeout = 18000 diff --git a/apps/starry/monitor/qemu-loongarch64.toml b/apps/starry/monitor/qemu-loongarch64.toml new file mode 100644 index 0000000000..d8b146a911 --- /dev/null +++ b/apps/starry/monitor/qemu-loongarch64.toml @@ -0,0 +1,19 @@ +# monitor loongarch64 -- Prometheus stack + glances carpet. +# Platform: DYNAMIC (axplat-dyn) -- build-loongarch64*.toml carries ax-driver/serial and does not opt +# out of dynamic platform mode; the retired static LoongArch path lacked the serial console binding +# ("/dev/console has no serial TTY binding" -> early exit). uefi=false / to_bin=true is the dynamic +# platform's raw-binary boot path. prometheus/node_exporter loong64 are Go-cross-compiled from the +# pinned source tag (upstream ships no loong64) -- see assets/MANIFEST.md; NOT a SKIP. +args = [ + "-machine", "virt", "-cpu", "la464", "-nographic", "-m", "2048M", "-smp", "1", + "-device", "virtio-blk-pci,drive=disk0", + "-drive", "id=disk0,if=none,format=raw,file=${workspace}/tmp/axbuild/rootfs/rootfs-loongarch64-alpine.img", + "-device", "virtio-net-pci,netdev=net0", "-netdev", "user,id=net0", +] +uefi = false +to_bin = true +shell_prefix = "root@starry:" +shell_init_cmd = "sh /usr/bin/run-monitor.sh" +success_regex = ['(?m)^MONITOR_OK=8/8', '(?m)^TEST PASSED\s*$'] +fail_regex = ['(?i)\bpanic(?:ked)?\b', '(?m)^TEST FAILED\s*$'] +timeout = 36000 diff --git a/apps/starry/monitor/qemu-riscv64.toml b/apps/starry/monitor/qemu-riscv64.toml new file mode 100644 index 0000000000..7c345afd94 --- /dev/null +++ b/apps/starry/monitor/qemu-riscv64.toml @@ -0,0 +1,15 @@ +# monitor riscv64 -- Prometheus stack + glances carpet. prometheus/node_exporter riscv64 are the +# OFFICIAL upstream release (upstream ships riscv64). uefi=false / to_bin=true = riscv64 raw boot. +args = [ + "-nographic", "-cpu", "rv64", "-m", "2048M", "-smp", "1", + "-device", "virtio-blk-pci,drive=disk0", + "-drive", "id=disk0,if=none,format=raw,file=${workspace}/tmp/axbuild/rootfs/rootfs-riscv64-alpine.img", + "-device", "virtio-net-pci,netdev=net0", "-netdev", "user,id=net0", +] +uefi = false +to_bin = true +shell_prefix = "root@starry:" +shell_init_cmd = "sh /usr/bin/run-monitor.sh" +success_regex = ['(?m)^MONITOR_OK=8/8', '(?m)^TEST PASSED\s*$'] +fail_regex = ['(?i)\bpanic(?:ked)?\b', '(?m)^TEST FAILED\s*$'] +timeout = 30000 diff --git a/apps/starry/monitor/qemu-x86_64.toml b/apps/starry/monitor/qemu-x86_64.toml new file mode 100644 index 0000000000..23daf75f92 --- /dev/null +++ b/apps/starry/monitor/qemu-x86_64.toml @@ -0,0 +1,18 @@ +# monitor x86_64 -- Prometheus stack (prometheus/promtool/node_exporter, static Go) + glances +# (musl CPython3: CLI / headless / pyte-verified TUI / client-server / web) carpet. +# -cpu Haswell exposes a modern userspace baseline (AVX2/XSAVE); the kernel CR4.OSXSAVE + XCR0 +# enable lives in dev. prometheus/node_exporter are CGO-free static Go (no libc), glances rides the +# injected musl closure. +args = [ + "-nographic", "-cpu", "Haswell", "-m", "2048M", "-smp", "1", + "-device", "virtio-blk-pci,drive=disk0", + "-drive", "id=disk0,if=none,format=raw,file=${workspace}/tmp/axbuild/rootfs/rootfs-x86_64-alpine.img", + "-device", "virtio-net-pci,netdev=net0", "-netdev", "user,id=net0", +] +uefi = false +to_bin = false +shell_prefix = "root@starry:" +shell_init_cmd = "sh /usr/bin/run-monitor.sh" +success_regex = ['(?m)^MONITOR_OK=8/8', '(?m)^TEST PASSED\s*$'] +fail_regex = ['(?i)\bpanic(?:ked)?\b', '(?m)^TEST FAILED\s*$'] +timeout = 15000