-
Notifications
You must be signed in to change notification settings - Fork 126
feat(starry): add pip functional test under apps #1002
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| # pip 测试. | ||
|
|
||
| 在 StarryOS (Alpine rootfs) 上验证 Python pip 的常用操作。 | ||
|
|
||
| ## 测试内容 | ||
|
|
||
| 25 个阶段,覆盖日常 pip 使用场景: | ||
|
|
||
| - venv 创建/激活/销毁 | ||
| - pip install / uninstall / upgrade / --dry-run | ||
| - pip list / show / freeze / check | ||
| - pip install -r requirements.txt | ||
| - pip install from local directory / local wheel | ||
| - pip wheel / pip download | ||
| - pip cache dir / info / purge | ||
| - pip install -e (editable) | ||
| - pip list --format=json / --outdated | ||
| - pip config / pip debug | ||
|
|
||
| ## 运行测试 | ||
|
|
||
| ```bash | ||
| cargo xtask starry app run -t pip | ||
| ``` | ||
|
|
||
| ## 在 Linux 主机上快速验证测试脚本 | ||
|
|
||
| ```bash | ||
| sh apps/starry/pip/test_pip.sh | ||
| ``` | ||
|
|
||
| ## 工作原理 | ||
|
|
||
| 1. `prebuild.sh` 被 xtask 框架调用,通过 `qemu-user-static` 在 staging rootfs 中执行 Alpine 原生 `apk` 安装 `python3 py3-pip`(使用 `--no-scripts` 跳过 busybox trigger) | ||
| 2. 通过 `readelf` 解析运行时依赖并复制到 overlay | ||
| 3. `test_pip.sh` 被复制到 overlay 的 `/usr/bin/` | ||
| 4. overlay 被注入到 rootfs 镜像中 | ||
| 5. QEMU 启动后执行 `/usr/bin/test_pip.sh` | ||
|
|
||
| ## 判定标准 | ||
|
|
||
| - **通过**: 输出匹配 `STARRY_PIP_TESTS_PASSED` | ||
| - **失败**: 输出匹配 `STARRY_PIP_STAGE_.*_FAILED`,或出现 panic / page fault / segmentation fault | ||
| - **超时**: 600 秒 | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| target = "x86_64-unknown-none" | ||
| env = { AX_IP = "10.0.2.15", AX_GW = "10.0.2.2" } | ||
| log = "Warn" | ||
| features = [ | ||
| "ax-hal/x86-pc", | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 所有其他 Starry app(git、openssh、redis、codex-cli、openrc)的 build config 均使用 如果 建议改为: features = ["qemu"]
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 建议简化为 |
||
| "qemu", | ||
| "ax-driver/plat-static", | ||
| "ax-driver/virtio-blk", | ||
| "ax-driver/virtio-net", | ||
| ] | ||
| plat_dyn = false | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,178 @@ | ||
| #!/usr/bin/env bash | ||
| set -euo pipefail | ||
|
|
||
| app_dir="${STARRY_APP_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" | ||
| arch="${STARRY_ARCH:-x86_64}" | ||
| base_rootfs="${STARRY_BASE_ROOTFS:-}" | ||
| staging_root="${STARRY_STAGING_ROOT:-}" | ||
| overlay_dir="${STARRY_OVERLAY_DIR:-}" | ||
| apk_cache="${STARRY_WORKSPACE:-$(cd "$app_dir/../../.." && pwd)}/target/pip-apk-cache" | ||
|
|
||
| require_env() { | ||
| local name="$1" | ||
| local value="$2" | ||
| if [[ -z "$value" ]]; then | ||
| echo "error: $name is required" >&2 | ||
| exit 1 | ||
| fi | ||
| } | ||
|
|
||
| ensure_host_packages() { | ||
| local missing=() | ||
|
|
||
| command -v debugfs >/dev/null 2>&1 || missing+=(e2fsprogs) | ||
| command -v install >/dev/null 2>&1 || missing+=(coreutils) | ||
| command -v readelf >/dev/null 2>&1 || missing+=(binutils) | ||
|
|
||
| case "$arch" in | ||
| aarch64) command -v qemu-aarch64-static >/dev/null 2>&1 || missing+=(qemu-user-static) ;; | ||
| riscv64) command -v qemu-riscv64-static >/dev/null 2>&1 || missing+=(qemu-user-static) ;; | ||
| x86_64) command -v qemu-x86_64-static >/dev/null 2>&1 || missing+=(qemu-user-static) ;; | ||
| loongarch64) command -v qemu-loongarch64-static >/dev/null 2>&1 || missing+=(qemu-user-static) ;; | ||
| esac | ||
|
|
||
| if [[ ${#missing[@]} -eq 0 ]]; then | ||
| return | ||
| fi | ||
|
|
||
| if ! command -v apt-get >/dev/null 2>&1; then | ||
| echo "error: missing required host packages and apt-get is unavailable: ${missing[*]}" >&2 | ||
| exit 1 | ||
| fi | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
但下方的 建议改用 |
||
|
|
||
| echo "installing missing host packages: ${missing[*]}" | ||
| apt-get update | ||
| apt-get install -y --no-install-recommends "${missing[@]}" | ||
| } | ||
|
|
||
| extract_base_rootfs() { | ||
| debugfs -R "rdump / $staging_root" "$base_rootfs" | ||
| } | ||
|
|
||
| install_pip_packages() { | ||
| local qemu_runner | ||
| case "$arch" in | ||
| aarch64) qemu_runner="qemu-aarch64-static" ;; | ||
| riscv64) qemu_runner="qemu-riscv64-static" ;; | ||
| x86_64) qemu_runner="qemu-x86_64-static" ;; | ||
| loongarch64) qemu_runner="qemu-loongarch64-static" ;; | ||
| *) echo "error: unsupported arch: $arch" >&2; exit 1 ;; | ||
| esac | ||
|
|
||
| if ! command -v "$qemu_runner" >/dev/null 2>&1; then | ||
| echo "error: $qemu_runner not found" >&2 | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Copy host DNS config so apk can resolve hostnames inside qemu-user. | ||
| if [[ -f /etc/resolv.conf ]]; then | ||
| cp /etc/resolv.conf "$staging_root/etc/resolv.conf" | ||
| fi | ||
|
|
||
| mkdir -p "$apk_cache" | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 宿主机没有 正确做法: "$qemu_runner" -L "$staging_root" \
"$staging_root/usr/bin/apk" \
--root / \
--cache-dir "$apk_cache" \
--update-cache --no-progress --no-scripts \
add python3 py3-pip
|
||
| echo "[pip prebuild] installing python3 and py3-pip via qemu-user apk..." | ||
| "$qemu_runner" -L "$staging_root" \ | ||
| "$staging_root/sbin/apk" \ | ||
| --root / \ | ||
| --cache-dir "$apk_cache" \ | ||
| --update-cache \ | ||
| --no-progress \ | ||
| --no-scripts \ | ||
| add python3 py3-pip | ||
| } | ||
|
|
||
| copy_file_to_overlay() { | ||
| local guest_path="$1" | ||
| local mode="$2" | ||
| local source="$staging_root${guest_path}" | ||
| local target="$overlay_dir${guest_path}" | ||
|
|
||
| if [[ ! -e "$source" ]]; then | ||
| echo "error: missing guest file after package install: $guest_path" >&2 | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [[ -L "$source" ]]; then | ||
| source="$(readlink -f "$source")" | ||
| fi | ||
|
|
||
| install -Dm"$mode" "$source" "$target" | ||
| } | ||
|
|
||
| find_library_path() { | ||
| local library="$1" | ||
| local dir | ||
|
|
||
| for dir in lib usr/lib usr/local/lib; do | ||
| if [[ -e "$staging_root/$dir/$library" ]]; then | ||
| printf '/%s/%s\n' "$dir" "$library" | ||
| return 0 | ||
| fi | ||
| done | ||
|
|
||
| return 1 | ||
| } | ||
|
|
||
| copy_runtime_dependencies() { | ||
| local pending=("$@") | ||
| local seen=" " | ||
| local guest_path library | ||
|
|
||
| while [[ ${#pending[@]} -gt 0 ]]; do | ||
| guest_path="${pending[0]}" | ||
| pending=("${pending[@]:1}") | ||
|
|
||
| if [[ "$seen" == *" $guest_path "* ]]; then | ||
| continue | ||
| fi | ||
| seen+="$guest_path " | ||
|
|
||
| while IFS= read -r library; do | ||
| local library_path | ||
| if ! library_path="$(find_library_path "$library")"; then | ||
| continue | ||
| fi | ||
| copy_file_to_overlay "$library_path" 0644 | ||
| pending+=("$library_path") | ||
| done < <( | ||
| readelf -d "$staging_root$guest_path" 2>/dev/null | | ||
| sed -n 's/.*Shared library: \[\(.*\)\].*/\1/p' | ||
| ) | ||
| done | ||
| } | ||
|
|
||
| populate_overlay() { | ||
| copy_file_to_overlay /usr/bin/python3 0755 | ||
| copy_file_to_overlay /usr/bin/pip3 0755 | ||
|
|
||
| if [[ -e "$staging_root/usr/bin/pip" ]]; then | ||
| copy_file_to_overlay /usr/bin/pip 0755 | ||
| fi | ||
|
|
||
| copy_runtime_dependencies /usr/bin/python3 | ||
|
|
||
| # Copy Python standard library | ||
| local pyver | ||
| pyver="$(ls "$staging_root/usr/lib/python3"* -d 2>/dev/null | head -1 | xargs basename)" | ||
| if [[ -n "$pyver" && -d "$staging_root/usr/lib/$pyver" ]]; then | ||
| mkdir -p "$overlay_dir/usr/lib/$pyver" | ||
| cp -a "$staging_root/usr/lib/$pyver/." "$overlay_dir/usr/lib/$pyver/" | ||
| fi | ||
|
|
||
| # Copy pip site-packages | ||
| if [[ -d "$staging_root/usr/lib/python3" ]]; then | ||
| mkdir -p "$overlay_dir/usr/lib/python3" | ||
| cp -a "$staging_root/usr/lib/python3/." "$overlay_dir/usr/lib/python3/" | ||
| fi | ||
|
|
||
| install -Dm0755 "$app_dir/test_pip.sh" "$overlay_dir/usr/bin/test_pip.sh" | ||
| } | ||
|
|
||
| require_env STARRY_BASE_ROOTFS "$base_rootfs" | ||
| require_env STARRY_STAGING_ROOT "$staging_root" | ||
| require_env STARRY_OVERLAY_DIR "$overlay_dir" | ||
|
|
||
| ensure_host_packages | ||
| extract_base_rootfs | ||
| install_pip_packages | ||
| populate_overlay | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| args = [ | ||
| "-machine", | ||
| "q35", | ||
| "-nographic", | ||
| "-m", | ||
| "1G", | ||
| "-device", | ||
| "virtio-blk-pci,drive=disk0", | ||
| "-drive", | ||
| "id=disk0,if=none,format=raw,file=${workspace}/tmp/axbuild/rootfs/rootfs-x86_64-pip.img", | ||
| "-device", | ||
| "virtio-net-pci,netdev=net0", | ||
| "-netdev", | ||
| "user,id=net0", | ||
| "-snapshot", | ||
| ] | ||
| uefi = false | ||
| to_bin = false | ||
| shell_prefix = "root@starry:" | ||
|
ZR233 marked this conversation as resolved.
|
||
| shell_init_cmd = "/usr/bin/test_pip.sh" | ||
| success_regex = ["(?m)^STARRY_PIP_TESTS_PASSED\\s*$"] | ||
| fail_regex = [ | ||
| "(?i)\\bpanic(?:ked)?\\b", | ||
| "(?i)page fault", | ||
| "(?i)segmentation fault", | ||
| "No such process", | ||
| "STARRY_PIP_STAGE_.*_FAILED", | ||
| ] | ||
| timeout = 600 | ||
Uh oh!
There was an error while loading. Please reload this page.