diff --git a/extensions/arm64-compat-vdso.sh b/extensions/arm64-compat-vdso.sh index 69857fce4a9a..8e601abadec2 100644 --- a/extensions/arm64-compat-vdso.sh +++ b/extensions/arm64-compat-vdso.sh @@ -1,4 +1,15 @@ # Enable 32-bit compat vDSO for arm64 kernels with GCC or clang. +# +# Builds the kernel with CONFIG_COMPAT + COMPAT_VDSO + ARM64_32BIT_EL0, letting +# an arm64 host running this kernel execute armhf (32-bit ARM) userspace +# natively at full speed. One concrete use case is Armbian's rootfs phase: on +# such a host, armhf chroot / package post-install steps run native instead of +# through qemu-user-static (~10× faster). See lib/functions/rootfs/qemu-static.sh +# and the PREFER_NATIVE_ARMHF build switch. +# +# Note: aarch64 silicon without 32-bit ARM userspace support at EL0 (notably +# Apple M-series) cannot run armhf even with this kernel option enabled. +# # Requirements: # - arm64 build target (ARCH=arm64, ARCHITECTURE=arm64). # - For GCC builds: a 32-bit ARM cross-compiler (default prefix arm-linux-gnueabi-), @@ -25,7 +36,7 @@ function host_dependencies_ready__arm64_compat_vdso() { fi local compat_gcc_prefix="${CROSS_COMPILE_COMPAT:-"arm-linux-gnueabi-"}" - if ! command -v "${compat_gcc_prefix}gcc" >/dev/null 2>&1; then + if ! command -v "${compat_gcc_prefix}gcc" > /dev/null 2>&1; then exit_with_error "Missing 32-bit compiler '${compat_gcc_prefix}gcc' for COMPAT_VDSO; install gcc-arm-linux-gnueabi or set CROSS_COMPILE_COMPAT" fi } @@ -46,7 +57,7 @@ function custom_kernel_config__arm64_compat_vdso() { opts_y+=("COMPAT" "COMPAT_VDSO" "ARM64_32BIT_EL0") if [[ -f .config ]]; then - kconfig_hit="$(grep -R -n -m1 "COMPAT_VDSO" arch/arm64 Kconfig* 2>/dev/null || true)" + kconfig_hit="$(grep -R -n -m1 "COMPAT_VDSO" arch/arm64 Kconfig* 2> /dev/null || true)" if [[ -z "${kconfig_hit}" ]]; then exit_with_error "Selected kernel tree lacks COMPAT_VDSO support for arm64" fi diff --git a/lib/functions/rootfs/qemu-static.sh b/lib/functions/rootfs/qemu-static.sh index 865f5ccec236..1824941fd084 100644 --- a/lib/functions/rootfs/qemu-static.sh +++ b/lib/functions/rootfs/qemu-static.sh @@ -193,35 +193,131 @@ function prepare_host_binfmt_qemu_cross() { } function prepare_host_binfmt_qemu_cross_arm64_host_armhf_target() { - display_alert "Trying to update binfmts - aarch64 mostly does 32-bit sans emulation, but Apple said no" "update-binfmts --enable qemu-${wanted_arch}" "debug" - run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" "&>" "/dev/null" "||" "true" # don't fail nor produce output, which can be misleading. + declare armhf_probe="/usr/arm-linux-gnueabihf/lib/ld-linux-armhf.so.3" + declare prefer_native="${PREFER_NATIVE_ARMHF:-yes}" + declare qemu_arm_was_enabled=0 + + # Snapshot qemu-arm state — drives both COMPAT probe and trust-existing. + if [[ -e /proc/sys/fs/binfmt_misc/qemu-arm ]] && + [[ "$(head -n1 /proc/sys/fs/binfmt_misc/qemu-arm 2> /dev/null)" == "enabled" ]]; then + qemu_arm_was_enabled=1 + fi + + # COMPAT probe must run with qemu-arm OFF, otherwise kernel routes + # armhf exec through qemu and the probe lies. Temp-disable, probe, + # restore on failure. `arch-test arm` is unreliable (probes ARMv5, + # COMPAT needs ≥v7); ld-linux-armhf comes from gcc-arm-linux-gnueabihf + # (armbian host dep for armhf|all). Toggle needs CAP_SYS_ADMIN — present + # in Armbian's docker_cli_prepare_launch; if /proc is read-only anyway, + # skip the probe gracefully and trust existing qemu-arm. + if [[ "${prefer_native}" == "yes" ]] && [[ -x "${armhf_probe}" ]]; then + declare toggle_ok=1 + if ((qemu_arm_was_enabled)); then + echo 0 > /proc/sys/fs/binfmt_misc/qemu-arm 2> /dev/null || toggle_ok=0 + fi + if ((toggle_ok)) && "${armhf_probe}" --help > /dev/null 2>&1; then + display_alert "Host kernel can run armhf natively (CONFIG_COMPAT)" "qemu-arm left disabled if it was on" "info" + return 0 + fi + if ((qemu_arm_was_enabled && toggle_ok)); then + echo 1 > /proc/sys/fs/binfmt_misc/qemu-arm || + exit_with_error "Failed to restore qemu-arm after failed native armhf probe" + fi + fi + + # Native COMPAT unavailable (or opt-out via PREFER_NATIVE_ARMHF=no). + # Trust existing qemu-arm registration only if it actually executes — + # `enabled` flag alone doesn't tell us the interpreter is runnable + # (stale path, removed package). Validate via arch-test. + if ((qemu_arm_was_enabled)); then + if command -v arch-test > /dev/null 2>&1 && arch-test arm > /dev/null 2>&1; then + display_alert "qemu-arm enabled and functional" "trusting existing setup" "debug" + return 0 + fi + display_alert "qemu-arm enabled but execution probe failed" "stale registration — reconfiguring" "warn" + fi + + # ld-linux-armhf may be absent on cross builds whose target isn't + # armhf (gcc-arm-linux-gnueabihf isn't pulled in then). Fall back to + # arch-test to avoid degraded host-capability detection on those + # flows; on Ampere CAX it reports false-negative but the probe above + # already covered the armhf-target case where it matters most. + if [[ ! -x "${armhf_probe}" ]] && command -v arch-test > /dev/null 2>&1 && arch-test arm > /dev/null 2>&1; then + display_alert "Host can run armhf (arch-test fallback)" "no qemu-arm setup needed" "debug" + return 0 + fi + + # No native COMPAT — need qemu-arm. Prefer a packaged descriptor + # (qemu-user-binfmt on resolute installs `/usr/bin/qemu-arm`; + # qemu-user-static elsewhere uses the -static suffix). Overwriting + # it would break the resolute interpreter path. + if [[ -f /usr/share/binfmts/qemu-arm ]]; then + # Three-step recovery: re-import the descriptor into binfmt-support's + # admin DB (handles stale/empty DB where --enable would fail with + # "not in database"); --enable activates the format; force-sync via + # /proc afterwards if the kernel entry was externally toggled to 0. + run_host_command_logged update-binfmts --import qemu-arm 2> /dev/null || true + run_host_command_logged update-binfmts --enable qemu-arm || true + [[ -e /proc/sys/fs/binfmt_misc/qemu-arm ]] && echo 1 > /proc/sys/fs/binfmt_misc/qemu-arm 2> /dev/null || true + if [[ -e /proc/sys/fs/binfmt_misc/qemu-arm ]] && + [[ "$(head -n1 /proc/sys/fs/binfmt_misc/qemu-arm 2> /dev/null)" == "enabled" ]]; then + _verify_qemu_arm_executes + display_alert "qemu-arm enabled via packaged descriptor" "leaving package-provided setup intact" "debug" + return 0 + fi + exit_with_error "/usr/share/binfmts/qemu-arm exists but qemu-arm could not be enabled — packaged interpreter likely missing. Reinstall qemu-user-binfmt (resolute) / qemu-user-static, or remove the descriptor and retry." + fi + + # Kernel entry exists but disabled, no descriptor on host. The kernel + # keeps the magic/mask in memory once registered; only the enabled + # flag toggles. Try `echo 1 > /proc/...` before giving up — that + # repairs the common "someone toggled it off" state without needing + # the descriptor file back. + if [[ -e /proc/sys/fs/binfmt_misc/qemu-arm ]]; then + echo 1 > /proc/sys/fs/binfmt_misc/qemu-arm 2> /dev/null || true + if [[ "$(head -n1 /proc/sys/fs/binfmt_misc/qemu-arm 2> /dev/null)" == "enabled" ]]; then + _verify_qemu_arm_executes + display_alert "qemu-arm re-enabled via /proc" "kernel state restored without descriptor" "debug" + return 0 + fi + exit_with_error "qemu-arm kernel entry present but cannot be re-enabled and no descriptor to re-register from. Reinstall qemu-user-binfmt / qemu-user-static and retry." + fi + + # Apple-Silicon-like (no COMPAT, no qemu pkg): hand-roll the descriptor. + display_alert "arm64 host can't run armhf natively (no CONFIG_COMPAT?)" "importing+enabling qemu-arm" "debug" + cat <<- BINFMT_ARM_MAGIC > /usr/share/binfmts/qemu-arm + package qemu-user-static + interpreter /usr/bin/qemu-arm-static + magic \x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00 + offset 0 + mask \xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff + credentials yes + fix_binary no + preserve yes + BINFMT_ARM_MAGIC + run_host_command_logged update-binfmts --import qemu-arm + run_host_command_logged update-binfmts --enable qemu-arm if [[ "${SHOW_DEBUG}" == "yes" ]]; then display_alert "Debugging arch-test" "full output" "debug" - run_host_command_logged arch-test "||" true + run_host_command_logged arch-test || true fi + _verify_qemu_arm_executes + display_alert "arm 32-bit emulation on arm64" "has been set up via qemu-arm" "cachehit" +} - # to check, we use arch-test; if will return 0 if _either_ the host can natively run armhf, or if qemu-arm is correctly working. - if arch-test arm; then - display_alert "Host can run armhf natively or emulation is correctly setup already" "no need to enable qemu-arm" "debug" - else - display_alert "arm64 host can't run armhf natively" "importing enabling qemu-arm" "debug" - cat <<-BINFMT_ARM_MAGIC >/usr/share/binfmts/qemu-arm - package qemu-user-static - interpreter /usr/bin/qemu-arm-static - magic \x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00 - offset 0 - mask \xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff - credentials yes - fix_binary no - preserve yes - BINFMT_ARM_MAGIC - run_host_command_logged update-binfmts --import "qemu-${wanted_arch}" - run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" - - # Test again using arch-test. - display_alert "Checking if arm 32-bit emulation on arm64 works after enabling" "qemu-arm emulation" "info" - run_host_command_logged arch-test arm - display_alert "arm 32-bit emulation on arm64" "has been correctly setup" "cachehit" +# Helper: confirm qemu-arm interpreter actually runs an armhf binary. +# `update-binfmts --enable` and an `enabled` flag in /proc only attest +# registration state, not runtime executability — a stale interpreter +# path or removed package slips through and fails later in chroot. Run +# `arch-test arm` here so we fail fast at host-prepare time. +function _verify_qemu_arm_executes() { + if ! command -v arch-test > /dev/null 2>&1; then + display_alert "qemu-arm runtime verification skipped" "arch-test not available on host" "warn" + return 0 + fi + if arch-test arm > /dev/null 2>&1; then + return 0 fi + exit_with_error "qemu-arm registered but armhf execution fails. Interpreter likely broken (stale path, removed package, missing qemu-arm-static). Reinstall qemu-user-binfmt (resolute) / qemu-user-static and retry." }