diff --git a/extensions/ccache.sh b/extensions/ccache.sh new file mode 100644 index 000000000000..a1ca739a096a --- /dev/null +++ b/extensions/ccache.sh @@ -0,0 +1,193 @@ +# +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2013-2026 Igor Pecovnik, igor@armbian.com +# This file is a part of the Armbian Build Framework https://github.com/armbian/build/ +# +# Compile-cache backend: ccache (https://ccache.dev/). +# +# Wraps compiler invocations for kernel / u-boot / ATF / Crust through the +# `ccache` binary and emits per-build hit/miss stats. Implements the generic +# `compile_prepare_vars` (env exports) and `compile_wrapper_pre/post` hooks +# (stats, also fired on interrupt). Per-artifact `*_make_config` hooks +# inject CCACHE_UMASK into the env-i make envs since the kernel/u-boot +# auto-passthrough only covers CCACHE_DIR. +# +# Auto-enabled by core when USE_CCACHE=yes / PRIVATE_CCACHE=yes is set, or +# when the ccache-remote extension is requested (see +# main_config_redefine_user_target in main-config.sh). Can also be enabled +# explicitly via ENABLE_EXTENSIONS=ccache regardless of the USE_CCACHE +# toggle. +# +# Mutually exclusive with other compile-cache extensions (sccache, …) — the +# mutex is enforced in extension_prepare_config__ccache. +# +# Ordering invariant: env wiring lives in compile_prepare_vars__ccache, +# which fires from prepare_compilation_vars (start-end.sh:20) — late +# enough that every extension_prepare_config_* and the userpatches / +# user_config phases have settled values like PRIVATE_CCACHE, and early +# enough that ${CCACHE} substitution and PATH prefix propagate into the +# arrays built by run_*_make_internal. Order is fixed by core, not by +# alphabetical hook name resolution. + +# Cross-hook state for SHOW_CCACHE pre→post (dir-size diff calculation). +declare -g __ext_ccache_dir_actual="" +declare -g __ext_ccache_dir_size_before="" + +# Mutually-exclusive list of compile-cache extensions. Update when a new +# backend extension is added (sccache, …). +declare -g -a __ext_ccache_conflicting_exts=("sccache") + +function extension_prepare_config__ccache() { + # Use the shared normalization (EXT fallback + comma/whitespace handling). + local _ext_list other + _ext_list="$(extension_list_normalized)" + for other in "${__ext_ccache_conflicting_exts[@]}"; do + if [[ "${_ext_list}" == *",${other},"* ]]; then + exit_with_error "${EXTENSION}: 'ccache' and '${other}' extensions are mutually exclusive — choose one compile-cache backend" + fi + done + # All env setup lives in compile_prepare_vars__ccache (called from + # prepare_compilation_vars). See ordering invariant in the file header. +} + +# Main env setup. Runs from prepare_compilation_vars — late enough that +# values set by other extensions (PRIVATE_CCACHE from ccache-remote) and +# by userpatches/lib.config / user_config hooks are settled, and early +# enough that ${CCACHE} substitution in run_*_make_internal sees the +# exported value. +function compile_prepare_vars__ccache() { + # Make the binary substitution available wherever scripts reference + # ${CCACHE} (kernel-make.sh:60, uboot.sh:259, atf.sh:86, crust.sh:55,59). + export CCACHE="ccache" + + # Drop a wrapper directory in front of PATH so bare `gcc` invocations + # also route through ccache via /usr/lib/ccache symlinks. This becomes + # part of the PATH captured by kernel-make.sh:23 and uboot.sh:243 when + # they build the env-i make environment. + export PATH="/usr/lib/ccache:${PATH}" + + # private ccache dir avoids permission issues when build is run as root + # while user invokes via sudo; see https://ccache.samba.org/manual.html#_sharing_a_cache + if [[ "${PRIVATE_CCACHE}" == "yes" && -z "${CCACHE_DIR}" ]]; then + export CCACHE_DIR="${SRC}/cache/ccache" + elif [[ -n "${CCACHE_DIR}" ]]; then + # CCACHE_DIR set by user or another extension: make sure it is exported. + export CCACHE_DIR + fi + + # Shared-cache mode: world-rw so multiple users on a build host can share + # the same directory. ccache reads CCACHE_UMASK from env at write time; + # the *_make_config hooks below inject it into env-i make envs since + # kernel/uboot auto-passthrough only covers CCACHE_DIR. + if [[ -z "${CCACHE_UMASK}" && "${PRIVATE_CCACHE}" != "yes" ]]; then + export CCACHE_UMASK=000 + elif [[ -n "${CCACHE_UMASK}" ]]; then + export CCACHE_UMASK + fi +} + +# Inject CCACHE_UMASK into the given env-i make envs array if set. Core +# kernel-make.sh / uboot.sh auto-pass CCACHE_DIR but not CCACHE_UMASK, +# and env -i would otherwise drop it so ccache writes fall back to the +# default umask — breaking world-rw guarantees for shared caches. +function _ext_ccache_inject_umask() { + local -n envs="$1" + [[ -n "${CCACHE_UMASK}" ]] && envs+=("CCACHE_UMASK=${CCACHE_UMASK@Q}") + return 0 +} + +function kernel_make_config__ccache() { _ext_ccache_inject_umask common_make_envs; } +function uboot_make_config__ccache() { _ext_ccache_inject_umask uboot_make_envs; } +# ATF / Crust don't use env -i (their make runs in the host shell), so +# CCACHE_UMASK propagates naturally from the export in compile_prepare_vars. +# No make_config hook needed for them. + +function compile_wrapper_pre__ccache() { + display_alert "Clearing ccache statistics" "ccache" "ccache" + run_host_command_logged ccache --zero-stats + + if [[ "${SHOW_CCACHE}" == "yes" ]]; then + display_alert "CCACHE_DIR" "${CCACHE_DIR:-"unset"}" "ccache" + display_alert "CCACHE_TEMPDIR" "${CCACHE_TEMPDIR:-"unset"}" "ccache" + + __ext_ccache_dir_actual="$(ccache --show-config | grep "cache_dir =" | cut -d "=" -f 2 | xargs echo)" + + # Guard against ccache versions / configs where cache_dir isn't reported + # yet (first-run, broken config). Empty path would feed `du -sb ""`, + # breaking the build under `set -e`; fall back to a zero baseline so + # the post-hook can still compute a (meaningless but harmless) diff. + local ccache_dir_size_before_human + if [[ -n "${__ext_ccache_dir_actual}" && -d "${__ext_ccache_dir_actual}" ]]; then + __ext_ccache_dir_size_before="$(du -sb "${__ext_ccache_dir_actual}" | cut -f 1)" + ccache_dir_size_before_human="$(numfmt --to=iec-i --suffix=B --format="%.2f" "${__ext_ccache_dir_size_before}")" + display_alert "ccache dir size before" "${ccache_dir_size_before_human}" "ccache" + else + __ext_ccache_dir_size_before="0" + display_alert "ccache dir size before" "unavailable (cache dir not resolved)" "wrn" + fi + + wait_for_disk_sync "before ccache config" + display_alert "ccache configuration" "ccache" "ccache" + run_host_command_logged ccache --show-config "&&" sync + fi + + display_alert "Running ccache'd build..." "ccache" "ccache" +} + +function compile_wrapper_post__ccache() { + local stats_output direct_hit direct_miss pct + stats_output=$(ccache --print-stats 2>&1 || true) + direct_hit=$(ccache_get_stat "${stats_output}" "direct_cache_hit") + direct_miss=$(ccache_get_stat "${stats_output}" "direct_cache_miss") + pct=$(ccache_hit_pct "${direct_hit}" "${direct_miss}") + display_alert "Ccache result" "hit=${direct_hit} miss=${direct_miss} (${pct}%)" "info" + + # Backward-compat: invoke the legacy ccache_post_compilation hook so + # 3rd-party extensions that listened to it (e.g. ccache-remote) continue + # to receive the event and can use ccache_get_stat / ccache_hit_pct. + call_extension_method "ccache_post_compilation" <<- 'CCACHE_POST_COMPILATION' + *called after ccache-wrapped compilation completes (success or failure)* + Legacy hook preserved for backward compatibility; new extensions should + implement compile_wrapper_post__ instead. + CCACHE_POST_COMPILATION + + if [[ "${SHOW_CCACHE}" == "yes" ]]; then + display_alert "Display ccache statistics" "ccache" "ccache" + run_host_command_logged ccache --show-stats --verbose + + # Mirror the pre-hook guard: skip diff if we never resolved the dir. + if [[ -n "${__ext_ccache_dir_actual}" && -d "${__ext_ccache_dir_actual}" ]]; then + local ccache_dir_size_after ccache_dir_size_diff ccache_dir_size_diff_human + ccache_dir_size_after="$(du -sb "${__ext_ccache_dir_actual}" | cut -f 1)" + ccache_dir_size_diff="$((ccache_dir_size_after - __ext_ccache_dir_size_before))" + ccache_dir_size_diff_human="$(numfmt --to=iec-i --suffix=B --format="%.2f" "${ccache_dir_size_diff}")" + display_alert "ccache dir size change" "${ccache_dir_size_diff_human}" "ccache" + else + display_alert "ccache dir size change" "unavailable (cache dir not resolved)" "wrn" + fi + fi +} + +# Parse a single numeric field from "ccache --print-stats" tab-separated +# output; returns 0 if field not found or not numeric. Public name preserved +# for backward compatibility with extensions/ccache-remote (which calls this +# from its ccache_post_compilation__show_remote_stats hook). +function ccache_get_stat() { + local stats_output="$1" field="$2" + local val + val=$(echo "${stats_output}" | grep "^${field}" | cut -f2 || true) + [[ "${val}" =~ ^[0-9]+$ ]] || val=0 + echo "${val}" +} + +# Hit percentage from hit and miss counts. Public name preserved for the +# same backward-compat reason as ccache_get_stat. +function ccache_hit_pct() { + local hit="$1" miss="$2" + local total=$((hit + miss)) + if [[ ${total} -gt 0 ]]; then + echo $((hit * 100 / total)) + else + echo 0 + fi +} diff --git a/lib/functions/compilation/atf.sh b/lib/functions/compilation/atf.sh index c8184037f77b..75e5fcfc32b3 100644 --- a/lib/functions/compilation/atf.sh +++ b/lib/functions/compilation/atf.sh @@ -83,7 +83,17 @@ compile_atf() { # - ENABLE_BACKTRACE="0" has been added to workaround a regression in ATF. Check: https://github.com/armbian/build/issues/1157 - run_host_command_logged "CROSS_COMPILE='ccache ${ATF_COMPILER}'" CCACHE_BASEDIR="$(pwd)" "CC='ccache ${ATF_COMPILER}gcc'" \ + call_extension_method "atf_make_config" <<- 'ATF_MAKE_CONFIG' + *Hook to customize ATF (TF-A) make environment before compilation* + Called right before invoking make for ATF. Extensions (compile-cache + wrappers like ccache/sccache, distcc setups, etc.) can modify env vars + exported by the make invocation; the actual wrap point is `${CCACHE}` + substituted into CROSS_COMPILE / CC below. + Available read-only variables: + - ATF_COMPILER, ATF, BOARD, BRANCH, LINUXFAMILY + ATF_MAKE_CONFIG + + run_host_command_logged "CROSS_COMPILE='${CCACHE} ${ATF_COMPILER}'" CCACHE_BASEDIR="$(pwd)" "CC='${CCACHE} ${ATF_COMPILER}gcc'" \ "CFLAGS='-fdiagnostics-color=always -Wno-error=attributes -Wno-error=incompatible-pointer-types'" \ "TF_LDFLAGS='${binutils_flags_atf}'" \ make ENABLE_BACKTRACE="0" LOG_LEVEL="40" BUILD_STRING="armbian" $target_make "${CTHREADS}" diff --git a/lib/functions/compilation/ccache.sh b/lib/functions/compilation/ccache.sh deleted file mode 100644 index a3a953adbf38..000000000000 --- a/lib/functions/compilation/ccache.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bash -# -# SPDX-License-Identifier: GPL-2.0 -# -# Copyright (c) 2013-2026 Igor Pecovnik, igor@armbian.com -# -# This file is a part of the Armbian Build Framework -# https://github.com/armbian/build/ - -# Parse a single numeric field from "ccache --print-stats" tab-separated output -# Returns 0 if field not found or not numeric -function ccache_get_stat() { - local stats_output="$1" field="$2" - local val - val=$(echo "$stats_output" | grep "^${field}" | cut -f2 || true) - [[ "${val}" =~ ^[0-9]+$ ]] || val=0 - echo "$val" -} - -# Calculate hit percentage from hit and miss counts -function ccache_hit_pct() { - local hit="$1" miss="$2" - local total=$(( hit + miss )) - if [[ $total -gt 0 ]]; then - echo $(( hit * 100 / total )) - else - echo 0 - fi -} - -# Helper function to show ccache stats - used as cleanup handler for interruption case -function ccache_show_compilation_stats() { - local stats_output direct_hit direct_miss pct - stats_output=$(ccache --print-stats 2>&1 || true) - direct_hit=$(ccache_get_stat "$stats_output" "direct_cache_hit") - direct_miss=$(ccache_get_stat "$stats_output" "direct_cache_miss") - pct=$(ccache_hit_pct "$direct_hit" "$direct_miss") - display_alert "Ccache result" "hit=${direct_hit} miss=${direct_miss} (${pct}%)" "info" - - # Hook for extensions to show additional stats (e.g., remote storage) - call_extension_method "ccache_post_compilation" <<- 'CCACHE_POST_COMPILATION' - *called after ccache-wrapped compilation completes (success or failure)* - Useful for displaying remote cache statistics or other post-build info. - CCACHE_POST_COMPILATION -} - -function do_with_ccache_statistics() { - - display_alert "Clearing ccache statistics" "ccache" "ccache" - run_host_command_logged ccache --zero-stats - - if [[ "${SHOW_CCACHE}" == "yes" ]]; then - # show value of CCACHE_DIR - display_alert "CCACHE_DIR" "${CCACHE_DIR:-"unset"}" "ccache" - display_alert "CCACHE_TEMPDIR" "${CCACHE_TEMPDIR:-"unset"}" "ccache" - - # determine what is the actual ccache_dir in use - local ccache_dir_actual - ccache_dir_actual="$(ccache --show-config | grep "cache_dir =" | cut -d "=" -f 2 | xargs echo)" - - # calculate the size of that dir, in bytes. - local ccache_dir_size_before ccache_dir_size_after ccache_dir_size_before_human - ccache_dir_size_before="$(du -sb "${ccache_dir_actual}" | cut -f 1)" - ccache_dir_size_before_human="$(numfmt --to=iec-i --suffix=B --format="%.2f" "${ccache_dir_size_before}")" - - # show the human-readable size of that dir, before we start. - display_alert "ccache dir size before" "${ccache_dir_size_before_human}" "ccache" - - # Show the ccache configuration - wait_for_disk_sync "before ccache config" - display_alert "ccache configuration" "ccache" "ccache" - run_host_command_logged ccache --show-config "&&" sync - fi - - # Register cleanup handler to show stats even if build is interrupted - add_cleanup_handler ccache_show_compilation_stats - - display_alert "Running ccache'd build..." "ccache" "ccache" - local build_exit_code=0 - "$@" || build_exit_code=$? - - # Show stats and remove from cleanup handlers (so it doesn't run twice on exit) - execute_and_remove_cleanup_handler ccache_show_compilation_stats - - # Re-raise the error if the build failed - if [[ ${build_exit_code} -ne 0 ]]; then - return ${build_exit_code} - fi - - if [[ "${SHOW_CCACHE}" == "yes" ]]; then - display_alert "Display ccache statistics" "ccache" "ccache" - run_host_command_logged ccache --show-stats --verbose - - # calculate the size of that dir, in bytes, after the compilation. - ccache_dir_size_after="$(du -sb "${ccache_dir_actual}" | cut -f 1)" - - # calculate the difference, in bytes. - local ccache_dir_size_diff - ccache_dir_size_diff="$((ccache_dir_size_after - ccache_dir_size_before))" - - # calculate the difference, in human-readable format; numfmt is from coreutils. - local ccache_dir_size_diff_human - ccache_dir_size_diff_human="$(numfmt --to=iec-i --suffix=B --format="%.2f" "${ccache_dir_size_diff}")" - - # display the difference - display_alert "ccache dir size change" "${ccache_dir_size_diff_human}" "ccache" - fi - - return 0 -} diff --git a/lib/functions/compilation/compile-wrapper.sh b/lib/functions/compilation/compile-wrapper.sh new file mode 100644 index 000000000000..1a77d5494949 --- /dev/null +++ b/lib/functions/compilation/compile-wrapper.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2013-2026 Igor Pecovnik, igor@armbian.com +# +# This file is a part of the Armbian Build Framework +# https://github.com/armbian/build/ + +# Generic compile-time wrapper, backend-agnostic. +# +# Wraps a compilation command (`make ...` invocation in run_host_command_logged +# or similar) with two extension hooks: +# +# compile_wrapper_pre — called before the build. Extensions zero stats, +# start long-lived helper servers, assert backend +# health, etc. +# compile_wrapper_post — called after the build (also on SIGINT/errexit via +# the registered cleanup handler). Extensions show +# stats, push final cache batches, etc. +# +# Extensions are identified by the conventional double-underscore suffix: +# compile_wrapper_pre__() +# compile_wrapper_post__() +# +# Default behavior with no extensions registered: passthrough — the command +# runs unchanged. This makes the wrapper safe to drop in everywhere a +# compilation invocation lives, without committing to any specific cache +# backend. +function do_with_compile_wrapper() { + # Register the post-hook as a cleanup handler BEFORE running pre hooks. + # A compile_wrapper_pre implementation may start a long-lived helper + # (e.g. `sccache --start-server`); if another pre hook then fails or the + # user interrupts during pre-hook execution, the post hook must still + # fire so backend cleanup (stop helpers, flush state) is not skipped. + add_cleanup_handler _do_with_compile_wrapper_run_post + + call_extension_method "compile_wrapper_pre" <<- 'COMPILE_WRAPPER_PRE' + *pre-compilation hook for cache wrappers (ccache, sccache, …) and + similar backend-agnostic setup* + Called once right before the wrapped compilation command runs. + Implementations may zero stats counters, start a long-lived helper + process, validate that a remote backend is reachable, etc. + The matching compile_wrapper_post hook is guaranteed to fire even + if a later pre hook fails or the build is interrupted, so cleanup + of resources started here is safe to rely on. + COMPILE_WRAPPER_PRE + + local build_exit_code=0 + "$@" || build_exit_code=$? + + # Remove and explicitly run the post-hook on the success path so it does + # not also fire from the cleanup handler at script exit. + execute_and_remove_cleanup_handler _do_with_compile_wrapper_run_post + + return ${build_exit_code} +} + +# Internal — invoked by do_with_compile_wrapper either directly on success or +# via the cleanup-handler chain on interruption. Kept as a separate function +# so it has a stable name to register with add_cleanup_handler. +function _do_with_compile_wrapper_run_post() { + call_extension_method "compile_wrapper_post" <<- 'COMPILE_WRAPPER_POST' + *post-compilation hook for cache wrappers and similar* + Called once after the wrapped compilation command completes (success + or failure) or is interrupted. Implementations may display stats, + flush a remote cache write buffer, shut down a helper server, etc. + COMPILE_WRAPPER_POST +} diff --git a/lib/functions/compilation/crust.sh b/lib/functions/compilation/crust.sh index d88a1dbeb7af..2f11ecd1dfa9 100644 --- a/lib/functions/compilation/crust.sh +++ b/lib/functions/compilation/crust.sh @@ -46,10 +46,20 @@ compile_crust() { return 0 fi - declare binutils_version binutils_flags_crust="" + declare binutils_version binutils_version=$(env or1k-elf-ld.bfd --version | head -1 | cut -d ")" -f 2 | xargs echo -n) display_alert "Binutils version for Crust" "${binutils_version}" "info" + call_extension_method "crust_make_config" <<- 'CRUST_MAKE_CONFIG' + *Hook to customize Crust make environment before compilation* + Called right before invoking make for Crust (both defconfig and target make). + Extensions (compile-cache wrappers like ccache/sccache, distcc setups, etc.) + can modify env vars exported by the make invocation; the actual wrap + point is `${CCACHE}` substituted into `CROSS_COMPILE` below. + Available read-only variables: + - CRUST_COMPILER, BOARD, BRANCH, LINUXFAMILY + CRUST_MAKE_CONFIG + run_host_command_logged CCACHE_BASEDIR="$(pwd)" \ "CFLAGS='-fdiagnostics-color=always -Wno-error=attributes -Wno-error=incompatible-pointer-types'" \ make ${CRUSTCONFIG} "${CTHREADS}" "CROSS_COMPILE='$CCACHE $CRUST_COMPILER'" diff --git a/lib/functions/compilation/kernel.sh b/lib/functions/compilation/kernel.sh index 13b35e86ae11..b5f828b986d6 100644 --- a/lib/functions/compilation/kernel.sh +++ b/lib/functions/compilation/kernel.sh @@ -256,11 +256,11 @@ function kernel_build() { cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}" display_alert "Building kernel" "${LINUXFAMILY} ${LINUXCONFIG} ${build_targets_build[*]}" "info" - do_with_ccache_statistics \ + do_with_compile_wrapper \ run_kernel_make_long_running "${install_make_params_quoted[@]@Q}" "${build_targets_build[@]}" # "V=1" # "-s" silent mode, "V=1" verbose mode display_alert "Installing kernel" "${LINUXFAMILY} ${LINUXCONFIG} ${build_targets_install[*]}" "info" - do_with_ccache_statistics \ + do_with_compile_wrapper \ run_kernel_make_long_running "${install_make_params_quoted[@]@Q}" "${build_targets_install[@]}" # "V=1" # "-s" silent mode, "V=1" verbose mode display_alert "Kernel built in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info" diff --git a/lib/functions/compilation/uboot.sh b/lib/functions/compilation/uboot.sh index 71a449cf9772..b09c4da1446a 100644 --- a/lib/functions/compilation/uboot.sh +++ b/lib/functions/compilation/uboot.sh @@ -270,7 +270,7 @@ function compile_uboot_target() { display_alert "${uboot_prefix}Compiling u-boot" "${version} ${target_make} with gcc '${gcc_version_main}'" "info" declare -g if_error_detail_message="${uboot_prefix}Failed to build u-boot ${version} ${target_make}" - do_with_ccache_statistics run_host_command_logged_long_running \ + do_with_compile_wrapper run_host_command_logged_long_running \ "env" "-i" "${uboot_make_envs[@]}" \ pipetty make "$target_make" "$CTHREADS" "${cross_compile}" diff --git a/lib/functions/configuration/compilation-config.sh b/lib/functions/configuration/compilation-config.sh index dfdaec04aea4..0cf2b190f459 100644 --- a/lib/functions/configuration/compilation-config.sh +++ b/lib/functions/configuration/compilation-config.sh @@ -8,22 +8,38 @@ # https://github.com/armbian/build/ function prepare_compilation_vars() { - # moved from config: rpardini: ccache belongs in compilation, not config. I think. - if [[ $USE_CCACHE == yes || ${PRIVATE_CCACHE} == yes ]]; then - display_alert "using CCACHE" "USE_CCACHE or PRIVATE_CCACHE is set to yes" "warn" - - CCACHE=ccache - export PATH="/usr/lib/ccache:$PATH" # this actually needs export'ing - # private ccache directory to avoid permission issues when using build script with "sudo" - # see https://ccache.samba.org/manual.html#_sharing_a_cache for alternative solution - [[ $PRIVATE_CCACHE == yes ]] && export CCACHE_DIR=$SRC/cache/ccache # actual export - - # Set default umask for ccache to allow write access for all users (enables cache sharing) - # CCACHE_UMASK=000 creates files with permissions 666 (rw-rw-rw-) and dirs with 777 (rwxrwxrwx) - # Only set this for shared cache, not for private cache - [[ -z "${CCACHE_UMASK}" && "${PRIVATE_CCACHE}" != "yes" ]] && export CCACHE_UMASK=000 - else - CCACHE="" + # Backend-agnostic dispatch point for compile-cache extensions + # (ccache, sccache, …). Runs after every extension_prepare_config_* + # (so values set late by other extensions — e.g. PRIVATE_CCACHE from + # ccache-remote — are visible) and before run_*_make_internal builds + # the env-i make arrays (so ${CCACHE} substitution and PATH prepend + # propagate correctly). Order-independent by construction. + # + # Reset CCACHE to empty first so a previous USE_CCACHE=yes build in + # the same shell (or a CCACHE exported in the user's environment) + # does not silently leak the wrapper into a USE_CCACHE=no run. Any + # enabled backend extension will assign CCACHE inside its hook. + declare -g CCACHE="" + call_extension_method "compile_prepare_vars" <<- 'COMPILE_PREPARE_VARS' + *compile-cache env setup hook for ccache / sccache / similar backends* + Called once early in default_build_start, after all extension + prepare_config hooks have run and before kernel/u-boot/ATF/Crust + make invocations begin. Implementations export the env vars their + backend needs (CCACHE, CCACHE_DIR, CCACHE_UMASK, SCCACHE_DIR, …) + so later array-building code captures them, and tweak PATH if a + wrapper prefix directory is needed. + COMPILE_PREPARE_VARS + + # Framework-level limitation: USE_CCACHE / PRIVATE_CCACHE set in + # userpatches/lib.config (sourced after initialize_extension_manager — + # see main-config.sh:443 "too late to define hook functions or add + # extensions in lib.config") cannot auto-enable the ccache extension. + # Warn the user so they can migrate; we deliberately do not duplicate + # the env-setup logic here — it lives in extensions/ccache.sh as the + # sole owner of the ccache backend. + if [[ ("${USE_CCACHE}" == "yes" || "${PRIVATE_CCACHE}" == "yes") && -z "${CCACHE}" ]]; then + display_alert "USE_CCACHE / PRIVATE_CCACHE set, but no compile-cache extension is active" \ + "likely set in userpatches/lib.config (framework limit: too late to enable extensions there). Add 'ccache' to ENABLE_EXTENSIONS / EXT in your CLI, env, or config file sourced earlier." "warn" fi # moved from config: this does not belong in configuration. it's a compilation thing. @@ -36,7 +52,7 @@ function prepare_compilation_vars() { # If CPUTHREADS is defined and a valid positive integer allow user to override CTHREADS # This is useful for limiting Armbian build to a specific number of threads, e.g. for build servers if [[ "$CPUTHREADS" =~ ^[1-9][0-9]*$ ]]; then - CTHREADS="-j$CPUTHREADS" + CTHREADS="-j$CPUTHREADS" echo "Using user-defined thread count: $CTHREADS" fi diff --git a/lib/functions/configuration/main-config.sh b/lib/functions/configuration/main-config.sh index 7dc2e2b730aa..93f37e6a6f33 100644 --- a/lib/functions/configuration/main-config.sh +++ b/lib/functions/configuration/main-config.sh @@ -47,7 +47,7 @@ function do_main_configuration() { unset VENDORSUPPORT,VENDORPRIVACY,VENDORBUGS,VENDORLOGO,ROOTPWD,MAINTAINER,MAINTAINERMAIL fi - [[ -z $VENDORCOLOR ]] && VENDORCOLOR="247;16;0" # RGB values for MOTD logo + [[ -z $VENDORCOLOR ]] && VENDORCOLOR="247;16;0" # RGB values for MOTD logo [[ -z $VENDORURL ]] && VENDORURL="https://duckduckgo.com/" [[ -z $VENDORSUPPORT ]] && VENDORSUPPORT="https://community.armbian.com/" [[ -z $VENDORPRIVACY ]] && VENDORPRIVACY="https://duckduckgo.com/" @@ -60,7 +60,7 @@ function do_main_configuration() { DEST_LANG="${DEST_LANG:-"en_US.UTF-8"}" # en_US.UTF-8 is default locale for target display_alert "DEST_LANG..." "DEST_LANG: ${DEST_LANG}" "debug" - declare -g USE_CCACHE="${USE_CCACHE:-no}" # stop using ccache as our worktree is more effective + declare -g USE_CCACHE="${USE_CCACHE:-no}" # stop using ccache as our worktree is more effective # Armbian config is central tool used in all builds. As its build externally, we have moved it to extension. Enable it here. enable_extension "armbian-config" @@ -173,7 +173,7 @@ function do_main_configuration() { # Support for LUKS / cryptroot if [[ $CRYPTROOT_ENABLE == yes ]]; then - enable_extension "fs-cryptroot-support" # add the tooling needed, cryptsetup + enable_extension "fs-cryptroot-support" # add the tooling needed, cryptsetup if [[ -z $CRYPTROOT_PASSPHRASE ]] && [[ -z $CRYPTROOT_AUTOUNLOCK ]]; then # a passphrase is mandatory if rootfs encryption is enabled, unless CRYPTROOT_AUTOUNLOCK is wanted exit_with_error "Root encryption is enabled but CRYPTROOT_PASSPHRASE or CRYPTROOT_AUTOUNLOCK is not set" fi @@ -326,7 +326,7 @@ function do_main_configuration() { ;; esac - # enable APA extension for Debian Unstable release + # enable APA extension for Debian Unstable release # loong64 is not supported now # [ "$RELEASE" = "sid" ] && [ "$ARCH" != "loong64" ] && enable_extension "apa" @@ -334,6 +334,40 @@ function do_main_configuration() { ## and (hopefully) not yet invoked any extension methods. So this is the perfect ## place to initialize the extension manager. It will create functions ## like the 'post_family_config' that is invoked below. + + # Auto-enable the ccache compile-cache extension when: + # - the legacy toggle is set (USE_CCACHE=yes / PRIVATE_CCACHE=yes), OR + # - the ccache-remote extension is requested (it sets USE_CCACHE=yes + # itself inside extension_prepare_config, too late for this + # shim to react to the toggle directly, so we trigger on the + # extension name instead). + # Skip when a competing compile-cache extension (sccache, …) is already + # requested. Mutex resolution lives inside the extensions themselves; + # the regex below just prevents redundant auto-enabling. + # + # Placed immediately before initialize_extension_manager so all + # in-do_main_configuration sources of USE_CCACHE / PRIVATE_CCACHE / + # ENABLE_EXTENSIONS are visible. The extension-list normalisation + # (EXT fallback + comma/whitespace handling) is shared with the + # per-extension mutex checks via extension_list_normalized. + # + # Note: USE_CCACHE set in userpatches/lib.config is a framework-level + # limitation — lib.config is sourced after initialize_extension_manager + # (see line ~443 'too late to define hook functions or add extensions + # in lib.config'). For that scenario prepare_compilation_vars emits a + # warning at compile phase nudging users to migrate to + # ENABLE_EXTENSIONS=ccache. + local _ext_list + _ext_list="$(extension_list_normalized)" + if [[ "${USE_CCACHE}" == "yes" || + "${PRIVATE_CCACHE}" == "yes" || + "${_ext_list}" == *,ccache-remote,* ]]; then + if [[ ! "${_ext_list}" == *,ccache,* && ! "${_ext_list}" == *,sccache,* ]]; then + enable_extension "ccache" + fi + fi + unset _ext_list + initialize_extension_manager call_extension_method "post_family_config" "config_tweaks_post_family_config" <<- 'POST_FAMILY_CONFIG' diff --git a/lib/functions/general/extensions.sh b/lib/functions/general/extensions.sh index fc65f37e09a0..5a9b6f5a2b77 100644 --- a/lib/functions/general/extensions.sh +++ b/lib/functions/general/extensions.sh @@ -24,6 +24,19 @@ function extension_hook_opt_out() { fi } +# Normalize the active extension list for substring matching. +# Mirrors initialize_extension_manager's resolution (ENABLE_EXTENSIONS:-EXT) +# and accepts both comma- and whitespace-separated forms. Emits a canonical +# `,name1,name2,…,` string suitable for `[[ "$list" == *,name,* ]]` checks. +# Stdout: the normalized list. Useful in both the in-core auto-enable shim +# and per-extension mutex checks between mutually-exclusive extensions. +function extension_list_normalized() { + local raw="${ENABLE_EXTENSIONS:-${EXT:-}}" + local list=",${raw// /,}," + while [[ "$list" == *,,* ]]; do list="${list//,,/,}"; done + echo "$list" +} + function extension_manager_declare_globals() { # global variables managing the state of the extension manager. treat as private. declare -g -A extension_function_info # maps a function name to a string with KEY=VALUEs information about the defining extension @@ -590,7 +603,7 @@ function enable_extensions_with_hostdeps_builtin_and_user() { display_alert "Extension search" "Searching in directory: \"${ext_dir}\"" "" if [[ -d "${ext_dir}" ]]; then declare -a ext_list_dir=() - mapfile -t ext_list_dir < <(find "${ext_dir}" -maxdepth 2 -type f -name "*.sh" -print0 | xargs -0 -r grep -l "${grep_args[@]}" 2>/dev/null || true) + mapfile -t ext_list_dir < <(find "${ext_dir}" -maxdepth 2 -type f -name "*.sh" -print0 | xargs -0 -r grep -l "${grep_args[@]}" 2> /dev/null || true) display_alert "Extension search result" "Found ${#ext_list_dir[@]} extensions in \"${ext_dir}\"" "" extension_list+=("${ext_list_dir[@]}") else diff --git a/lib/library-functions.sh b/lib/library-functions.sh index adc4bc44a1f8..8a40582facc7 100644 --- a/lib/library-functions.sh +++ b/lib/library-functions.sh @@ -285,9 +285,9 @@ source "${SRC}"/lib/functions/compilation/atf.sh #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled set -o errtrace # trace ERR through - enabled set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled -### lib/functions/compilation/ccache.sh -# shellcheck source=lib/functions/compilation/ccache.sh -source "${SRC}"/lib/functions/compilation/ccache.sh +### lib/functions/compilation/compile-wrapper.sh +# shellcheck source=lib/functions/compilation/compile-wrapper.sh +source "${SRC}"/lib/functions/compilation/compile-wrapper.sh # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon"