From da2d2bff28d052a11ce97cef906a97ca3feab998 Mon Sep 17 00:00:00 2001 From: Till0196 <16399842+Till0196@users.noreply.github.com> Date: Tue, 19 May 2026 18:23:34 +0900 Subject: [PATCH 1/4] k8s: ship sysbox-runc as a containerd drop-in on k3s / RKE2 k3s and RKE2 with containerd 2.x regenerate /etc/containerd/config.toml on every restart, so the existing dasel-based config.toml editing path is wiped out on the next reboot. The supported extension point is the config-v3 drop-in directory (config-v3.toml.d), which is read after the generated base config and merged on top of it. Detect k3s / RKE2 via systemd, resolve the per-distro drop-in dir under /var/lib/rancher//agent/etc/containerd/, and write a minimal sysbox-runc runtime block as a standalone drop-in. The drop-in body is shipped as a template (config/containerd-sysbox- dropin.toml.tmpl) and rendered with a sed substitution at install time so the TOML lives next to the other k8s config artifacts rather than embedded in the deploy script. Vanilla containerd nodes keep their existing config.toml-editing path. unconfig_containerd_for_sysbox is updated to mirror the new path: on k3s / RKE2 it simply removes the drop-in and restarts the wrapper service. --- k8s/Dockerfile.sysbox-ce | 2 + k8s/config/containerd-sysbox-dropin.toml.tmpl | 5 + k8s/scripts/sysbox-deploy-k8s.sh | 102 +++++++++++++++++- 3 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 k8s/config/containerd-sysbox-dropin.toml.tmpl diff --git a/k8s/Dockerfile.sysbox-ce b/k8s/Dockerfile.sysbox-ce index 56bf435..2971946 100644 --- a/k8s/Dockerfile.sysbox-ce +++ b/k8s/Dockerfile.sysbox-ce @@ -75,6 +75,8 @@ COPY scripts/sysbox-deploy-k8s.sh /opt/sysbox/scripts/sysbox-deploy-k8s.sh COPY scripts/sysbox-installer-helper.sh /opt/sysbox/scripts/sysbox-installer-helper.sh COPY scripts/sysbox-removal-helper.sh /opt/sysbox/scripts/sysbox-removal-helper.sh +COPY config/containerd-sysbox-dropin.toml.tmpl /opt/sysbox/config/containerd-sysbox-dropin.toml.tmpl + # # Load CRI-O installation artifacts # diff --git a/k8s/config/containerd-sysbox-dropin.toml.tmpl b/k8s/config/containerd-sysbox-dropin.toml.tmpl new file mode 100644 index 0000000..5155008 --- /dev/null +++ b/k8s/config/containerd-sysbox-dropin.toml.tmpl @@ -0,0 +1,5 @@ +[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.sysbox-runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.sysbox-runc.options] + BinaryName = "@SYSBOX_RUNC_PATH@" + SystemdCgroup = true diff --git a/k8s/scripts/sysbox-deploy-k8s.sh b/k8s/scripts/sysbox-deploy-k8s.sh index 8409df3..88b8c3e 100755 --- a/k8s/scripts/sysbox-deploy-k8s.sh +++ b/k8s/scripts/sysbox-deploy-k8s.sh @@ -39,6 +39,9 @@ sysbox_version=$(echo "$SYSBOX_VERSION" | sed '/-[0-9]/!s/.*/&-0/') sysbox_artifacts="/opt/sysbox" crio_artifacts="/opt/crio-deploy" +# Template for the containerd drop-in used on k3s / RKE2. +containerd_sysbox_dropin_tmpl="${sysbox_artifacts}/config/containerd-sysbox-dropin.toml.tmpl" + # The daemonset spec will set up these mounts. host_systemd="/mnt/host/lib/systemd/system" host_sysctl="/mnt/host/lib/sysctl.d" @@ -671,20 +674,93 @@ function unconfig_crio_for_sysbox() { # Containerd Configuration Functions # -function config_containerd_for_sysbox() { - echo "Adding Sysbox to containerd config ..." +# Returns the name of the k3s / RKE2 distribution that owns containerd on +# this node ("k3s" or "rke2"), or empty if the node runs vanilla containerd. +# Detection is done via systemd because the kubelet-reported runtime version +# string ("containerd://X.Y.Z-k3sN") cannot distinguish k3s from rke2. +function k8s_dist_owning_containerd() { + if systemctl is-active --quiet rke2-agent || systemctl is-active --quiet rke2-server; then + echo "rke2" + elif systemctl is-active --quiet k3s-agent || systemctl is-active --quiet k3s; then + echo "k3s" + fi +} + +# Returns the containerd config-v3 drop-in directory used by k3s / RKE2 on +# this node, or empty for vanilla containerd. k3s and RKE2 with containerd +# 2.x read all *.toml files in this directory and merge them on top of the +# generated base config, so shipping the Sysbox runtime as a standalone +# drop-in avoids overwriting the distro's generated config.toml. +function k8s_containerd_dropin_dir() { + local dist + dist="$(k8s_dist_owning_containerd)" + [ -z "${dist}" ] && return + echo "${host_var_lib}/rancher/${dist}/agent/etc/containerd/config-v3.toml.d" +} + +# Restart whichever service owns containerd on this node. For k3s / RKE2 the +# wrapper service must be restarted because it manages an embedded containerd; +# vanilla nodes restart containerd directly. +function restart_container_runtime() { + if systemctl is-active --quiet rke2-agent; then + systemctl restart rke2-agent + elif systemctl is-active --quiet rke2-server; then + systemctl restart rke2-server + elif systemctl is-active --quiet k3s-agent; then + systemctl restart k3s-agent + elif systemctl is-active --quiet k3s; then + systemctl restart k3s + else + systemctl restart containerd + fi +} - # Backup the original containerd config if not already backed up - if [ ! -f "${host_containerd_conf_file_backup}" ]; then - cp "${host_containerd_conf_file}" "${host_containerd_conf_file_backup}" +# Write the Sysbox containerd drop-in used by k3s / RKE2. Emits only the +# sysbox-runc runtime block so the distro's generated base config is left +# untouched. Uses the containerd 2.x config-v3 plugin key. +function write_containerd_sysbox_dropin() { + local dropin_dir="$1" + local sysbox_runc_path="$2" + local dropin_file="${dropin_dir}/sysbox.toml" + + if [ ! -f "${containerd_sysbox_dropin_tmpl}" ]; then + echo "Error: containerd drop-in template not found at ${containerd_sysbox_dropin_tmpl}" + return 1 fi + echo "Writing Sysbox containerd drop-in to ${dropin_file} ..." + mkdir -p "${dropin_dir}" + sed "s|@SYSBOX_RUNC_PATH@|${sysbox_runc_path}|g" \ + "${containerd_sysbox_dropin_tmpl}" >"${dropin_file}" +} + +function config_containerd_for_sysbox() { + echo "Adding Sysbox to containerd config ..." + # Determine the correct sysbox-runc path local sysbox_runc_path="/usr/bin/sysbox-runc" if host_flatcar_distro; then sysbox_runc_path="/opt/bin/sysbox-runc" fi + # k3s / RKE2 ship containerd with a generated config.toml that is + # rewritten on every restart; the supported extension point is the + # config-v3 drop-in directory. Use it when present and skip the + # /etc/containerd/config.toml path entirely. + local dropin_dir + dropin_dir="$(k8s_containerd_dropin_dir)" + if [ -n "${dropin_dir}" ]; then + write_containerd_sysbox_dropin "${dropin_dir}" "${sysbox_runc_path}" + echo "Restarting container runtime to apply changes ..." + restart_container_runtime + return + fi + + # Backup the original containerd config if not already backed up + if [ ! -f "${host_containerd_conf_file_backup}" ]; then + cp "${host_containerd_conf_file}" "${host_containerd_conf_file_backup}" + fi + # Check if sysbox-runc runtime section already exists if grep -q "runtimes.sysbox-runc" "${host_containerd_conf_file}"; then echo "sysbox-runc runtime already configured in containerd config" @@ -714,6 +790,22 @@ function config_containerd_for_sysbox() { function unconfig_containerd_for_sysbox() { echo "Removing Sysbox from containerd config ..." + # k3s / RKE2: just delete the drop-in we created. + local dropin_dir + dropin_dir="$(k8s_containerd_dropin_dir)" + if [ -n "${dropin_dir}" ]; then + local dropin_file="${dropin_dir}/sysbox.toml" + if [ -f "${dropin_file}" ]; then + echo "Removing Sysbox containerd drop-in ${dropin_file} ..." + rm -f "${dropin_file}" + echo "Restarting container runtime to apply changes ..." + restart_container_runtime + else + echo "sysbox-runc drop-in not found" + fi + return + fi + if [ -f "${host_containerd_conf_file}" ]; then # Check if sysbox-runc runtime configuration exists if grep -q "runtimes.sysbox-runc" "${host_containerd_conf_file}"; then From f78bf65e67aa937d99c87b3dedc841364051f5cc Mon Sep 17 00:00:00 2001 From: Till0196 <16399842+Till0196@users.noreply.github.com> Date: Tue, 19 May 2026 18:26:18 +0900 Subject: [PATCH 2/4] k8s: ship containerd drop-in as plain TOML The drop-in only ever substituted the sysbox-runc binary path. Carrying it as a .tmpl with an @SYSBOX_RUNC_PATH@ placeholder is heavier than needed: rename it to a plain .toml that is valid on its own and hard-code BinaryName to /usr/bin/sysbox-runc (the only path the installer ever writes on non-Flatcar nodes). Simplify write_containerd_sysbox_dropin() to a straight cp. config_containerd_for_sysbox() still computes sysbox_runc_path for the legacy dasel-based containerd.conf branch. --- k8s/Dockerfile.sysbox-ce | 2 +- ...n.toml.tmpl => containerd-sysbox-dropin.toml} | 2 +- k8s/scripts/sysbox-deploy-k8s.sh | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) rename k8s/config/{containerd-sysbox-dropin.toml.tmpl => containerd-sysbox-dropin.toml} (84%) diff --git a/k8s/Dockerfile.sysbox-ce b/k8s/Dockerfile.sysbox-ce index 2971946..4bfbf1e 100644 --- a/k8s/Dockerfile.sysbox-ce +++ b/k8s/Dockerfile.sysbox-ce @@ -75,7 +75,7 @@ COPY scripts/sysbox-deploy-k8s.sh /opt/sysbox/scripts/sysbox-deploy-k8s.sh COPY scripts/sysbox-installer-helper.sh /opt/sysbox/scripts/sysbox-installer-helper.sh COPY scripts/sysbox-removal-helper.sh /opt/sysbox/scripts/sysbox-removal-helper.sh -COPY config/containerd-sysbox-dropin.toml.tmpl /opt/sysbox/config/containerd-sysbox-dropin.toml.tmpl +COPY config/containerd-sysbox-dropin.toml /opt/sysbox/config/containerd-sysbox-dropin.toml # # Load CRI-O installation artifacts diff --git a/k8s/config/containerd-sysbox-dropin.toml.tmpl b/k8s/config/containerd-sysbox-dropin.toml similarity index 84% rename from k8s/config/containerd-sysbox-dropin.toml.tmpl rename to k8s/config/containerd-sysbox-dropin.toml index 5155008..3d93d07 100644 --- a/k8s/config/containerd-sysbox-dropin.toml.tmpl +++ b/k8s/config/containerd-sysbox-dropin.toml @@ -1,5 +1,5 @@ [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.sysbox-runc] runtime_type = "io.containerd.runc.v2" [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.sysbox-runc.options] - BinaryName = "@SYSBOX_RUNC_PATH@" + BinaryName = "/usr/bin/sysbox-runc" SystemdCgroup = true diff --git a/k8s/scripts/sysbox-deploy-k8s.sh b/k8s/scripts/sysbox-deploy-k8s.sh index 88b8c3e..cdf64f2 100755 --- a/k8s/scripts/sysbox-deploy-k8s.sh +++ b/k8s/scripts/sysbox-deploy-k8s.sh @@ -39,8 +39,10 @@ sysbox_version=$(echo "$SYSBOX_VERSION" | sed '/-[0-9]/!s/.*/&-0/') sysbox_artifacts="/opt/sysbox" crio_artifacts="/opt/crio-deploy" -# Template for the containerd drop-in used on k3s / RKE2. -containerd_sysbox_dropin_tmpl="${sysbox_artifacts}/config/containerd-sysbox-dropin.toml.tmpl" +# Containerd drop-in used on k3s / RKE2. Ships with /usr/bin/sysbox-runc; on +# Flatcar do_distro_adjustments() rewrites this artifact to /opt/bin/sysbox-runc +# up-front, so the install path can copy it verbatim. +containerd_sysbox_dropin_src="${sysbox_artifacts}/config/containerd-sysbox-dropin.toml" # The daemonset spec will set up these mounts. host_systemd="/mnt/host/lib/systemd/system" @@ -720,18 +722,16 @@ function restart_container_runtime() { # untouched. Uses the containerd 2.x config-v3 plugin key. function write_containerd_sysbox_dropin() { local dropin_dir="$1" - local sysbox_runc_path="$2" local dropin_file="${dropin_dir}/sysbox.toml" - if [ ! -f "${containerd_sysbox_dropin_tmpl}" ]; then - echo "Error: containerd drop-in template not found at ${containerd_sysbox_dropin_tmpl}" + if [ ! -f "${containerd_sysbox_dropin_src}" ]; then + echo "Error: containerd drop-in source not found at ${containerd_sysbox_dropin_src}" return 1 fi echo "Writing Sysbox containerd drop-in to ${dropin_file} ..." mkdir -p "${dropin_dir}" - sed "s|@SYSBOX_RUNC_PATH@|${sysbox_runc_path}|g" \ - "${containerd_sysbox_dropin_tmpl}" >"${dropin_file}" + cp "${containerd_sysbox_dropin_src}" "${dropin_file}" } function config_containerd_for_sysbox() { @@ -750,7 +750,7 @@ function config_containerd_for_sysbox() { local dropin_dir dropin_dir="$(k8s_containerd_dropin_dir)" if [ -n "${dropin_dir}" ]; then - write_containerd_sysbox_dropin "${dropin_dir}" "${sysbox_runc_path}" + write_containerd_sysbox_dropin "${dropin_dir}" echo "Restarting container runtime to apply changes ..." restart_container_runtime return From 317828c265da40ee4681e85268c071454984df27 Mon Sep 17 00:00:00 2001 From: Till0196 <16399842+Till0196@users.noreply.github.com> Date: Tue, 19 May 2026 18:23:34 +0900 Subject: [PATCH 3/4] k8s: skip shiftfs install on Flatcar 4593+ (kernel 6.x) Flatcar Container Linux 4593 and later ship a 6.x kernel where the sysbox-flatcar-preview shiftfs.ko no longer applies; the kernel provides idmap mounts instead, which sysbox-runc uses transparently. Mirror the non-Flatcar branch's kernel-version gate in install_sysbox_deps so shiftfs installation is skipped above the shiftfs_max_kernel_ver. Also fall back to the generic binary directory for Sysbox-CE on Flatcar, since the per-release flatcar-* artifact dir only exists in the EE build that depends on shiftfs. --- k8s/scripts/sysbox-deploy-k8s.sh | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/k8s/scripts/sysbox-deploy-k8s.sh b/k8s/scripts/sysbox-deploy-k8s.sh index cdf64f2..6779b8f 100755 --- a/k8s/scripts/sysbox-deploy-k8s.sh +++ b/k8s/scripts/sysbox-deploy-k8s.sh @@ -287,8 +287,16 @@ function get_artifacts_dir() { [[ "$distro" =~ "debian" ]]; then artifacts_dir="${sysbox_artifacts}/bin/generic" elif [[ "$distro" =~ "flatcar" ]]; then - local release=$(echo $distro | cut -d"-" -f2) - artifacts_dir="${sysbox_artifacts}/bin/flatcar-${release}" + if [[ ${sysbox_edition} == "Sysbox" ]]; then + # Sysbox-CE (sysbox_edition="Sysbox") ships only the generic + # binaries; Flatcar 4593+ runs a 6.x kernel with idmap mounts so + # the EE-only shiftfs build that normally lives under + # bin/flatcar- is not required. + artifacts_dir="${sysbox_artifacts}/bin/generic" + else + local release=$(echo $distro | cut -d"-" -f2) + artifacts_dir="${sysbox_artifacts}/bin/flatcar-${release}" + fi fi echo $artifacts_dir @@ -498,7 +506,16 @@ function install_sysbox_deps() { fi if host_flatcar_distro; then - install_sysbox_deps_flatcar + # Mirror the non-Flatcar branch: only attempt to install shiftfs when + # the host kernel is in the supported range. Flatcar 4593+ ships + # kernel 6.x where shiftfs is unavailable (and unnecessary, as the + # kernel provides idmap mounts), so the prebuilt shiftfs.ko from + # sysbox-flatcar-preview no longer applies. + if semver_lt $kversion 6.3; then + install_sysbox_deps_flatcar + else + echo "Skipping shiftfs installation (kernel version $kversion is above the max required for shiftfs ($shiftfs_max_kernel_ver))." + fi else if semver_ge $kversion 5.4 && semver_lt $kversion 5.8; then cp -r "/opt/shiftfs-k5.4" "$host_run/shiftfs-dkms" From 9e3e06861c10eeb8a2304ffc4f87ac0a17b350cc Mon Sep 17 00:00:00 2001 From: Till0196 <16399842+Till0196@users.noreply.github.com> Date: Tue, 19 May 2026 18:27:23 +0900 Subject: [PATCH 4/4] k8s: allow Sysbox-CE on Flatcar 4593+ (kernel 6.x) Drop the EE-only guard in do_distro_adjustments(). The guard was added when shiftfs was the only path to make sysbox work on Flatcar, and shiftfs was an EE-exclusive preview build. On kernel 6.x Flatcar ships mainline idmap mount support, which sysbox-runc uses transparently; the EE-only justification no longer holds, and the companion change to skip install_sysbox_deps_flatcar() above the shiftfs ceiling already makes the install path correct for CE on 6.x. Also fold the Flatcar-specific BinaryName rewrite for the containerd drop-in into do_distro_adjustments(), matching how every other Flatcar path override (systemd units, crio.conf, helper scripts) is handled in the same function. Addresses: nestybox/sysbox#995 --- k8s/scripts/sysbox-deploy-k8s.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/k8s/scripts/sysbox-deploy-k8s.sh b/k8s/scripts/sysbox-deploy-k8s.sh index 6779b8f..d7a6c74 100755 --- a/k8s/scripts/sysbox-deploy-k8s.sh +++ b/k8s/scripts/sysbox-deploy-k8s.sh @@ -1276,11 +1276,6 @@ function do_distro_adjustments() { return fi - # Ensure that Flatcar installation proceeds only in Sysbox-EE case. - if [[ ${sysbox_edition} != "Sysbox-EE" ]]; then - die "Flatcar OS distribution is only supported on Sysbox Enterprise-Edition. Exiting ..." - fi - # Adjust global vars. host_bin="/mnt/host/opt/bin" host_local_bin="/mnt/host/opt/local/bin" @@ -1311,6 +1306,9 @@ function do_distro_adjustments() { sed -i '/^ExecStart=/ s@/usr/local/bin@/opt/local/bin@g' ${sysbox_artifacts}/systemd/sysbox-installer-helper.service sed -i '/^ExecStart=/ s@/usr/local/bin@/opt/local/bin@g' ${sysbox_artifacts}/systemd/sysbox-removal-helper.service + # Adjust the containerd drop-in used on k3s / RKE2. + sed -i 's@/usr/bin/sysbox-runc@/opt/bin/sysbox-runc@' ${sysbox_artifacts}/config/containerd-sysbox-dropin.toml + # Sysctl adjustments. sed -i '/^kernel.unprivileged_userns_clone/ s/^#*/# /' ${sysbox_artifacts}/systemd/99-sysbox-sysctl.conf }