From 3fa496b29c748ddf6285ed5f65a8be7f870817a2 Mon Sep 17 00:00:00 2001 From: "Wesierski, Dawid" Date: Fri, 29 May 2026 10:03:40 +0000 Subject: [PATCH 1/3] Add: continuous isolation monitor with spike analysis --- script/isolation_monitor.sh | 222 ++++++++++++++++++++++++++++++++++++ script/isolation_report.sh | 4 + 2 files changed, 226 insertions(+) create mode 100755 script/isolation_monitor.sh diff --git a/script/isolation_monitor.sh b/script/isolation_monitor.sh new file mode 100755 index 000000000..8097c863c --- /dev/null +++ b/script/isolation_monitor.sh @@ -0,0 +1,222 @@ +#!/bin/bash +# isolation_monitor.sh — continuous isolation monitoring with spike analysis +# +# Runs isolation_report.sh in a tight loop (default 30s samples), +# logs every iteration with timestamps, and on Ctrl+C prints a +# spike analysis showing when the worst isolation violations occurred. +# +# Usage: +# sudo ./isolation_monitor.sh [sample_duration] [cpu_min] [cpu_max] +# sudo ./isolation_monitor.sh # 5s samples, auto-detect cores +# sudo ./isolation_monitor.sh 3 # 3s samples +# sudo ./isolation_monitor.sh 5 2 7 # 5s samples, CPUs 2-7 + +set -euo pipefail + +SAMPLE_DURATION=${1:-30} +CPU_ARGS=("${@:2}") + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPORT_SCRIPT="${SCRIPT_DIR}/isolation_report.sh" +LOG_DIR="/tmp/isolation_monitor_$$" +SUMMARY_CSV="${LOG_DIR}/summary.csv" + +mkdir -p "$LOG_DIR" + +# ── state ── +ITERATION=0 +declare -a TIMESTAMPS=() +declare -a TOTALS=() +declare -a INTRUDERS=() +declare -a ALLOWED_COUNTS=() +declare -a EXPECTED_COUNTS=() +declare -a INTRUDER_DETAILS=() +declare -a LOG_FILES=() + +# ── cleanup & analysis on exit ── +finish() { + echo "" + echo "" + local S="================================================================" + + # Use actual completed samples, not ITERATION (which may have been + # incremented before the last sample finished). + local completed=${#TIMESTAMPS[@]} + + if ((completed == 0)); then + echo "No samples collected." + rm -rf "$LOG_DIR" + exit 0 + fi + + printf '\n%s\n SPIKE ANALYSIS (%d samples collected)\n%s\n\n' "$S" "$completed" "$S" + + # CSV header + printf '%-6s %-24s %10s %10s %10s %10s\n' \ + "#" "TIMESTAMP" "TOTAL_SW" "EXPECTED" "ALLOWED" "INTRUDER" + printf ' %s\n' "$(printf '%.0s-' {1..80})" + + local max_intruder=0 max_intruder_idx=0 + local max_total=0 max_total_idx=0 + + for ((i = 0; i < completed; i++)); do + printf '%-6d %-24s %10d %10d %10d %10d' \ + "$((i + 1))" "${TIMESTAMPS[$i]}" "${TOTALS[$i]}" \ + "${EXPECTED_COUNTS[$i]}" "${ALLOWED_COUNTS[$i]}" "${INTRUDERS[$i]}" + + # track worst intruder spike + if ((INTRUDERS[i] > max_intruder)); then + max_intruder=${INTRUDERS[$i]} + max_intruder_idx=$i + fi + # track worst total spike + if ((TOTALS[i] > max_total)); then + max_total=${TOTALS[$i]} + max_total_idx=$i + fi + + # mark spikes inline + if ((INTRUDERS[i] > 0)); then + printf ' ← INTRUDERS!' + fi + printf '\n' + done + + printf '\n%s\n WORST SPIKES\n%s\n\n' "$S" "$S" + + printf ' Biggest TOTAL switch spike:\n' + printf ' Sample #%d | %s | %d total switches\n\n' \ + "$((max_total_idx + 1))" "${TIMESTAMPS[$max_total_idx]}" "$max_total" + + printf ' Biggest INTRUDER spike:\n' + if ((max_intruder > 0)); then + printf ' Sample #%d | %s | %d intruder switches\n' \ + "$((max_intruder_idx + 1))" "${TIMESTAMPS[$max_intruder_idx]}" "$max_intruder" + if [[ -n "${INTRUDER_DETAILS[$max_intruder_idx]:-}" ]]; then + printf ' Intruders: %s\n' "${INTRUDER_DETAILS[$max_intruder_idx]}" + fi + else + printf ' None — no intruders observed across all samples!\n' + fi + + # compute averages + local sum_total=0 sum_intruder=0 sum_allowed=0 + for ((i = 0; i < completed; i++)); do + sum_total=$((sum_total + TOTALS[i])) + sum_intruder=$((sum_intruder + INTRUDERS[i])) + sum_allowed=$((sum_allowed + ALLOWED_COUNTS[i])) + done + + printf '\n%s\n AVERAGES (per %ds sample)\n%s\n\n' "$S" "$SAMPLE_DURATION" "$S" + printf ' Avg total switches : %d\n' "$((sum_total / completed))" + printf ' Avg allowed switches: %d\n' "$((sum_allowed / completed))" + printf ' Avg intruder switches: %d\n' "$((sum_intruder / completed))" + + # rename max spike files with _MAX suffix + if ((max_intruder > 0)); then + local src="${LOG_FILES[$max_intruder_idx]}" + local dst="${src%.log}_MAX_INTRUDER.log" + mv "$src" "$dst" 2>/dev/null && printf '\n Renamed worst intruder log → %s\n' "$(basename "$dst")" + fi + if ((max_total > 0)); then + local src="${LOG_FILES[$max_total_idx]}" + if [[ -f "$src" ]]; then + local dst="${src%.log}_MAX_TOTAL.log" + mv "$src" "$dst" 2>/dev/null && printf ' Renamed worst total log → %s\n' "$(basename "$dst")" + else + # already renamed as intruder max — add total tag too + local prev="${src%.log}_MAX_INTRUDER.log" + local dst="${src%.log}_MAX_INTRUDER_MAX_TOTAL.log" + mv "$prev" "$dst" 2>/dev/null && printf ' Renamed worst total log → %s\n' "$(basename "$dst")" + fi + fi + + printf '\n Full logs saved in: %s/\n\n' "$LOG_DIR" + printf '%s\n Done — %d samples over %s\n%s\n\n' \ + "$S" "$completed" \ + "$(date -u -d @$((completed * SAMPLE_DURATION)) +%H:%M:%S)" \ + "$S" +} + +trap finish EXIT + +# ── parse one report output, extract counts ── +parse_report() { + local log_file="$1" + local total=0 expected=0 allowed=0 intruder=0 + local intruder_names="" + + while IFS= read -r line; do + if [[ "$line" =~ Total\ switches\ observed\ :\ ([0-9]+) ]]; then + total="${BASH_REMATCH[1]}" + elif [[ "$line" =~ EXPECTED\ switches.*:\ ([0-9]+) ]]; then + expected="${BASH_REMATCH[1]}" + elif [[ "$line" =~ ALLOWED.*switches.*:\ ([0-9]+) ]]; then + allowed="${BASH_REMATCH[1]}" + elif [[ "$line" =~ INTRUDER\ switches.*:\ ([0-9]+) ]]; then + intruder="${BASH_REMATCH[1]}" + elif [[ "$line" =~ ^[[:space:]]+X[[:space:]]+(.+) ]]; then + local name="${BASH_REMATCH[1]}" + if [[ -n "$intruder_names" ]]; then + intruder_names+="; ${name}" + else + intruder_names="${name}" + fi + fi + done <"$log_file" + + echo "${total} ${expected} ${allowed} ${intruder}" + # stash intruder details via global + _PARSED_INTRUDER_DETAILS="$intruder_names" +} + +# ── validate inputs ── +if ! [[ "$SAMPLE_DURATION" =~ ^[0-9]+$ ]] || ((SAMPLE_DURATION < 1)); then + echo "Error: sample duration must be a positive integer (got: '$SAMPLE_DURATION')" >&2 + exit 1 +fi + +if [[ ! -x "$REPORT_SCRIPT" ]]; then + echo "Error: cannot find or execute $REPORT_SCRIPT" >&2 + exit 1 +fi + +echo "================================================================" +echo " Continuous Isolation Monitor" +echo " Sample duration : ${SAMPLE_DURATION}s" +echo " CPU args : ${CPU_ARGS[*]:-auto-detect}" +echo " Log directory : ${LOG_DIR}/" +echo " Press Ctrl+C to stop and see spike analysis" +echo "================================================================" +echo "" + +# ── main loop ── +while true; do + ITERATION=$((ITERATION + 1)) + ts=$(date '+%Y-%m-%d %H:%M:%S') + ts_file=$(date '+%Y-%m-%d_%H-%M-%S') + log_file="${LOG_DIR}/${ts_file}.log" + + printf '\n──── Sample #%d | %s ────\n' "$ITERATION" "$ts" + + # run the report, capture output + "$REPORT_SCRIPT" "$SAMPLE_DURATION" "${CPU_ARGS[@]}" 2>&1 | tee "$log_file" + + # parse results + _PARSED_INTRUDER_DETAILS="" + read -r total expected allowed intruder <<<"$(parse_report "$log_file")" + + TIMESTAMPS+=("$ts") + TOTALS+=("${total:-0}") + EXPECTED_COUNTS+=("${expected:-0}") + ALLOWED_COUNTS+=("${allowed:-0}") + INTRUDERS+=("${intruder:-0}") + INTRUDER_DETAILS+=("$_PARSED_INTRUDER_DETAILS") + LOG_FILES+=("$log_file") + + # quick inline status + printf ' → Sample #%d: total=%d expected=%d allowed=%d intruder=%d\n' \ + "$ITERATION" "${total:-0}" "${expected:-0}" "${allowed:-0}" "${intruder:-0}" + + # no sleep — loop immediately for minimal downtime +done diff --git a/script/isolation_report.sh b/script/isolation_report.sh index 43f842ff1..68ea4f6ab 100755 --- a/script/isolation_report.sh +++ b/script/isolation_report.sh @@ -88,6 +88,10 @@ HELPEOF case "${1:-}" in -h | --help) usage ;; esac DURATION=${1:-5} +if ! [[ "$DURATION" =~ ^[0-9]+$ ]] || ((DURATION < 1)); then + echo "Error: duration must be a positive integer (got: '$DURATION')" >&2 + exit 1 +fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RAW_LOG=$(mktemp /tmp/sched_audit_XXXXXX.log) trap 'rm -f "$RAW_LOG"' EXIT From aa3812469a0c3fb09dcbb2e7dc73aeec347f5049 Mon Sep 17 00:00:00 2001 From: "Wesierski, Dawid" Date: Fri, 29 May 2026 10:45:28 +0000 Subject: [PATCH 2/3] Add: support for flexible cpu schema --- script/isolation_monitor.sh | 8 ++-- script/isolation_report.sh | 91 +++++++++++++++++++++++++------------ 2 files changed, 66 insertions(+), 33 deletions(-) diff --git a/script/isolation_monitor.sh b/script/isolation_monitor.sh index 8097c863c..b3f97f48a 100755 --- a/script/isolation_monitor.sh +++ b/script/isolation_monitor.sh @@ -6,10 +6,12 @@ # spike analysis showing when the worst isolation violations occurred. # # Usage: -# sudo ./isolation_monitor.sh [sample_duration] [cpu_min] [cpu_max] -# sudo ./isolation_monitor.sh # 5s samples, auto-detect cores +# sudo ./isolation_monitor.sh [sample_duration] [cpu_spec...] +# sudo ./isolation_monitor.sh # 30s samples, auto-detect cores # sudo ./isolation_monitor.sh 3 # 3s samples -# sudo ./isolation_monitor.sh 5 2 7 # 5s samples, CPUs 2-7 +# sudo ./isolation_monitor.sh 5 2-7 # 5s samples, CPUs 2-7 +# sudo ./isolation_monitor.sh 5 2 5 7 # 5s samples, CPUs 2, 5, 7 +# sudo ./isolation_monitor.sh 5 2-4,7 # 5s samples, CPUs 2,3,4,7 set -euo pipefail diff --git a/script/isolation_report.sh b/script/isolation_report.sh index 68ea4f6ab..263ba15ea 100755 --- a/script/isolation_report.sh +++ b/script/isolation_report.sh @@ -6,28 +6,32 @@ # INTRUDER — should not be on isolated cores # # Usage: -# sudo ./sched_audit.sh [options] [duration_sec] [cpu_min] [cpu_max] -# sudo ./sched_audit.sh 5 2 3 # 5s sample on CPUs 2-3 (overrides auto-detect) -# sudo ./sched_audit.sh 10 # 10s sample, auto-detect isolated range -# sudo ./sched_audit.sh -h # show help +# sudo ./sched_audit.sh [options] [duration_sec] [cpu_spec...] +# sudo ./sched_audit.sh 5 2 3 # 5s sample on CPUs 2-3 (range) +# sudo ./sched_audit.sh 5 2-7 # 5s sample on CPUs 2-7 (range notation) +# sudo ./sched_audit.sh 5 2 5 7 # 5s sample on CPUs 2, 5, 7 (discrete) +# sudo ./sched_audit.sh 5 2-4,7 # 5s sample on CPUs 2,3,4,7 (mixed) +# sudo ./sched_audit.sh 10 # 10s sample, auto-detect isolated range +# sudo ./sched_audit.sh -h # show help set -euo pipefail usage() { cat <<'HELPEOF' -Usage: sudo isolation_report.sh [options] [duration_sec] [cpu_min] [cpu_max] +Usage: sudo isolation_report.sh [options] [duration_sec] [cpu_spec...] Run a bpftrace sched_switch audit on isolated (or specified) CPU cores, then classify every observed task as EXPECTED, ALLOWED, or INTRUDER. POSITIONAL ARGUMENTS: duration_sec Sampling duration in seconds (default: 5) - cpu_min First CPU core to monitor (default: auto-detect) - cpu_max Last CPU core to monitor (default: auto-detect) - - If cpu_min and cpu_max are given they OVERRIDE the auto-detected - isolcpus range. If omitted, the script reads isolcpus from - /sys/devices/system/cpu/isolated or /proc/cmdline. + cpu_spec CPU cores to monitor. Supports: + 2 7 range min-max (legacy) + 2-7 range notation + 2,5,7 comma-separated + 2-4,7,9-11 mixed ranges and singles + 2 5 7 space-separated discrete list + (default: auto-detect from isolcpus) OPTIONS: -h, --help Show this help message and exit @@ -35,8 +39,10 @@ OPTIONS: EXAMPLES: sudo ./isolation_report.sh # 5s, auto-detect isolated cores sudo ./isolation_report.sh 10 # 10s, auto-detect - sudo ./isolation_report.sh 5 2 3 # 5s, force CPUs 2-3 - sudo ./isolation_report.sh 10 4 7 # 10s, force CPUs 4-7 + sudo ./isolation_report.sh 5 2 3 # 5s, force CPUs 2-3 (range) + sudo ./isolation_report.sh 5 2-7 # 5s, force CPUs 2-7 + sudo ./isolation_report.sh 5 2 5 7 # 5s, force CPUs 2,5,7 (discrete) + sudo ./isolation_report.sh 5 2-4,7 # 5s, CPUs 2,3,4,7 PATTERN FILES (RC files): Every thread observed on the monitored cores is classified by matching @@ -103,35 +109,60 @@ detect_isolated() { echo "$iso" } +# Expand a cpulist spec (e.g. "2-4,7,9-11") into a sorted, deduplicated +# space-separated list of individual CPU numbers. expand_cpulist() { python3 -c " -s='$1'; r=[] +import sys +s='$1'; r=set() for p in s.split(','): - a,_,b=p.partition('-'); r.extend(range(int(a),int(b or a)+1)) -print(min(r), max(r)) + p=p.strip() + if not p: continue + a,_,b=p.partition('-') + r.update(range(int(a),int(b or a)+1)) +print(' '.join(str(x) for x in sorted(r))) " } -if [[ -n "${2:-}" && -n "${3:-}" ]]; then - # Manual override — always wins - CPU_MIN=$2 - CPU_MAX=$3 - echo "✓ Manual override — monitoring CPUs ${CPU_MIN}-${CPU_MAX}" +# Build CPU_LIST (space-separated individual CPUs) from arguments +if [[ -n "${2:-}" ]]; then + # Join all args after duration with commas, then expand + raw="${*:2}" + # Replace spaces with commas so "2 5 7" becomes "2,5,7" + raw="${raw// /,}" + read -ra CPU_LIST <<<"$(expand_cpulist "$raw")" + echo "✓ Manual override — monitoring CPUs: ${CPU_LIST[*]}" else ISO=$(detect_isolated) if [[ -z "$ISO" ]]; then - CPU_MIN=${2:-2} - CPU_MAX=${3:-3} - echo "⚠ No isolcpus found — using CPU${CPU_MIN}-CPU${CPU_MAX}" + CPU_LIST=(2 3) + echo "⚠ No isolcpus found — using CPUs: ${CPU_LIST[*]}" else - read -r CPU_MIN CPU_MAX <<<"$(expand_cpulist "$ISO")" - echo "✓ Detected isolated cores: $ISO (min=$CPU_MIN max=$CPU_MAX)" + read -ra CPU_LIST <<<"$(expand_cpulist "$ISO")" + echo "✓ Detected isolated cores: $ISO → CPUs: ${CPU_LIST[*]}" fi fi +# Derive min/max for display +CPU_MIN=${CPU_LIST[0]} +CPU_MAX=${CPU_LIST[-1]} + +# Build bpftrace CPU filter expression +if ((CPU_MAX - CPU_MIN + 1 == ${#CPU_LIST[@]})); then + # Contiguous range — use efficient >= && <= + BPF_CPU_FILTER="cpu >= $CPU_MIN && cpu <= $CPU_MAX" +else + # Non-contiguous — use explicit || checks + filter_parts=() + for c in "${CPU_LIST[@]}"; do + filter_parts+=("cpu == $c") + done + BPF_CPU_FILTER=$(IFS='|'; echo "${filter_parts[*]}" | sed 's/|/ || /g') +fi + S="================================================================" -printf '\n%s\n sched_switch audit | cores %d-%d | %ds sample\n%s\n\n' \ - "$S" "$CPU_MIN" "$CPU_MAX" "$DURATION" "$S" +printf '\n%s\n sched_switch audit | cores [%s] | %ds sample\n%s\n\n' \ + "$S" "${CPU_LIST[*]}" "$DURATION" "$S" # Load patterns from RC files (skip blank lines and comments) load_patterns() { @@ -178,7 +209,7 @@ classify() { BTRACE_PROG=' tracepoint:sched:sched_switch -/ cpu >= '"$CPU_MIN"' && cpu <= '"$CPU_MAX"' / +/ '"$BPF_CPU_FILTER"' / { @sw[cpu, args->next_comm, args->next_pid] = count(); } @@ -192,7 +223,7 @@ interval:s:'"$DURATION"' ' echo "Running bpftrace for ${DURATION}s..." -echo "(watching CPUs ${CPU_MIN}–${CPU_MAX} for sched_switch events)" +echo "(watching CPUs [${CPU_LIST[*]}] for sched_switch events)" echo "" sudo bpftrace -e "$BTRACE_PROG" 2>/dev/null | tee "$RAW_LOG" From d660415ba1f2fcbece57a9101555b623c30b75fd Mon Sep 17 00:00:00 2001 From: "Wesierski, Dawid" Date: Tue, 14 Jul 2026 09:26:33 +0000 Subject: [PATCH 3/3] Fix: Remove code-injection and insecure tempdir in isolation scripts expand_cpulist() interpolated the untrusted cpu_spec argument directly into a python3 -c source string, letting a crafted spec break out of the string literal and execute arbitrary Python. Pass it through the CPU_SPEC environment variable instead. isolation_monitor.sh created its log directory at a predictable /tmp/isolation_monitor_$$ path with a plain mkdir, which is subject to a pre-creation symlink attack. Use mktemp -d for an unpredictable, owner-only directory. Also fix a stale sched_audit.sh reference in the usage header left over from a prior rename. Signed-off-by: Wesierski, Dawid --- script/isolation_monitor.sh | 5 +---- script/isolation_report.sh | 28 +++++++++++++++++----------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/script/isolation_monitor.sh b/script/isolation_monitor.sh index b3f97f48a..d56db5ece 100755 --- a/script/isolation_monitor.sh +++ b/script/isolation_monitor.sh @@ -20,10 +20,7 @@ CPU_ARGS=("${@:2}") SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPORT_SCRIPT="${SCRIPT_DIR}/isolation_report.sh" -LOG_DIR="/tmp/isolation_monitor_$$" -SUMMARY_CSV="${LOG_DIR}/summary.csv" - -mkdir -p "$LOG_DIR" +LOG_DIR=$(mktemp -d /tmp/isolation_monitor_XXXXXX) # ── state ── ITERATION=0 diff --git a/script/isolation_report.sh b/script/isolation_report.sh index 263ba15ea..80383b4f3 100755 --- a/script/isolation_report.sh +++ b/script/isolation_report.sh @@ -6,13 +6,13 @@ # INTRUDER — should not be on isolated cores # # Usage: -# sudo ./sched_audit.sh [options] [duration_sec] [cpu_spec...] -# sudo ./sched_audit.sh 5 2 3 # 5s sample on CPUs 2-3 (range) -# sudo ./sched_audit.sh 5 2-7 # 5s sample on CPUs 2-7 (range notation) -# sudo ./sched_audit.sh 5 2 5 7 # 5s sample on CPUs 2, 5, 7 (discrete) -# sudo ./sched_audit.sh 5 2-4,7 # 5s sample on CPUs 2,3,4,7 (mixed) -# sudo ./sched_audit.sh 10 # 10s sample, auto-detect isolated range -# sudo ./sched_audit.sh -h # show help +# sudo ./isolation_report.sh [options] [duration_sec] [cpu_spec...] +# sudo ./isolation_report.sh 5 2 3 # 5s sample on CPUs 2-3 (range) +# sudo ./isolation_report.sh 5 2-7 # 5s sample on CPUs 2-7 (range notation) +# sudo ./isolation_report.sh 5 2 5 7 # 5s sample on CPUs 2, 5, 7 (discrete) +# sudo ./isolation_report.sh 5 2-4,7 # 5s sample on CPUs 2,3,4,7 (mixed) +# sudo ./isolation_report.sh 10 # 10s sample, auto-detect isolated range +# sudo ./isolation_report.sh -h # show help set -euo pipefail @@ -111,10 +111,12 @@ detect_isolated() { # Expand a cpulist spec (e.g. "2-4,7,9-11") into a sorted, deduplicated # space-separated list of individual CPU numbers. +# The spec is passed via CPU_SPEC env var (not interpolated into the +# Python source) to avoid code injection from untrusted input. expand_cpulist() { - python3 -c " -import sys -s='$1'; r=set() + CPU_SPEC="$1" python3 -c " +import os +s=os.environ['CPU_SPEC']; r=set() for p in s.split(','): p=p.strip() if not p: continue @@ -157,7 +159,11 @@ else for c in "${CPU_LIST[@]}"; do filter_parts+=("cpu == $c") done - BPF_CPU_FILTER=$(IFS='|'; echo "${filter_parts[*]}" | sed 's/|/ || /g') + joined=$( + IFS='|' + echo "${filter_parts[*]}" + ) + BPF_CPU_FILTER=${joined//|/ || } fi S="================================================================"