diff --git a/script/isolation_monitor.sh b/script/isolation_monitor.sh new file mode 100755 index 000000000..d56db5ece --- /dev/null +++ b/script/isolation_monitor.sh @@ -0,0 +1,221 @@ +#!/bin/bash +# isolation_monitor.sh — continuous isolation monitoring with spike analysis +# +# Runs isolation_report.sh in a tight loop (default 30s samples), +# logs every iteration with timestamps, and on Ctrl+C prints a +# spike analysis showing when the worst isolation violations occurred. +# +# Usage: +# sudo ./isolation_monitor.sh [sample_duration] [cpu_spec...] +# sudo ./isolation_monitor.sh # 30s samples, auto-detect cores +# sudo ./isolation_monitor.sh 3 # 3s samples +# sudo ./isolation_monitor.sh 5 2-7 # 5s samples, CPUs 2-7 +# sudo ./isolation_monitor.sh 5 2 5 7 # 5s samples, CPUs 2, 5, 7 +# sudo ./isolation_monitor.sh 5 2-4,7 # 5s samples, CPUs 2,3,4,7 + +set -euo pipefail + +SAMPLE_DURATION=${1:-30} +CPU_ARGS=("${@:2}") + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPORT_SCRIPT="${SCRIPT_DIR}/isolation_report.sh" +LOG_DIR=$(mktemp -d /tmp/isolation_monitor_XXXXXX) + +# ── state ── +ITERATION=0 +declare -a TIMESTAMPS=() +declare -a TOTALS=() +declare -a INTRUDERS=() +declare -a ALLOWED_COUNTS=() +declare -a EXPECTED_COUNTS=() +declare -a INTRUDER_DETAILS=() +declare -a LOG_FILES=() + +# ── cleanup & analysis on exit ── +finish() { + echo "" + echo "" + local S="================================================================" + + # Use actual completed samples, not ITERATION (which may have been + # incremented before the last sample finished). + local completed=${#TIMESTAMPS[@]} + + if ((completed == 0)); then + echo "No samples collected." + rm -rf "$LOG_DIR" + exit 0 + fi + + printf '\n%s\n SPIKE ANALYSIS (%d samples collected)\n%s\n\n' "$S" "$completed" "$S" + + # CSV header + printf '%-6s %-24s %10s %10s %10s %10s\n' \ + "#" "TIMESTAMP" "TOTAL_SW" "EXPECTED" "ALLOWED" "INTRUDER" + printf ' %s\n' "$(printf '%.0s-' {1..80})" + + local max_intruder=0 max_intruder_idx=0 + local max_total=0 max_total_idx=0 + + for ((i = 0; i < completed; i++)); do + printf '%-6d %-24s %10d %10d %10d %10d' \ + "$((i + 1))" "${TIMESTAMPS[$i]}" "${TOTALS[$i]}" \ + "${EXPECTED_COUNTS[$i]}" "${ALLOWED_COUNTS[$i]}" "${INTRUDERS[$i]}" + + # track worst intruder spike + if ((INTRUDERS[i] > max_intruder)); then + max_intruder=${INTRUDERS[$i]} + max_intruder_idx=$i + fi + # track worst total spike + if ((TOTALS[i] > max_total)); then + max_total=${TOTALS[$i]} + max_total_idx=$i + fi + + # mark spikes inline + if ((INTRUDERS[i] > 0)); then + printf ' ← INTRUDERS!' + fi + printf '\n' + done + + printf '\n%s\n WORST SPIKES\n%s\n\n' "$S" "$S" + + printf ' Biggest TOTAL switch spike:\n' + printf ' Sample #%d | %s | %d total switches\n\n' \ + "$((max_total_idx + 1))" "${TIMESTAMPS[$max_total_idx]}" "$max_total" + + printf ' Biggest INTRUDER spike:\n' + if ((max_intruder > 0)); then + printf ' Sample #%d | %s | %d intruder switches\n' \ + "$((max_intruder_idx + 1))" "${TIMESTAMPS[$max_intruder_idx]}" "$max_intruder" + if [[ -n "${INTRUDER_DETAILS[$max_intruder_idx]:-}" ]]; then + printf ' Intruders: %s\n' "${INTRUDER_DETAILS[$max_intruder_idx]}" + fi + else + printf ' None — no intruders observed across all samples!\n' + fi + + # compute averages + local sum_total=0 sum_intruder=0 sum_allowed=0 + for ((i = 0; i < completed; i++)); do + sum_total=$((sum_total + TOTALS[i])) + sum_intruder=$((sum_intruder + INTRUDERS[i])) + sum_allowed=$((sum_allowed + ALLOWED_COUNTS[i])) + done + + printf '\n%s\n AVERAGES (per %ds sample)\n%s\n\n' "$S" "$SAMPLE_DURATION" "$S" + printf ' Avg total switches : %d\n' "$((sum_total / completed))" + printf ' Avg allowed switches: %d\n' "$((sum_allowed / completed))" + printf ' Avg intruder switches: %d\n' "$((sum_intruder / completed))" + + # rename max spike files with _MAX suffix + if ((max_intruder > 0)); then + local src="${LOG_FILES[$max_intruder_idx]}" + local dst="${src%.log}_MAX_INTRUDER.log" + mv "$src" "$dst" 2>/dev/null && printf '\n Renamed worst intruder log → %s\n' "$(basename "$dst")" + fi + if ((max_total > 0)); then + local src="${LOG_FILES[$max_total_idx]}" + if [[ -f "$src" ]]; then + local dst="${src%.log}_MAX_TOTAL.log" + mv "$src" "$dst" 2>/dev/null && printf ' Renamed worst total log → %s\n' "$(basename "$dst")" + else + # already renamed as intruder max — add total tag too + local prev="${src%.log}_MAX_INTRUDER.log" + local dst="${src%.log}_MAX_INTRUDER_MAX_TOTAL.log" + mv "$prev" "$dst" 2>/dev/null && printf ' Renamed worst total log → %s\n' "$(basename "$dst")" + fi + fi + + printf '\n Full logs saved in: %s/\n\n' "$LOG_DIR" + printf '%s\n Done — %d samples over %s\n%s\n\n' \ + "$S" "$completed" \ + "$(date -u -d @$((completed * SAMPLE_DURATION)) +%H:%M:%S)" \ + "$S" +} + +trap finish EXIT + +# ── parse one report output, extract counts ── +parse_report() { + local log_file="$1" + local total=0 expected=0 allowed=0 intruder=0 + local intruder_names="" + + while IFS= read -r line; do + if [[ "$line" =~ Total\ switches\ observed\ :\ ([0-9]+) ]]; then + total="${BASH_REMATCH[1]}" + elif [[ "$line" =~ EXPECTED\ switches.*:\ ([0-9]+) ]]; then + expected="${BASH_REMATCH[1]}" + elif [[ "$line" =~ ALLOWED.*switches.*:\ ([0-9]+) ]]; then + allowed="${BASH_REMATCH[1]}" + elif [[ "$line" =~ INTRUDER\ switches.*:\ ([0-9]+) ]]; then + intruder="${BASH_REMATCH[1]}" + elif [[ "$line" =~ ^[[:space:]]+X[[:space:]]+(.+) ]]; then + local name="${BASH_REMATCH[1]}" + if [[ -n "$intruder_names" ]]; then + intruder_names+="; ${name}" + else + intruder_names="${name}" + fi + fi + done <"$log_file" + + echo "${total} ${expected} ${allowed} ${intruder}" + # stash intruder details via global + _PARSED_INTRUDER_DETAILS="$intruder_names" +} + +# ── validate inputs ── +if ! [[ "$SAMPLE_DURATION" =~ ^[0-9]+$ ]] || ((SAMPLE_DURATION < 1)); then + echo "Error: sample duration must be a positive integer (got: '$SAMPLE_DURATION')" >&2 + exit 1 +fi + +if [[ ! -x "$REPORT_SCRIPT" ]]; then + echo "Error: cannot find or execute $REPORT_SCRIPT" >&2 + exit 1 +fi + +echo "================================================================" +echo " Continuous Isolation Monitor" +echo " Sample duration : ${SAMPLE_DURATION}s" +echo " CPU args : ${CPU_ARGS[*]:-auto-detect}" +echo " Log directory : ${LOG_DIR}/" +echo " Press Ctrl+C to stop and see spike analysis" +echo "================================================================" +echo "" + +# ── main loop ── +while true; do + ITERATION=$((ITERATION + 1)) + ts=$(date '+%Y-%m-%d %H:%M:%S') + ts_file=$(date '+%Y-%m-%d_%H-%M-%S') + log_file="${LOG_DIR}/${ts_file}.log" + + printf '\n──── Sample #%d | %s ────\n' "$ITERATION" "$ts" + + # run the report, capture output + "$REPORT_SCRIPT" "$SAMPLE_DURATION" "${CPU_ARGS[@]}" 2>&1 | tee "$log_file" + + # parse results + _PARSED_INTRUDER_DETAILS="" + read -r total expected allowed intruder <<<"$(parse_report "$log_file")" + + TIMESTAMPS+=("$ts") + TOTALS+=("${total:-0}") + EXPECTED_COUNTS+=("${expected:-0}") + ALLOWED_COUNTS+=("${allowed:-0}") + INTRUDERS+=("${intruder:-0}") + INTRUDER_DETAILS+=("$_PARSED_INTRUDER_DETAILS") + LOG_FILES+=("$log_file") + + # quick inline status + printf ' → Sample #%d: total=%d expected=%d allowed=%d intruder=%d\n' \ + "$ITERATION" "${total:-0}" "${expected:-0}" "${allowed:-0}" "${intruder:-0}" + + # no sleep — loop immediately for minimal downtime +done diff --git a/script/isolation_report.sh b/script/isolation_report.sh index 43f842ff1..80383b4f3 100755 --- a/script/isolation_report.sh +++ b/script/isolation_report.sh @@ -6,28 +6,32 @@ # INTRUDER — should not be on isolated cores # # Usage: -# sudo ./sched_audit.sh [options] [duration_sec] [cpu_min] [cpu_max] -# sudo ./sched_audit.sh 5 2 3 # 5s sample on CPUs 2-3 (overrides auto-detect) -# sudo ./sched_audit.sh 10 # 10s sample, auto-detect isolated range -# sudo ./sched_audit.sh -h # show help +# sudo ./isolation_report.sh [options] [duration_sec] [cpu_spec...] +# sudo ./isolation_report.sh 5 2 3 # 5s sample on CPUs 2-3 (range) +# sudo ./isolation_report.sh 5 2-7 # 5s sample on CPUs 2-7 (range notation) +# sudo ./isolation_report.sh 5 2 5 7 # 5s sample on CPUs 2, 5, 7 (discrete) +# sudo ./isolation_report.sh 5 2-4,7 # 5s sample on CPUs 2,3,4,7 (mixed) +# sudo ./isolation_report.sh 10 # 10s sample, auto-detect isolated range +# sudo ./isolation_report.sh -h # show help set -euo pipefail usage() { cat <<'HELPEOF' -Usage: sudo isolation_report.sh [options] [duration_sec] [cpu_min] [cpu_max] +Usage: sudo isolation_report.sh [options] [duration_sec] [cpu_spec...] Run a bpftrace sched_switch audit on isolated (or specified) CPU cores, then classify every observed task as EXPECTED, ALLOWED, or INTRUDER. POSITIONAL ARGUMENTS: duration_sec Sampling duration in seconds (default: 5) - cpu_min First CPU core to monitor (default: auto-detect) - cpu_max Last CPU core to monitor (default: auto-detect) - - If cpu_min and cpu_max are given they OVERRIDE the auto-detected - isolcpus range. If omitted, the script reads isolcpus from - /sys/devices/system/cpu/isolated or /proc/cmdline. + cpu_spec CPU cores to monitor. Supports: + 2 7 range min-max (legacy) + 2-7 range notation + 2,5,7 comma-separated + 2-4,7,9-11 mixed ranges and singles + 2 5 7 space-separated discrete list + (default: auto-detect from isolcpus) OPTIONS: -h, --help Show this help message and exit @@ -35,8 +39,10 @@ OPTIONS: EXAMPLES: sudo ./isolation_report.sh # 5s, auto-detect isolated cores sudo ./isolation_report.sh 10 # 10s, auto-detect - sudo ./isolation_report.sh 5 2 3 # 5s, force CPUs 2-3 - sudo ./isolation_report.sh 10 4 7 # 10s, force CPUs 4-7 + sudo ./isolation_report.sh 5 2 3 # 5s, force CPUs 2-3 (range) + sudo ./isolation_report.sh 5 2-7 # 5s, force CPUs 2-7 + sudo ./isolation_report.sh 5 2 5 7 # 5s, force CPUs 2,5,7 (discrete) + sudo ./isolation_report.sh 5 2-4,7 # 5s, CPUs 2,3,4,7 PATTERN FILES (RC files): Every thread observed on the monitored cores is classified by matching @@ -88,6 +94,10 @@ HELPEOF case "${1:-}" in -h | --help) usage ;; esac DURATION=${1:-5} +if ! [[ "$DURATION" =~ ^[0-9]+$ ]] || ((DURATION < 1)); then + echo "Error: duration must be a positive integer (got: '$DURATION')" >&2 + exit 1 +fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RAW_LOG=$(mktemp /tmp/sched_audit_XXXXXX.log) trap 'rm -f "$RAW_LOG"' EXIT @@ -99,35 +109,66 @@ detect_isolated() { echo "$iso" } +# Expand a cpulist spec (e.g. "2-4,7,9-11") into a sorted, deduplicated +# space-separated list of individual CPU numbers. +# The spec is passed via CPU_SPEC env var (not interpolated into the +# Python source) to avoid code injection from untrusted input. expand_cpulist() { - python3 -c " -s='$1'; r=[] + CPU_SPEC="$1" python3 -c " +import os +s=os.environ['CPU_SPEC']; r=set() for p in s.split(','): - a,_,b=p.partition('-'); r.extend(range(int(a),int(b or a)+1)) -print(min(r), max(r)) + p=p.strip() + if not p: continue + a,_,b=p.partition('-') + r.update(range(int(a),int(b or a)+1)) +print(' '.join(str(x) for x in sorted(r))) " } -if [[ -n "${2:-}" && -n "${3:-}" ]]; then - # Manual override — always wins - CPU_MIN=$2 - CPU_MAX=$3 - echo "✓ Manual override — monitoring CPUs ${CPU_MIN}-${CPU_MAX}" +# Build CPU_LIST (space-separated individual CPUs) from arguments +if [[ -n "${2:-}" ]]; then + # Join all args after duration with commas, then expand + raw="${*:2}" + # Replace spaces with commas so "2 5 7" becomes "2,5,7" + raw="${raw// /,}" + read -ra CPU_LIST <<<"$(expand_cpulist "$raw")" + echo "✓ Manual override — monitoring CPUs: ${CPU_LIST[*]}" else ISO=$(detect_isolated) if [[ -z "$ISO" ]]; then - CPU_MIN=${2:-2} - CPU_MAX=${3:-3} - echo "⚠ No isolcpus found — using CPU${CPU_MIN}-CPU${CPU_MAX}" + CPU_LIST=(2 3) + echo "⚠ No isolcpus found — using CPUs: ${CPU_LIST[*]}" else - read -r CPU_MIN CPU_MAX <<<"$(expand_cpulist "$ISO")" - echo "✓ Detected isolated cores: $ISO (min=$CPU_MIN max=$CPU_MAX)" + read -ra CPU_LIST <<<"$(expand_cpulist "$ISO")" + echo "✓ Detected isolated cores: $ISO → CPUs: ${CPU_LIST[*]}" fi fi +# Derive min/max for display +CPU_MIN=${CPU_LIST[0]} +CPU_MAX=${CPU_LIST[-1]} + +# Build bpftrace CPU filter expression +if ((CPU_MAX - CPU_MIN + 1 == ${#CPU_LIST[@]})); then + # Contiguous range — use efficient >= && <= + BPF_CPU_FILTER="cpu >= $CPU_MIN && cpu <= $CPU_MAX" +else + # Non-contiguous — use explicit || checks + filter_parts=() + for c in "${CPU_LIST[@]}"; do + filter_parts+=("cpu == $c") + done + joined=$( + IFS='|' + echo "${filter_parts[*]}" + ) + BPF_CPU_FILTER=${joined//|/ || } +fi + S="================================================================" -printf '\n%s\n sched_switch audit | cores %d-%d | %ds sample\n%s\n\n' \ - "$S" "$CPU_MIN" "$CPU_MAX" "$DURATION" "$S" +printf '\n%s\n sched_switch audit | cores [%s] | %ds sample\n%s\n\n' \ + "$S" "${CPU_LIST[*]}" "$DURATION" "$S" # Load patterns from RC files (skip blank lines and comments) load_patterns() { @@ -174,7 +215,7 @@ classify() { BTRACE_PROG=' tracepoint:sched:sched_switch -/ cpu >= '"$CPU_MIN"' && cpu <= '"$CPU_MAX"' / +/ '"$BPF_CPU_FILTER"' / { @sw[cpu, args->next_comm, args->next_pid] = count(); } @@ -188,7 +229,7 @@ interval:s:'"$DURATION"' ' echo "Running bpftrace for ${DURATION}s..." -echo "(watching CPUs ${CPU_MIN}–${CPU_MAX} for sched_switch events)" +echo "(watching CPUs [${CPU_LIST[*]}] for sched_switch events)" echo "" sudo bpftrace -e "$BTRACE_PROG" 2>/dev/null | tee "$RAW_LOG"