Skip to content

Add: continuous isolation monitor with spike analysis #1591

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DawidWesierski4 merged 3 commits into from
Jul 14, 2026
Merged

Add: continuous isolation monitor with spike analysis #1591

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3fa496b
Add: continuous isolation monitor with spike analysis
DawidWesierski4 May 29, 2026
aa38124
Add: support for flexible cpu schema
DawidWesierski4 May 29, 2026
d660415
Fix: Remove code-injection and insecure tempdir in isolation scripts
DawidWesierski4 Jul 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
221 changes: 221 additions & 0 deletions script/isolation_monitor.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,221 @@
#!/bin/bash
# isolation_monitor.sh — continuous isolation monitoring with spike analysis
#
# Runs isolation_report.sh in a tight loop (default 30s samples),
# logs every iteration with timestamps, and on Ctrl+C prints a
# spike analysis showing when the worst isolation violations occurred.
#
# Usage:
# sudo ./isolation_monitor.sh [sample_duration] [cpu_spec...]
# sudo ./isolation_monitor.sh # 30s samples, auto-detect cores
# sudo ./isolation_monitor.sh 3 # 3s samples
# sudo ./isolation_monitor.sh 5 2-7 # 5s samples, CPUs 2-7
# sudo ./isolation_monitor.sh 5 2 5 7 # 5s samples, CPUs 2, 5, 7
# sudo ./isolation_monitor.sh 5 2-4,7 # 5s samples, CPUs 2,3,4,7

set -euo pipefail

SAMPLE_DURATION=${1:-30}
CPU_ARGS=("${@:2}")

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPORT_SCRIPT="${SCRIPT_DIR}/isolation_report.sh"
LOG_DIR=$(mktemp -d /tmp/isolation_monitor_XXXXXX)

# ── state ──
ITERATION=0
declare -a TIMESTAMPS=()
declare -a TOTALS=()
declare -a INTRUDERS=()
declare -a ALLOWED_COUNTS=()
declare -a EXPECTED_COUNTS=()
declare -a INTRUDER_DETAILS=()
declare -a LOG_FILES=()

# ── cleanup & analysis on exit ──
finish() {
echo ""
echo ""
local S="================================================================"

# Use actual completed samples, not ITERATION (which may have been
# incremented before the last sample finished).
local completed=${#TIMESTAMPS[@]}

if ((completed == 0)); then
echo "No samples collected."
rm -rf "$LOG_DIR"
exit 0
fi

printf '\n%s\n SPIKE ANALYSIS (%d samples collected)\n%s\n\n' "$S" "$completed" "$S"

# CSV header
printf '%-6s %-24s %10s %10s %10s %10s\n' \
"#" "TIMESTAMP" "TOTAL_SW" "EXPECTED" "ALLOWED" "INTRUDER"
printf ' %s\n' "$(printf '%.0s-' {1..80})"

local max_intruder=0 max_intruder_idx=0
local max_total=0 max_total_idx=0

for ((i = 0; i < completed; i++)); do
printf '%-6d %-24s %10d %10d %10d %10d' \
"$((i + 1))" "${TIMESTAMPS[$i]}" "${TOTALS[$i]}" \
"${EXPECTED_COUNTS[$i]}" "${ALLOWED_COUNTS[$i]}" "${INTRUDERS[$i]}"

# track worst intruder spike
if ((INTRUDERS[i] > max_intruder)); then
max_intruder=${INTRUDERS[$i]}
max_intruder_idx=$i
fi
# track worst total spike
if ((TOTALS[i] > max_total)); then
max_total=${TOTALS[$i]}
max_total_idx=$i
fi

# mark spikes inline
if ((INTRUDERS[i] > 0)); then
printf ' ← INTRUDERS!'
fi
printf '\n'
done

printf '\n%s\n WORST SPIKES\n%s\n\n' "$S" "$S"

printf ' Biggest TOTAL switch spike:\n'
printf ' Sample #%d | %s | %d total switches\n\n' \
"$((max_total_idx + 1))" "${TIMESTAMPS[$max_total_idx]}" "$max_total"

printf ' Biggest INTRUDER spike:\n'
if ((max_intruder > 0)); then
printf ' Sample #%d | %s | %d intruder switches\n' \
"$((max_intruder_idx + 1))" "${TIMESTAMPS[$max_intruder_idx]}" "$max_intruder"
if [[ -n "${INTRUDER_DETAILS[$max_intruder_idx]:-}" ]]; then
printf ' Intruders: %s\n' "${INTRUDER_DETAILS[$max_intruder_idx]}"
fi
else
printf ' None — no intruders observed across all samples!\n'
fi

# compute averages
local sum_total=0 sum_intruder=0 sum_allowed=0
for ((i = 0; i < completed; i++)); do
sum_total=$((sum_total + TOTALS[i]))
sum_intruder=$((sum_intruder + INTRUDERS[i]))
sum_allowed=$((sum_allowed + ALLOWED_COUNTS[i]))
done

printf '\n%s\n AVERAGES (per %ds sample)\n%s\n\n' "$S" "$SAMPLE_DURATION" "$S"
printf ' Avg total switches : %d\n' "$((sum_total / completed))"
printf ' Avg allowed switches: %d\n' "$((sum_allowed / completed))"
printf ' Avg intruder switches: %d\n' "$((sum_intruder / completed))"

# rename max spike files with _MAX suffix
if ((max_intruder > 0)); then
local src="${LOG_FILES[$max_intruder_idx]}"
local dst="${src%.log}_MAX_INTRUDER.log"
mv "$src" "$dst" 2>/dev/null && printf '\n Renamed worst intruder log → %s\n' "$(basename "$dst")"
fi
if ((max_total > 0)); then
local src="${LOG_FILES[$max_total_idx]}"
if [[ -f "$src" ]]; then
local dst="${src%.log}_MAX_TOTAL.log"
mv "$src" "$dst" 2>/dev/null && printf ' Renamed worst total log → %s\n' "$(basename "$dst")"
else
# already renamed as intruder max — add total tag too
local prev="${src%.log}_MAX_INTRUDER.log"
local dst="${src%.log}_MAX_INTRUDER_MAX_TOTAL.log"
mv "$prev" "$dst" 2>/dev/null && printf ' Renamed worst total log → %s\n' "$(basename "$dst")"
fi
fi

printf '\n Full logs saved in: %s/\n\n' "$LOG_DIR"
printf '%s\n Done — %d samples over %s\n%s\n\n' \
"$S" "$completed" \
"$(date -u -d @$((completed * SAMPLE_DURATION)) +%H:%M:%S)" \
"$S"
}

trap finish EXIT

# ── parse one report output, extract counts ──
parse_report() {
local log_file="$1"
local total=0 expected=0 allowed=0 intruder=0
local intruder_names=""

while IFS= read -r line; do
if [[ "$line" =~ Total\ switches\ observed\ :\ ([0-9]+) ]]; then
total="${BASH_REMATCH[1]}"
elif [[ "$line" =~ EXPECTED\ switches.*:\ ([0-9]+) ]]; then
expected="${BASH_REMATCH[1]}"
elif [[ "$line" =~ ALLOWED.*switches.*:\ ([0-9]+) ]]; then
allowed="${BASH_REMATCH[1]}"
elif [[ "$line" =~ INTRUDER\ switches.*:\ ([0-9]+) ]]; then
intruder="${BASH_REMATCH[1]}"
elif [[ "$line" =~ ^[[:space:]]+X[[:space:]]+(.+) ]]; then
local name="${BASH_REMATCH[1]}"
if [[ -n "$intruder_names" ]]; then
intruder_names+="; ${name}"
else
intruder_names="${name}"
fi
fi
done <"$log_file"

echo "${total} ${expected} ${allowed} ${intruder}"
# stash intruder details via global
_PARSED_INTRUDER_DETAILS="$intruder_names"
}

# ── validate inputs ──
if ! [[ "$SAMPLE_DURATION" =~ ^[0-9]+$ ]] || ((SAMPLE_DURATION < 1)); then
echo "Error: sample duration must be a positive integer (got: '$SAMPLE_DURATION')" >&2
exit 1
fi

if [[ ! -x "$REPORT_SCRIPT" ]]; then
echo "Error: cannot find or execute $REPORT_SCRIPT" >&2
exit 1
fi

echo "================================================================"
echo " Continuous Isolation Monitor"
echo " Sample duration : ${SAMPLE_DURATION}s"
echo " CPU args : ${CPU_ARGS[*]:-auto-detect}"
echo " Log directory : ${LOG_DIR}/"
echo " Press Ctrl+C to stop and see spike analysis"
echo "================================================================"
echo ""

# ── main loop ──
while true; do
ITERATION=$((ITERATION + 1))
ts=$(date '+%Y-%m-%d %H:%M:%S')
ts_file=$(date '+%Y-%m-%d_%H-%M-%S')
log_file="${LOG_DIR}/${ts_file}.log"

printf '\n──── Sample #%d | %s ────\n' "$ITERATION" "$ts"

# run the report, capture output
"$REPORT_SCRIPT" "$SAMPLE_DURATION" "${CPU_ARGS[@]}" 2>&1 | tee "$log_file"

# parse results
_PARSED_INTRUDER_DETAILS=""
read -r total expected allowed intruder <<<"$(parse_report "$log_file")"

TIMESTAMPS+=("$ts")
TOTALS+=("${total:-0}")
EXPECTED_COUNTS+=("${expected:-0}")
ALLOWED_COUNTS+=("${allowed:-0}")
INTRUDERS+=("${intruder:-0}")
INTRUDER_DETAILS+=("$_PARSED_INTRUDER_DETAILS")
LOG_FILES+=("$log_file")

# quick inline status
printf ' → Sample #%d: total=%d expected=%d allowed=%d intruder=%d\n' \
"$ITERATION" "${total:-0}" "${expected:-0}" "${allowed:-0}" "${intruder:-0}"

# no sleep — loop immediately for minimal downtime
done
103 changes: 72 additions & 31 deletions script/isolation_report.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,37 +6,43 @@
# INTRUDER — should not be on isolated cores
#
# Usage:
# sudo ./sched_audit.sh [options] [duration_sec] [cpu_min] [cpu_max]
# sudo ./sched_audit.sh 5 2 3 # 5s sample on CPUs 2-3 (overrides auto-detect)
# sudo ./sched_audit.sh 10 # 10s sample, auto-detect isolated range
# sudo ./sched_audit.sh -h # show help
# sudo ./isolation_report.sh [options] [duration_sec] [cpu_spec...]
# sudo ./isolation_report.sh 5 2 3 # 5s sample on CPUs 2-3 (range)
# sudo ./isolation_report.sh 5 2-7 # 5s sample on CPUs 2-7 (range notation)
# sudo ./isolation_report.sh 5 2 5 7 # 5s sample on CPUs 2, 5, 7 (discrete)
# sudo ./isolation_report.sh 5 2-4,7 # 5s sample on CPUs 2,3,4,7 (mixed)
# sudo ./isolation_report.sh 10 # 10s sample, auto-detect isolated range
# sudo ./isolation_report.sh -h # show help

set -euo pipefail

usage() {
cat <<'HELPEOF'
Usage: sudo isolation_report.sh [options] [duration_sec] [cpu_min] [cpu_max]
Usage: sudo isolation_report.sh [options] [duration_sec] [cpu_spec...]

Run a bpftrace sched_switch audit on isolated (or specified) CPU cores,
then classify every observed task as EXPECTED, ALLOWED, or INTRUDER.

POSITIONAL ARGUMENTS:
duration_sec Sampling duration in seconds (default: 5)
cpu_min First CPU core to monitor (default: auto-detect)
cpu_max Last CPU core to monitor (default: auto-detect)

If cpu_min and cpu_max are given they OVERRIDE the auto-detected
isolcpus range. If omitted, the script reads isolcpus from
/sys/devices/system/cpu/isolated or /proc/cmdline.
cpu_spec CPU cores to monitor. Supports:
2 7 range min-max (legacy)
2-7 range notation
2,5,7 comma-separated
2-4,7,9-11 mixed ranges and singles
2 5 7 space-separated discrete list
(default: auto-detect from isolcpus)

OPTIONS:
-h, --help Show this help message and exit

EXAMPLES:
sudo ./isolation_report.sh # 5s, auto-detect isolated cores
sudo ./isolation_report.sh 10 # 10s, auto-detect
sudo ./isolation_report.sh 5 2 3 # 5s, force CPUs 2-3
sudo ./isolation_report.sh 10 4 7 # 10s, force CPUs 4-7
sudo ./isolation_report.sh 5 2 3 # 5s, force CPUs 2-3 (range)
sudo ./isolation_report.sh 5 2-7 # 5s, force CPUs 2-7
sudo ./isolation_report.sh 5 2 5 7 # 5s, force CPUs 2,5,7 (discrete)
sudo ./isolation_report.sh 5 2-4,7 # 5s, CPUs 2,3,4,7

PATTERN FILES (RC files):
Every thread observed on the monitored cores is classified by matching
Expand Down Expand Up @@ -88,6 +94,10 @@ HELPEOF
case "${1:-}" in -h | --help) usage ;; esac

DURATION=${1:-5}
if ! [[ "$DURATION" =~ ^[0-9]+$ ]] || ((DURATION < 1)); then
echo "Error: duration must be a positive integer (got: '$DURATION')" >&2
exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
RAW_LOG=$(mktemp /tmp/sched_audit_XXXXXX.log)
trap 'rm -f "$RAW_LOG"' EXIT
Expand All @@ -99,35 +109,66 @@ detect_isolated() {
echo "$iso"
}

# Expand a cpulist spec (e.g. "2-4,7,9-11") into a sorted, deduplicated
# space-separated list of individual CPU numbers.
# The spec is passed via CPU_SPEC env var (not interpolated into the
# Python source) to avoid code injection from untrusted input.
expand_cpulist() {
python3 -c "
s='$1'; r=[]
CPU_SPEC="$1" python3 -c "
import os
s=os.environ['CPU_SPEC']; r=set()
for p in s.split(','):
a,_,b=p.partition('-'); r.extend(range(int(a),int(b or a)+1))
print(min(r), max(r))
p=p.strip()
if not p: continue
a,_,b=p.partition('-')
r.update(range(int(a),int(b or a)+1))
print(' '.join(str(x) for x in sorted(r)))
"
}

if [[ -n "${2:-}" && -n "${3:-}" ]]; then
# Manual override — always wins
CPU_MIN=$2
CPU_MAX=$3
echo "✓ Manual override — monitoring CPUs ${CPU_MIN}-${CPU_MAX}"
# Build CPU_LIST (space-separated individual CPUs) from arguments
if [[ -n "${2:-}" ]]; then
# Join all args after duration with commas, then expand
raw="${*:2}"
# Replace spaces with commas so "2 5 7" becomes "2,5,7"
raw="${raw// /,}"
read -ra CPU_LIST <<<"$(expand_cpulist "$raw")"
echo "✓ Manual override — monitoring CPUs: ${CPU_LIST[*]}"
else
ISO=$(detect_isolated)
if [[ -z "$ISO" ]]; then
CPU_MIN=${2:-2}
CPU_MAX=${3:-3}
echo "⚠ No isolcpus found — using CPU${CPU_MIN}-CPU${CPU_MAX}"
CPU_LIST=(2 3)
echo "⚠ No isolcpus found — using CPUs: ${CPU_LIST[*]}"
else
read -r CPU_MIN CPU_MAX <<<"$(expand_cpulist "$ISO")"
echo "✓ Detected isolated cores: $ISO (min=$CPU_MIN max=$CPU_MAX)"
read -ra CPU_LIST <<<"$(expand_cpulist "$ISO")"
echo "✓ Detected isolated cores: $ISO → CPUs: ${CPU_LIST[*]}"
fi
fi

# Derive min/max for display
CPU_MIN=${CPU_LIST[0]}
CPU_MAX=${CPU_LIST[-1]}

# Build bpftrace CPU filter expression
if ((CPU_MAX - CPU_MIN + 1 == ${#CPU_LIST[@]})); then
# Contiguous range — use efficient >= && <=
BPF_CPU_FILTER="cpu >= $CPU_MIN && cpu <= $CPU_MAX"
else
# Non-contiguous — use explicit || checks
filter_parts=()
for c in "${CPU_LIST[@]}"; do
filter_parts+=("cpu == $c")
done
joined=$(
IFS='|'
echo "${filter_parts[*]}"
)
BPF_CPU_FILTER=${joined//|/ || }
fi

S="================================================================"
printf '\n%s\n sched_switch audit | cores %d-%d | %ds sample\n%s\n\n' \
"$S" "$CPU_MIN" "$CPU_MAX" "$DURATION" "$S"
printf '\n%s\n sched_switch audit | cores [%s] | %ds sample\n%s\n\n' \
"$S" "${CPU_LIST[*]}" "$DURATION" "$S"

# Load patterns from RC files (skip blank lines and comments)
load_patterns() {
Expand Down Expand Up @@ -174,7 +215,7 @@ classify() {

BTRACE_PROG='
tracepoint:sched:sched_switch
/ cpu >= '"$CPU_MIN"' && cpu <= '"$CPU_MAX"' /
/ '"$BPF_CPU_FILTER"' /
{
@sw[cpu, args->next_comm, args->next_pid] = count();
}
Expand All @@ -188,7 +229,7 @@ interval:s:'"$DURATION"'
'

echo "Running bpftrace for ${DURATION}s..."
echo "(watching CPUs ${CPU_MIN}–${CPU_MAX} for sched_switch events)"
echo "(watching CPUs [${CPU_LIST[*]}] for sched_switch events)"
echo ""

sudo bpftrace -e "$BTRACE_PROG" 2>/dev/null | tee "$RAW_LOG"
Expand Down
Loading