diff --git a/Cargo.lock b/Cargo.lock index e398f45a77e92..8ee3d85b99a1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5525,6 +5525,7 @@ dependencies = [ "snapbox", "solar-compiler", "soldeer-core", + "strum", "tempfile", "thiserror 2.0.18", "toml 0.9.12+spec-1.1.0", diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 39c281e81be25..a1bdd88a2114b 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -48,6 +48,7 @@ tracing.workspace = true walkdir.workspace = true yansi.workspace = true clap = { version = "4", features = ["derive"] } +strum = { workspace = true, features = ["derive"] } [target.'cfg(target_os = "windows")'.dependencies] path-slash = "0.2" diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index b4423353141e7..b582e0f37c93a 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -113,6 +113,9 @@ pub use fuzz::{FuzzConfig, FuzzCorpusConfig, FuzzDictionaryConfig}; mod invariant; pub use invariant::InvariantConfig; +pub mod mutation; +pub use mutation::{MutationConfig, MutatorType}; + mod inline; pub use inline::{InlineConfig, InlineConfigError, NatSpec}; @@ -351,6 +354,8 @@ pub struct Config { pub coverage_pattern_inverse: Option, /// Path where last test run failures are recorded. pub test_failures_file: PathBuf, + /// Path where mutation tests are cached, to resume running them + pub mutation_dir: PathBuf, /// Max concurrent threads to use. pub threads: Option, /// Whether to show test execution progress. @@ -359,6 +364,8 @@ pub struct Config { pub fuzz: FuzzConfig, /// Configuration for invariant testing pub invariant: InvariantConfig, + /// Configuration for mutation testing + pub mutation: MutationConfig, /// Whether to allow ffi cheatcodes in test pub ffi: bool, /// Whether to show `console.log` outputs in realtime during script/test execution @@ -710,6 +717,7 @@ impl Config { "doc", "fuzz", "invariant", + "mutation", "labels", "dependencies", "soldeer", @@ -1329,6 +1337,9 @@ impl Config { )); } + // Remove mutation test cache directory + let _ = fs::remove_dir_all(project.root().join(&self.mutation_dir)); + // Remove fuzz and invariant cache directories. let mut remove_test_dir = |test_dir: &Option| { if let Some(test_dir) = test_dir { @@ -2684,10 +2695,12 @@ impl Default for Config { path_pattern_inverse: None, coverage_pattern_inverse: None, test_failures_file: "cache/test-failures".into(), + mutation_dir: "cache/mutation".into(), threads: None, show_progress: false, fuzz: FuzzConfig::new("cache/fuzz".into()), invariant: InvariantConfig::new("cache/invariant".into()), + mutation: MutationConfig::default(), always_use_create_2_factory: false, ffi: false, live_logs: false, @@ -6953,6 +6966,9 @@ mod tests { runs = 256 unknown_invariant_key = "should_warn" + [mutation] + unknown_mutation_key = "should_warn" + [vyper] unknown_vyper_key = "should_warn" @@ -6981,6 +6997,9 @@ mod tests { runs = 512 unknown_nested_invariant_key = "should_warn" + [profile.default.mutation] + unknown_nested_mutation_key = "should_warn" + [profile.default.vyper] unknown_nested_vyper_key = "should_warn" @@ -7019,6 +7038,7 @@ mod tests { ("unknown_doc_key", "doc"), ("unknown_fuzz_key", "fuzz"), ("unknown_invariant_key", "invariant"), + ("unknown_mutation_key", "mutation"), ("unknown_vyper_key", "vyper"), ("unknown_bind_json_key", "bind_json"), ]; @@ -7044,6 +7064,7 @@ mod tests { ("unknown_nested_doc_key", "doc"), ("unknown_nested_fuzz_key", "fuzz"), ("unknown_nested_invariant_key", "invariant"), + ("unknown_nested_mutation_key", "mutation"), ("unknown_nested_vyper_key", "vyper"), ("unknown_nested_bind_json_key", "bind_json"), ]; @@ -7092,11 +7113,11 @@ mod tests { }) .collect(); - // 1 profile key + 7 standalone + 7 nested + 2 array = 17 total + // 1 profile key + 8 standalone + 8 nested + 2 array = 19 total assert_eq!( unknown_key_warnings.len(), - 17, - "Expected 17 unknown key warnings (1 profile + 7 standalone + 7 nested + 2 array), got {}: {:?}", + 19, + "Expected 19 unknown key warnings (1 profile + 8 standalone + 8 nested + 2 array), got {}: {:?}", unknown_key_warnings.len(), unknown_key_warnings ); diff --git a/crates/config/src/mutation.rs b/crates/config/src/mutation.rs new file mode 100644 index 0000000000000..58dd7548e6ad9 --- /dev/null +++ b/crates/config/src/mutation.rs @@ -0,0 +1,80 @@ +//! Configuration for mutation testing. + +use serde::{Deserialize, Serialize}; +use strum::IntoEnumIterator; + +/// Represents each available mutation operator. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + strum::Display, + strum::EnumString, + strum::EnumIter, +)] +#[serde(rename_all = "kebab-case")] +#[strum(serialize_all = "kebab-case")] +pub enum MutatorType { + Assembly, + Assignment, + BinaryOp, + DeleteExpression, + ElimDelegate, + Require, + UnaryOp, +} + +impl MutatorType { + /// Returns a list of all available mutator types. + pub fn all() -> Vec { + Self::iter().collect() + } + + /// Returns the operators that are excluded by default. + pub const fn default_excluded() -> Vec { + Vec::new() + } +} + +/// Configuration for mutation testing. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct MutationConfig { + /// Re-enable operators that are excluded by default. + pub include_operators: Vec, + /// Exclude additional operators beyond the defaults. + pub exclude_operators: Vec, + /// Per-mutant wall-clock timeout, in seconds. + /// + /// When set, each mutant's compile-and-test work is bounded by this + /// duration; mutants that exceed it are recorded as `TimedOut`. This is + /// the analog of `invariant.timeout` for mutation campaigns. + /// + /// Note: enforcement is best-effort. Background work for a timed-out + /// mutant may continue briefly until the underlying compile / test loop + /// reaches a checkpoint, but the worker slot is freed immediately so + /// other mutants can proceed. Cleanup backlog is bounded by the configured + /// mutation worker count. + pub timeout: Option, +} + +impl MutationConfig { + /// Returns the list of operators that are currently enabled. + /// + /// Effective set: `all() - default_excluded - exclude_operators + include_operators` + pub fn enabled_operators(&self) -> Vec { + let default_excluded = MutatorType::default_excluded(); + MutatorType::all() + .into_iter() + .filter(|op| { + let excluded = default_excluded.contains(op) || self.exclude_operators.contains(op); + let included = self.include_operators.contains(op); + !excluded || included + }) + .collect() + } +} diff --git a/crates/forge/Cargo.toml b/crates/forge/Cargo.toml index 667da6b442ca1..203c318724222 100644 --- a/crates/forge/Cargo.toml +++ b/crates/forge/Cargo.toml @@ -65,8 +65,6 @@ alloy-transport.workspace = true tempo-alloy.workspace = true -revm.workspace = true - clap = { version = "4", features = ["derive", "env", "unicode", "wrap_help"] } clap_complete.workspace = true dunce.workspace = true @@ -75,13 +73,14 @@ inferno = { version = "0.12", default-features = false } itertools.workspace = true parking_lot.workspace = true regex = { workspace = true, default-features = false } +revm.workspace = true semver.workspace = true serde_json.workspace = true similar = { version = "2", features = ["inline"] } solar.workspace = true strum = { workspace = true, features = ["derive"] } thiserror.workspace = true -tokio = { workspace = true, features = ["time"] } +tokio = { workspace = true, features = ["time", "signal"] } toml_edit.workspace = true watchexec = "8.0" watchexec-events = "6.0" @@ -101,6 +100,7 @@ opener = "0.8" soldeer-commands.workspace = true soldeer-core.workspace = true quick-junit = "0.5.2" +tempfile.workspace = true [dev-dependencies] alloy-hardforks.workspace = true diff --git a/crates/forge/src/cmd/test/mod.rs b/crates/forge/src/cmd/test/mod.rs index cc2e2b7e9b411..90d294ad52760 100644 --- a/crates/forge/src/cmd/test/mod.rs +++ b/crates/forge/src/cmd/test/mod.rs @@ -1,14 +1,11 @@ -use super::{ - install, - test::filter::{ProjectPathsAwareFilter, RerunFailure, RerunFailures}, - watch::WatchArgs, -}; +use super::{install, watch::WatchArgs}; use crate::{ MultiContractRunner, MultiContractRunnerBuilder, decode::decode_console_logs, diagnostic::build::SOLC_ERROR, gas_report::GasReport, multi_runner::{MultiNetworkConfig, ShowmapConfig, matches_artifact}, + mutation::{MutationRunConfig, run_mutation_testing}, result::{SuiteResult, TestKindReport, TestOutcome, TestResult, TestStatus}, traces::{ CallTraceDecoderBuilder, InternalTraceMode, TraceKind, @@ -27,7 +24,9 @@ use foundry_cli::{ opts::{BuildOpts, EvmArgs, GlobalArgs}, utils::{self, LoadConfig}, }; -use foundry_common::{EmptyTestFilter, TestFunctionExt, compile::ProjectCompiler, fs, shell}; +use foundry_common::{ + EmptyTestFilter, TestFilter, TestFunctionExt, compile::ProjectCompiler, fs, shell, +}; use foundry_compilers::{ CompilationError, ProjectCompileOutput, artifacts::{Libraries, output_selection::OutputSelection}, @@ -72,7 +71,8 @@ use yansi::Paint; mod filter; mod summary; use crate::{result::TestKind, traces::render_trace_arena_inner}; -pub use filter::FilterArgs; +pub use filter::{FilterArgs, ProjectPathsAwareFilter}; +use filter::{RerunFailure, RerunFailures}; use quick_junit::{NonSuccessKind, Report, TestCase, TestCaseStatus, TestSuite}; use summary::{TestSummaryReport, format_invariant_metrics_table}; @@ -302,6 +302,37 @@ pub struct TestArgs { #[command(flatten)] pub watch: WatchArgs, + + /// Enable mutation testing. + /// If passed with file paths, only those files will be tested. + #[arg(long, num_args(0..), value_name = "PATH")] + pub mutate: Option>, + + /// Specify which files to mutate with glob pattern matching. + /// + /// Mutually exclusive with passing explicit paths to `--mutate`; either + /// supply paths to `--mutate` or use this glob filter, not both. + #[arg(long, value_name = "PATTERN", requires = "mutate", conflicts_with = "mutate_contract")] + pub mutate_path: Option, + + /// Only mutate contracts whose name matches the specified regex pattern. + /// + /// Mutually exclusive with `--mutate-path`. + #[arg(long, value_name = "REGEX", requires = "mutate")] + pub mutate_contract: Option, + + /// Number of parallel workers for mutation testing. + /// Defaults to the number of CPU cores. + #[arg(long, value_name = "JOBS", requires = "mutate")] + pub mutation_jobs: Option, + + /// Best-effort per-mutant wall-clock timeout in seconds. Mutants that + /// exceed it are recorded as "timed out" and cleanup continues in the + /// background with bounded pending workers. + /// + /// Analogous to `--invariant-timeout` for invariant campaigns. + #[arg(long, value_name = "TIMEOUT", requires = "mutate")] + pub mutation_timeout: Option, } impl TestArgs { @@ -348,6 +379,7 @@ impl TestArgs { ("--list", self.list), ("--junit", self.junit), ("--show-progress", self.show_progress), + ("--mutate", self.mutate.is_some()), // `--live-logs` writes console.log straight to stdout; the // `live_logs = true` config equivalent is overridden in // `compile_and_run`. @@ -394,6 +426,20 @@ impl TestArgs { .collect()); } + let filter_args = test_filter.args(); + let has_contract_or_test_filter = filter_args.test_pattern.is_some() + || filter_args.test_pattern_inverse.is_some() + || filter_args.contract_pattern.is_some() + || filter_args.contract_pattern_inverse.is_some(); + if !has_contract_or_test_filter { + return Ok(source_files_iter(&config.src, MultiCompilerLanguage::FILE_EXTENSIONS) + .chain( + source_files_iter(&config.test, MultiCompilerLanguage::FILE_EXTENSIONS) + .filter(|path| test_filter.matches_path(path)), + ) + .collect()); + } + let mut project = config.create_project(true, true)?; project.update_output_selection(|selection| { *selection = OutputSelection::common_output_selection(["abi".to_string()]); @@ -439,6 +485,14 @@ impl TestArgs { // Merge all configs. let (mut config, evm_opts) = self.load_config_and_evm_opts()?; + let should_mutate = self.mutate.is_some(); + + // Force dyn test linking for mutation testing + if should_mutate { + config.dynamic_test_linking = true; + config.cache = true; + } + // Override foundry.toml knobs that would print outside the NDJSON // stream or bail mid-suite; the CLI equivalents are rejected in // `reject_machine_unsupported_flags`. @@ -503,6 +557,45 @@ impl TestArgs { } let invariant_workers = config.invariant.workers; + // Mutation testing has bespoke orchestration (per-mutant temp + // workspaces, baseline + N mutants, aggregated mutation report). It is + // not compatible with the single-run debug / flame / list / junit + // modes — running them together would either mix incompatible output + // formats, or run the secondary mode against the baseline tests and + // then silently continue into mutation testing. Reject up front with a + // clear error rather than do the wrong thing. + if self.mutate.is_some() { + let mut conflicts = Vec::new(); + if self.list { + conflicts.push("--list"); + } + if self.debug { + conflicts.push("--debug"); + } + if self.flamegraph { + conflicts.push("--flamegraph"); + } + if self.flamechart { + conflicts.push("--flamechart"); + } + if self.junit { + conflicts.push("--junit"); + } + if coverage { + conflicts.push("coverage"); + } + if self.showmap_out.is_some() { + conflicts.push("--showmap-out"); + } + if !conflicts.is_empty() { + bail!( + "`--mutate` cannot be combined with: {}. Re-run without those flags to use \ + mutation testing.", + conflicts.join(", ") + ); + } + } + // Explicitly enable isolation for gas reports for more correct gas accounting. if self.gas_report { evm_opts.isolate = true; @@ -544,6 +637,10 @@ impl TestArgs { // Auto-detect network from fork chain ID when not explicitly configured. evm_opts.infer_network_from_fork().await; + // Clone config and evm_opts before dispatch (needed for mutation testing). + let config_for_mutation = config.clone(); + let evm_opts_for_mutation = evm_opts.clone(); + // Parse inline config early to detect per-test network annotations. let inline_config = InlineConfig::new_parsed(output, &config)?; let override_networks = inline_config.referenced_override_networks(&config.profile); @@ -706,6 +803,234 @@ impl TestArgs { } } + // All tests have been run once before reaching this point + if let Some(mutate) = &self.mutate { + // Check outcome here, stop if any test failed + if outcome.failed() > 0 { + eyre::bail!("Cannot run mutation testing with failed tests"); + } + + // A green baseline that ran zero non-skipped tests is not useful: + // every compileable mutant would be reported as `Alive` (no test + // failed, so nothing killed it), which produces a wildly + // misleading mutation report. Hard-error so users get an actual + // signal that their filter / path / setup matched nothing. + if outcome.successes().next().is_none() { + eyre::bail!( + "Mutation testing requires at least one passing baseline test; the current \ + filter/path selection matched zero non-skipped tests. Loosen `--match-test` / \ + `--match-contract` / `--match-path` or check the project layout." + ); + } + + // Explicit paths on --mutate cannot be combined with the --mutate-path + // glob filter: clap can't express this directly because --mutate takes + // an optional list of paths. + if !mutate.is_empty() && self.mutate_path.is_some() { + eyre::bail!( + "`--mutate-path ` cannot be combined with explicit paths passed to `--mutate`; pass either paths or a glob pattern, not both" + ); + } + + // The mutation runner builds a single-pass `MultiContractRunner` + // (`runner.rs::compile_and_test_inner`) and does not honor inline + // per-test network annotations. If the project declares network + // overrides, running mutation testing would silently execute those + // tests on the wrong network and produce false survivors / kills. + // Bail with a clear error rather than do the wrong thing silently. + if !override_networks.is_empty() { + eyre::bail!( + "Mutation testing does not yet support inline per-test network overrides \ + (found {} annotated network(s)). Re-run without `--mutate` or remove the \ + per-test network annotations.", + override_networks.len() + ); + } + + // The mutation runner symlinks dependency directories (`lib`, + // `node_modules`, `dependencies`) into each per-mutant TempDir for + // performance — see `workspace::copy_project`. That isolation + // breaks down if tests can write to those shared trees, either via + // `vm.writeFile` (broad `fs_permissions`) or arbitrary `ffi` calls. + // Detect both up front so users aren't surprised by races or + // corruption of their real dependency tree. + use foundry_config::fs_permissions::FsAccessPermission; + if config_for_mutation.ffi { + eyre::bail!( + "Mutation testing is unsafe with `ffi = true`: per-mutant workspaces share \ + symlinked dependency directories, and arbitrary FFI commands run by tests \ + can race or corrupt the real `lib`/`node_modules`/`dependencies` trees. \ + Disable ffi in your foundry.toml to run mutation tests." + ); + } + + // Only refuse write-capable `fs_permissions` whose path can actually + // reach one of the symlinked dependency trees. Scoped writes (e.g. + // `./out`, `./snapshots`) are safe because they target paths that + // never resolve into the shared `lib`/`node_modules`/`dependencies` + // trees. + let root = &config_for_mutation.root; + let canonicalize_through_existing_ancestor = |path: &Path| -> PathBuf { + let resolved = + if path.is_absolute() { path.to_path_buf() } else { root.join(path) }; + if let Ok(canon) = dunce::canonicalize(&resolved) { + return canon; + } + + let mut missing = Vec::new(); + let mut ancestor = resolved.as_path(); + while !ancestor.exists() { + let Some(name) = ancestor.file_name() else { break }; + missing.push(name.to_owned()); + let Some(parent) = ancestor.parent() else { break }; + ancestor = parent; + } + + let mut canon = dunce::canonicalize(ancestor).unwrap_or_else(|_| ancestor.into()); + for component in missing.iter().rev() { + canon.push(component); + } + canon + }; + + let mut shared_dep_dirs: Vec = config_for_mutation + .libs + .iter() + .filter(|p| p.exists()) + .map(|p| canonicalize_through_existing_ancestor(p)) + .collect(); + for dep_dir in ["node_modules", "dependencies"] { + let dep_path = root.join(dep_dir); + if dep_path.exists() && dep_path.is_dir() { + shared_dep_dirs.push(canonicalize_through_existing_ancestor(&dep_path)); + } + } + + let effective_permission = |path: &Path| -> Option { + let mut max_path_len = 0; + let mut highest_permission = FsAccessPermission::None; + + for perm in &config_for_mutation.fs_permissions.permissions { + let permission_path = canonicalize_through_existing_ancestor(&perm.path); + if path.starts_with(&permission_path) { + let path_len = permission_path.components().count(); + if path_len > max_path_len { + max_path_len = path_len; + highest_permission = perm.access; + } else if path_len == max_path_len { + highest_permission = match (highest_permission, perm.access) { + (FsAccessPermission::ReadWrite, _) + | (FsAccessPermission::Read, FsAccessPermission::Write) + | (FsAccessPermission::Write, FsAccessPermission::Read) => { + FsAccessPermission::ReadWrite + } + (FsAccessPermission::None, perm) => perm, + (existing_perm, _) => existing_perm, + }; + } + } + } + + (max_path_len > 0).then_some(highest_permission) + }; + + let grants_write = |path: &Path| { + matches!( + effective_permission(path), + Some(FsAccessPermission::Write | FsAccessPermission::ReadWrite) + ) + }; + + let unsafe_write_paths: Vec<&Path> = config_for_mutation + .fs_permissions + .permissions + .iter() + .filter(|perm| { + matches!(perm.access, FsAccessPermission::Write | FsAccessPermission::ReadWrite) + }) + .filter(|perm| { + let perm_path = canonicalize_through_existing_ancestor(&perm.path); + shared_dep_dirs.iter().any(|dep| { + if perm_path.starts_with(dep) { + grants_write(&perm_path) + } else if dep.starts_with(&perm_path) { + grants_write(dep) + } else { + false + } + }) + }) + .map(|p| p.path.as_path()) + .collect(); + + if !unsafe_write_paths.is_empty() { + let paths = unsafe_write_paths + .iter() + .map(|p| format!(" - {}", p.display())) + .collect::>() + .join("\n"); + eyre::bail!( + "Mutation testing is unsafe with write-capable `fs_permissions` that can \ + reach the symlinked dependency trees (`lib`/`node_modules`/`dependencies`); \ + per-mutant workspaces share those trees, so `vm.writeFile` calls would race \ + against or corrupt your real dependencies. Restrict the following \ + `fs_permissions` entries to read-only or scope them away from dependency \ + paths:\n{paths}" + ); + } + + let json_output = shell::is_json(); + let selected_sources_relative = self + .get_sources_to_compile(&config_for_mutation, filter)? + .into_iter() + .filter_map(|path| { + path.strip_prefix(&config_for_mutation.root).ok().map(PathBuf::from) + }) + .collect::>(); + + let mutation_config = MutationRunConfig { + mutate_paths: mutate.clone(), + mutate_path_pattern: self.mutate_path.clone(), + mutate_contract_pattern: self.mutate_contract.clone(), + num_workers: self.mutation_jobs.unwrap_or(0), + show_progress: self.show_progress, + json_output, + // Carry the same filter args (--match-test, --match-contract, + // --match-path, positional path shorthand, --rerun, ...) and + // isolation flag the baseline actually used, so every mutant + // exercises the exact same test set under the same execution + // model. We pull from the materialized `filter`, not the raw + // CLI flags on `self`, because the baseline applies extras: + // the positional `forge test ` shorthand is folded into + // `path_pattern`, and `--rerun` injects last-run failures + // into `test_pattern`. Using `self.filter.clone()` would lose + // those and let mutant runs silently diverge from baseline. + filter_args: filter.args().clone(), + selected_sources_relative, + isolate: evm_opts_for_mutation.isolate, + }; + + let result = run_mutation_testing( + Arc::new(config_for_mutation.clone()), + output, + evm_opts_for_mutation.clone(), + mutation_config, + ) + .await?; + + if result.cancelled { + std::process::exit(130); + } + + // Output JSON if requested + if json_output { + let json_output = result.summary.to_json_output(result.duration_secs); + sh_println!("{}", serde_json::to_string(&json_output)?)?; + } + + outcome = TestOutcome::empty(None, true); + } + Ok(outcome) } @@ -822,8 +1147,10 @@ impl TestArgs { // If we need to render to a serialized format, we should not print anything else to stdout. // Machine mode is also a structured stream and must not interleave human output. - let silent = - machine_mode || self.gas_report && shell::is_json() || self.summary && shell::is_json(); + let silent = machine_mode + || self.gas_report && shell::is_json() + || self.summary && shell::is_json() + || self.mutate.is_some() && shell::is_json(); let num_filtered = runner.matching_test_functions(filter).count(); @@ -881,7 +1208,12 @@ impl TestArgs { // Run tests in a non-streaming fashion and collect results for serialization. // Agent stream wins over `--json`. - if !machine_mode && !self.gas_report && !self.summary && shell::is_json() { + if self.mutate.is_none() + && !machine_mode + && !self.gas_report + && !self.summary + && shell::is_json() + { let mut results = runner.test_collect(filter)?; for suite_result in results.values_mut() { for test_result in suite_result.test_results.values_mut() { @@ -1491,6 +1823,13 @@ impl Provider for TestArgs { dict.insert("show_progress".to_string(), true.into()); } + // Mutation-testing CLI overrides + if let Some(timeout) = self.mutation_timeout { + let mut mutation_dict = Dict::default(); + mutation_dict.insert("timeout".to_string(), timeout.into()); + dict.insert("mutation".to_string(), mutation_dict.into()); + } + Ok(Map::from([(Config::selected_profile(), dict)])) } } diff --git a/crates/forge/src/lib.rs b/crates/forge/src/lib.rs index 9e84e6b80821a..463ce18717b02 100644 --- a/crates/forge/src/lib.rs +++ b/crates/forge/src/lib.rs @@ -26,6 +26,10 @@ pub mod gas_report; pub mod multi_runner; pub use multi_runner::{MultiContractRunner, MultiContractRunnerBuilder}; +pub mod mutation; + +pub mod workspace; + mod runner; pub use runner::ContractRunner; diff --git a/crates/forge/src/mutation/mod.rs b/crates/forge/src/mutation/mod.rs new file mode 100644 index 0000000000000..c2026e6761621 --- /dev/null +++ b/crates/forge/src/mutation/mod.rs @@ -0,0 +1,666 @@ +use std::{ + collections::{BTreeMap, HashSet, hash_map::DefaultHasher}, + hash::{Hash, Hasher}, + path::{Path, PathBuf}, + sync::Arc, +}; + +use crate::mutation::{ + mutant::{Mutant, MutationResult}, + visitor::MutantVisitor, +}; +pub use crate::mutation::{ + orchestrator::{MutationRunConfig, MutationRunResult, run_mutation_testing}, + progress::MutationProgress, + reporter::MutationReporter, + runner::run_mutations_parallel_with_progress, +}; +use eyre::eyre; +use foundry_common::sh_warn; +use serde::{Deserialize, Serialize}; +use solar::{ + ast::{ + Span, + interface::{Session, source_map::FileName}, + visit::Visit, + }, + parse::Parser, +}; + +fn failed_to_parse(path: &Path) -> eyre::Report { + eyre!("failed to parse {}", path.display()) +} + +#[derive(Clone, Copy)] +enum CacheKind<'a> { + Mutants, + Results { execution_key: &'a str }, + Survived { execution_key: &'a str }, +} + +#[derive(Serialize, Deserialize)] +struct CachedMutationResults { + mutant_count: usize, + mutant_hash: u64, + results: Vec<(Mutant, MutationResult)>, +} + +fn mutant_set_hash(mutants: &[Mutant]) -> u64 { + let mut entries: Vec<_> = mutants + .iter() + .map(|mutant| { + ( + mutant.span.lo().0, + mutant.span.hi().0, + mutant.mutation.to_string(), + mutant.original.clone(), + ) + }) + .collect(); + entries.sort(); + + let mut hasher = DefaultHasher::new(); + for entry in entries { + entry.hash(&mut hasher); + } + hasher.finish() +} + +pub mod mutant; +mod mutators; +pub mod orchestrator; +pub mod progress; +mod reporter; +pub mod runner; +mod visitor; + +pub struct MutationsSummary { + dead: Vec, + survived: Vec, + invalid: Vec, + skipped: Vec, + /// Mutants whose compile-and-test work exceeded the configured timeout. + /// Tracked separately so they are not counted toward survived/killed. + timed_out: Vec, +} + +impl Default for MutationsSummary { + fn default() -> Self { + Self::new() + } +} + +impl MutationsSummary { + pub const fn new() -> Self { + Self { + dead: Vec::new(), + survived: Vec::new(), + invalid: Vec::new(), + skipped: Vec::new(), + timed_out: Vec::new(), + } + } + + pub fn update_invalid_mutant(&mut self, mutant: Mutant) { + self.invalid.push(mutant); + } + + pub fn add_dead_mutant(&mut self, mutant: Mutant) { + self.dead.push(mutant); + } + + pub fn add_survived_mutant(&mut self, mutant: Mutant) { + self.survived.push(mutant); + } + + pub fn add_skipped_mutant(&mut self, mutant: Mutant) { + self.skipped.push(mutant); + } + + pub fn add_timed_out_mutant(&mut self, mutant: Mutant) { + self.timed_out.push(mutant); + } + + pub const fn total_mutants(&self) -> usize { + self.dead.len() + + self.survived.len() + + self.invalid.len() + + self.skipped.len() + + self.timed_out.len() + } + + pub const fn total_dead(&self) -> usize { + self.dead.len() + } + + pub const fn total_survived(&self) -> usize { + self.survived.len() + } + + pub const fn total_invalid(&self) -> usize { + self.invalid.len() + } + + pub const fn total_skipped(&self) -> usize { + self.skipped.len() + } + + pub const fn total_timed_out(&self) -> usize { + self.timed_out.len() + } + + pub const fn get_dead(&self) -> &Vec { + &self.dead + } + + pub const fn get_survived(&self) -> &Vec { + &self.survived + } + + pub const fn get_invalid(&self) -> &Vec { + &self.invalid + } + + pub const fn get_timed_out(&self) -> &Vec { + &self.timed_out + } + + /// Merge another MutationsSummary into this one + pub fn merge(&mut self, other: &Self) { + self.dead.extend(other.dead.clone()); + self.survived.extend(other.survived.clone()); + self.invalid.extend(other.invalid.clone()); + self.skipped.extend(other.skipped.clone()); + self.timed_out.extend(other.timed_out.clone()); + } + + /// Calculate mutation score (percentage of dead mutants out of valid mutants) + /// Higher scores indicate better test coverage + pub fn mutation_score(&self) -> f64 { + let valid_mutants = self.dead.len() + self.survived.len(); + if valid_mutants == 0 { 0.0 } else { self.dead.len() as f64 / valid_mutants as f64 * 100.0 } + } + + /// Convert to JSON output format. + /// + /// Output is sorted deterministically: files in lexicographic order + /// (`BTreeMap` keys), and survived mutants within each file sorted by + /// `(line, column, original, mutant)`. Without this, parallel worker + /// completion order leaks into the JSON and breaks downstream diffing, + /// snapshot tests, and reproducibility. + pub fn to_json_output(&self, duration_secs: f64) -> MutationJsonOutput { + let mut survived_mutants: BTreeMap> = BTreeMap::new(); + + for mutant in &self.survived { + let file_path = mutant.relative_path(); + let entry = survived_mutants.entry(file_path).or_default(); + entry.push(SurvivedMutantJson::from_mutant(mutant)); + } + + for entries in survived_mutants.values_mut() { + entries.sort_by(|a, b| { + (a.line, a.column, &a.original, &a.mutant).cmp(&( + b.line, + b.column, + &b.original, + &b.mutant, + )) + }); + } + + MutationJsonOutput { + summary: MutationSummaryJson { + total: self.total_mutants(), + killed: self.total_dead(), + survived: self.total_survived(), + invalid: self.total_invalid(), + skipped: self.total_skipped(), + timed_out: self.total_timed_out(), + mutation_score: self.mutation_score(), + duration_secs, + }, + survived_mutants, + } + } +} + +/// JSON output for mutation testing results. +/// +/// Uses [`BTreeMap`] for `survived_mutants` so file ordering in the emitted +/// JSON is deterministic. +#[derive(Debug, Clone, Serialize)] +pub struct MutationJsonOutput { + pub summary: MutationSummaryJson, + pub survived_mutants: BTreeMap>, +} + +/// Summary section of JSON output +#[derive(Debug, Clone, Serialize)] +pub struct MutationSummaryJson { + pub total: usize, + pub killed: usize, + pub survived: usize, + pub invalid: usize, + pub skipped: usize, + pub timed_out: usize, + pub mutation_score: f64, + pub duration_secs: f64, +} + +/// Individual survived mutant in JSON output +#[derive(Debug, Clone, Serialize)] +pub struct SurvivedMutantJson { + pub line: usize, + pub column: usize, + pub original: String, + pub mutant: String, +} + +impl SurvivedMutantJson { + /// Create from a Mutant, using the full original expression + pub fn from_mutant(mutant: &Mutant) -> Self { + Self { + line: mutant.line_number, + column: mutant.column_number, + original: mutant.original.clone(), + mutant: mutant.mutation.to_string(), + } + } +} + +/// Tracks spans where mutations have survived (weren't killed by tests). +/// Used for adaptive mutation testing to skip redundant mutations. +#[derive(Debug, Clone, Default)] +pub struct SurvivedSpans { + spans: HashSet<(u32, u32)>, // (lo, hi) byte positions +} + +impl SurvivedSpans { + pub fn new() -> Self { + Self { spans: HashSet::new() } + } + + /// Mark a span as having a surviving mutation + pub fn mark_survived(&mut self, span: Span) { + self.spans.insert((span.lo().0, span.hi().0)); + } + + /// Check if any survived parent span contains this span. + /// + /// Exact span matches are not skipped: a persisted survived-span cache only + /// records byte ranges, not which mutant at that range survived. Re-testing + /// exact spans after an interrupted run keeps known survivors from being + /// converted into `Skipped` results in the next complete cache. + pub fn should_skip(&self, span: Span) -> bool { + let (lo, hi) = (span.lo().0, span.hi().0); + + self.spans.iter().any(|&(parent_lo, parent_hi)| { + parent_lo <= lo && hi <= parent_hi && (parent_lo != lo || parent_hi != hi) + }) + } + + /// Check if any survived span contains this span, including exact matches. + /// + /// Live workers know exact same-span mutants are siblings in the current + /// run, so once one survives the remaining siblings can be skipped. + pub fn should_skip_in_live_run(&self, span: Span) -> bool { + let (lo, hi) = (span.lo().0, span.hi().0); + + self.spans.iter().any(|&(parent_lo, parent_hi)| parent_lo <= lo && hi <= parent_hi) + } + + /// Serialize to a list of (lo, hi) pairs for caching + fn to_vec(&self) -> Vec<(u32, u32)> { + self.spans.iter().copied().collect() + } + + /// Deserialize from a list of (lo, hi) pairs + fn from_vec(pairs: Vec<(u32, u32)>) -> Self { + Self { spans: pairs.into_iter().collect() } + } +} + +pub struct MutationHandler { + contract_to_mutate: PathBuf, + pub src: Arc, + pub mutations: Vec, + config: Arc, + report: MutationsSummary, + survived_spans: SurvivedSpans, + /// Optional regex used to restrict mutation to specific contracts within + /// the file (matches against contract name). + contract_filter: Option, +} + +impl MutationHandler { + pub fn new(contract_to_mutate: PathBuf, config: Arc) -> Self { + Self { + contract_to_mutate, + src: Arc::default(), + mutations: vec![], + config, + report: MutationsSummary::new(), + survived_spans: SurvivedSpans::new(), + contract_filter: None, + } + } + + /// Restrict mutation to contracts whose name matches `filter`. + pub fn with_contract_filter(mut self, filter: regex::Regex) -> Self { + self.contract_filter = Some(filter); + self + } + + pub fn read_source_contract(&mut self) -> Result<(), std::io::Error> { + let content = std::fs::read_to_string(&self.contract_to_mutate)?; + self.src = Arc::new(content); + Ok(()) + } + + /// Add a dead mutant to the report + pub fn add_dead_mutant(&mut self, mutant: Mutant) { + self.report.add_dead_mutant(mutant); + } + + /// Add a survived mutant to the report + pub fn add_survived_mutant(&mut self, mutant: Mutant) { + self.report.add_survived_mutant(mutant); + } + + /// Add an invalid mutant to the report + pub fn add_invalid_mutant(&mut self, mutant: Mutant) { + self.report.update_invalid_mutant(mutant); + } + + pub fn add_skipped_mutant(&mut self, mutant: Mutant) { + self.report.add_skipped_mutant(mutant); + } + + pub fn add_timed_out_mutant(&mut self, mutant: Mutant) { + self.report.add_timed_out_mutant(mutant); + } + + /// Get a reference to the current report + pub const fn get_report(&self) -> &MutationsSummary { + &self.report + } + + // Note: we now get the build hash directly from the recent compile output (see test flow) + + /// Returns the cache file path for the given build hash and cache kind. + /// The filename encodes a hash of the full contract path to prevent collisions + /// between files with the same stem in different directories, and a hash of + /// the active mutation config so changes to enabled operators invalidate + /// previously cached mutants. Result-like caches also include an execution + /// key so stale outcomes are not reused after test/config/EVM changes. + fn cache_file_path(&self, hash: &str, kind: CacheKind<'_>) -> PathBuf { + let mut hasher = DefaultHasher::new(); + self.contract_to_mutate.hash(&mut hasher); + let path_hash = hasher.finish(); + + // Hash the effective set of enabled mutation operators so mutant cache + // entries are invalidated when the user changes `include_operators` / + // `exclude_operators` in their config. + // + // Also fold in the active `--mutate-contract` regex pattern, because + // running with vs. without that filter produces a different mutant set + // for the same file. + let mut mutant_cfg_hasher = DefaultHasher::new(); + // Version salt for this mutant-set cache schema. Bump this if the + // inputs that define generated mutants change. + "mutant-set-v2".hash(&mut mutant_cfg_hasher); + for op in self.config.mutation.enabled_operators() { + op.to_string().hash(&mut mutant_cfg_hasher); + } + match self.contract_filter.as_ref() { + Some(re) => { + "filter:".hash(&mut mutant_cfg_hasher); + re.as_str().hash(&mut mutant_cfg_hasher); + } + None => "nofilter".hash(&mut mutant_cfg_hasher), + } + let mutant_cfg_hash = mutant_cfg_hasher.finish(); + + let (ext, execution_suffix) = match kind { + CacheKind::Mutants => ("mutants", String::new()), + CacheKind::Results { execution_key } => ("results", format!("_{execution_key}")), + CacheKind::Survived { execution_key } => ("survived", format!("_{execution_key}")), + }; + + let stem = + self.contract_to_mutate.file_stem().and_then(|s| s.to_str()).unwrap_or("unknown"); + self.config.root.join(&self.config.mutation_dir).join(format!( + "{hash}_{stem}_{path_hash:x}_{mutant_cfg_hash:x}{execution_suffix}.{ext}" + )) + } + + /// Persists cached mutants using build hash for cache invalidation. + pub fn persist_cached_mutants(&self, hash: &str, mutants: &[Mutant]) -> std::io::Result<()> { + let cache_file = self.cache_file_path(hash, CacheKind::Mutants); + if let Some(dir) = cache_file.parent() { + std::fs::create_dir_all(dir)?; + } + let json = serde_json::to_string_pretty(mutants).map_err(std::io::Error::other)?; + std::fs::write(cache_file, json) + } + + /// Persists results for mutants using build hash for cache invalidation. + pub fn persist_cached_results( + &self, + hash: &str, + execution_key: &str, + mutants: &[Mutant], + results: &[(Mutant, crate::mutation::mutant::MutationResult)], + ) -> std::io::Result<()> { + let cache_file = self.cache_file_path(hash, CacheKind::Results { execution_key }); + if let Some(dir) = cache_file.parent() { + std::fs::create_dir_all(dir)?; + } + let cached = CachedMutationResults { + mutant_count: mutants.len(), + mutant_hash: mutant_set_hash(mutants), + results: results.to_vec(), + }; + let json = serde_json::to_string_pretty(&cached).map_err(std::io::Error::other)?; + std::fs::write(cache_file, json) + } + + /// Read a source string, and for each contract found, gets its ast and visit it to list + /// all mutations to conduct. + pub async fn generate_ast(&mut self) -> eyre::Result<()> { + let path = &self.contract_to_mutate; + let target_content = Arc::clone(&self.src); + let sess = Session::builder().with_silent_emitter(None).build(); + + let contract_filter = self.contract_filter.clone(); + + let result = sess.enter(|| -> eyre::Result> { + let arena = solar::ast::Arena::new(); + let mut parser = + Parser::from_lazy_source_code(&sess, &arena, FileName::from(path.clone()), || { + Ok((*target_content).clone()) + }) + .map_err(|_e| failed_to_parse(path))?; + + let ast = parser.parse_file().map_err(|e| { + e.emit(); + failed_to_parse(path) + })?; + + let operators = self.config.mutation.enabled_operators(); + let mut mutant_visitor = MutantVisitor::with_operators(path.clone(), &operators) + .with_source(&target_content); + + if let Some(filter) = contract_filter { + mutant_visitor = + mutant_visitor.with_contract_filter(move |name| filter.is_match(name)); + } + let _ = mutant_visitor.visit_source_unit(&ast); + + for err in mutant_visitor.take_errors() { + let _ = sh_warn!("{err:?}"); + } + + Ok(mutant_visitor.mutation_to_conduct) + }); + + match result { + Ok(mutations) => { + self.mutations.extend(mutations); + Ok(()) + } + Err(err) => Err(err), + } + } + + /// Retrieves cached mutants using build hash. + pub fn retrieve_cached_mutants(&self, hash: &str) -> Option> { + let cache_file = self.cache_file_path(hash, CacheKind::Mutants); + let data = std::fs::read_to_string(cache_file).ok()?; + serde_json::from_str(&data).ok() + } + + /// Retrieves cached results using build hash. + pub fn retrieve_cached_mutant_results( + &self, + hash: &str, + execution_key: &str, + mutants: &[Mutant], + ) -> Option> { + let cache_file = self.cache_file_path(hash, CacheKind::Results { execution_key }); + let data = std::fs::read_to_string(cache_file).ok()?; + let cached: CachedMutationResults = serde_json::from_str(&data).ok()?; + (cached.mutant_count == mutants.len() && cached.mutant_hash == mutant_set_hash(mutants)) + .then_some(cached.results) + } + + /// Mark a span as having a surviving mutation + pub fn mark_span_survived(&mut self, span: Span) { + self.survived_spans.mark_survived(span); + } + + /// Check if a span should be skipped (has survived mutation or is child of survived span) + pub fn should_skip_span(&self, span: Span) -> bool { + self.survived_spans.should_skip(span) + } + + /// Persist survived spans to cache for adaptive mutation testing. + pub fn persist_survived_spans(&self, hash: &str, execution_key: &str) -> std::io::Result<()> { + let cache_file = self.cache_file_path(hash, CacheKind::Survived { execution_key }); + if let Some(dir) = cache_file.parent() { + std::fs::create_dir_all(dir)?; + } + let spans = self.survived_spans.to_vec(); + let json = serde_json::to_string_pretty(&spans).map_err(std::io::Error::other)?; + std::fs::write(cache_file, json) + } + + /// Retrieve survived spans from cache. + pub fn retrieve_survived_spans(&mut self, hash: &str, execution_key: &str) -> bool { + let cache_file = self.cache_file_path(hash, CacheKind::Survived { execution_key }); + + if let Ok(data) = std::fs::read_to_string(cache_file) + && let Ok(pairs) = serde_json::from_str::>(&data) + { + self.survived_spans = SurvivedSpans::from_vec(pairs); + return true; + } + + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + use foundry_config::Config; + use solar::ast::interface::BytePos; + use tempfile::TempDir; + + fn test_handler(config: Config) -> MutationHandler { + let source = config.root.join("src").join("Counter.sol"); + MutationHandler::new(source, Arc::new(config)) + } + + fn test_config() -> (TempDir, Config) { + let temp = TempDir::new().unwrap(); + let config = Config { + root: temp.path().to_path_buf(), + mutation_dir: "cache/mutation".into(), + ..Default::default() + }; + (temp, config) + } + + fn mutant(lo: u32, hi: u32, original: &str) -> Mutant { + Mutant { + path: PathBuf::from("src/Counter.sol"), + span: Span::new(BytePos(lo), BytePos(hi)), + mutation: mutant::MutationType::DeleteExpression, + original: original.to_string(), + source_line: "number++;".to_string(), + line_number: 1, + column_number: 1, + } + } + + #[test] + fn result_cache_path_includes_execution_key() { + let (_temp, config) = test_config(); + let handler = test_handler(config); + + let first = + handler.cache_file_path("build", CacheKind::Results { execution_key: "exec-a" }); + let second = + handler.cache_file_path("build", CacheKind::Results { execution_key: "exec-b" }); + let mutants = handler.cache_file_path("build", CacheKind::Mutants); + + assert_ne!(first, second); + assert_ne!(first, mutants); + assert_ne!(second, mutants); + } + + #[test] + fn survived_span_cache_path_includes_execution_key() { + let (_temp, config) = test_config(); + let handler = test_handler(config); + + let first = + handler.cache_file_path("build", CacheKind::Survived { execution_key: "exec-a" }); + let second = + handler.cache_file_path("build", CacheKind::Survived { execution_key: "exec-b" }); + + assert_ne!(first, second); + } + + #[test] + fn mutant_cache_path_ignores_execution_only_timeout() { + let (_temp, mut first_config) = test_config(); + let mut second_config = first_config.clone(); + + first_config.mutation.timeout = Some(1); + second_config.mutation.timeout = Some(99); + + let first = test_handler(first_config).cache_file_path("build", CacheKind::Mutants); + let second = test_handler(second_config).cache_file_path("build", CacheKind::Mutants); + + assert_eq!(first, second); + } + + #[test] + fn result_cache_validates_current_mutant_set() { + let (_temp, config) = test_config(); + let handler = test_handler(config); + let mutants = vec![mutant(10, 20, "number++")]; + let results = vec![(mutants[0].clone(), MutationResult::Dead)]; + + handler.persist_cached_results("build", "exec", &mutants, &results).unwrap(); + + assert!(handler.retrieve_cached_mutant_results("build", "exec", &mutants).is_some()); + + let changed_mutants = vec![mutant(10, 20, "number--")]; + assert!( + handler.retrieve_cached_mutant_results("build", "exec", &changed_mutants).is_none() + ); + } +} diff --git a/crates/forge/src/mutation/mutant.rs b/crates/forge/src/mutation/mutant.rs new file mode 100644 index 0000000000000..1b6352f5a7b8b --- /dev/null +++ b/crates/forge/src/mutation/mutant.rs @@ -0,0 +1,392 @@ +use std::{fmt::Display, path::PathBuf}; + +use serde::{Deserialize, Serialize}; +use solar::{ + interface::BytePos, + parse::ast::{BinOpKind, LitKind, Span, StrKind, UnOpKind}, +}; + +use super::visitor::AssignVarTypes; + +/// Wraps an unary operator mutated, to easily store pre/post-fix op swaps +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UnaryOpMutated { + /// String containing the whole new expression (operator and its target) + /// eg `a++` + new_expression: String, + + /// The underlying operator used by this mutant + #[serde(serialize_with = "serialize_unop_kind", deserialize_with = "deserialize_unop_kind")] + pub resulting_op_kind: UnOpKind, +} + +// Custom serialization for UnOpKind +fn serialize_unop_kind(value: &UnOpKind, serializer: S) -> Result +where + S: serde::Serializer, +{ + let s = format!("{value:?}"); + serializer.serialize_str(&s) +} + +fn deserialize_unop_kind<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + match s.as_str() { + "PreInc" => Ok(UnOpKind::PreInc), + "PostInc" => Ok(UnOpKind::PostInc), + "PreDec" => Ok(UnOpKind::PreDec), + "PostDec" => Ok(UnOpKind::PostDec), + "Not" => Ok(UnOpKind::Not), + "BitNot" => Ok(UnOpKind::BitNot), + "Neg" => Ok(UnOpKind::Neg), + other => Err(serde::de::Error::custom(format!("Unknown UnOpKind: {other}"))), + } +} + +impl UnaryOpMutated { + pub const fn new(new_expression: String, resulting_op_kind: UnOpKind) -> Self { + Self { new_expression, resulting_op_kind } + } +} + +impl Display for UnaryOpMutated { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.new_expression) + } +} + +// Custom serialization for BinOpKind +fn serialize_binop(value: &BinOpKind, serializer: S) -> Result +where + S: serde::Serializer, +{ + let s = format!("{value:?}"); + serializer.serialize_str(&s) +} + +fn deserialize_binop<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + match s.as_str() { + "Add" => Ok(BinOpKind::Add), + "Sub" => Ok(BinOpKind::Sub), + "Mul" => Ok(BinOpKind::Mul), + "Div" => Ok(BinOpKind::Div), + "And" => Ok(BinOpKind::And), + "Or" => Ok(BinOpKind::Or), + "Eq" => Ok(BinOpKind::Eq), + "Ne" => Ok(BinOpKind::Ne), + "Lt" => Ok(BinOpKind::Lt), + "Le" => Ok(BinOpKind::Le), + "Gt" => Ok(BinOpKind::Gt), + "Ge" => Ok(BinOpKind::Ge), + "BitAnd" => Ok(BinOpKind::BitAnd), + "BitOr" => Ok(BinOpKind::BitOr), + "BitXor" => Ok(BinOpKind::BitXor), + "Shl" => Ok(BinOpKind::Shl), + "Shr" => Ok(BinOpKind::Shr), + "Sar" => Ok(BinOpKind::Sar), + "Pow" => Ok(BinOpKind::Pow), + "Rem" => Ok(BinOpKind::Rem), + other => Err(serde::de::Error::custom(format!("Unknown BinOpKind: {other}"))), + } +} + +// @todo add a mutation from universalmutator: line swap (swap two lines of code, as it +// could theoretically uncover untested reentrancies +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum OwnedStrKind { + Str, + Unicode, + Hex, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum OwnedLiteral { + Str { + kind: OwnedStrKind, + text: String, + }, + Number(alloy_primitives::U256), + Rational(String), + Address(String), + Bool(bool), + Err(String), + /// Signed-negation of a numeric literal (e.g. `-123`). We cannot represent + /// negative values inside `Number(U256)` (the cast wraps via two's + /// complement and renders as a huge unsigned literal), so we carry the + /// negation textually and render it as `-{val}`. + NegatedNumber(alloy_primitives::U256), +} + +impl From<&LitKind<'_>> for OwnedLiteral { + fn from(lit_kind: &LitKind<'_>) -> Self { + match lit_kind { + LitKind::Bool(b) => Self::Bool(*b), + LitKind::Number(n) => Self::Number(*n), + LitKind::Rational(r) => Self::Rational(r.to_string()), + LitKind::Address(addr) => Self::Address(addr.to_string()), + LitKind::Str(sk, bytesym, _extras) => { + let text = String::from_utf8_lossy(bytesym.as_byte_str()).into_owned(); + let kind = match sk { + StrKind::Str => OwnedStrKind::Str, + StrKind::Unicode => OwnedStrKind::Unicode, + StrKind::Hex => OwnedStrKind::Hex, + }; + Self::Str { kind, text } + } + LitKind::Err(_) => Self::Err("parse_error".to_string()), + } + } +} + +impl Display for OwnedLiteral { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Bool(val) => write!(f, "{val}"), + Self::Number(val) => write!(f, "{val}"), + Self::NegatedNumber(val) => write!(f, "-{val}"), + Self::Rational(s) => write!(f, "{s}"), + Self::Address(s) => write!(f, "{s}"), + Self::Str { kind, text } => match kind { + OwnedStrKind::Str => write!(f, "\"{text}\""), + OwnedStrKind::Unicode => write!(f, "unicode\"{text}\""), + OwnedStrKind::Hex => write!(f, "hex\"{text}\""), + }, + Self::Err(s) => write!(f, "{s}"), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum MutationType { + // Note: Solar's LitKind::Number(U256) doesn't differentiate int vs uint - it only stores the + // numeric value without signedness info. For now we generate mutations for both and let solc + // filter out invalid ones (e.g., -x on uint). Future improvement: track variable types in a + // symbol table to avoid generating invalid mutants. + /// For an initializer x, of type + /// bool: replace x with !x + /// uint: replace x with 0 + /// int: replace x with 0; replace x with -x (temp: this is mutated for uint as well) + /// + /// For a binary op y: apply BinaryOp(y) + Assignment(AssignVarTypes), + + /// For a binary op y in BinOpKind ("+", "-", ">=", etc) + /// replace y with each non-y in op (legacy, kept for cache compatibility) + #[serde(serialize_with = "serialize_binop", deserialize_with = "deserialize_binop")] + BinaryOp(BinOpKind), + + /// Binary operator mutation with full expression context + /// Stores both the new operator and the full mutated expression for display + BinaryOpExpr { + #[serde(serialize_with = "serialize_binop", deserialize_with = "deserialize_binop")] + new_op: BinOpKind, + mutated_expr: String, + }, + + /// For a delete expr x `delete foo`, replace x with `assert(true)` + DeleteExpression, + + /// replace "delegatecall" with "call" + ElimDelegate, + + /// Gambit doesn't implement nor define it? + FunctionCall, + + // /// For a if(x) condition x: + // /// replace x with true; replace x with false + // This mutation is not used anymore, as we mutate the condition as an expression, + // which will creates true/false mutant as well as more complex conditions (eg if(foo++ > + // --bar) ) IfStatementMutation, + /// For a require(x) condition: + /// replace x with true; replace x with false + // Same as for IfStatementMutation, the expression inside the require is mutated as an + // expression to handle increment etc + Require, + + /// For require(condition)/assert(condition), mutate the condition: + /// - require(x) -> require(true) (always passes - security critical!) + /// - require(x) -> require(false) (always fails) + /// - require(x) -> require(!x) (inverted condition) + RequireCondition { + /// The mutated full call expression + mutated_call: String, + }, + + // @todo review if needed -> this might creates *a lot* of combinations for super-polyadic fn + // tho only swapping same type (to avoid obvious compilation failure), but should + // take into account implicit casting too... + /// For 2 args of the same type x,y in a function args: + /// swap(x, y) + SwapArgumentsFunction, + + // @todo same remark as above, might end up in a space too big to explore + filtering out + // based on type + /// For an expr taking 2 expression x, y (x+y, x-y, x = x + ...): + /// swap(x, y) + SwapArgumentsOperator, + + /// For an unary operator x in UnOpKind (eg "++", "--", "~", "!"): + /// replace x with all other operator in op + /// Pre or post- are different UnOp + UnaryOperator(UnaryOpMutated), + + YulOpcode { + original_opcode: String, + new_opcode: String, + mutated_expr: String, + }, +} + +impl Display for MutationType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Assignment(kind) => match kind { + AssignVarTypes::Literal(lit) => write!(f, "{lit}"), + AssignVarTypes::Identifier(ident) => write!(f, "{ident}"), + }, + Self::BinaryOp(kind) => write!(f, "{}", kind.to_str()), + Self::BinaryOpExpr { mutated_expr, .. } => write!(f, "{mutated_expr}"), + Self::DeleteExpression => write!(f, "assert(true)"), + Self::ElimDelegate => write!(f, "call"), + Self::UnaryOperator(mutated) => write!(f, "{mutated}"), + Self::RequireCondition { mutated_call } => write!(f, "{mutated_call}"), + + Self::YulOpcode { mutated_expr, .. } => write!(f, "{mutated_expr}"), + + Self::FunctionCall + | Self::Require + | Self::SwapArgumentsFunction + | Self::SwapArgumentsOperator => write!(f, ""), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum MutationResult { + Dead, + Alive, + Invalid, + Skipped, + /// The mutant's compile-and-test run exceeded the configured timeout. + /// Treated as unresolved: not counted toward survived or killed. + TimedOut, +} + +impl MutationResult { + /// Short uppercase label used in progress / reporter output. + pub const fn label(&self) -> &'static str { + match self { + Self::Dead => "KILLED", + Self::Alive => "SURVIVED", + Self::Invalid => "INVALID", + Self::Skipped => "SKIPPED", + Self::TimedOut => "TIMED OUT", + } + } +} + +/// A given mutation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Mutant { + /// The path to the project root where this mutant (tries to) live + pub path: PathBuf, + #[serde(serialize_with = "serialize_span", deserialize_with = "deserialize_span")] + pub span: Span, + pub mutation: MutationType, + /// The original source text that will be replaced by this mutation (full expression) + #[serde(default)] + pub original: String, + /// The full source line for context (e.g., "uint256 x = a * b;") + #[serde(default)] + pub source_line: String, + /// Line number in the source file (1-indexed) + #[serde(default)] + pub line_number: usize, + /// Column number in the source file (1-indexed) + #[serde(default)] + pub column_number: usize, +} + +// Custom serialization for Span (since solar::parse::ast::Span doesn't implement Serialize) +fn serialize_span(span: &Span, serializer: S) -> Result +where + S: serde::Serializer, +{ + use serde::Serialize; + #[derive(Serialize)] + struct SpanHelper { + lo: u32, + hi: u32, + } + SpanHelper { lo: span.lo().0, hi: span.hi().0 }.serialize(serializer) +} + +fn deserialize_span<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + use serde::Deserialize; + #[derive(Deserialize)] + struct SpanHelper { + lo: u32, + hi: u32, + } + let helper = SpanHelper::deserialize(deserializer)?; + Ok(Span::new(BytePos(helper.lo), BytePos(helper.hi))) +} + +impl Mutant { + /// Returns a relative path string. + /// + /// Walks ancestor components looking for a well-known directory root + /// (`src`, `test`, `lib`, `contracts`) so the output is cross-platform and + /// does not rely on OS-specific path separators. + pub fn relative_path(&self) -> String { + let components: Vec<_> = self.path.components().collect(); + for (i, comp) in components.iter().enumerate() { + if let std::path::Component::Normal(name) = comp { + let s = name.to_string_lossy(); + if matches!(s.as_ref(), "src" | "test" | "script" | "lib" | "contracts") { + let parts: Vec<_> = components[i..] + .iter() + .filter_map(|c| match c { + std::path::Component::Normal(s) => Some(s.to_string_lossy()), + _ => None, + }) + .collect(); + return parts.join("/"); + } + } + } + self.path.file_name().and_then(|n| n.to_str()).unwrap_or("unknown").to_string() + } + + /// Returns a concise one-line description of the mutation (full original code) + pub fn short_description(&self) -> String { + let original = if self.original.is_empty() { + "".to_string() + } else { + self.original.trim().to_string() + }; + let mutated = self.mutation.to_string(); + + format!("`{}` → `{}`", original, mutated.trim()) + } +} + +impl Display for Mutant { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.line_number > 0 { + write!(f, "{}:{}: {}", self.relative_path(), self.line_number, self.short_description()) + } else { + write!(f, "{}: {}", self.relative_path(), self.short_description()) + } + } +} diff --git a/crates/forge/src/mutation/mutators/assembly_mutator.rs b/crates/forge/src/mutation/mutators/assembly_mutator.rs new file mode 100644 index 0000000000000..bd62a617adac1 --- /dev/null +++ b/crates/forge/src/mutation/mutators/assembly_mutator.rs @@ -0,0 +1,242 @@ +use std::collections::HashMap; + +use eyre::Result; +use solar::ast::yul; + +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType}; + +pub struct AssemblyMutator { + opcode_mutations: HashMap<&'static str, Vec<&'static str>>, +} + +impl Default for AssemblyMutator { + fn default() -> Self { + Self::new() + } +} + +impl AssemblyMutator { + pub fn new() -> Self { + let mut opcode_mutations: HashMap<&'static str, Vec<&'static str>> = HashMap::new(); + + // Arithmetic — stay within arithmetic family + opcode_mutations.insert("add", vec!["sub", "mul"]); + opcode_mutations.insert("sub", vec!["add", "mul", "div"]); + opcode_mutations.insert("mul", vec!["add", "div"]); + opcode_mutations.insert("div", vec!["mul", "sub", "mod"]); + opcode_mutations.insert("sdiv", vec!["smod", "mul"]); + opcode_mutations.insert("mod", vec!["div", "mul"]); + opcode_mutations.insert("smod", vec!["sdiv", "mod"]); + opcode_mutations.insert("exp", vec!["mul", "add"]); + opcode_mutations.insert("addmod", vec!["mulmod"]); + opcode_mutations.insert("mulmod", vec!["addmod"]); + + // Comparisons — stay within comparison family + opcode_mutations.insert("lt", vec!["gt", "eq", "slt"]); + opcode_mutations.insert("gt", vec!["lt", "eq", "sgt"]); + opcode_mutations.insert("slt", vec!["sgt", "lt"]); + opcode_mutations.insert("sgt", vec!["slt", "gt"]); + opcode_mutations.insert("eq", vec!["lt", "gt"]); + + // Bitwise — stay within bitwise family + opcode_mutations.insert("and", vec!["or", "xor"]); + opcode_mutations.insert("or", vec!["and", "xor"]); + opcode_mutations.insert("xor", vec!["and", "or"]); + + // Shifts — stay within shift family + opcode_mutations.insert("shl", vec!["shr", "sar"]); + opcode_mutations.insert("shr", vec!["shl", "sar"]); + opcode_mutations.insert("sar", vec!["shr", "shl"]); + + Self { opcode_mutations } + } + + pub fn get_mutations(&self, opcode: &str) -> Option<&[&'static str]> { + self.opcode_mutations.get(opcode).map(|v| v.as_slice()) + } +} + +impl Mutator for AssemblyMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let yul_expr = context.yul_expr.ok_or_else(|| eyre::eyre!("No Yul expression"))?; + + let call = match &yul_expr.kind { + yul::ExprKind::Call(call) => call, + _ => return Ok(vec![]), + }; + + let opcode_name = call.name.as_str(); + + let alternatives = match self.get_mutations(opcode_name) { + Some(alts) => alts, + None => return Ok(vec![]), + }; + + let original = context.original_text(); + if original.is_empty() { + return Ok(vec![]); + } + + let expected_len = (context.span.hi().0 - context.span.lo().0) as usize; + if original.len() != expected_len { + return Ok(vec![]); + } + + let source_line = context.source_line(); + let line_number = context.line_number(); + let column_number = context.column_number(); + + let name_span = call.name.span; + + let mutants = alternatives + .iter() + .filter_map(|&new_opcode| { + let mutated = + replace_at_span(&original, context.span, name_span, opcode_name, new_opcode)?; + Some(Mutant { + span: context.span, + mutation: MutationType::YulOpcode { + original_opcode: opcode_name.to_string(), + new_opcode: new_opcode.to_string(), + mutated_expr: mutated, + }, + path: context.path.clone(), + original: original.clone(), + source_line: source_line.clone(), + line_number, + column_number, + }) + }) + .collect(); + + Ok(mutants) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if let Some(yul_expr) = ctxt.yul_expr + && let yul::ExprKind::Call(call) = &yul_expr.kind + { + return self.opcode_mutations.contains_key(call.name.as_str()); + } + false + } +} + +fn replace_at_span( + original: &str, + outer_span: solar::ast::Span, + target_span: solar::ast::Span, + expected_opcode: &str, + replacement: &str, +) -> Option { + let outer_lo = outer_span.lo().0 as usize; + let target_lo = target_span.lo().0 as usize; + let target_hi = target_span.hi().0 as usize; + + let rel_lo = target_lo.checked_sub(outer_lo)?; + let rel_hi = target_hi.checked_sub(outer_lo)?; + + if rel_lo > rel_hi || rel_hi > original.len() { + return None; + } + + let prefix = original.get(..rel_lo)?; + let replaced = original.get(rel_lo..rel_hi)?; + let suffix = original.get(rel_hi..)?; + + if replaced != expected_opcode { + return None; + } + + Some(format!("{prefix}{replacement}{suffix}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_opcode_mutations_exist() { + let mutator = AssemblyMutator::new(); + + assert!(mutator.get_mutations("add").unwrap().contains(&"sub")); + assert!(mutator.get_mutations("mul").unwrap().contains(&"div")); + + assert!(mutator.get_mutations("lt").unwrap().contains(&"gt")); + assert!(mutator.get_mutations("slt").unwrap().contains(&"sgt")); + + assert!(mutator.get_mutations("and").unwrap().contains(&"or")); + assert!(mutator.get_mutations("shl").unwrap().contains(&"shr")); + } + + #[test] + fn test_no_cross_family_mutations() { + let mutator = AssemblyMutator::new(); + let add_alts = mutator.get_mutations("add").unwrap(); + assert!(!add_alts.contains(&"xor"), "add should not mutate to xor (cross-family)"); + assert!(!add_alts.contains(&"and"), "add should not mutate to and (cross-family)"); + + let mul_alts = mutator.get_mutations("mul").unwrap(); + assert!(!mul_alts.contains(&"and"), "mul should not mutate to and (cross-family)"); + } + + #[test] + fn test_no_iszero_not_mapping() { + let mutator = AssemblyMutator::new(); + assert!(mutator.get_mutations("iszero").is_none(), "iszero should not be mutated"); + assert!(mutator.get_mutations("not").is_none(), "not should not be mutated"); + } + + #[test] + fn test_no_mload_sload_mapping() { + let mutator = AssemblyMutator::new(); + assert!(mutator.get_mutations("mload").is_none()); + assert!(mutator.get_mutations("sload").is_none()); + } + + #[test] + fn test_replace_at_span_valid() { + use solar::interface::BytePos; + let original = "add(a, b)"; + let outer = solar::ast::Span::new(BytePos(10), BytePos(19)); + let target = solar::ast::Span::new(BytePos(10), BytePos(13)); + let result = replace_at_span(original, outer, target, "add", "sub"); + assert_eq!(result, Some("sub(a, b)".to_string())); + } + + #[test] + fn test_replace_at_span_target_outside_outer() { + use solar::interface::BytePos; + let original = "add(a, b)"; + let outer = solar::ast::Span::new(BytePos(20), BytePos(29)); + let target = solar::ast::Span::new(BytePos(10), BytePos(13)); + assert!(replace_at_span(original, outer, target, "add", "sub").is_none()); + } + + #[test] + fn test_replace_at_span_target_exceeds_length() { + use solar::interface::BytePos; + let original = "add(a, b)"; + let outer = solar::ast::Span::new(BytePos(10), BytePos(19)); + let target = solar::ast::Span::new(BytePos(10), BytePos(30)); + assert!(replace_at_span(original, outer, target, "add", "sub").is_none()); + } + + #[test] + fn test_replace_at_span_opcode_mismatch() { + use solar::interface::BytePos; + let original = "mul(a, b)"; + let outer = solar::ast::Span::new(BytePos(10), BytePos(19)); + let target = solar::ast::Span::new(BytePos(10), BytePos(13)); + assert!(replace_at_span(original, outer, target, "add", "sub").is_none()); + } + + #[test] + fn test_replace_at_span_empty_original() { + use solar::interface::BytePos; + let outer = solar::ast::Span::new(BytePos(10), BytePos(19)); + let target = solar::ast::Span::new(BytePos(10), BytePos(13)); + assert!(replace_at_span("", outer, target, "add", "sub").is_none()); + } +} diff --git a/crates/forge/src/mutation/mutators/assignment_mutator.rs b/crates/forge/src/mutation/mutators/assignment_mutator.rs new file mode 100644 index 0000000000000..cfc71f809ab43 --- /dev/null +++ b/crates/forge/src/mutation/mutators/assignment_mutator.rs @@ -0,0 +1,148 @@ +use alloy_primitives::U256; +use eyre::Result; +use solar::ast::{ExprKind, Span}; + +use crate::mutation::{ + mutant::{Mutant, MutationType, OwnedLiteral}, + mutators::{MutationContext, Mutator}, + visitor::AssignVarTypes, +}; + +pub struct AssignmentMutator; + +impl Mutator for AssignmentMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let (assign_var_type, replacement_span) = match extract_rhs_info(context) { + Some(info) => info, + None => return Ok(vec![]), // is_applicable should filter this + }; + + let original = context.original_text(); + let source_line = context.source_line(); + let line_number = context.line_number(); + let column_number = context.column_number(); + + match assign_var_type { + AssignVarTypes::Literal(ref lit) => match lit { + OwnedLiteral::Bool(val) => Ok(vec![Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal( + OwnedLiteral::Bool(!val), + )), + path: context.path.clone(), + original, + source_line, + line_number, + column_number, + }]), + OwnedLiteral::Number(val) if *val == U256::ZERO => Ok(vec![]), + OwnedLiteral::Number(val) => Ok(vec![ + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal( + OwnedLiteral::Number(U256::ZERO), + )), + path: context.path.clone(), + original: original.clone(), + source_line: source_line.clone(), + line_number, + column_number, + }, + // Negation of a numeric literal must be carried textually: + // applying `-*val` on a `U256` wraps via two's complement + // and would render as a huge unsigned literal (e.g. `1` + // becomes `2^256 - 1`), producing wrong-source mutants. + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal( + OwnedLiteral::NegatedNumber(*val), + )), + path: context.path.clone(), + original, + source_line, + line_number, + column_number, + }, + ]), + OwnedLiteral::Str { .. } => Ok(vec![]), + OwnedLiteral::Rational(_) => Ok(vec![]), + OwnedLiteral::Address(_) => Ok(vec![]), + OwnedLiteral::Err(_) => Ok(vec![]), + // `NegatedNumber` is only ever constructed *as* a mutant; it + // does not appear as an original literal in the source AST, + // so there is nothing to mutate here. + OwnedLiteral::NegatedNumber(_) => Ok(vec![]), + }, + AssignVarTypes::Identifier(ref ident) => Ok(vec![ + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal( + OwnedLiteral::Number(U256::ZERO), + )), + path: context.path.clone(), + original: original.clone(), + source_line: source_line.clone(), + line_number, + column_number, + }, + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Identifier(format!( + "-{ident}" + ))), + path: context.path.clone(), + original, + source_line, + line_number, + column_number, + }, + ]), + } + } + + /// Match is the expr is an assign with a var definition having a literal or identifier as + /// initializer + fn is_applicable(&self, context: &MutationContext<'_>) -> bool { + if let Some(expr) = context.expr { + if let ExprKind::Assign(_lhs, _op_opt, rhs_actual_expr) = &expr.kind { + matches!(rhs_actual_expr.kind, ExprKind::Lit(..) | ExprKind::Ident(..)) + } else { + false // Not an assign + } + } else if let Some(var_definition) = context.var_definition { + if let Some(init) = &var_definition.initializer { + matches!(&init.kind, ExprKind::Lit(..) | ExprKind::Ident(..)) + } else { + false // No initializer + } + } else { + false // Not an expression or var_definition + } + } +} + +fn extract_rhs_info<'ast>(context: &MutationContext<'ast>) -> Option<(AssignVarTypes, Span)> { + let relevant_expr_for_rhs = if let Some(var_definition) = context.var_definition { + var_definition.initializer.as_ref()? + } else { + let expr = context.expr?; + match &expr.kind { + ExprKind::Assign(_lhs, _op_opt, rhs_actual_expr) => &**rhs_actual_expr, + // If the context.expr is already what we want to get the type from + // (e.g. a simple Lit or Ident being passed directly, though is_applicable filters this) + ExprKind::Lit(..) | ExprKind::Ident(..) => expr, + _ => return None, + } + }; + + match &relevant_expr_for_rhs.kind { + ExprKind::Lit(kind, _) => { + let owned = OwnedLiteral::from(&kind.kind); + Some((AssignVarTypes::Literal(owned), relevant_expr_for_rhs.span)) + } + ExprKind::Ident(val) => { + Some((AssignVarTypes::Identifier(val.to_string()), relevant_expr_for_rhs.span)) + } + _ => None, + } +} diff --git a/crates/forge/src/mutation/mutators/binary_op_mutator.rs b/crates/forge/src/mutation/mutators/binary_op_mutator.rs new file mode 100644 index 0000000000000..7bf90815df9fb --- /dev/null +++ b/crates/forge/src/mutation/mutators/binary_op_mutator.rs @@ -0,0 +1,131 @@ +use eyre::{OptionExt, Result}; +use solar::ast::{BinOp, BinOpKind, Expr, ExprKind, Span}; + +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType}; + +pub struct BinaryOpMutator; + +impl Mutator for BinaryOpMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let expr = context.expr.ok_or_eyre("BinaryOpMutator: no expression")?; + let (bin_op, _op_span, lhs, rhs, compound_assignment) = get_bin_op_parts(expr)?; + let op = bin_op.kind; + + let operations_bools = vec![ + BinOpKind::Lt, + BinOpKind::Le, + BinOpKind::Gt, + BinOpKind::Ge, + BinOpKind::Eq, + BinOpKind::Ne, + BinOpKind::Or, + BinOpKind::And, + ]; + + let operations_num_bitwise = vec![ + BinOpKind::Shr, + BinOpKind::Shl, + BinOpKind::Sar, + BinOpKind::BitAnd, + BinOpKind::BitOr, + BinOpKind::BitXor, + BinOpKind::Add, + BinOpKind::Sub, + BinOpKind::Pow, + BinOpKind::Mul, + BinOpKind::Div, + BinOpKind::Rem, + ]; + + let operations = + if operations_bools.contains(&op) { operations_bools } else { operations_num_bitwise }; + + // Extract LHS and RHS text from source + let source = context.source.unwrap_or(""); + let lhs_text = extract_span_text(source, lhs.span); + let rhs_text = extract_span_text(source, rhs.span); + let op_str = op.to_str(); + + let original_expr = if compound_assignment { + format!("{lhs_text} {op_str}= {rhs_text}") + } else { + format!("{lhs_text} {op_str} {rhs_text}") + }; + + // Use the full expression span for the mutation (not just the operator span) + let expr_span = context.span; + + // Get line context + let source_line = context.source_line(); + let line_number = context.line_number(); + let column_number = context.column_number(); + + Ok(operations + .into_iter() + .filter(|&kind| kind != op) + .filter(|&kind| !compound_assignment || is_valid_compound_assignment_op(kind)) + .map(|kind| { + let mutated_expr = if compound_assignment { + format!("{} {}= {}", lhs_text, kind.to_str(), rhs_text) + } else { + format!("{} {} {}", lhs_text, kind.to_str(), rhs_text) + }; + Mutant { + span: expr_span, + mutation: MutationType::BinaryOpExpr { new_op: kind, mutated_expr }, + path: context.path.clone(), + original: original_expr.clone(), + source_line: source_line.clone(), + line_number, + column_number, + } + }) + .collect()) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if ctxt.expr.is_none() { + return false; + } + + matches!( + ctxt.expr.unwrap().kind, + ExprKind::Binary(_, _, _) | ExprKind::Assign(_, Some(_), _) + ) + } +} + +const fn is_valid_compound_assignment_op(kind: BinOpKind) -> bool { + matches!( + kind, + BinOpKind::BitOr + | BinOpKind::BitXor + | BinOpKind::BitAnd + | BinOpKind::Shl + | BinOpKind::Shr + | BinOpKind::Add + | BinOpKind::Sub + | BinOpKind::Mul + | BinOpKind::Div + | BinOpKind::Rem + ) +} + +/// Extract the binary operator, its span, and LHS/RHS expressions +fn get_bin_op_parts<'a>( + expr: &'a Expr<'a>, +) -> Result<(BinOp, Span, &'a Expr<'a>, &'a Expr<'a>, bool)> { + match &expr.kind { + ExprKind::Assign(lhs, Some(op), rhs) => Ok((*op, op.span, lhs, rhs, true)), + ExprKind::Binary(lhs, op, rhs) => Ok((*op, op.span, lhs, rhs, false)), + _ => eyre::bail!("BinaryOpMutator: unexpected expression kind"), + } +} + +/// Extract text from source given a span +fn extract_span_text(source: &str, span: Span) -> String { + let lo = span.lo().0 as usize; + let hi = span.hi().0 as usize; + source.get(lo..hi).map(|s| s.trim().to_string()).unwrap_or_default() +} diff --git a/crates/forge/src/mutation/mutators/delete_expression_mutator.rs b/crates/forge/src/mutation/mutators/delete_expression_mutator.rs new file mode 100644 index 0000000000000..d4bea4adf7296 --- /dev/null +++ b/crates/forge/src/mutation/mutators/delete_expression_mutator.rs @@ -0,0 +1,26 @@ +use eyre::Result; +use solar::ast::ExprKind; + +use super::{MutationContext, Mutator}; + +use crate::mutation::mutant::{Mutant, MutationType}; + +pub struct DeleteExpressionMutator; + +impl Mutator for DeleteExpressionMutator { + fn generate_mutants(&self, ctxt: &MutationContext<'_>) -> Result> { + Ok(vec![Mutant { + span: ctxt.span, + mutation: MutationType::DeleteExpression, + path: ctxt.path.clone(), + original: ctxt.original_text(), + source_line: ctxt.source_line(), + line_number: ctxt.line_number(), + column_number: ctxt.column_number(), + }]) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if let Some(expr) = ctxt.expr { matches!(expr.kind, ExprKind::Delete(_)) } else { false } + } +} diff --git a/crates/forge/src/mutation/mutators/elim_delegate_mutator.rs b/crates/forge/src/mutation/mutators/elim_delegate_mutator.rs new file mode 100644 index 0000000000000..bc36f10bb790a --- /dev/null +++ b/crates/forge/src/mutation/mutators/elim_delegate_mutator.rs @@ -0,0 +1,62 @@ +use std::fmt::Display; + +use eyre::Result; +use solar::ast::ExprKind; + +use super::{MutationContext, Mutator}; + +use crate::mutation::mutant::{Mutant, MutationType}; + +pub struct ElimDelegateMutator; + +impl Mutator for ElimDelegateMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + // Narrow the span to just the `delegatecall` identifier so the replacement + // text ("call") does not clobber the surrounding call expression + // (e.g. `target.delegatecall(data)`). + let ident_span = context + .expr + .as_ref() + .and_then(|expr| match &expr.kind { + ExprKind::Call(callee, _) => Some(callee), + _ => None, + }) + .and_then(|callee| match &callee.kind { + ExprKind::Member(_, ident) => Some(ident.span), + _ => None, + }) + .unwrap_or(context.span); + + Ok(vec![Mutant { + span: ident_span, + mutation: MutationType::ElimDelegate, + path: context.path.clone(), + // Use the narrowed identifier as the "original" text so the diff line + // ("- delegatecall" / "+ call") matches the actual textual replacement. + original: "delegatecall".to_string(), + source_line: context.source_line(), + line_number: context.line_number(), + column_number: context.column_number(), + }]) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + ctxt.expr + .as_ref() + .and_then(|expr| match &expr.kind { + ExprKind::Call(callee, _) => Some(callee), + _ => None, + }) + .and_then(|callee| match &callee.kind { + ExprKind::Member(_, ident) => Some(ident), + _ => None, + }) + .is_some_and(|ident| ident.to_string() == "delegatecall") + } +} + +impl Display for ElimDelegateMutator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "") + } +} diff --git a/crates/forge/src/mutation/mutators/mod.rs b/crates/forge/src/mutation/mutators/mod.rs new file mode 100644 index 0000000000000..20af2f8b155a2 --- /dev/null +++ b/crates/forge/src/mutation/mutators/mod.rs @@ -0,0 +1,167 @@ +use std::path::PathBuf; + +use eyre::Result; +use solar::ast::{Expr, Span, VariableDefinition, yul}; + +use crate::mutation::Mutant; + +pub mod assembly_mutator; +pub mod assignment_mutator; +pub mod binary_op_mutator; +pub mod delete_expression_mutator; +pub mod elim_delegate_mutator; +pub mod mutator_registry; +pub mod require_mutator; +pub mod unary_op_mutator; + +pub trait Mutator: Send + Sync { + /// Generate all mutant corresponding to a given context + fn generate_mutants(&self, ctxt: &MutationContext<'_>) -> Result>; + /// True if a mutator can be applied to an expression/node + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool; +} + +#[derive(Debug)] +pub struct MutationContext<'a> { + pub path: PathBuf, + pub span: Span, + /// The expression to mutate + pub expr: Option<&'a Expr<'a>>, + pub var_definition: Option<&'a VariableDefinition<'a>>, + /// Yul expression for assembly block mutations + pub yul_expr: Option<&'a yul::Expr<'a>>, + /// The full source code (used to extract original text for mutations) + pub source: Option<&'a str>, +} + +impl MutationContext<'_> { + /// Extract the original source text covered by this context's span + pub fn original_text(&self) -> String { + self.source + .and_then(|src| { + let lo = self.span.lo().0 as usize; + let hi = self.span.hi().0 as usize; + src.get(lo..hi).map(|s| s.to_string()) + }) + .unwrap_or_default() + } + + /// Get the line number (1-indexed) for this context's span + pub fn line_number(&self) -> usize { + self.source + .map(|src| { + let pos = self.span.lo().0 as usize; + src.get(..pos).map(|s| s.lines().count()).unwrap_or(0).max(1) + }) + .unwrap_or(1) + } + + /// Get the column number (1-indexed) for this context's span + pub fn column_number(&self) -> usize { + self.source + .map(|src| { + let pos = self.span.lo().0 as usize; + let line_start = + src.get(..pos).and_then(|s| s.rfind('\n')).map(|i| i + 1).unwrap_or(0); + pos - line_start + 1 + }) + .unwrap_or(1) + } + + /// Get the full source line containing this span + pub fn source_line(&self) -> String { + self.source + .and_then(|src| { + let pos = self.span.lo().0 as usize; + // Find line start + let line_start = src[..pos].rfind('\n').map(|i| i + 1).unwrap_or(0); + // Find line end + let line_end = src[pos..].find('\n').map(|i| pos + i).unwrap_or(src.len()); + src.get(line_start..line_end).map(|s| s.trim().to_string()) + }) + .unwrap_or_default() + } +} + +impl<'a> MutationContext<'a> { + #[must_use] + pub const fn builder() -> MutationContextBuilder<'a> { + MutationContextBuilder::new() + } +} + +pub struct MutationContextBuilder<'a> { + path: Option, + span: Option, + expr: Option<&'a Expr<'a>>, + var_definition: Option<&'a VariableDefinition<'a>>, + yul_expr: Option<&'a yul::Expr<'a>>, + source: Option<&'a str>, +} + +impl<'a> MutationContextBuilder<'a> { + // Create a new empty builder + pub const fn new() -> Self { + MutationContextBuilder { + path: None, + span: None, + expr: None, + var_definition: None, + yul_expr: None, + source: None, + } + } + + // Required + pub fn with_path(mut self, path: PathBuf) -> Self { + self.path = Some(path); + self + } + + // Required + pub const fn with_span(mut self, span: Span) -> Self { + self.span = Some(span); + self + } + + // Optional + pub const fn with_expr(mut self, expr: &'a Expr<'a>) -> Self { + self.expr = Some(expr); + self + } + + // Optional + pub const fn with_var_definition(mut self, var_definition: &'a VariableDefinition<'a>) -> Self { + self.var_definition = Some(var_definition); + self + } + + // Optional + pub const fn with_yul_expr(mut self, yul_expr: &'a yul::Expr<'a>) -> Self { + self.yul_expr = Some(yul_expr); + self + } + + // Optional - provide source code for extracting original text + pub const fn with_source(mut self, source: &'a str) -> Self { + self.source = Some(source); + self + } + + pub fn build(self) -> Result, &'static str> { + let span = self.span.ok_or("Span is required for MutationContext")?; + let path = self.path.ok_or("Path is required for MutationContext")?; + + Ok(MutationContext { + path, + span, + expr: self.expr, + var_definition: self.var_definition, + yul_expr: self.yul_expr, + source: self.source, + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/forge/src/mutation/mutators/mutator_registry.rs b/crates/forge/src/mutation/mutators/mutator_registry.rs new file mode 100644 index 0000000000000..9c0cda6b53fae --- /dev/null +++ b/crates/forge/src/mutation/mutators/mutator_registry.rs @@ -0,0 +1,113 @@ +use crate::mutation::mutant::Mutant; +use eyre::Report; +use foundry_config::MutatorType; + +use super::{ + MutationContext, Mutator, assembly_mutator, assignment_mutator, binary_op_mutator, + delete_expression_mutator, elim_delegate_mutator, require_mutator, unary_op_mutator, +}; + +/// Registry of all available mutators (ie implementing the Mutator trait) +pub struct MutatorRegistry { + mutators: Vec>, +} + +pub struct MutationGenerationResult { + pub mutations: Vec, + pub errors: Vec, +} + +impl MutatorRegistry { + #[cfg(test)] + pub fn default() -> Self { + Self::from_enabled(&MutatorType::all()) + } + + pub fn from_enabled(enabled: &[MutatorType]) -> Self { + let mut registry = Self { mutators: Vec::new() }; + + for ty in enabled { + match ty { + MutatorType::Assembly => { + registry.mutators.push(Box::new(assembly_mutator::AssemblyMutator::new())); + } + MutatorType::Assignment => { + registry.mutators.push(Box::new(assignment_mutator::AssignmentMutator)); + } + MutatorType::BinaryOp => { + registry.mutators.push(Box::new(binary_op_mutator::BinaryOpMutator)); + } + MutatorType::DeleteExpression => { + registry + .mutators + .push(Box::new(delete_expression_mutator::DeleteExpressionMutator)); + } + MutatorType::ElimDelegate => { + registry.mutators.push(Box::new(elim_delegate_mutator::ElimDelegateMutator)); + } + MutatorType::Require => { + registry.mutators.push(Box::new(require_mutator::RequireMutator)); + } + MutatorType::UnaryOp => { + registry.mutators.push(Box::new(unary_op_mutator::UnaryOpMutator)); + } + } + } + + registry + } + + #[cfg(test)] + pub fn new_with_mutators(mutators: Vec>) -> Self { + Self { mutators } + } + + /// Find all applicable mutators for a given context and return the corresponding mutations + /// and any mutator errors encountered while generating them. + pub fn generate_mutations(&self, context: &MutationContext<'_>) -> MutationGenerationResult { + let mut mutations = Vec::new(); + let mut errors = Vec::new(); + for mutator in self.mutators.iter().filter(|mutator| mutator.is_applicable(context)) { + match mutator.generate_mutants(context) { + Ok(generated) => mutations.extend(generated), + Err(err) => errors.push(err), + } + } + MutationGenerationResult { mutations, errors } + } +} + +#[cfg(test)] +mod tests { + use eyre::{Result, eyre}; + use solar::ast::Span; + + use super::*; + + struct FailingMutator; + + impl Mutator for FailingMutator { + fn generate_mutants(&self, _ctxt: &MutationContext<'_>) -> Result> { + Err(eyre!("synthetic mutator failure")) + } + + fn is_applicable(&self, _ctxt: &MutationContext<'_>) -> bool { + true + } + } + + #[test] + fn generate_mutations_collects_mutator_errors() { + let registry = MutatorRegistry::new_with_mutators(vec![Box::new(FailingMutator)]); + let context = MutationContext::builder() + .with_path("test.sol".into()) + .with_span(Span::DUMMY) + .build() + .unwrap(); + + let result = registry.generate_mutations(&context); + assert!(result.mutations.is_empty()); + let err = result.errors.into_iter().next().unwrap(); + assert!(err.to_string().contains("synthetic mutator failure")); + } +} diff --git a/crates/forge/src/mutation/mutators/require_mutator.rs b/crates/forge/src/mutation/mutators/require_mutator.rs new file mode 100644 index 0000000000000..6b2e8ae420ea0 --- /dev/null +++ b/crates/forge/src/mutation/mutators/require_mutator.rs @@ -0,0 +1,146 @@ +//! Require/Assert condition mutator. +//! +//! This mutator targets security-critical validation patterns in Solidity: +//! - `require(condition)` / `require(condition, "message")` +//! - `assert(condition)` +//! +//! Mutations generated: +//! - `require(x)` -> `require(true)` - Always passes (security critical!) +//! - `require(x)` -> `require(false)` - Always fails +//! - `require(x)` -> `require(!x)` - Inverted condition +//! +//! These mutations are particularly valuable for security testing because: +//! - Access control checks (onlyOwner patterns) +//! - Input validation (bounds checking, address validation) +//! - State preconditions (reentrancy guards, paused checks) + +use eyre::Result; +use solar::ast::{CallArgsKind, Expr, ExprKind, UnOpKind}; + +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType}; + +pub struct RequireMutator; + +impl Mutator for RequireMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let expr = context.expr.ok_or_else(|| eyre::eyre!("RequireMutator: no expression"))?; + + // Extract function name and arguments + let (func_name, args_exprs) = match &expr.kind { + ExprKind::Call(callee, call_args) => { + let name = match &callee.kind { + ExprKind::Ident(ident) => ident.to_string(), + _ => return Ok(vec![]), + }; + // Extract the expressions from CallArgs + let exprs = match &call_args.kind { + CallArgsKind::Unnamed(exprs) => exprs, + CallArgsKind::Named(_) => return Ok(vec![]), // Named args not supported + }; + (name, exprs) + } + _ => return Ok(vec![]), + }; + + // Only handle require and assert + if func_name != "require" && func_name != "assert" { + return Ok(vec![]); + } + + // Need at least one argument (the condition) + if args_exprs.is_empty() { + return Ok(vec![]); + } + + let condition_expr = &args_exprs[0]; + let original = context.original_text(); + let source_line = context.source_line(); + let line_number = context.line_number(); + let column_number = context.column_number(); + + let source = context.source.unwrap_or(""); + + // Build the rest of the call (message and other arguments after the condition, + // if any) using span-based extraction so that commas inside the condition + // expression (e.g. `require(foo(a, b))`) do not break splitting. + let rest_args = if args_exprs.len() > 1 { + // Extract from the character right after the condition expression up to + // the last argument's end. + let start = condition_expr.span.hi().0 as usize; + let end = args_exprs.last().map(|e| e.span.hi().0 as usize).unwrap_or(start); + source.get(start..end).map(|s| s.to_string()).unwrap_or_default() + } else { + String::new() + }; + + let mut mutants = Vec::new(); + let mut push_mutant = |mutated_call: String, original: String, source_line: String| { + if mutated_call.trim() == original.trim() { + return; + } + mutants.push(Mutant { + span: expr.span, + mutation: MutationType::RequireCondition { mutated_call }, + path: context.path.clone(), + original, + source_line, + line_number, + column_number, + }); + }; + + // Mutation 1: require(x) -> require(true) + // This is security-critical: if tests pass, the condition was never actually needed + let mutated_true = format!("{func_name}(true{rest_args})"); + push_mutant(mutated_true, original.clone(), source_line.clone()); + + let mutated_false = format!("{func_name}(false{rest_args})"); + push_mutant(mutated_false, original.clone(), source_line.clone()); + + let inverted_condition = invert_condition_text(source, condition_expr); + let mutated_inverted = format!("{func_name}({inverted_condition}{rest_args})"); + push_mutant(mutated_inverted, original, source_line); + + Ok(mutants) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + ctxt.expr + .as_ref() + .and_then(|expr| match &expr.kind { + ExprKind::Call(callee, call_args) => { + // Must have at least one argument + match &call_args.kind { + CallArgsKind::Unnamed(exprs) if !exprs.is_empty() => {} + _ => return None, + } + match &callee.kind { + ExprKind::Ident(ident) => Some(ident.to_string()), + _ => None, + } + } + _ => None, + }) + .is_some_and(|name| name == "require" || name == "assert") + } +} + +/// Extract text from source given a span +fn extract_span_text(source: &str, span: solar::ast::Span) -> String { + let lo = span.lo().0 as usize; + let hi = span.hi().0 as usize; + source.get(lo..hi).map(|s| s.to_string()).unwrap_or_default() +} + +fn invert_condition_text(source: &str, condition_expr: &Expr<'_>) -> String { + match &condition_expr.kind { + ExprKind::Unary(op, inner) if op.kind == UnOpKind::Not => { + extract_span_text(source, inner.span) + } + _ => { + let condition = extract_span_text(source, condition_expr.span); + format!("!({})", condition.trim()) + } + } +} diff --git a/crates/forge/src/mutation/mutators/tests/assembly_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/assembly_mutator_test.rs new file mode 100644 index 0000000000000..4c6bef88cd18b --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/assembly_mutator_test.rs @@ -0,0 +1,487 @@ +//! Integration tests for the AssemblyMutator using real-world Solady-inspired patterns. + +use std::path::PathBuf; + +use solar::{ + ast::{Arena, interface::source_map::FileName, visit::Visit}, + parse::Parser, +}; + +use crate::mutation::{Session, mutant::MutationType, visitor::MutantVisitor}; + +/// Solady FixedPointMathLib: `fullMulDivUnchecked` uses mul, sub, div, lt, and, xor. +#[test] +fn test_solady_full_mul_div_unchecked() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library FixedPointMathLib { + function fullMulDivUnchecked(uint256 x, uint256 y, uint256 d) + internal pure returns (uint256 z) + { + assembly { + z := mul(x, y) + let mm := mulmod(x, y, not(0)) + let p1 := sub(mm, add(z, lt(mm, z))) + let t := and(d, sub(0, d)) + d := div(d, t) + let inv := xor(2, mul(3, d)) + inv := mul(inv, sub(2, mul(d, inv))) + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "mul", "add"); + assert_has_opcode_mutation(&yul, "mul", "div"); + assert_has_opcode_mutation(&yul, "sub", "add"); + assert_has_opcode_mutation(&yul, "div", "mul"); + assert_has_opcode_mutation(&yul, "lt", "gt"); + assert_has_opcode_mutation(&yul, "and", "or"); + assert_has_opcode_mutation(&yul, "xor", "and"); + assert_has_opcode_mutation(&yul, "mulmod", "addmod"); +} + +/// Solady: `divUp` uses iszero, mod, div, add. +#[test] +fn test_solady_div_up() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library FixedPointMathLib { + function divUp(uint256 x, uint256 d) internal pure returns (uint256 z) { + assembly { + if iszero(d) { revert(0, 0) } + z := add(iszero(iszero(mod(x, d))), div(x, d)) + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "add", "sub"); + assert_has_opcode_mutation(&yul, "div", "mul"); + assert_has_opcode_mutation(&yul, "mod", "div"); +} + +/// Solady: `zeroFloorSub` / `saturatingSub` uses gt, sub, mul. +#[test] +fn test_solady_zero_floor_sub() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library FixedPointMathLib { + function zeroFloorSub(uint256 x, uint256 y) internal pure returns (uint256 z) { + assembly { + z := mul(gt(x, y), sub(x, y)) + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "gt", "lt"); + assert_has_opcode_mutation(&yul, "sub", "add"); + assert_has_opcode_mutation(&yul, "mul", "add"); +} + +/// Solady: `min(uint256)` uses lt, xor, mul. +#[test] +fn test_solady_min_unsigned() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library FixedPointMathLib { + function min(uint256 x, uint256 y) internal pure returns (uint256 z) { + assembly { + z := xor(x, mul(xor(x, y), lt(y, x))) + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "lt", "gt"); + assert_has_opcode_mutation(&yul, "lt", "eq"); + assert_has_opcode_mutation(&yul, "xor", "and"); +} + +/// Solady: `max(int256)` uses sgt — tests signed comparison mutations. +#[test] +fn test_solady_max_signed() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library FixedPointMathLib { + function max(int256 x, int256 y) internal pure returns (int256 z) { + assembly { + z := xor(x, mul(xor(x, y), sgt(y, x))) + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "sgt", "slt"); + assert_has_opcode_mutation(&yul, "sgt", "gt"); +} + +/// Solady: `dist(int256)` uses sgt, sub, xor, add — signed distance. +#[test] +fn test_solady_dist_signed() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library FixedPointMathLib { + function dist(int256 x, int256 y) internal pure returns (uint256 z) { + assembly { + z := add(xor(sub(0, sgt(x, y)), sub(y, x)), sgt(x, y)) + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "sgt", "slt"); + assert_has_opcode_mutation(&yul, "sub", "add"); + assert_has_opcode_mutation(&yul, "add", "sub"); + assert_has_opcode_mutation(&yul, "xor", "or"); +} + +/// Solady: `saturatingAdd` uses or, sub, lt, add. +#[test] +fn test_solady_saturating_add() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library FixedPointMathLib { + function saturatingAdd(uint256 x, uint256 y) internal pure returns (uint256 z) { + assembly { + z := or(sub(0, lt(add(x, y), x)), add(x, y)) + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "or", "and"); + assert_has_opcode_mutation(&yul, "lt", "gt"); +} + +/// Solady: `log256` uses shl, shr, lt, or — shift mutations. +#[test] +fn test_solady_log256_shifts() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library FixedPointMathLib { + function log256(uint256 x) internal pure returns (uint256 r) { + assembly { + r := shl(7, lt(0xffffffffffffffffffffffffffffffff, x)) + r := or(r, shl(6, lt(0xffffffffffffffff, shr(r, x)))) + r := or(r, shl(5, lt(0xffffffff, shr(r, x)))) + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "shl", "shr"); + assert_has_opcode_mutation(&yul, "shr", "shl"); + assert_has_opcode_mutation(&yul, "or", "and"); +} + +/// Solady: `rawAddMod` / `rawMulMod` — addmod ↔ mulmod swaps. +#[test] +fn test_solady_addmod_mulmod() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library FixedPointMathLib { + function rawAddMod(uint256 x, uint256 y, uint256 d) internal pure returns (uint256 z) { + assembly { + z := addmod(x, y, d) + } + } + + function rawMulMod(uint256 x, uint256 y, uint256 d) internal pure returns (uint256 z) { + assembly { + z := mulmod(x, y, d) + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "addmod", "mulmod"); + assert_has_opcode_mutation(&yul, "mulmod", "addmod"); +} + +/// Opcodes NOT in the mapping should produce zero mutations. +#[test] +fn test_unmapped_opcodes_not_mutated() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +contract Example { + function test() external view returns (uint256) { + assembly { + let x := mload(0x40) + mstore(0, caller()) + let h := keccak256(0, 32) + sstore(h, x) + } + } +} +"#; + + let yul = yul_mutations(source); + let bad_opcodes = ["mload", "mstore", "caller", "keccak256", "sstore"]; + for opcode in bad_opcodes { + assert!( + !yul.iter().any(|m| m.original_opcode == opcode), + "'{opcode}' should NOT be mutated" + ); + } +} + +/// Empty assembly block should produce zero Yul mutations. +#[test] +fn test_empty_assembly_block() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +contract Example { + function test() external pure { + assembly {} + } +} +"#; + + let yul = yul_mutations(source); + assert!(yul.is_empty(), "Empty assembly should produce no mutations"); +} + +/// Assembly inside a library function body is traversed. +#[test] +fn test_library_assembly_traversal() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library AssemblyLib { + function addAsm(uint256 a, uint256 b) internal pure returns (uint256 result) { + assembly { + result := add(a, b) + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "add", "sub"); + assert_has_opcode_mutation(&yul, "add", "mul"); +} + +/// Nested calls: only the outermost opcode at each visit should be mutated, +/// inner calls get their own visit. +#[test] +fn test_nested_calls_mutated_independently() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +contract Example { + function test(uint256 x, uint256 y) external pure returns (uint256) { + assembly { + mstore(0, add(mul(x, y), div(x, y))) + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "add", "sub"); + assert_has_opcode_mutation(&yul, "mul", "div"); + assert_has_opcode_mutation(&yul, "div", "mul"); +} + +/// Solidity code without assembly should produce zero Yul mutations. +#[test] +fn test_no_assembly_no_yul_mutations() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +contract PureSolidity { + function add(uint256 a, uint256 b) external pure returns (uint256) { + return a + b; + } + + function compare(uint256 a, uint256 b) external pure returns (bool) { + return a < b; + } +} +"#; + + let yul = yul_mutations(source); + assert!(yul.is_empty(), "Pure Solidity should produce no Yul mutations"); +} + +/// Solady: `invMod` uses a for-loop with div, sub, mul, eq, slt, mod. +#[test] +fn test_solady_inv_mod_loop() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library FixedPointMathLib { + function invMod(uint256 a, uint256 n) internal pure returns (uint256 x) { + assembly { + let g := n + let r := mod(a, n) + for { let y := 1 } 1 {} { + let q := div(g, r) + let t := g + g := r + r := sub(t, mul(r, q)) + let u := x + x := y + y := sub(u, mul(y, q)) + if iszero(r) { break } + } + x := mul(eq(g, 1), add(x, mul(slt(x, 0), n))) + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "div", "mul"); + assert_has_opcode_mutation(&yul, "sub", "add"); + assert_has_opcode_mutation(&yul, "mul", "add"); + assert_has_opcode_mutation(&yul, "mod", "div"); + assert_has_opcode_mutation(&yul, "eq", "lt"); + assert_has_opcode_mutation(&yul, "slt", "sgt"); +} + +/// Solady: `rpow` uses exp — exponentiation mutation. +#[test] +fn test_solady_rpow_exp() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +library FixedPointMathLib { + function rpow(uint256 x, uint256 y, uint256 b) internal pure returns (uint256 z) { + assembly { + z := mul(b, iszero(y)) + if x { + z := xor(b, mul(xor(b, x), and(y, 1))) + let half := shr(1, b) + for { y := shr(1, y) } y { y := shr(1, y) } { + let xx := mul(x, x) + let xxRound := add(xx, half) + if or(lt(xxRound, xx), shr(128, x)) { + revert(0, 0) + } + x := shr(1, add(mul(xxRound, xxRound), half)) + } + } + } + } +} +"#; + + let yul = yul_mutations(source); + assert_has_opcode_mutation(&yul, "shr", "shl"); + assert_has_opcode_mutation(&yul, "xor", "and"); + assert_has_opcode_mutation(&yul, "or", "and"); +} + +/// Span-based replacement correctly handles the exact opcode token, +/// verified by checking the mutated expression text. +#[test] +fn test_span_replacement_correctness() { + let source = r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +contract Example { + function test(uint256 a, uint256 b) external pure returns (uint256 r) { + assembly { + r := add(a, b) + } + } +} +"#; + + let yul = yul_mutations(source); + + let add_to_sub: Vec<_> = + yul.iter().filter(|m| m.original_opcode == "add" && m.new_opcode == "sub").collect(); + + assert_eq!(add_to_sub.len(), 1, "Should have exactly one add->sub mutation"); + assert_eq!(add_to_sub[0].mutated_expr, "sub(a, b)"); +} + +struct YulMutation { + original_opcode: String, + new_opcode: String, + mutated_expr: String, +} + +fn yul_mutations(source: &str) -> Vec { + let sess = Session::builder().with_silent_emitter(None).build(); + + sess.enter(|| -> solar::interface::Result> { + let arena = Arena::new(); + let mut parser = Parser::from_lazy_source_code( + &sess, + &arena, + FileName::Real(PathBuf::from("test.sol")), + || Ok(source.to_string()), + )?; + + let ast = parser.parse_file().map_err(|e| e.emit())?; + + let mut visitor = MutantVisitor::default(PathBuf::from("test.sol")).with_source(source); + let _ = visitor.visit_source_unit(&ast); + + Ok(visitor + .mutation_to_conduct + .into_iter() + .filter_map(|m| match m.mutation { + MutationType::YulOpcode { original_opcode, new_opcode, mutated_expr } => { + Some(YulMutation { original_opcode, new_opcode, mutated_expr }) + } + _ => None, + }) + .collect()) + }) + .unwrap_or_default() +} + +fn assert_has_opcode_mutation(mutations: &[YulMutation], from: &str, to: &str) { + assert!( + mutations.iter().any(|m| m.original_opcode == from && m.new_opcode == to), + "Expected mutation {from} -> {to}. Got: [{}]", + mutations + .iter() + .map(|m| format!("{} -> {}", m.original_opcode, m.new_opcode)) + .collect::>() + .join(", ") + ); +} diff --git a/crates/forge/src/mutation/mutators/tests/assignment_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/assignment_mutator_test.rs new file mode 100644 index 0000000000000..dbbfdc412e37c --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/assignment_mutator_test.rs @@ -0,0 +1,17 @@ +use crate::mutation::mutators::{ + assignment_mutator::AssignmentMutator, tests::helper::mutator_tests, +}; + +// Each emitted mutation only carries the *replacement text* for the RHS +// span — not the full statement. So `x = 123` mutates to `0` (zero) and +// `-123` (signed-negation), not `x = 0` / `x = -123`. +mutator_tests!(AssignmentMutator; + assign_lit: "x = y" => Some(vec!["0", "-y"]); + assign_number: "x = 123" => Some(vec!["0", "-123"]); + assign_zero: "x = 0" => None; + assign_true: "x = true" => Some(vec!["false"]); + assign_false: "x = false" => Some(vec!["true"]); + assign_declare: "uint256 x = 123" => Some(vec!["0", "-123"]); + declare_zero: "uint256 x = 0" => None; + non_assign: "a = b + c" => None; +); diff --git a/crates/forge/src/mutation/mutators/tests/binary_op_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/binary_op_mutator_test.rs new file mode 100644 index 0000000000000..c4f5bf1bb8e2a --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/binary_op_mutator_test.rs @@ -0,0 +1,20 @@ +use crate::mutation::mutators::{binary_op_mutator::BinaryOpMutator, tests::helper::mutator_tests}; + +mutator_tests!(BinaryOpMutator; + add: "x + y" => Some(vec!["x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]); + sub: "x - y" => Some(vec!["x + y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]); + mul: "x * y" => Some(vec!["x + y", "x - y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]); + div: "x / y" => Some(vec!["x + y", "x - y", "x * y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]); + modulus: "x % y" => Some(vec!["x + y", "x - y", "x * y", "x / y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]); + pow: "x ** y" => Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]); + bit_shift_left: "x << y" => Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]); + bit_shift_right: "x >> y" => Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >>> y", "x & y", "x | y", "x ^ y"]); + bit_shift_right_unsigned: "x >>> y" => Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x & y", "x | y", "x ^ y"]); + bit_and: "x & y" => Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x | y", "x ^ y"]); + bit_or: "x | y" => Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x ^ y"]); + bit_xor: "x ^ y" => Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y"]); + non_binary: "a = true" => None; + compound_assign_add: "a += b" => Some(vec!["a >>= b", "a <<= b", "a &= b", "a |= b", "a ^= b", "a -= b", "a *= b", "a /= b", "a %= b"]); + compound_assign_sub: "a -= b" => Some(vec!["a >>= b", "a <<= b", "a &= b", "a |= b", "a ^= b", "a += b", "a *= b", "a /= b", "a %= b"]); + compound_assign_mul: "a *= b" => Some(vec!["a >>= b", "a <<= b", "a &= b", "a |= b", "a ^= b", "a += b", "a -= b", "a /= b", "a %= b"]); +); diff --git a/crates/forge/src/mutation/mutators/tests/delete_expression_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/delete_expression_mutator_test.rs new file mode 100644 index 0000000000000..3d68858757319 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/delete_expression_mutator_test.rs @@ -0,0 +1,11 @@ +use crate::mutation::mutators::{ + delete_expression_mutator::DeleteExpressionMutator, tests::helper::mutator_tests, +}; + +// `delete x` is replaced by `assert(true)` (a no-op statement) — the test +// expects the mutation's *replacement text*, not the original expression +// stripped of the `delete` keyword. +mutator_tests!(DeleteExpressionMutator; + delete_expr: "delete x" => Some(vec!["assert(true)"]); + non_delete: "a = b + c" => None; +); diff --git a/crates/forge/src/mutation/mutators/tests/elim_delegate_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/elim_delegate_mutator_test.rs new file mode 100644 index 0000000000000..913aef0ef7bf8 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/elim_delegate_mutator_test.rs @@ -0,0 +1,13 @@ +use crate::mutation::mutators::{ + elim_delegate_mutator::ElimDelegateMutator, tests::helper::mutator_tests, +}; + +// The mutator narrows the replacement span to just the `delegatecall` +// identifier and rewrites it to `call`, so the emitted mutation text is +// just `"call"`. It only matches plain `.delegatecall(args)` Call +// expressions; the variant with `{value: ...}` parses as a `CallOptions` +// wrapper and is intentionally left to a follow-up. +mutator_tests!(ElimDelegateMutator; + delegate_expr: "target.delegatecall(data)" => Some(vec!["call"]); + non_delegate: "target.call(data)" => None; +); diff --git a/crates/forge/src/mutation/mutators/tests/helper.rs b/crates/forge/src/mutation/mutators/tests/helper.rs new file mode 100644 index 0000000000000..88d0ca477192d --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/helper.rs @@ -0,0 +1,127 @@ +use solar::{ + ast::{Arena, interface::source_map::FileName, visit::Visit}, + parse::Parser, +}; + +use std::path::PathBuf; + +use crate::mutation::{Session, mutators::Mutator, visitor::MutantVisitor}; + +pub struct MutatorTestCase<'a> { + /// Source code to test - should be valid Solidity code + /// e.g., `"// SPDX-License-Identifier: MIT\npragma solidity ^0.8.0;\ncontract C { function f() + /// { x = 1; } }"` + pub input: &'a str, + /// All the mutations expected for this input, using this mutator + pub expected_mutations: Option>, +} + +pub trait MutatorTester { + fn test_mutator(mutator: M, test_case: MutatorTestCase<'_>) { + // Wrap the snippet in a minimal but valid Solidity source unit so the + // parser/visitor actually walks into expression contexts. Bare + // fragments like `"x + y"` or `"a += b"` are not parseable on their + // own; without wrapping, the parser would emit errors that the test + // harness used to swallow, silently making mutator tests vacuous. + let wrapped = format!( + "// SPDX-License-Identifier: MIT\n\ + pragma solidity ^0.8.0;\n\ + contract __TestC {{\n\ + function __test() public {{\n\ + {input};\n\ + }}\n\ + }}\n", + input = test_case.input, + ); + + let sess = Session::builder().with_silent_emitter(None).build(); + + let outcome = sess.enter(|| -> solar::interface::Result> { + let arena = Arena::new(); + + let mut parser = Parser::from_lazy_source_code( + &sess, + &arena, + FileName::Real(PathBuf::from("test.sol")), + || Ok(wrapped.clone()), + )?; + + let ast = parser.parse_file().map_err(|e| e.emit())?; + + let mut mutant_visitor = MutantVisitor::new_with_mutators( + PathBuf::from("test.sol"), + vec![Box::new(mutator)], + ) + .with_source(&wrapped); + + let _ = mutant_visitor.visit_source_unit(&ast); + + Ok(mutant_visitor + .mutation_to_conduct + .into_iter() + .map(|m| m.mutation.to_string()) + .collect()) + }); + + // Surface parse/visit errors instead of silently passing the test. + let mutations = outcome.unwrap_or_else(|_| { + panic!( + "mutator test input failed to parse/visit; wrapped source was:\n{wrapped}\n\ + raw input: {input:?}", + input = test_case.input, + ) + }); + + if let Some(expected) = test_case.expected_mutations { + let mut actual = mutations; + actual.sort(); + + let mut expected = expected.into_iter().map(str::to_string).collect::>(); + expected.sort(); + + assert_eq!(actual, expected, "Unexpected mutation set for input {:?}", test_case.input); + } else { + assert!( + mutations.is_empty(), + "Expected no mutations, got {}: {:?}", + mutations.len(), + mutations, + ); + } + } +} + +// Implement for unit test module +impl MutatorTester for () {} + +/// Generates one `#[test]` function per case for a [`Mutator`]. +/// +/// Each case becomes a standalone test (parallel execution, individual reporting, +/// IDE run buttons), without pulling in `rstest`. +/// +/// # Example +/// +/// ```ignore +/// mutator_tests!(UnaryOpMutator; +/// pre_inc: "++x" => Some(vec!["--x", "~x", "-x", "x++", "x--"]); +/// non_unary: "a = b + c" => None; +/// ); +/// ``` +macro_rules! mutator_tests { + ($mutator:expr; $($name:ident: $input:expr => $expected:expr);+ $(;)?) => { + $( + #[test] + fn $name() { + <() as $crate::mutation::mutators::tests::helper::MutatorTester>::test_mutator( + $mutator, + $crate::mutation::mutators::tests::helper::MutatorTestCase { + input: $input, + expected_mutations: $expected, + }, + ); + } + )+ + }; +} + +pub(crate) use mutator_tests; diff --git a/crates/forge/src/mutation/mutators/tests/mod.rs b/crates/forge/src/mutation/mutators/tests/mod.rs new file mode 100644 index 0000000000000..b1c8b4f85996f --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/mod.rs @@ -0,0 +1,9 @@ +mod assembly_mutator_test; +mod assignment_mutator_test; +mod binary_op_mutator_test; +mod delete_expression_mutator_test; +mod elim_delegate_mutator_test; +mod require_mutator_test; +mod unary_op_mutator_test; + +mod helper; diff --git a/crates/forge/src/mutation/mutators/tests/require_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/require_mutator_test.rs new file mode 100644 index 0000000000000..a363cd82caa8a --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/require_mutator_test.rs @@ -0,0 +1,16 @@ +use crate::mutation::mutators::{require_mutator::RequireMutator, tests::helper::mutator_tests}; + +mutator_tests!(RequireMutator; + require_true: "require(true)" => Some(vec!["require(false)", "require(!(true))"]); + require_false: "require(false)" => Some(vec!["require(true)", "require(!(false))"]); + require_not: "require(!paused, \"paused\")" => Some(vec![ + "require(false, \"paused\")", + "require(paused, \"paused\")", + "require(true, \"paused\")", + ]); + require_not_composite: "require(!paused && isOwner, \"paused\")" => Some(vec![ + "require(!(!paused && isOwner), \"paused\")", + "require(false, \"paused\")", + "require(true, \"paused\")", + ]); +); diff --git a/crates/forge/src/mutation/mutators/tests/unary_op_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/unary_op_mutator_test.rs new file mode 100644 index 0000000000000..2f06c5c2459f1 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/unary_op_mutator_test.rs @@ -0,0 +1,34 @@ +use crate::mutation::mutators::{tests::helper::mutator_tests, unary_op_mutator::UnaryOpMutator}; + +mutator_tests!(UnaryOpMutator; + pre_inc: "++x" => Some(vec!["--x", "~x", "-x", "x++", "x--"]); + pre_dec: "--x" => Some(vec!["++x", "~x", "-x", "x++", "x--"]); + neg: "-x" => Some(vec!["++x", "--x", "~x", "x++", "x--"]); + bit_not: "~x" => Some(vec!["++x", "--x", "-x", "x++", "x--"]); + post_inc: "x++" => Some(vec!["++x", "--x", "~x", "-x", "x--"]); + post_dec: "x--" => Some(vec!["++x", "--x", "~x", "-x", "x++"]); + bool_not: "!x" => Some(vec!["x"]); + indexed_post_inc: "arr[i]++" => Some(vec![ + "++arr[i]", + "--arr[i]", + "~arr[i]", + "-arr[i]", + "arr[i]--", + ]); + member_post_inc: "boxValue.value++" => Some(vec![ + "++boxValue.value", + "--boxValue.value", + "~boxValue.value", + "-boxValue.value", + "boxValue.value--", + ]); + chained_member_post_inc: "foo().bar++" => Some(vec![ + "++foo().bar", + "--foo().bar", + "~foo().bar", + "-foo().bar", + "foo().bar--", + ]); + not_parenthesized_binary: "!(a == b)" => Some(vec!["(a == b)"]); + non_unary: "a = b + c" => None; +); diff --git a/crates/forge/src/mutation/mutators/unary_op_mutator.rs b/crates/forge/src/mutation/mutators/unary_op_mutator.rs new file mode 100644 index 0000000000000..4817d0bd1552a --- /dev/null +++ b/crates/forge/src/mutation/mutators/unary_op_mutator.rs @@ -0,0 +1,117 @@ +use eyre::Result; +use solar::ast::{ExprKind, Span, UnOpKind}; + +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType, UnaryOpMutated}; + +pub struct UnaryOpMutator; + +impl Mutator for UnaryOpMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let operations = vec![ + UnOpKind::PreInc, // number + UnOpKind::PreDec, // n + UnOpKind::Neg, // n @todo filter this one only for int + UnOpKind::BitNot, // n + ]; + + let post_fixed_operations = vec![UnOpKind::PostInc, UnOpKind::PostDec]; + + let expr = context.expr.unwrap(); + + let op; + let target_span; + + match &expr.kind { + ExprKind::Unary(un_op, target) => { + op = un_op.kind; + target_span = target.span; + } + _ => unreachable!(), + }; + + let target_content = extract_span_text(context.source.unwrap_or(""), target_span); + if target_content.is_empty() { + return Ok(vec![]); + } + + let original = context.original_text(); + let source_line = context.source_line(); + let line_number = context.line_number(); + let column_number = context.column_number(); + + // Bool has only the Not operator as possible target -> we try removing it + if op == UnOpKind::Not { + return Ok(vec![Mutant { + span: expr.span, + mutation: MutationType::UnaryOperator(UnaryOpMutated::new( + target_content, + UnOpKind::Not, + )), + path: context.path.clone(), + original, + source_line, + line_number, + column_number, + }]); + } + + let mut mutations: Vec; + + mutations = operations + .into_iter() + .filter(|&kind| kind != op) + .map(|kind| { + let new_expression = format!("{}{}", kind.to_str(), target_content); + + let mutated = UnaryOpMutated::new(new_expression, kind); + + Mutant { + span: expr.span, + mutation: MutationType::UnaryOperator(mutated), + path: context.path.clone(), + original: original.clone(), + source_line: source_line.clone(), + line_number, + column_number, + } + }) + .collect(); + + mutations.extend(post_fixed_operations.into_iter().filter(|&kind| kind != op).map( + |kind| { + let new_expression = format!("{}{}", target_content, kind.to_str()); + + let mutated = UnaryOpMutated::new(new_expression, kind); + + Mutant { + span: expr.span, + mutation: MutationType::UnaryOperator(mutated), + path: context.path.clone(), + original: original.clone(), + source_line: source_line.clone(), + line_number, + column_number, + } + }, + )); + + Ok(mutations) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if let Some(expr) = ctxt.expr + && let ExprKind::Unary(_, _) = &expr.kind + { + return true; + } + + false + } +} + +fn extract_span_text(source: &str, span: Span) -> String { + let lo = span.lo().0 as usize; + let hi = span.hi().0 as usize; + source.get(lo..hi).map(str::trim).unwrap_or_default().to_string() +} diff --git a/crates/forge/src/mutation/orchestrator.rs b/crates/forge/src/mutation/orchestrator.rs new file mode 100644 index 0000000000000..abdc0c144f8a0 --- /dev/null +++ b/crates/forge/src/mutation/orchestrator.rs @@ -0,0 +1,779 @@ +//! Mutation testing orchestrator. +//! +//! This module coordinates the mutation testing workflow, including: +//! - Filtering source files for mutation +//! - Managing mutation handlers per file +//! - Running mutations in parallel with caching +//! - Aggregating results and reporting + +use std::{ + collections::HashSet, + path::PathBuf, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Instant, +}; + +use alloy_primitives::keccak256; +use eyre::{Result, WrapErr}; +use foundry_cli::utils::FoundryPathExt; +use foundry_common::{compile::ProjectCompiler, sh_println}; +use foundry_compilers::{ + Language, ProjectCompileOutput, + compilers::multi::{MultiCompiler, MultiCompilerLanguage}, + utils::source_files_iter, +}; +use foundry_config::{Config, filter::GlobMatcher}; +use foundry_evm::opts::EvmOpts; + +use crate::{ + cmd::test::FilterArgs, + mutation::{ + MutationHandler, MutationProgress, MutationReporter, MutationsSummary, + mutant::{Mutant, MutationResult}, + runner::run_mutations_parallel_with_progress, + }, +}; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, serde::Serialize)] +struct ArtifactCacheFingerprint { + source: String, + name: String, + version: String, + build_id: String, + profile: String, +} + +#[derive(serde::Serialize)] +struct ExecutionCacheFingerprint<'a> { + schema: &'static str, + config: &'a Config, + evm_opts: &'a EvmOpts, + filter_args: FilterArgsFingerprint<'a>, + num_workers: usize, + artifacts: &'a [ArtifactCacheFingerprint], +} + +#[derive(serde::Serialize)] +struct FilterArgsFingerprint<'a> { + test_pattern: Option<&'a str>, + test_pattern_inverse: Option<&'a str>, + contract_pattern: Option<&'a str>, + contract_pattern_inverse: Option<&'a str>, + path_pattern: Option<&'a str>, + path_pattern_inverse: Option<&'a str>, +} + +/// Configuration for mutation testing run. +pub struct MutationRunConfig { + /// Paths to mutate (if empty, use all source files). + pub mutate_paths: Vec, + /// Optional glob pattern to filter paths. + pub mutate_path_pattern: Option, + /// Optional contract regex pattern to filter contracts. + pub mutate_contract_pattern: Option, + /// Number of parallel workers (0 = auto-detect). + pub num_workers: usize, + /// Whether to show progress display. + pub show_progress: bool, + /// Whether to output JSON (suppress all other output). + pub json_output: bool, + /// Test filter (`--match-test`, `--match-contract`, `--match-path`, ...) + /// applied identically to baseline and every mutant run so they exercise + /// the same test set. + pub filter_args: FilterArgs, + /// Project-relative source files selected for the baseline compile. + /// Re-rooted into each per-mutant workspace so compilation and execution + /// honor the same filtered test universe. + pub selected_sources_relative: Vec, + /// EVM isolation flag — mirrors the canonical `forge test` runner so + /// baseline and mutant runs use the same execution model. + pub isolate: bool, +} + +impl MutationRunConfig { + /// Determine number of workers, using auto-detection if 0. + pub fn effective_workers(&self) -> usize { + if self.num_workers == 0 { + std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1) + } else { + self.num_workers + } + } +} + +/// Result of a mutation testing run. +pub struct MutationRunResult { + /// Summary of all mutations across all files. + pub summary: MutationsSummary, + /// Whether the run was cancelled (e.g., Ctrl+C). + pub cancelled: bool, + /// Duration of the mutation testing run in seconds. + pub duration_secs: f64, +} + +/// Run mutation testing on the project. +/// +/// This function encapsulates the mutation testing logic that was previously +/// in the test command. It handles: +/// - Filtering source files based on patterns +/// - Per-file mutation handling with caching +/// - Parallel mutation execution +/// - Result aggregation and reporting +pub async fn run_mutation_testing( + config: Arc, + output: &ProjectCompileOutput, + evm_opts: EvmOpts, + mutation_config: MutationRunConfig, +) -> Result { + let num_workers = mutation_config.effective_workers(); + let json_output = mutation_config.json_output; + + // Determine which paths to mutate + let mutate_paths = resolve_mutate_paths(&config, output, &mutation_config)?; + let execution_cache_output = ProjectCompiler::new() + .dynamic_test_linking(config.dynamic_test_linking) + .quiet(json_output) + .files( + mutation_config + .selected_sources_relative + .iter() + .map(|path| config.root.join(path)) + .filter(|path| path.exists()) + .collect::>(), + ) + .compile(&config.project()?)?; + let execution_cache_key = mutation_execution_cache_key( + &config, + &execution_cache_output, + &evm_opts, + &mutation_config.filter_args, + num_workers, + )?; + + if !mutation_config.show_progress && !json_output { + sh_println!("Running mutation tests with {} parallel workers...", num_workers)?; + } + + let mut mutation_summary = MutationsSummary::new(); + let mut cancelled = false; + let start_time = Instant::now(); + let cancellation_requested = Arc::new(AtomicBool::new(false)); + let ctrlc_handle = { + let cancellation_requested = Arc::clone(&cancellation_requested); + tokio::spawn(async move { + if tokio::signal::ctrl_c().await.is_ok() { + cancellation_requested.store(true, Ordering::SeqCst); + } + }) + }; + + for path in mutate_paths { + if cancellation_requested.load(Ordering::SeqCst) { + cancelled = true; + break; + } + + if !mutation_config.show_progress && !json_output { + sh_println!("Running mutation tests for {}", path.display())?; + } + + // Create handler for this file, optionally restricting to a subset of + // contracts by name when --mutate-contract is provided. + let mut handler = MutationHandler::new(path.clone(), config.clone()); + if let Some(filter) = &mutation_config.mutate_contract_pattern { + handler = handler.with_contract_filter(filter.clone()); + } + handler.read_source_contract()?; + + // Get build ID for caching + let build_id = output + .artifact_ids() + .find_map(|(id, _)| (id.source == path).then_some(id.build_id)) + .unwrap_or_default(); + + // Load persisted survived spans before generating/loading mutants so + // resumed runs can retain adaptively skipped points as Skipped results + // while only executing mutants whose spans still need coverage. + handler.retrieve_survived_spans(&build_id, &execution_cache_key); + + // Generate or load cached mutants. Adaptive resume happens after the + // full mutant set is known so skipped points are still counted and + // reported as Skipped instead of disappearing from totals. + let mut mutants = if let Some(ms) = handler.retrieve_cached_mutants(&build_id) { + ms + } else { + handler.generate_ast().await?; + handler.mutations.clone() + }; + + if mutants.is_empty() { + if !mutation_config.show_progress && !json_output { + sh_println!(" No mutants generated for {}", path.display())?; + } + continue; + } + + // Check for cached results only after the current mutant set is known. + // The result cache carries a count/hash of that set so stale or partial + // caches cannot suppress newly generated mutants. + if let Some(prior) = + handler.retrieve_cached_mutant_results(&build_id, &execution_cache_key, &mutants) + { + if !mutation_config.show_progress && !json_output { + sh_println!(" Using cached results for {} mutants", prior.len())?; + } + for (mutant, status) in prior { + match status { + MutationResult::Dead => handler.add_dead_mutant(mutant), + MutationResult::Alive => handler.add_survived_mutant(mutant), + MutationResult::Invalid => handler.add_invalid_mutant(mutant), + MutationResult::Skipped => handler.add_skipped_mutant(mutant), + MutationResult::TimedOut => handler.add_timed_out_mutant(mutant), + } + } + mutation_summary.merge(handler.get_report()); + continue; + } + + // Sort mutations by span for optimal adaptive testing + mutants.sort_by(|a, b| { + a.span.lo().0.cmp(&b.span.lo().0).then_with(|| b.span.hi().0.cmp(&a.span.hi().0)) + }); + + let (mutants_to_test, skipped_results) = + partition_adaptively_skipped_mutants(&mut handler, &mutants); + + // Create progress display if enabled (not in JSON mode) + let progress = if mutation_config.show_progress && !json_output { + let p = MutationProgress::with_timeout( + mutants_to_test.len(), + num_workers, + config.mutation.timeout, + ); + // Show relative path from project root + let display_path = + path.strip_prefix(&config.root).unwrap_or(&path).display().to_string(); + p.set_current_file(&display_path); + Some(p) + } else if !json_output { + sh_println!( + " Generated {} mutants; testing {}, adaptively skipped {}", + mutants.len(), + mutants_to_test.len(), + skipped_results.len() + )?; + None + } else { + None + }; + + // Run mutations in parallel using isolated workspaces + let batch = run_mutations_parallel_with_progress( + mutants_to_test.clone(), + path.clone(), + handler.src.clone(), + config.clone(), + evm_opts.clone(), + num_workers, + progress.clone(), + json_output, + mutation_config.filter_args.clone(), + Arc::new(mutation_config.selected_sources_relative.clone()), + mutation_config.isolate, + Arc::clone(&cancellation_requested), + )?; + let file_cancelled = batch.cancelled; + + // Collect results for caching + let mut results_vec = Vec::with_capacity(skipped_results.len() + batch.results.len()); + results_vec.extend(skipped_results); + for result in batch.results { + results_vec.push((result.mutant.clone(), result.result.clone())); + match result.result { + MutationResult::Dead => handler.add_dead_mutant(result.mutant), + MutationResult::Alive => { + handler.mark_span_survived(result.mutant.span); + handler.add_survived_mutant(result.mutant); + } + MutationResult::Invalid => handler.add_invalid_mutant(result.mutant), + MutationResult::Skipped => handler.add_skipped_mutant(result.mutant), + MutationResult::TimedOut => handler.add_timed_out_mutant(result.mutant), + } + } + + // Detect cancellation early so we can decide whether the result set is + // complete before persisting it. Without this guard a Ctrl+C mid-run + // would write a *partial* results vector to the cache and the next run + // would treat that subset as the full answer for this file. + let complete_run = !file_cancelled && results_vec.len() == mutants.len(); + + // Persist results for caching only when the run for this file is + // complete. Partial caches are silent correctness bugs: + // - cancelled runs would be reloaded as authoritative + // - non-cancelled-but-short result vectors indicate a bug, not a hit + // The mutants list itself is fine to persist (it's deterministic from + // the AST + operator set) and so are survived spans (best-effort hint). + // + // Sort the persisted result vector by mutant span so the on-disk + // cache is independent of rayon worker completion order; otherwise + // the cache file changes content-hash run-to-run even when the + // outcomes are identical, defeating diffing and reproducibility. + results_vec.sort_by(|(a, _), (b, _)| { + a.span.lo().0.cmp(&b.span.lo().0).then_with(|| a.span.hi().0.cmp(&b.span.hi().0)) + }); + if !mutants.is_empty() && !build_id.is_empty() { + let _ = handler.persist_cached_mutants(&build_id, &mutants); + if complete_run { + let _ = handler.persist_cached_results( + &build_id, + &execution_cache_key, + &mutants, + &results_vec, + ); + } + let _ = handler.persist_survived_spans(&build_id, &execution_cache_key); + } + + mutation_summary.merge(handler.get_report()); + + // If cancelled, break out of the loop + if file_cancelled { + cancelled = true; + break; + } + } + cancelled |= cancellation_requested.load(Ordering::SeqCst); + + // Report results + let duration = start_time.elapsed(); + let duration_secs = duration.as_secs_f64(); + + // Only show human-readable report if not in JSON mode + if !json_output { + MutationReporter::new().report(&mutation_summary, duration); + } + + ctrlc_handle.abort(); + + Ok(MutationRunResult { summary: mutation_summary, cancelled, duration_secs }) +} + +/// Build the cache discriminator for mutation *results*. +/// +/// Mutant generation only depends on the source build + selected mutators, but +/// result correctness depends on the compiled test universe and execution +/// settings. Hashing the full serialized config intentionally includes fuzz / +/// invariant settings, test filters, fs permissions, sender/balance/env values, +/// and future config fields unless explicitly skipped by `Config` itself. The +/// artifact fingerprint covers the same filter-selected source and test build +/// IDs that baseline and mutant runs compile. Worker count is included because +/// adaptive span skipping is concurrency-sensitive. +fn mutation_execution_cache_key( + config: &Config, + output: &ProjectCompileOutput, + evm_opts: &EvmOpts, + filter_args: &FilterArgs, + num_workers: usize, +) -> Result { + let artifacts = output + .artifact_ids() + .map(|(id, _)| ArtifactCacheFingerprint { + source: id.source.display().to_string(), + name: id.name, + version: id.version.to_string(), + build_id: id.build_id, + profile: id.profile, + }) + .collect::>(); + mutation_execution_cache_key_from_parts(config, evm_opts, filter_args, num_workers, artifacts) +} + +fn mutation_execution_cache_key_from_parts( + config: &Config, + evm_opts: &EvmOpts, + filter_args: &FilterArgs, + num_workers: usize, + mut artifacts: Vec, +) -> Result { + artifacts.sort(); + let fingerprint = ExecutionCacheFingerprint { + schema: "mutation-results-v1", + config, + evm_opts, + filter_args: filter_args_fingerprint(filter_args), + num_workers, + artifacts: &artifacts, + }; + let encoded = serde_json::to_vec(&fingerprint) + .wrap_err("failed to encode mutation execution cache key")?; + + Ok(keccak256(encoded).to_string()) +} + +fn filter_args_fingerprint(filter_args: &FilterArgs) -> FilterArgsFingerprint<'_> { + FilterArgsFingerprint { + test_pattern: filter_args.test_pattern.as_ref().map(|re| re.as_str()), + test_pattern_inverse: filter_args.test_pattern_inverse.as_ref().map(|re| re.as_str()), + contract_pattern: filter_args.contract_pattern.as_ref().map(|re| re.as_str()), + contract_pattern_inverse: filter_args + .contract_pattern_inverse + .as_ref() + .map(|re| re.as_str()), + path_pattern: filter_args.path_pattern.as_ref().map(|glob| glob.as_str()), + path_pattern_inverse: filter_args.path_pattern_inverse.as_ref().map(|glob| glob.as_str()), + } +} + +fn partition_adaptively_skipped_mutants( + handler: &mut MutationHandler, + mutants: &[Mutant], +) -> (Vec, Vec<(Mutant, MutationResult)>) { + let mut skipped_results = Vec::new(); + let mutants_to_test = mutants + .iter() + .filter_map(|mutant| { + if handler.should_skip_span(mutant.span) { + handler.add_skipped_mutant(mutant.clone()); + skipped_results.push((mutant.clone(), MutationResult::Skipped)); + None + } else { + Some(mutant.clone()) + } + }) + .collect(); + + (mutants_to_test, skipped_results) +} + +/// Resolve which paths to mutate based on configuration. +/// +/// Resolution order: +/// 1. Pick the *base* set of candidate files: +/// - `--mutate-path ` → all source files matching the glob, OR +/// - explicit `--mutate PATH...` → those validated files, OR +/// - default → every Solidity file under `config.src`. +/// 2. If `--mutate-contract ` is set, intersect the base set with files that contain at +/// least one contract whose name matches the regex. The per-file contract filter still +/// re-applies inside the handler. +fn resolve_mutate_paths( + config: &Config, + output: &ProjectCompileOutput, + mutation_config: &MutationRunConfig, +) -> Result> { + // 1. Base path set. + let base: Vec = if let Some(pattern) = &mutation_config.mutate_path_pattern { + let paths: Vec<_> = source_files_iter(&config.src, MultiCompilerLanguage::FILE_EXTENSIONS) + .filter(|entry| entry.is_sol() && !entry.is_sol_test() && pattern.is_match(entry)) + .collect(); + if paths.is_empty() { + eyre::bail!("no source matched --mutate-path pattern `{pattern}`"); + } + paths + } else if !mutation_config.mutate_paths.is_empty() { + let root_canon = + config.root.canonicalize().wrap_err("failed to canonicalize project root")?; + let mut validated = Vec::with_capacity(mutation_config.mutate_paths.len()); + for path in &mutation_config.mutate_paths { + let resolved = if path.is_relative() { config.root.join(path) } else { path.clone() }; + if !resolved.exists() { + eyre::bail!("mutate path does not exist: {}", resolved.display()); + } + if !resolved.is_file() { + eyre::bail!("mutate path is not a file: {}", resolved.display()); + } + let canon = resolved + .canonicalize() + .wrap_err_with(|| format!("failed to canonicalize: {}", resolved.display()))?; + if !canon.starts_with(&root_canon) { + eyre::bail!("mutate path is outside the project root: {}", resolved.display()); + } + if !canon.is_sol() { + eyre::bail!("mutate path is not a Solidity file: {}", resolved.display()); + } + if canon.is_sol_test() { + eyre::bail!( + "mutate path is a test file, not a source file: {}", + resolved.display() + ); + } + validated.push(canon); + } + validated + } else { + source_files_iter(&config.src, MultiCompilerLanguage::FILE_EXTENSIONS) + .filter(|entry| entry.is_sol() && !entry.is_sol_test()) + .collect() + }; + + // 2. Intersect with `--mutate-contract` if set, so explicit `--mutate ` combined with + // `--mutate-contract ` does the principled thing (the listed files, restricted to + // those containing a matching contract) instead of silently expanding to every source file. + let paths = if let Some(contract_pattern) = &mutation_config.mutate_contract_pattern { + let matching_sources: HashSet = output + .artifact_ids() + .filter_map(|(id, _)| contract_pattern.is_match(&id.name).then_some(id.source.clone())) + .collect(); + let paths: Vec<_> = + base.into_iter().filter(|entry| matching_sources.contains(entry)).collect(); + if paths.is_empty() { + if mutation_config.mutate_paths.is_empty() + && mutation_config.mutate_path_pattern.is_none() + { + eyre::bail!("no source matched --mutate-contract pattern `{contract_pattern}`"); + } + eyre::bail!("no source matched --mutate-contract within the selected mutation paths"); + } + paths + } else { + base + }; + + Ok(paths) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + + use crate::mutation::mutant::MutationType; + use solar::{ast::Span, interface::BytePos}; + + fn artifact(build_id: &str) -> ArtifactCacheFingerprint { + ArtifactCacheFingerprint { + source: "src/Counter.sol".to_string(), + name: "Counter".to_string(), + version: "0.8.30".to_string(), + build_id: build_id.to_string(), + profile: "default".to_string(), + } + } + + fn filter_args() -> FilterArgs { + FilterArgs { + test_pattern: None, + test_pattern_inverse: None, + contract_pattern: None, + contract_pattern_inverse: None, + path_pattern: None, + path_pattern_inverse: None, + coverage_pattern_inverse: None, + } + } + + fn mutant(lo: u32, hi: u32) -> Mutant { + Mutant { + path: PathBuf::from("src/Counter.sol"), + span: Span::new(BytePos(lo), BytePos(hi)), + mutation: MutationType::DeleteExpression, + original: "number++".to_string(), + source_line: "number++;".to_string(), + line_number: 1, + column_number: 1, + } + } + + #[test] + fn execution_cache_key_changes_when_fuzz_config_changes() { + let first = Config::default(); + let mut second = first.clone(); + second.fuzz.runs += 1; + + let evm_opts = EvmOpts::default(); + let filter_args = filter_args(); + let artifacts = vec![artifact("build-a")]; + + let first_key = mutation_execution_cache_key_from_parts( + &first, + &evm_opts, + &filter_args, + 1, + artifacts.clone(), + ) + .unwrap(); + let second_key = + mutation_execution_cache_key_from_parts(&second, &evm_opts, &filter_args, 1, artifacts) + .unwrap(); + + assert_ne!(first_key, second_key); + } + + #[test] + fn execution_cache_key_changes_when_evm_options_change() { + let config = Config::default(); + let first = EvmOpts::default(); + let mut second = first.clone(); + second.memory_limit = first.memory_limit + 1; + + let filter_args = filter_args(); + let artifacts = vec![artifact("build-a")]; + + let first_key = mutation_execution_cache_key_from_parts( + &config, + &first, + &filter_args, + 1, + artifacts.clone(), + ) + .unwrap(); + let second_key = + mutation_execution_cache_key_from_parts(&config, &second, &filter_args, 1, artifacts) + .unwrap(); + + assert_ne!(first_key, second_key); + } + + #[test] + fn execution_cache_key_changes_when_compiled_artifacts_change() { + let config = Config::default(); + let evm_opts = EvmOpts::default(); + let filter_args = filter_args(); + + let first_key = mutation_execution_cache_key_from_parts( + &config, + &evm_opts, + &filter_args, + 1, + vec![artifact("build-a")], + ) + .unwrap(); + let second_key = mutation_execution_cache_key_from_parts( + &config, + &evm_opts, + &filter_args, + 1, + vec![artifact("build-b")], + ) + .unwrap(); + + assert_ne!(first_key, second_key); + } + + #[test] + fn execution_cache_key_sorts_artifacts_before_hashing() { + let config = Config::default(); + let evm_opts = EvmOpts::default(); + let filter_args = filter_args(); + + let first = vec![artifact("build-a"), artifact("build-b")]; + let second = vec![artifact("build-b"), artifact("build-a")]; + + let first_key = + mutation_execution_cache_key_from_parts(&config, &evm_opts, &filter_args, 1, first) + .unwrap(); + let second_key = + mutation_execution_cache_key_from_parts(&config, &evm_opts, &filter_args, 1, second) + .unwrap(); + + assert_eq!(first_key, second_key); + } + + #[test] + fn execution_cache_key_changes_when_worker_count_changes() { + let config = Config::default(); + let evm_opts = EvmOpts::default(); + let filter_args = filter_args(); + let artifacts = vec![artifact("build-a")]; + + let first_key = mutation_execution_cache_key_from_parts( + &config, + &evm_opts, + &filter_args, + 1, + artifacts.clone(), + ) + .unwrap(); + let second_key = + mutation_execution_cache_key_from_parts(&config, &evm_opts, &filter_args, 4, artifacts) + .unwrap(); + + assert_ne!(first_key, second_key); + } + + #[test] + fn execution_cache_key_changes_when_match_test_filter_changes() { + let config = Config::default(); + let evm_opts = EvmOpts::default(); + let mut first_filter = filter_args(); + let mut second_filter = filter_args(); + first_filter.test_pattern = Some(regex::Regex::new("testA|testAlpha").unwrap()); + second_filter.test_pattern = Some(regex::Regex::new("testB|testBeta").unwrap()); + let artifacts = vec![artifact("build-a")]; + + let first_key = mutation_execution_cache_key_from_parts( + &config, + &evm_opts, + &first_filter, + 1, + artifacts.clone(), + ) + .unwrap(); + let second_key = mutation_execution_cache_key_from_parts( + &config, + &evm_opts, + &second_filter, + 1, + artifacts, + ) + .unwrap(); + + assert_ne!(first_key, second_key); + } + + #[test] + fn execution_cache_key_changes_when_match_path_filter_changes() { + let config = Config::default(); + let evm_opts = EvmOpts::default(); + let mut first_filter = filter_args(); + let mut second_filter = filter_args(); + first_filter.path_pattern = Some(GlobMatcher::from_str("test/A.t.sol").unwrap()); + second_filter.path_pattern = Some(GlobMatcher::from_str("test/B.t.sol").unwrap()); + let artifacts = vec![artifact("build-a")]; + + let first_key = mutation_execution_cache_key_from_parts( + &config, + &evm_opts, + &first_filter, + 1, + artifacts.clone(), + ) + .unwrap(); + let second_key = mutation_execution_cache_key_from_parts( + &config, + &evm_opts, + &second_filter, + 1, + artifacts, + ) + .unwrap(); + + assert_ne!(first_key, second_key); + } + + #[test] + fn resumed_adaptive_skips_are_reported_as_skipped_results() { + let mut handler = + MutationHandler::new(PathBuf::from("src/Counter.sol"), Arc::new(Config::default())); + handler.mark_span_survived(Span::new(BytePos(10), BytePos(20))); + + let exact_survivor = mutant(10, 20); + let skipped_child = mutant(12, 18); + let unrelated = mutant(30, 40); + let (mutants_to_test, skipped_results) = partition_adaptively_skipped_mutants( + &mut handler, + &[exact_survivor.clone(), skipped_child.clone(), unrelated.clone()], + ); + + assert_eq!(mutants_to_test.len(), 2); + assert_eq!(mutants_to_test[0].span, exact_survivor.span); + assert_eq!(mutants_to_test[1].span, unrelated.span); + assert_eq!(skipped_results.len(), 1); + assert!(matches!(skipped_results[0].1, MutationResult::Skipped)); + assert_eq!(skipped_results[0].0.span, skipped_child.span); + assert_eq!(handler.get_report().total_skipped(), 1); + assert_eq!(handler.get_report().total_mutants(), 1); + } +} diff --git a/crates/forge/src/mutation/progress.rs b/crates/forge/src/mutation/progress.rs new file mode 100644 index 0000000000000..51b39fc4a58a8 --- /dev/null +++ b/crates/forge/src/mutation/progress.rs @@ -0,0 +1,321 @@ +//! Progress display for mutation testing. + +use std::{ + collections::HashMap, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::{Duration, Instant}, +}; + +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; +use parking_lot::Mutex; +use yansi::Paint; + +use crate::mutation::mutant::{Mutant, MutationResult}; + +/// Live tally of mutant outcomes, rendered into the overall progress bar. +#[derive(Debug, Default, Clone, Copy)] +struct LiveCounts { + killed: usize, + survived: usize, + invalid: usize, + skipped: usize, + timed_out: usize, +} + +impl LiveCounts { + const fn record(&mut self, result: &MutationResult) { + match result { + MutationResult::Dead => self.killed += 1, + MutationResult::Alive => self.survived += 1, + MutationResult::Invalid => self.invalid += 1, + MutationResult::Skipped => self.skipped += 1, + MutationResult::TimedOut => self.timed_out += 1, + } + } +} + +/// State stored per active mutant so we can show per-mutant elapsed time and +/// remove the correct row when a mutant completes (rather than FIFO which +/// breaks under parallel completion). +#[derive(Debug)] +struct ActiveMutant { + pb: ProgressBar, + started_at: Instant, +} + +/// State for mutation testing progress display. +#[derive(Debug)] +pub struct MutationProgressState { + multi: MultiProgress, + overall_progress: ProgressBar, + /// Active mutant progress bars keyed by a stable identifier (path + span + + /// mutation string) so completion correctly removes the right row. + active_mutants: HashMap, + /// Running per-result counts displayed on the overall bar. + counts: LiveCounts, + /// Optional per-mutant timeout (seconds), shown next to each active row. + timeout_secs: Option, + /// Number of parallel workers, used in the prefix. + num_workers: usize, +} + +impl MutationProgressState { + pub fn new(total_mutants: usize, num_workers: usize) -> Self { + Self::with_timeout(total_mutants, num_workers, None) + } + + pub fn with_timeout( + total_mutants: usize, + num_workers: usize, + timeout_secs: Option, + ) -> Self { + let multi = MultiProgress::new(); + + // Overall progress bar: includes elapsed wall-clock plus a running + // tally (killed / survived / invalid / timed-out / skipped). + let overall_progress = multi.add(ProgressBar::new(total_mutants as u64)); + overall_progress.set_style( + ProgressStyle::with_template( + "{bar:40.cyan/blue} {pos:>4}/{len:4} mutants ({prefix} jobs) [{elapsed_precise}] {wide_msg}", + ) + .unwrap() + .progress_chars("##-"), + ); + overall_progress.set_prefix(num_workers.to_string()); + overall_progress.enable_steady_tick(Duration::from_millis(100)); + + Self { + multi, + overall_progress, + active_mutants: HashMap::with_capacity(num_workers), + counts: LiveCounts::default(), + timeout_secs, + num_workers, + } + } + + /// Set the current file being tested. Renders as part of the overall + /// bar's message together with the running tally. + pub fn set_current_file(&mut self, file: &str) { + // Re-emit message so the file is reflected immediately. + let msg = self.format_message(file); + self.overall_progress.set_message(msg); + } + + fn format_message(&self, file: &str) -> String { + let counts = self.counts; + let timeout_suffix = match self.timeout_secs { + Some(t) => format!(" · timeout {t}s/mutant"), + None => String::new(), + }; + format!( + "k:{} s:{} i:{} t:{} sk:{}{} · {}", + counts.killed, + counts.survived, + counts.invalid, + counts.timed_out, + counts.skipped, + timeout_suffix, + file, + ) + } + + fn refresh_message(&self) { + // Preserve the file segment from the last-rendered message if any. + let current = self.overall_progress.message(); + let file = current.rsplit(" · ").next().unwrap_or(""); + self.overall_progress.set_message(self.format_message(file)); + } + + /// Stable identifier for a mutant — used as the key in `active_mutants`. + fn mutant_key(mutant: &Mutant) -> String { + format!( + "{}:{}-{}:{}", + mutant.path.display(), + mutant.span.lo().0, + mutant.span.hi().0, + mutant.mutation, + ) + } + + /// Add a mutant being tested + pub fn add_mutant_progress(&mut self, mutant: &Mutant) { + let pb = self.multi.add(ProgressBar::new_spinner()); + pb.set_style( + ProgressStyle::with_template(" {spinner} {wide_msg}").unwrap().tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ "), + ); + pb.enable_steady_tick(Duration::from_millis(100)); + + let display = format!( + "line {}: `{}` → `{}`", + mutant.line_number, + truncate_str(&mutant.original, 40), + truncate_str(&mutant.mutation.to_string(), 40), + ); + pb.set_message(display); + + self.active_mutants + .insert(Self::mutant_key(mutant), ActiveMutant { pb, started_at: Instant::now() }); + } + + /// Complete a mutant and show result. Prints a one-line summary above the + /// bars (via `multi.suspend`) before clearing that mutant's spinner. + pub fn complete_mutant(&mut self, mutant: &Mutant, result: &MutationResult) { + self.counts.record(result); + self.overall_progress.inc(1); + + let elapsed = self + .active_mutants + .remove(&Self::mutant_key(mutant)) + .map(|am| { + let el = am.started_at.elapsed(); + am.pb.finish_and_clear(); + el + }) + .unwrap_or_default(); + + // Only emit per-result completion lines for things the user cares + // about (kills, survivors, timeouts). Invalid and skipped are noisy. + // Pad the raw label *before* applying color so ANSI escapes don't + // throw off alignment. + let raw_label = format!("{:9}", result.label()); + let label = match result { + MutationResult::Dead => Paint::green(&raw_label).bold().to_string(), + MutationResult::Alive => Paint::red(&raw_label).bold().to_string(), + MutationResult::TimedOut => Paint::yellow(&raw_label).bold().to_string(), + MutationResult::Invalid | MutationResult::Skipped => { + self.refresh_message(); + return; + } + }; + + let line = format!( + " {label} line {ln}: `{orig}` → `{mut_}` ({elapsed:.1?})", + ln = mutant.line_number, + orig = truncate_str(&mutant.original, 40), + mut_ = truncate_str(&mutant.mutation.to_string(), 40), + elapsed = elapsed, + ); + self.multi.suspend(|| { + let _ = foundry_common::sh_println!("{line}"); + }); + self.refresh_message(); + } + + /// Clear all progress bars + pub fn clear(&mut self) { + for (_, am) in self.active_mutants.drain() { + am.pb.finish_and_clear(); + } + self.overall_progress.finish_and_clear(); + let _ = self.multi.clear(); + } + + /// Finish with a message + pub fn finish(&mut self, message: &str) { + for (_, am) in self.active_mutants.drain() { + am.pb.finish_and_clear(); + } + self.overall_progress.finish_with_message(message.to_string()); + } + + /// Used for tests / introspection. + #[allow(dead_code)] + pub const fn num_workers(&self) -> usize { + self.num_workers + } +} + +/// Thread-safe wrapper for mutation progress +#[derive(Debug, Clone)] +pub struct MutationProgress { + pub inner: Arc>, + pub cancelled: Arc, + pub completed: Arc, + pub total: usize, +} + +impl MutationProgress { + pub fn new(total_mutants: usize, num_workers: usize) -> Self { + Self::with_timeout(total_mutants, num_workers, None) + } + + pub fn with_timeout( + total_mutants: usize, + num_workers: usize, + timeout_secs: Option, + ) -> Self { + Self { + inner: Arc::new(Mutex::new(MutationProgressState::with_timeout( + total_mutants, + num_workers, + timeout_secs, + ))), + cancelled: Arc::new(AtomicBool::new(false)), + completed: Arc::new(AtomicUsize::new(0)), + total: total_mutants, + } + } + + /// Check if testing was cancelled (Ctrl+C) + pub fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::SeqCst) + } + + /// Signal cancellation + pub fn cancel(&self) { + self.cancelled.store(true, Ordering::SeqCst); + } + + /// Set the current file + pub fn set_current_file(&self, file: &str) { + self.inner.lock().set_current_file(file); + } + + /// Record a mutant starting + pub fn start_mutant(&self, mutant: &Mutant) { + self.inner.lock().add_mutant_progress(mutant); + } + + /// Record a mutant completing + pub fn complete_mutant(&self, mutant: &Mutant, result: &MutationResult) -> usize { + let completed = self.completed.fetch_add(1, Ordering::SeqCst) + 1; + self.inner.lock().complete_mutant(mutant, result); + completed + } + + /// Clear progress display + pub fn clear(&self) { + MutationProgressState::clear(&mut self.inner.lock()); + } + + /// Finish with message + pub fn finish(&self, message: &str) { + self.inner.lock().finish(message); + } +} + +/// Truncate a string to max length, centering around the middle (where the operator typically is) +fn truncate_str(s: &str, max_len: usize) -> String { + let s = s.trim(); + if s.len() <= max_len { + return s.to_string(); + } + + // Center the truncation around the middle of the string + let half = max_len.saturating_sub(3) / 2; // Leave room for "..." + let mid = s.len() / 2; + let start = mid.saturating_sub(half); + let end = (start + max_len.saturating_sub(3)).min(s.len()); + + if start == 0 { + format!("{}...", &s[..end]) + } else if end == s.len() { + format!("...{}", &s[start..]) + } else { + format!("...{}...", &s[start..end]) + } +} diff --git a/crates/forge/src/mutation/reporter.rs b/crates/forge/src/mutation/reporter.rs new file mode 100644 index 0000000000000..3dd4228c7c489 --- /dev/null +++ b/crates/forge/src/mutation/reporter.rs @@ -0,0 +1,190 @@ +use comfy_table::{Cell, Color, Row, Table, modifiers::UTF8_ROUND_CORNERS, presets::UTF8_FULL}; +use std::time::Duration; +use yansi::Paint; + +use crate::mutation::{MutationsSummary, mutant::Mutant}; + +pub struct MutationReporter { + table: Table, +} + +impl Default for MutationReporter { + fn default() -> Self { + Self::new() + } +} + +impl MutationReporter { + pub fn new() -> Self { + let mut table = Table::new(); + table.load_preset(UTF8_FULL); + table.apply_modifier(UTF8_ROUND_CORNERS); + + table.set_header(vec![ + Cell::new("Status"), + Cell::new("# Mutants"), + Cell::new("% of Total"), + ]); + + Self { table } + } + + pub fn report(&mut self, summary: &MutationsSummary, duration: Duration) { + let total = summary.total_mutants(); + if total == 0 { + let _ = sh_println!("No mutants were generated."); + return; + } + + // Summary table + self.add_row("Survived", summary.total_survived(), total, Color::Red); + self.add_row("Killed", summary.total_dead(), total, Color::Green); + self.add_row("Invalid", summary.total_invalid(), total, Color::DarkGrey); + self.add_row("Skipped", summary.total_skipped(), total, Color::Yellow); + if summary.total_timed_out() > 0 { + self.add_row("Timed out", summary.total_timed_out(), total, Color::Magenta); + } + + let _ = sh_println!("\n{}", "═".repeat(60)); + let _ = sh_println!("{}", Paint::bold("MUTATION TESTING RESULTS")); + let _ = sh_println!("{}", "═".repeat(60)); + + let _ = sh_println!("\n{}\n", self.table); + + // Legend: short, factual definitions of each status. + let _ = sh_println!("{}", Paint::dim("Legend:")); + let _ = sh_println!(" {} - tests did not catch the mutation", Paint::red("Survived")); + let _ = sh_println!(" {} - tests caught the mutation", Paint::green("Killed")); + let _ = sh_println!(" {} - mutation produced a compilation error", Paint::dim("Invalid")); + let _ = sh_println!( + " {} - redundant mutation on the same expression", + Paint::yellow("Skipped") + ); + let _ = sh_println!( + " {} - compile/test exceeded the configured timeout\n", + Paint::magenta("Timed out") + ); + + // Mutation score with color + let score = summary.mutation_score(); + let score_display = format!("{score:.1}%"); + let score_colored = if score >= 80.0 { + Paint::green(&score_display).bold() + } else if score >= 60.0 { + Paint::yellow(&score_display).bold() + } else { + Paint::red(&score_display).bold() + }; + + // Format duration similar to test output + let duration_str = if duration.as_secs() >= 60 { + format!("{}m {:.2}s", duration.as_secs() / 60, duration.as_secs_f64() % 60.0) + } else { + format!("{:.2}s", duration.as_secs_f64()) + }; + + let _ = sh_println!( + "Mutation Score: {} ({}/{} mutants killed); finished in {}", + score_colored, + summary.total_dead(), + summary.total_dead() + summary.total_survived(), + duration_str + ); + + // Survived mutants section - the most important for developers. + if !summary.get_survived().is_empty() { + let _ = sh_println!("\n{}", "─".repeat(60)); + let _ = sh_println!("{}", Paint::red("Survived mutants").bold()); + let _ = sh_println!("{}", "─".repeat(60)); + + // Sort by (file, line, column, span, mutation text) so the + // reported order is deterministic across runs / worker counts. + // Workers complete in arbitrary order, so without this every run + // can permute the report. + let mut survived: Vec<&Mutant> = summary.get_survived().iter().collect(); + survived.sort_by(|a, b| { + ( + a.relative_path(), + a.line_number, + a.column_number, + a.span.lo().0, + a.span.hi().0, + a.mutation.to_string(), + ) + .cmp(&( + b.relative_path(), + b.line_number, + b.column_number, + b.span.lo().0, + b.span.hi().0, + b.mutation.to_string(), + )) + }); + for (i, mutant) in survived.iter().enumerate() { + self.print_survived_mutant(i + 1, mutant); + } + } + + // Killed mutants (collapsed: just count). + if !summary.get_dead().is_empty() { + let _ = sh_println!("\n{}", "─".repeat(60)); + let _ = sh_println!("{} mutants {}", summary.total_dead(), Paint::green("killed")); + } + + // Invalid mutants (collapsed: just count). + if !summary.get_invalid().is_empty() { + let _ = sh_println!("\n{}", "─".repeat(60)); + let _ = sh_println!("{} mutants {}", summary.total_invalid(), Paint::dim("invalid")); + } + + // Timed-out mutants (collapsed: just count). + if !summary.get_timed_out().is_empty() { + let _ = sh_println!("\n{}", "─".repeat(60)); + let _ = sh_println!( + "{} mutants {}", + summary.total_timed_out(), + Paint::magenta("timed out") + ); + } + + let _ = sh_println!("\n{}", "═".repeat(60)); + } + + fn add_row(&mut self, status: &str, count: usize, total: usize, color: Color) { + let pct = if total > 0 { count as f64 / total as f64 * 100.0 } else { 0.0 }; + + let mut row = Row::new(); + row.add_cell(Cell::new(status).fg(color)) + .add_cell(Cell::new(count.to_string())) + .add_cell(Cell::new(format!("{pct:.1}%"))); + self.table.add_row(row); + } + + fn print_survived_mutant(&self, index: usize, mutant: &Mutant) { + // Show file:line + let location = if mutant.line_number > 0 { + format!("{}:{}", mutant.relative_path(), mutant.line_number) + } else { + mutant.relative_path() + }; + + let _ = sh_println!("\n {}. {}", Paint::red(&index).bold(), Paint::bold(&location)); + + // Show the source line context if available + if !mutant.source_line.is_empty() { + let _ = sh_println!(" {}", Paint::dim(&mutant.source_line)); + } + + // Show the diff + let _ = sh_println!(" {}", Paint::dim("Mutation:")); + let original = if mutant.original.is_empty() { + "".to_string() + } else { + mutant.original.clone() + }; + let mutated = mutant.mutation.to_string(); + + let _ = sh_println!(" {} {}", Paint::red("-"), Paint::red(original.trim())); + let _ = sh_println!(" {} {}", Paint::green("+"), Paint::green(&mutated)); + } +} diff --git a/crates/forge/src/mutation/runner.rs b/crates/forge/src/mutation/runner.rs new file mode 100644 index 0000000000000..ad60566e57661 --- /dev/null +++ b/crates/forge/src/mutation/runner.rs @@ -0,0 +1,799 @@ +//! Parallel mutation testing runner. +//! +//! This module provides high-performance parallel execution of mutation tests. +//! Each mutant is tested in an isolated temporary workspace to enable concurrent execution. + +use std::{ + collections::BTreeMap, + fs, + panic::{self, AssertUnwindSafe}, + path::{Path, PathBuf}, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicUsize, Ordering}, + mpsc, + }, + thread::JoinHandle, + time::Duration, +}; + +use eyre::Result; +use foundry_common::{compile::ProjectCompiler, sh_eprintln, sh_println}; +use foundry_compilers::compilers::multi::MultiCompiler; +use foundry_config::Config; +#[cfg(feature = "optimism")] +use foundry_evm::core::evm::OpEvmNetwork; +use foundry_evm::{ + core::evm::{ + BlockEnvFor, EthEvmNetwork, FoundryEvmNetwork, SpecFor, TempoEvmNetwork, TxEnvFor, + }, + opts::EvmOpts, +}; +use rayon::prelude::*; +use tempfile::TempDir; + +use crate::{ + MultiContractRunnerBuilder, + cmd::test::FilterArgs, + mutation::{ + SurvivedSpans, + mutant::{Mutant, MutationResult}, + progress::MutationProgress, + }, + result::SuiteResult, + workspace, +}; + +/// Result of testing a single mutant. +#[derive(Debug, Clone)] +pub struct MutantTestResult { + pub mutant: Mutant, + pub result: MutationResult, +} + +/// Result of a parallel mutation batch. +#[derive(Debug, Clone)] +pub struct MutationBatchResult { + pub results: Vec, + pub cancelled: bool, +} + +/// Tracks progress and adaptive span skipping across parallel workers. +pub struct SharedMutationState { + /// Spans where mutations have survived - shared across workers for adaptive skipping. + pub survived_spans: Mutex, + /// Progress counter. + pub completed: AtomicUsize, + pub total: AtomicUsize, + /// Cancellation flag (Ctrl+C) + pub cancelled: Arc, + /// Optional progress display + pub progress: Option, + /// Whether to suppress all output (for JSON mode) + pub silent: bool, + /// Worker threads spawned for timed-out mutants. We keep these handles + /// alive (and the `TempDir` they own) so that: + /// 1. The `TempDir` is *not* dropped while the worker is still touching it. + /// 2. We can join the threads at the end of the run and surface leaks. + pub pending_workers: Mutex>>, + /// Maximum number of timed-out worker handles to keep pending at once. + /// Older handles are joined before parking more, bounding cleanup backlog. + max_pending_workers: AtomicUsize, +} + +impl SharedMutationState { + pub fn new( + cancelled: Arc, + silent: bool, + progress: Option, + ) -> Self { + Self { + survived_spans: Mutex::new(SurvivedSpans::new()), + completed: AtomicUsize::new(0), + total: AtomicUsize::new(0), + cancelled, + progress, + silent, + pending_workers: Mutex::new(Vec::new()), + max_pending_workers: AtomicUsize::new(usize::MAX), + } + } + + pub fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::SeqCst) + } + + pub fn cancel(&self) { + self.cancelled.store(true, Ordering::SeqCst); + if let Some(ref progress) = self.progress { + progress.cancel(); + } + } + + pub fn should_skip_span(&self, span: solar::ast::Span) -> bool { + // Handle mutex poisoning gracefully - don't skip if we can't check + self.survived_spans.lock().map(|guard| guard.should_skip_in_live_run(span)).unwrap_or(false) + } + + pub fn mark_span_survived(&self, span: solar::ast::Span) { + // Handle mutex poisoning gracefully - just skip marking if poisoned + if let Ok(mut guard) = self.survived_spans.lock() { + guard.mark_survived(span); + } + } + + pub fn increment_completed(&self) -> usize { + self.completed.fetch_add(1, Ordering::SeqCst) + 1 + } + + pub fn set_max_pending_workers(&self, max: usize) { + self.max_pending_workers.store(max.max(1), Ordering::SeqCst); + } + + fn park_timed_out_worker(&self, handle: JoinHandle<()>) { + let mut pending = match self.pending_workers.lock() { + Ok(pending) => pending, + Err(_) => { + let _ = handle.join(); + return; + } + }; + + let max_pending = self.max_pending_workers.load(Ordering::SeqCst).max(1); + while pending.len() >= max_pending { + let old_handle = pending.remove(0); + drop(pending); + let _ = old_handle.join(); + pending = match self.pending_workers.lock() { + Ok(pending) => pending, + Err(_) => { + let _ = handle.join(); + return; + } + }; + } + + pending.push(handle); + } +} + +impl Default for SharedMutationState { + fn default() -> Self { + Self::new(Arc::new(AtomicBool::new(false)), false, None) + } +} + +/// Run mutation tests in parallel with optional progress display. +#[allow(clippy::too_many_arguments)] +pub fn run_mutations_parallel_with_progress( + mutants: Vec, + source_path: PathBuf, + original_source: Arc, + config: Arc, + evm_opts: EvmOpts, + num_workers: usize, + progress: Option, + silent: bool, + filter_args: FilterArgs, + selected_sources_relative: Arc>, + isolate: bool, + cancellation_requested: Arc, +) -> Result { + let total = mutants.len(); + if total == 0 { + return Ok(MutationBatchResult { results: vec![], cancelled: false }); + } + + // Default to available parallelism if num_workers is 0 + let num_workers = if num_workers == 0 { + std::thread::available_parallelism().map(|p| p.get()).unwrap_or(1) + } else { + num_workers + }; + + let shared_state = Arc::new(SharedMutationState::new(cancellation_requested, silent, progress)); + shared_state.total.store(total, Ordering::SeqCst); + shared_state.set_max_pending_workers(num_workers); + + // Only print if no progress bar and not silent + if shared_state.progress.is_none() && !shared_state.silent { + let _ = sh_println!("Running {} mutants in parallel with {} workers", total, num_workers); + } + + // Get relative path of source within project - MUST be relative for safety + // Canonicalize paths to handle relative vs absolute path comparisons + let source_abs = + if source_path.is_absolute() { source_path } else { config.root.join(&source_path) }; + + let source_relative = source_abs + .strip_prefix(&config.root) + .map_err(|_| { + eyre::eyre!( + "Source path {} is not under project root {}", + source_abs.display(), + config.root.display() + ) + })? + .to_path_buf(); + + workspace::ensure_safe_relative_path(&source_relative, "source", &source_abs)?; + + // Configure rayon thread pool + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(num_workers) + .stack_size(16 * 1024 * 1024) // 16MB stack to avoid overflow in deep call chains + .build() + .map_err(|e| eyre::eyre!("Failed to create thread pool: {}", e))?; + + // Use a thread-safe collection to store results as they complete + let completed_results: Arc>> = + Arc::new(Mutex::new(Vec::with_capacity(total))); + + let filter_args = Arc::new(filter_args); + + pool.install(|| { + mutants.into_par_iter().for_each(|mutant| { + // Skip if cancelled + if shared_state.is_cancelled() { + return; + } + + // Wrap in catch_unwind to prevent one panic from aborting the entire run + let mutant_clone = mutant.clone(); + let result = panic::catch_unwind(AssertUnwindSafe(|| { + test_single_mutant_isolated( + mutant, + &source_relative, + &original_source, + &config, + &evm_opts, + &shared_state, + &filter_args, + &selected_sources_relative, + isolate, + ) + })); + + let test_result = match result { + Ok(r) => r, + Err(_) => { + if shared_state.progress.is_none() { + let _ = sh_eprintln!("Panic while testing mutant: {}", mutant_clone); + } + MutantTestResult { mutant: mutant_clone, result: MutationResult::Invalid } + } + }; + + // Store result immediately + if let Ok(mut results) = completed_results.lock() { + results.push(test_result); + } + }); + }); + + // Extract results + let results = Arc::try_unwrap(completed_results) + .map(|m| m.into_inner().unwrap_or_default()) + .unwrap_or_default(); + + // Drain and join any worker threads that were left running by a + // wall-clock `TimedOut`. Each worker owns its own `TempDir`, so joining + // here is what actually deletes the per-mutant workspace from disk. This + // is the difference between a clean shutdown and stale `forge_mutation_*` + // directories piling up under `$TMPDIR`. + // + // We intentionally block: by this point all rayon work is done, the + // wall-clock budget has already been spent, and the only thing left to do + // is reclaim cleanup. The inner `fuzz.timeout` / `invariant.timeout` + // values we propagated earlier bound how long any individual worker can + // actually run. + let pending = shared_state + .pending_workers + .lock() + .map(|mut g| std::mem::take(&mut *g)) + .unwrap_or_default(); + let pending_count = pending.len(); + if pending_count > 0 && !shared_state.silent && shared_state.progress.is_none() { + let _ = sh_println!("Waiting for {pending_count} timed-out worker(s) to finish cleanup..."); + } + for handle in pending { + let _ = handle.join(); + } + + let cancelled = shared_state.is_cancelled(); + + // Clear progress and handle cancellation + if let Some(ref progress) = shared_state.progress { + progress.clear(); + } + if cancelled && !shared_state.silent { + let _ = sh_println!( + "\nMutation testing cancelled. Showing results for {} completed mutants.\n", + results.len() + ); + } + + Ok(MutationBatchResult { results, cancelled }) +} + +/// Test a single mutant in an isolated temporary workspace. +#[allow(clippy::too_many_arguments)] +fn test_single_mutant_isolated( + mutant: Mutant, + source_relative: &PathBuf, + original_source: &Arc, + config: &Arc, + evm_opts: &EvmOpts, + shared_state: &Arc, + filter_args: &Arc, + selected_sources_relative: &Arc>, + isolate: bool, +) -> MutantTestResult { + // Check if we should skip this mutant based on adaptive span tracking + if shared_state.should_skip_span(mutant.span) { + if let Some(ref progress) = shared_state.progress { + progress.complete_mutant(&mutant, &MutationResult::Skipped); + } else if !shared_state.silent { + let completed = shared_state.increment_completed(); + let total = shared_state.total.load(Ordering::SeqCst); + let _ = sh_println!( + "[{}/{}] Skipping mutant (adaptive: span already has surviving mutation)", + completed, + total + ); + } + return MutantTestResult { mutant, result: MutationResult::Skipped }; + } + + // Show progress or log + if let Some(ref progress) = shared_state.progress { + progress.start_mutant(&mutant); + } else if !shared_state.silent { + let completed = shared_state.increment_completed(); + let total = shared_state.total.load(Ordering::SeqCst); + let _ = sh_println!("[{}/{}] Testing mutant: {}", completed, total, mutant); + } + + // Create isolated workspace using TempDir for automatic cleanup on drop + let temp_dir = match TempDir::with_prefix("forge_mutation_") { + Ok(dir) => dir, + Err(e) => { + let _ = sh_eprintln!("Failed to create temp directory: {}", e); + return MutantTestResult { mutant, result: MutationResult::Invalid }; + } + }; + + // Copy project to temp directory + if let Err(e) = workspace::copy_project(config, temp_dir.path()) { + let _ = sh_eprintln!("Failed to copy project: {}", e); + return MutantTestResult { mutant, result: MutationResult::Invalid }; + } + + // Apply mutation - source_relative is guaranteed to be relative at this point + let mutated_source_path = temp_dir.path().join(source_relative); + if let Err(e) = apply_mutation(&mutant, original_source, &mutated_source_path) { + let _ = sh_eprintln!("Failed to apply mutation: {}", e); + return MutantTestResult { mutant, result: MutationResult::Invalid }; + } + + let temp_path = temp_dir.path().to_path_buf(); + let temp_config = temp_config_for_mutation(config, &temp_path); + let temp_config = Arc::new(temp_config); + + // Compile and test, optionally bounded by a wall-clock timeout. + // + // Lifetime contract: `temp_dir` (the `TempDir`) must live *at least* as + // long as the worker thread that reads from `temp_path`. Dropping the + // `TempDir` early would delete the workspace while a worker still touches + // it, which is a real correctness bug (random compile/test failures and + // dangling fs handles on Windows). + // + // To satisfy that contract we move `temp_dir` ownership into the worker + // thread. If the wall-clock budget fires the outer call returns + // `TimedOut`, but the `TempDir` only drops when the worker thread itself + // exits. The `JoinHandle` is stored in `shared_state.pending_workers` and + // joined at the end of the parallel run. + let timeout = config.mutation.timeout.map(|s| Duration::from_secs(s as u64)); + + let result = match timeout { + Some(budget) => run_compile_and_test_with_timeout( + temp_config, + evm_opts, + budget, + temp_dir, + shared_state, + filter_args.clone(), + selected_sources_relative.clone(), + isolate, + ), + None => { + let res = match compile_and_test( + &temp_config, + evm_opts, + filter_args, + selected_sources_relative, + isolate, + ) { + Ok(true) => MutationResult::Dead, + Ok(false) => MutationResult::Alive, + Err(_) => MutationResult::Invalid, + }; + drop(temp_dir); // explicit: workspace is only safe to remove now + res + } + }; + + // Track adaptive survived spans only for genuinely Alive mutants; TimedOut + // is unresolved and must not mask other mutations on the same span. + if matches!(result, MutationResult::Alive) { + shared_state.mark_span_survived(mutant.span); + } + + // Update progress + if let Some(ref progress) = shared_state.progress { + progress.complete_mutant(&mutant, &result); + } + + MutantTestResult { mutant, result } +} + +/// Run `compile_and_test` on a worker thread and wait at most `budget` for it +/// to complete. Returns `TimedOut` on overrun and `Invalid` on infrastructure +/// errors / panics. +/// +/// The worker takes ownership of `temp_dir` so the underlying workspace +/// directory is only dropped when the worker thread actually exits. On +/// timeout the `JoinHandle` is parked in `shared_state.pending_workers` +/// and joined at the end of the parallel run. +#[allow(clippy::too_many_arguments)] +fn run_compile_and_test_with_timeout( + config: Arc, + evm_opts: &EvmOpts, + budget: Duration, + temp_dir: TempDir, + shared_state: &Arc, + filter_args: Arc, + selected_sources_relative: Arc>, + isolate: bool, +) -> MutationResult { + let (tx, rx) = mpsc::channel::>(); + let opts = evm_opts.clone(); + // Move `temp_dir` into the worker so its `Drop` only runs after the worker + // thread exits. Do NOT capture by reference — the worker may outlive this + // function on timeout. + let cfg = Arc::clone(&config); + let filter_for_worker = Arc::clone(&filter_args); + let selected_sources_for_worker = Arc::clone(&selected_sources_relative); + + let spawn_result = std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .name("mutation-worker".to_string()) + .spawn(move || { + let res = panic::catch_unwind(AssertUnwindSafe(|| { + compile_and_test( + &cfg, + &opts, + &filter_for_worker, + &selected_sources_for_worker, + isolate, + ) + })) + .unwrap_or_else(|_| Err(eyre::eyre!("worker panicked"))); + let _ = tx.send(res); + // Keep `temp_dir` alive until *after* the worker is done with the + // workspace. Dropping here (vs at function entry on timeout) + // guarantees no use-after-free of the filesystem. + drop(temp_dir); + }); + + let handle = match spawn_result { + Ok(h) => h, + Err(_) => return MutationResult::Invalid, + }; + + match rx.recv_timeout(budget) { + Ok(Ok(true)) => { + // Worker finished and sent a result; join briefly so the TempDir + // is actually cleaned up before we return. + let _ = handle.join(); + MutationResult::Dead + } + Ok(Ok(false)) => { + let _ = handle.join(); + MutationResult::Alive + } + Ok(Err(_)) => { + let _ = handle.join(); + MutationResult::Invalid + } + Err(_) => { + // Timeout fired. The worker is still running and still owns the + // TempDir; park the handle so we can join (and reclaim cleanup) + // at the end of the parallel run instead of leaking it. + shared_state.park_timed_out_worker(handle); + MutationResult::TimedOut + } + } +} + +/// Apply a mutation to a source file. +fn apply_mutation(mutant: &Mutant, original_source: &str, dest_path: &Path) -> Result<()> { + let span = mutant.span; + let replacement = mutant.mutation.to_string(); + let start_pos = span.lo().0 as usize; + let end_pos = span.hi().0 as usize; + + // Use checked slicing to avoid panics on invalid spans or non-UTF8 boundaries + let before = original_source.get(..start_pos).ok_or_else(|| { + eyre::eyre!( + "Invalid mutation span: start {} is out of bounds for source length {}", + start_pos, + original_source.len() + ) + })?; + + let after = original_source.get(end_pos..).ok_or_else(|| { + eyre::eyre!( + "Invalid mutation span: end {} is out of bounds for source length {}", + end_pos, + original_source.len() + ) + })?; + + let mut new_content = String::with_capacity(before.len() + replacement.len() + after.len()); + new_content.push_str(before); + new_content.push_str(&replacement); + new_content.push_str(after); + + // Ensure parent directory exists + if let Some(parent) = dest_path.parent() { + fs::create_dir_all(parent)?; + } + + fs::write(dest_path, new_content)?; + Ok(()) +} + +/// Build the config used inside a per-mutant temp workspace. +/// +/// Start from the already materialized baseline config instead of reloading +/// `foundry.toml`, so CLI overrides and runtime normalization stay identical +/// between the baseline run and every mutant run. +fn temp_config_for_mutation(config: &Config, temp_path: &Path) -> Config { + let mut temp_config = config.clone(); + temp_config.root = temp_path.to_path_buf(); + temp_config.src = rebase_project_path(&config.root, temp_path, &config.src); + temp_config.test = rebase_project_path(&config.root, temp_path, &config.test); + temp_config.script = rebase_project_path(&config.root, temp_path, &config.script); + temp_config.out = rebase_project_path(&config.root, temp_path, &config.out); + temp_config.cache_path = rebase_project_path(&config.root, temp_path, &config.cache_path); + temp_config.snapshots = rebase_project_path(&config.root, temp_path, &config.snapshots); + temp_config.broadcast = rebase_project_path(&config.root, temp_path, &config.broadcast); + temp_config.mutation_dir = rebase_project_path(&config.root, temp_path, &config.mutation_dir); + temp_config.libs = + config.libs.iter().map(|lib| rebase_project_path(&config.root, temp_path, lib)).collect(); + temp_config.include_paths = config + .include_paths + .iter() + .map(|path| rebase_project_path(&config.root, temp_path, path)) + .collect(); + temp_config.allow_paths = config + .allow_paths + .iter() + .map(|path| rebase_project_path(&config.root, temp_path, path)) + .collect(); + + if let Some(path) = &config.fuzz.failure_persist_dir { + temp_config.fuzz.failure_persist_dir = + Some(rebase_project_path(&config.root, temp_path, path)); + } + if let Some(path) = &config.invariant.failure_persist_dir { + temp_config.invariant.failure_persist_dir = + Some(rebase_project_path(&config.root, temp_path, path)); + } + + // Propagate the per-mutant timeout into the inner fuzz/invariant harness + // so the hot test loop itself bails out at the deadline. Without this the + // outer `recv_timeout` would only stop *waiting* — the leaked worker + // thread would keep running expensive fuzz/invariant runs and starve the + // pool. We never raise an existing user-configured value. + if let Some(mutation_timeout) = config.mutation.timeout { + temp_config.fuzz.timeout = Some(match temp_config.fuzz.timeout { + Some(existing) => existing.min(mutation_timeout), + None => mutation_timeout, + }); + temp_config.invariant.timeout = Some(match temp_config.invariant.timeout { + Some(existing) => existing.min(mutation_timeout), + None => mutation_timeout, + }); + } + + temp_config +} + +fn rebase_project_path(root: &Path, temp_path: &Path, path: &Path) -> PathBuf { + let rel = workspace::relative_to_root(root, path); + if rel.is_absolute() { path.to_path_buf() } else { temp_path.join(rel) } +} + +/// Compile the project and run tests, returning true if any test failed (mutant killed). +/// +/// Dispatches to the correct network type based on `evm_opts.networks`. +fn compile_and_test( + config: &Arc, + evm_opts: &EvmOpts, + filter_args: &FilterArgs, + selected_sources_relative: &[PathBuf], + isolate: bool, +) -> Result { + if evm_opts.networks.is_tempo() { + compile_and_test_inner::( + config, + evm_opts, + filter_args, + selected_sources_relative, + isolate, + ) + } else { + #[cfg(feature = "optimism")] + if evm_opts.networks.is_optimism() { + return compile_and_test_inner::( + config, + evm_opts, + filter_args, + selected_sources_relative, + isolate, + ); + } + compile_and_test_inner::( + config, + evm_opts, + filter_args, + selected_sources_relative, + isolate, + ) + } +} + +fn compile_and_test_inner( + config: &Arc, + evm_opts: &EvmOpts, + filter_args: &FilterArgs, + selected_sources_relative: &[PathBuf], + isolate: bool, +) -> Result { + // Compile + let files = selected_sources_relative + .iter() + .map(|path| config.root.join(path)) + .filter(|path| path.exists()) + .collect::>(); + let compiler = ProjectCompiler::new() + .dynamic_test_linking(config.dynamic_test_linking) + .quiet(true) + .files(files); + + let compile_output = compiler.compile(&config.project()?)?; + + // Rebuild the per-mutant test filter so `--match-test`, `--match-contract`, + // `--match-path`, ... are honored against the temp workspace's paths + // (not the original project root). Without this the mutant runs would + // ignore user filters and execute a different test set than the baseline. + let filter = filter_args.clone().merge_with_config(config); + + // Run tests - need a multi-threaded Tokio runtime since test() uses rayon internally + // with par_iter, and rayon workers need tokio handle access + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) // Minimize overhead, tests use rayon for parallelism + .enable_all() + .build() + .map_err(|e| eyre::eyre!("Failed to create tokio runtime: {}", e))?; + + // Use block_on to run within the runtime context + let results: BTreeMap = rt.block_on(async { + let (evm_env, tx_env, fork_block) = + evm_opts.env::, BlockEnvFor, TxEnvFor>().await?; + + // Build test runner mirroring the canonical `forge test` runner: same + // isolation flag, same fail-fast semantics for mutation, and same + // filter so kept/skipped tests stay consistent across baseline and + // mutant runs. + let mut runner = MultiContractRunnerBuilder::new(config.clone()) + .set_debug(false) + .initial_balance(evm_opts.initial_balance) + .sender(evm_opts.sender) + .with_fork(evm_opts.get_fork(config, evm_env.cfg_env.chain_id, fork_block)) + .enable_isolation(isolate) + .fail_fast(true) + .build::(&compile_output, evm_env, tx_env, evm_opts.clone())?; + + runner.test_collect(&filter) + })?; + + // Check if any test failed (mutant killed) + let killed = results.values().any(|suite| suite.failed() > 0); + + Ok(killed) +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::U256; + + #[test] + fn park_timed_out_worker_bounds_pending_handles() { + let state = SharedMutationState::default(); + state.set_max_pending_workers(1); + + state.park_timed_out_worker(std::thread::spawn(|| {})); + assert_eq!(state.pending_workers.lock().unwrap().len(), 1); + + state.park_timed_out_worker(std::thread::spawn(|| {})); + assert_eq!(state.pending_workers.lock().unwrap().len(), 1); + + let pending = std::mem::take(&mut *state.pending_workers.lock().unwrap()); + for handle in pending { + handle.join().unwrap(); + } + } + + #[test] + fn temp_config_preserves_materialized_overrides_and_rebases_paths() { + let project = TempDir::new().unwrap(); + let temp = TempDir::new().unwrap(); + let root = project.path(); + + let mut config = Config { + root: root.to_path_buf(), + src: root.join("contracts"), + test: root.join("checks"), + script: root.join("deploy"), + out: root.join("custom-out"), + cache_path: root.join("custom-cache"), + snapshots: root.join("custom-snapshots"), + broadcast: root.join("custom-broadcast"), + mutation_dir: root.join("custom-cache/mutation"), + libs: vec![root.join("vendor")], + include_paths: vec![root.join("shared")], + allow_paths: vec![root.join("fixtures")], + dynamic_test_linking: true, + cache: true, + ..Default::default() + }; + config.fuzz.seed = Some(U256::from(42)); + config.fuzz.timeout = Some(90); + config.invariant.timeout = Some(80); + config.fuzz.failure_persist_dir = Some(root.join("custom-cache/fuzz")); + config.invariant.failure_persist_dir = Some(root.join("custom-cache/invariant")); + config.mutation.timeout = Some(5); + + let temp_config = temp_config_for_mutation(&config, temp.path()); + + assert_eq!(temp_config.root, temp.path()); + assert_eq!(temp_config.src, temp.path().join("contracts")); + assert_eq!(temp_config.test, temp.path().join("checks")); + assert_eq!(temp_config.script, temp.path().join("deploy")); + assert_eq!(temp_config.out, temp.path().join("custom-out")); + assert_eq!(temp_config.cache_path, temp.path().join("custom-cache")); + assert_eq!(temp_config.snapshots, temp.path().join("custom-snapshots")); + assert_eq!(temp_config.broadcast, temp.path().join("custom-broadcast")); + assert_eq!(temp_config.mutation_dir, temp.path().join("custom-cache/mutation")); + assert_eq!(temp_config.libs, vec![temp.path().join("vendor")]); + assert_eq!(temp_config.include_paths, vec![temp.path().join("shared")]); + assert_eq!(temp_config.allow_paths, vec![temp.path().join("fixtures")]); + assert_eq!( + temp_config.fuzz.failure_persist_dir, + Some(temp.path().join("custom-cache/fuzz")) + ); + assert_eq!( + temp_config.invariant.failure_persist_dir, + Some(temp.path().join("custom-cache/invariant")) + ); + assert!(temp_config.dynamic_test_linking); + assert!(temp_config.cache); + assert_eq!(temp_config.fuzz.seed, Some(U256::from(42))); + assert_eq!(temp_config.fuzz.timeout, Some(5)); + assert_eq!(temp_config.invariant.timeout, Some(5)); + } +} diff --git a/crates/forge/src/mutation/visitor.rs b/crates/forge/src/mutation/visitor.rs new file mode 100644 index 0000000000000..3997df316da7f --- /dev/null +++ b/crates/forge/src/mutation/visitor.rs @@ -0,0 +1,290 @@ +use std::{ops::ControlFlow, path::PathBuf}; + +use eyre::Report; +use solar::ast::{Expr, ItemContract, VariableDefinition, visit::Visit, yul}; + +#[cfg(test)] +use crate::mutation::mutators::Mutator; +use crate::mutation::{ + mutant::{Mutant, OwnedLiteral}, + mutators::{MutationContext, mutator_registry::MutatorRegistry}, +}; +use foundry_config::MutatorType; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum AssignVarTypes { + Literal(OwnedLiteral), + Identifier(String), +} + +/// A visitor which collect all expression to mutate as well as the mutation types +#[allow(clippy::type_complexity)] +pub struct MutantVisitor<'src> { + pub mutation_to_conduct: Vec, + errors: Vec, + pub mutator_registry: MutatorRegistry, + pub path: PathBuf, + pub source: Option<&'src str>, + /// Optional per-contract name filter. When `Some`, mutations are only collected + /// from contracts whose name matches the predicate. + pub contract_filter: Option bool>>, + /// Whether the currently-visited contract is allowed by `contract_filter`. + /// `true` when no filter is set or when we are visiting a contract whose name + /// matched the filter. Top-level items (outside any contract) are always + /// considered "allowed". + in_allowed_contract: bool, +} + +impl<'src> MutantVisitor<'src> { + /// Create a visitor with the specified mutator operators enabled + pub fn with_operators(path: PathBuf, operators: &[MutatorType]) -> Self { + Self { + mutation_to_conduct: Vec::new(), + errors: Vec::new(), + mutator_registry: MutatorRegistry::from_enabled(operators), + path, + source: None, + contract_filter: None, + in_allowed_contract: true, + } + } + + /// Use all mutators from registry (all operators enabled) + #[cfg(test)] + pub fn default(path: PathBuf) -> Self { + Self { + mutation_to_conduct: Vec::new(), + errors: Vec::new(), + mutator_registry: MutatorRegistry::default(), + path, + source: None, + contract_filter: None, + in_allowed_contract: true, + } + } + + /// Use only a set of mutators + #[cfg(test)] + pub fn new_with_mutators(path: PathBuf, mutators: Vec>) -> Self { + Self { + mutation_to_conduct: Vec::new(), + errors: Vec::new(), + mutator_registry: MutatorRegistry::new_with_mutators(mutators), + path, + source: None, + contract_filter: None, + in_allowed_contract: true, + } + } + + /// Set the source code for extracting original text + pub const fn with_source(mut self, source: &'src str) -> Self { + self.source = Some(source); + self + } + + /// Set a contract-name filter; only contracts whose name matches the + /// predicate will have their bodies mutated. + pub fn with_contract_filter(mut self, filter: F) -> Self + where + F: Fn(&str) -> bool + 'static, + { + self.contract_filter = Some(Box::new(filter)); + self + } + + pub fn take_errors(&mut self) -> Vec { + std::mem::take(&mut self.errors) + } + + fn collect_mutations(&mut self, context: &MutationContext<'_>) { + let result = self.mutator_registry.generate_mutations(context); + self.mutation_to_conduct.extend(result.mutations); + + for err in result.errors { + self.errors.push(err.wrap_err(format!( + "failed to generate mutations for {}:{}:{}", + self.path.display(), + context.line_number(), + context.column_number() + ))); + } + } +} + +impl<'ast> Visit<'ast> for MutantVisitor<'ast> { + type BreakValue = (); + + fn visit_item_contract( + &mut self, + contract: &'ast ItemContract<'ast>, + ) -> ControlFlow { + // When a contract name filter is configured, only descend into matching + // contracts. We toggle `in_allowed_contract` for the duration of the + // walk so nested visit_expr / visit_variable_definition calls can gate + // mutant collection accordingly. + let prev = self.in_allowed_contract; + self.in_allowed_contract = match &self.contract_filter { + Some(filter) => filter(contract.name.as_str()), + None => true, + }; + let res = self.walk_item_contract(contract); + self.in_allowed_contract = prev; + res + } + + fn visit_variable_definition( + &mut self, + var: &'ast VariableDefinition<'ast>, + ) -> ControlFlow { + // Skip entirely when the surrounding contract is filtered out. + if !self.in_allowed_contract { + return self.walk_variable_definition(var); + } + + let mut builder = MutationContext::builder() + .with_path(self.path.clone()) + .with_span(var.span) + .with_var_definition(var); + + if let Some(src) = self.source { + builder = builder.with_source(src); + } + + let context = builder + .build() + .expect("MutationContext requires both path and span for variable definition"); + + self.collect_mutations(&context); + self.walk_variable_definition(var) + } + + fn visit_expr(&mut self, expr: &'ast Expr<'ast>) -> ControlFlow { + // Skip entirely when the surrounding contract is filtered out. + if !self.in_allowed_contract { + return self.walk_expr(expr); + } + + let mut builder = MutationContext::builder() + .with_path(self.path.clone()) + .with_span(expr.span) + .with_expr(expr); + + if let Some(src) = self.source { + builder = builder.with_source(src); + } + + let context = + builder.build().expect("MutationContext requires both path and span for expression"); + + self.collect_mutations(&context); + self.walk_expr(expr) + } + + fn visit_yul_expr(&mut self, expr: &'ast yul::Expr<'ast>) -> ControlFlow { + // Skip entirely when the surrounding contract is filtered out. + if !self.in_allowed_contract { + return self.walk_yul_expr(expr); + } + + let mut builder = MutationContext::builder() + .with_path(self.path.clone()) + .with_span(expr.span) + .with_yul_expr(expr); + + if let Some(src) = self.source { + builder = builder.with_source(src); + } + + let context = builder + .build() + .expect("MutationContext requires both path and span for yul expression"); + + self.collect_mutations(&context); + self.walk_yul_expr(expr) + } +} + +#[cfg(test)] +mod tests { + use eyre::{Result, eyre}; + use solar::{ + ast::{Arena, interface::source_map::FileName}, + parse::Parser, + }; + + use super::*; + use crate::mutation::{Session, mutant::MutationType}; + + struct FailingExprMutator; + + impl Mutator for FailingExprMutator { + fn generate_mutants(&self, _ctxt: &MutationContext<'_>) -> Result> { + Err(eyre!("synthetic visitor failure")) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + ctxt.expr.is_some() + } + } + + struct PassingExprMutator; + + impl Mutator for PassingExprMutator { + fn generate_mutants(&self, ctxt: &MutationContext<'_>) -> Result> { + Ok(vec![Mutant { + path: ctxt.path.clone(), + span: ctxt.span, + mutation: MutationType::DeleteExpression, + original: ctxt.original_text(), + source_line: ctxt.source_line(), + line_number: ctxt.line_number(), + column_number: ctxt.column_number(), + }]) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + ctxt.expr.is_some() + } + } + + #[test] + fn visitor_collects_mutations_and_surfaces_mutator_errors() { + let source = "\ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; +contract Test { + function test() public { + uint256 x = 1 + 2; + } +} +"; + let path = PathBuf::from("test.sol"); + let sess = Session::builder().with_silent_emitter(None).build(); + + sess.enter(|| { + let arena = Arena::new(); + let mut parser = + Parser::from_lazy_source_code(&sess, &arena, FileName::Real(path.clone()), || { + Ok(source.to_string()) + }) + .unwrap(); + let ast = parser.parse_file().map_err(|e| e.emit()).unwrap(); + let mut visitor = MutantVisitor::new_with_mutators( + path, + vec![Box::new(FailingExprMutator), Box::new(PassingExprMutator)], + ) + .with_source(source); + + let _ = visitor.visit_source_unit(&ast); + let errors = visitor.take_errors(); + + assert!(!visitor.mutation_to_conduct.is_empty()); + assert!(!errors.is_empty()); + + let err = format!("{:?}", errors[0]); + assert!(err.contains("failed to generate mutations for test.sol:")); + assert!(err.contains("synthetic visitor failure")); + }); + } +} diff --git a/crates/forge/src/workspace.rs b/crates/forge/src/workspace.rs new file mode 100644 index 0000000000000..6d783fadc9ac2 --- /dev/null +++ b/crates/forge/src/workspace.rs @@ -0,0 +1,697 @@ +//! Shared utilities for creating isolated project workspaces. +//! +//! Used by both mutation testing and brutalization to copy a project +//! to a temporary directory for safe source-level modifications. + +use std::{ + fs, + path::{Component, Path, PathBuf}, +}; + +use eyre::Result; +use foundry_config::Config; + +/// Check if a path is safe for use as a relative path within a workspace. +/// Rejects absolute paths, parent directory components (..), and other unsafe patterns. +pub fn is_safe_relative_path(p: &Path) -> bool { + !p.is_absolute() + && p.components().all(|c| matches!(c, Component::Normal(_) | Component::CurDir)) +} + +/// Validates that `rel` is a safe relative path. Returns an error mentioning `label` and `orig` +/// if the path contains `..`, is absolute, or otherwise escapes the project root. +pub fn ensure_safe_relative_path(rel: &Path, label: &str, orig: &Path) -> Result<()> { + if !is_safe_relative_path(rel) { + eyre::bail!("requires {label} directory under project root, got: {}", orig.display()); + } + Ok(()) +} + +/// Compute relative path of `path` under `root`, or return the path unchanged if not under root. +pub fn relative_to_root(root: &Path, path: &Path) -> PathBuf { + path.strip_prefix(root).map(|p| p.to_path_buf()).unwrap_or_else(|_| path.to_path_buf()) +} + +/// Verify that `candidate` resolves (after following symlinks) to a path that lives +/// inside `allowed_root`. Protects against `src`/`test`/`lib`/etc. being symlinks +/// that escape the project root. +/// +/// `label` and `orig` are only used for error messages. +fn ensure_within_root( + allowed_root: &Path, + candidate: &Path, + label: &str, + orig: &Path, +) -> Result<()> { + // If the path doesn't exist yet, lexical containment is the best we can do. + if !candidate.exists() { + return Ok(()); + } + let canon_root = allowed_root.canonicalize().map_err(|e| { + eyre::eyre!("failed to canonicalize project root {}: {e}", allowed_root.display()) + })?; + let canon_candidate = candidate.canonicalize().map_err(|e| { + eyre::eyre!("failed to canonicalize {label} path {}: {e}", candidate.display()) + })?; + if !canon_candidate.starts_with(&canon_root) { + eyre::bail!( + "{label} path {} escapes project root {} (resolved to {})", + orig.display(), + allowed_root.display(), + canon_candidate.display() + ); + } + Ok(()) +} + +/// Copy essential project files to a temp workspace. +/// +/// Copies src and test directories, symlinks library directories (read-only), +/// and copies config files (foundry.toml, remappings.txt). +pub fn copy_project(config: &Config, temp_dir: &Path) -> Result<()> { + let src_rel = relative_to_root(&config.root, &config.src); + ensure_safe_relative_path(&src_rel, "src", &config.src)?; + ensure_within_root(&config.root, &config.src, "src", &config.src)?; + + let test_rel = relative_to_root(&config.root, &config.test); + ensure_safe_relative_path(&test_rel, "test", &config.test)?; + ensure_within_root(&config.root, &config.test, "test", &config.test)?; + + copy_dir_recursive(&config.src, &temp_dir.join(&src_rel))?; + + if config.test != config.src { + copy_dir_recursive(&config.test, &temp_dir.join(&test_rel))?; + } + + let handled_extra_roots = handled_project_roots(config)?; + for extra_path in config.include_paths.iter().chain(config.allow_paths.iter()) { + copy_extra_project_path(&config.root, temp_dir, extra_path, &handled_extra_roots)?; + } + + // Copy `script/` too when present and distinct from src/test. Many real + // projects keep helper contracts, deployment scripts, or fixtures under + // `script/` and reference them from tests via relative imports. Without + // this, baselines that compile fine produce a sea of `Invalid` mutants + // for purely-environmental reasons. + if config.script.exists() && config.script != config.src && config.script != config.test { + let script_rel = relative_to_root(&config.root, &config.script); + ensure_safe_relative_path(&script_rel, "script", &config.script)?; + ensure_within_root(&config.root, &config.script, "script", &config.script)?; + copy_dir_recursive(&config.script, &temp_dir.join(&script_rel))?; + } + + for lib_path in &config.libs { + if lib_path.exists() { + let lib_rel = relative_to_root(&config.root, lib_path); + ensure_safe_relative_path(&lib_rel, "lib", lib_path)?; + ensure_within_root(&config.root, lib_path, "lib", lib_path)?; + let target = temp_dir.join(&lib_rel); + + if !target.exists() { + if let Some(parent) = target.parent() { + fs::create_dir_all(parent)?; + } + if symlink_dir(lib_path, &target).is_err() { + copy_dir_recursive(lib_path, &target)?; + } + } + + symlink_nested_libs(lib_path, &target, 0)?; + } + } + + for dep_dir in ["node_modules", "dependencies"] { + let dep_path = config.root.join(dep_dir); + if dep_path.exists() && dep_path.is_dir() { + // Reject if the project-root entry is a symlink that escapes the root. + ensure_within_root(&config.root, &dep_path, dep_dir, &dep_path)?; + let target = temp_dir.join(dep_dir); + if !target.exists() && symlink_dir(&dep_path, &target).is_err() { + copy_dir_recursive(&dep_path, &target)?; + } + } + } + + let foundry_toml = config.root.join("foundry.toml"); + if foundry_toml.exists() { + fs::copy(&foundry_toml, temp_dir.join("foundry.toml"))?; + } + + let remappings = config.root.join("remappings.txt"); + if remappings.exists() { + fs::copy(&remappings, temp_dir.join("remappings.txt"))?; + } + + Ok(()) +} + +fn handled_project_roots(config: &Config) -> Result> { + let mut roots = Vec::new(); + push_handled_project_root(&mut roots, &config.root, &config.src, "src")?; + push_handled_project_root(&mut roots, &config.root, &config.test, "test")?; + + if config.script.exists() && config.script != config.src && config.script != config.test { + push_handled_project_root(&mut roots, &config.root, &config.script, "script")?; + } + + for lib_path in &config.libs { + if lib_path.exists() { + push_handled_project_root(&mut roots, &config.root, lib_path, "lib")?; + } + } + + for dep_dir in ["node_modules", "dependencies"] { + let dep_path = config.root.join(dep_dir); + if dep_path.exists() && dep_path.is_dir() { + roots.push(PathBuf::from(dep_dir)); + } + } + + Ok(roots) +} + +fn push_handled_project_root( + roots: &mut Vec, + root: &Path, + path: &Path, + label: &str, +) -> Result<()> { + let rel = relative_to_root(root, path); + ensure_safe_relative_path(&rel, label, path)?; + ensure_within_root(root, path, label, path)?; + roots.push(rel); + Ok(()) +} + +fn is_covered_by_handled_root(rel: &Path, handled_roots: &[PathBuf]) -> bool { + handled_roots.iter().any(|root| !root.as_os_str().is_empty() && rel.starts_with(root)) +} + +fn copy_extra_project_path( + root: &Path, + temp_dir: &Path, + path: &Path, + handled_roots: &[PathBuf], +) -> Result<()> { + let resolved = if path.is_absolute() { path.to_path_buf() } else { root.join(path) }; + let rel = relative_to_root(root, &resolved); + ensure_safe_relative_path(&rel, "include/allow", path)?; + ensure_within_root(root, &resolved, "include/allow", path)?; + + if is_covered_by_handled_root(&rel, handled_roots) { + return Ok(()); + } + + if !resolved.exists() { + return Ok(()); + } + + let target = temp_dir.join(rel); + if resolved.is_dir() { + copy_dir_recursive(&resolved, &target) + } else { + if let Some(parent) = target.parent() { + fs::create_dir_all(parent)?; + } + fs::copy(&resolved, target)?; + Ok(()) + } +} + +/// Create a symlink to a directory (cross-platform). +pub fn symlink_dir(src: &Path, dst: &Path) -> Result<()> { + #[cfg(unix)] + { + std::os::unix::fs::symlink(src, dst)?; + } + #[cfg(windows)] + { + std::os::windows::fs::symlink_dir(src, dst)?; + } + Ok(()) +} + +/// Maximum recursion depth for nested lib symlinks to prevent infinite loops. +const MAX_SYMLINK_DEPTH: usize = 10; + +/// Recursively symlink nested lib directories within a library. +fn symlink_nested_libs(lib_src: &Path, lib_dst: &Path, depth: usize) -> Result<()> { + if depth >= MAX_SYMLINK_DEPTH { + return Ok(()); + } + + let nested_lib_dirs: Vec = + if let Ok(config) = Config::load_with_root_and_fallback(lib_src) { + config.libs + } else { + vec![PathBuf::from("lib")] + }; + + for nested_lib_dir in nested_lib_dirs { + // A dependency's foundry.toml is untrusted input. Reject any nested lib + // path that is absolute or contains `..`, then verify the resolved path + // doesn't escape the dependency root via symlink. + if !is_safe_relative_path(&nested_lib_dir) { + continue; + } + let nested_lib = lib_src.join(&nested_lib_dir); + if !nested_lib.exists() { + continue; + } + // Use symlink_metadata so we don't follow a symlinked nested lib root. + let Ok(meta) = fs::symlink_metadata(&nested_lib) else { continue }; + if meta.file_type().is_symlink() || !meta.is_dir() { + continue; + } + if ensure_within_root(lib_src, &nested_lib, "nested lib", &nested_lib).is_err() { + continue; + } + process_nested_lib_dir(&nested_lib, lib_dst, &nested_lib_dir, depth)?; + } + + Ok(()) +} + +fn process_nested_lib_dir( + nested_lib: &Path, + lib_dst: &Path, + lib_rel: &Path, + depth: usize, +) -> Result<()> { + if !nested_lib.exists() || !nested_lib.is_dir() { + return Ok(()); + } + + let entries = match fs::read_dir(nested_lib) { + Ok(e) => e, + Err(_) => return Ok(()), + }; + + for entry in entries.flatten() { + // Use file_type() (does not follow symlinks) so a symlinked entry in a + // dependency's lib dir cannot be silently followed and re-symlinked + // outside the workspace. + let Ok(file_type) = entry.file_type() else { continue }; + if file_type.is_symlink() || !file_type.is_dir() { + continue; + } + + let entry_path = entry.path(); + let entry_name = entry.file_name(); + let nested_dst = lib_dst.join(lib_rel).join(&entry_name); + + if !nested_dst.exists() { + if let Some(parent) = nested_dst.parent() { + let _ = fs::create_dir_all(parent); + } + let _ = symlink_dir(&entry_path, &nested_dst); + } + + symlink_nested_libs(&entry_path, &nested_dst, depth + 1)?; + } + + Ok(()) +} + +/// Recursively copy a directory, skipping symlinked directories for safety. +pub fn copy_dir_recursive(src: &Path, dst: &Path) -> Result<()> { + if !src.exists() { + return Ok(()); + } + + fs::create_dir_all(dst)?; + + for entry in fs::read_dir(src)? { + let entry = entry?; + let path = entry.path(); + let dest_path = dst.join(entry.file_name()); + + let meta = fs::symlink_metadata(&path)?; + + if meta.file_type().is_symlink() { + if path.is_dir() { + continue; + } + fs::copy(&path, &dest_path)?; + } else if meta.is_dir() { + copy_dir_recursive(&path, &dest_path)?; + } else { + fs::copy(&path, &dest_path)?; + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn create_test_dir_structure(base: &Path, structure: &[&str]) { + for path in structure { + let full_path = base.join(path); + if path.ends_with('/') { + fs::create_dir_all(&full_path).unwrap(); + } else { + if let Some(parent) = full_path.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(&full_path, format!("// {path}")).unwrap(); + } + } + } + + #[test] + fn test_symlink_dir_creates_symlink() { + let temp = TempDir::new().unwrap(); + let src = temp.path().join("source_dir"); + let dst = temp.path().join("target_link"); + + fs::create_dir(&src).unwrap(); + fs::write(src.join("file.txt"), "content").unwrap(); + + symlink_dir(&src, &dst).unwrap(); + + assert!(dst.exists()); + assert!(dst.is_symlink()); + assert!(dst.join("file.txt").exists()); + } + + #[test] + fn test_symlink_nested_libs_single_level() { + let temp = TempDir::new().unwrap(); + + let lib_src = temp.path().join("lib_src"); + create_test_dir_structure( + &lib_src, + &[ + "src/Contract.sol", + "lib/", + "lib/openzeppelin/contracts/token/ERC20.sol", + "lib/solmate/src/tokens/ERC20.sol", + ], + ); + + let lib_dst = temp.path().join("lib_dst"); + fs::create_dir(&lib_dst).unwrap(); + + symlink_nested_libs(&lib_src, &lib_dst, 0).unwrap(); + + assert!(lib_dst.join("lib/openzeppelin").exists()); + assert!(lib_dst.join("lib/solmate").exists()); + assert!(lib_dst.join("lib/openzeppelin/contracts/token/ERC20.sol").exists()); + assert!(lib_dst.join("lib/solmate/src/tokens/ERC20.sol").exists()); + } + + #[test] + fn test_symlink_nested_libs_deeply_nested() { + let temp = TempDir::new().unwrap(); + + let lib_src = temp.path().join("lib_src"); + create_test_dir_structure( + &lib_src, + &[ + "src/Main.sol", + "lib/", + "lib/dep-a/src/A.sol", + "lib/dep-a/lib/", + "lib/dep-a/lib/dep-b/src/B.sol", + "lib/dep-a/lib/dep-b/lib/", + "lib/dep-a/lib/dep-b/lib/dep-c/src/C.sol", + ], + ); + + let lib_dst = temp.path().join("lib_dst"); + fs::create_dir(&lib_dst).unwrap(); + + symlink_nested_libs(&lib_src, &lib_dst, 0).unwrap(); + + assert!(lib_dst.join("lib/dep-a").exists()); + assert!(lib_dst.join("lib/dep-a/lib/dep-b").exists()); + assert!(lib_dst.join("lib/dep-a/lib/dep-b/lib/dep-c").exists()); + assert!(lib_dst.join("lib/dep-a/lib/dep-b/lib/dep-c/src/C.sol").exists()); + } + + #[test] + fn test_symlink_nested_libs_no_nested_lib_dir() { + let temp = TempDir::new().unwrap(); + + let lib_src = temp.path().join("lib_src"); + create_test_dir_structure(&lib_src, &["src/Contract.sol", "test/Test.sol"]); + + let lib_dst = temp.path().join("lib_dst"); + fs::create_dir(&lib_dst).unwrap(); + + symlink_nested_libs(&lib_src, &lib_dst, 0).unwrap(); + + assert!(!lib_dst.join("lib").exists()); + } + + #[test] + fn test_symlink_nested_libs_skips_existing() { + let temp = TempDir::new().unwrap(); + + let lib_src = temp.path().join("lib_src"); + create_test_dir_structure(&lib_src, &["lib/", "lib/existing/src/File.sol"]); + + let lib_dst = temp.path().join("lib_dst"); + fs::create_dir_all(lib_dst.join("lib/existing")).unwrap(); + fs::write(lib_dst.join("lib/existing/marker.txt"), "pre-existing").unwrap(); + + symlink_nested_libs(&lib_src, &lib_dst, 0).unwrap(); + + assert!(lib_dst.join("lib/existing/marker.txt").exists()); + } + + #[test] + fn test_copy_dir_recursive_basic() { + let temp = TempDir::new().unwrap(); + + let src = temp.path().join("src"); + create_test_dir_structure( + &src, + &["file1.sol", "subdir/file2.sol", "subdir/nested/file3.sol"], + ); + + let dst = temp.path().join("dst"); + copy_dir_recursive(&src, &dst).unwrap(); + + assert!(dst.join("file1.sol").exists()); + assert!(dst.join("subdir/file2.sol").exists()); + assert!(dst.join("subdir/nested/file3.sol").exists()); + } + + #[test] + fn test_copy_dir_recursive_skips_symlinked_dirs() { + let temp = TempDir::new().unwrap(); + + let src = temp.path().join("src"); + let external = temp.path().join("external"); + + fs::create_dir_all(&external).unwrap(); + fs::write(external.join("secret.txt"), "should not be copied").unwrap(); + + fs::create_dir_all(&src).unwrap(); + fs::write(src.join("file.sol"), "content").unwrap(); + + symlink_dir(&external, &src.join("external_link")).unwrap(); + + let dst = temp.path().join("dst"); + copy_dir_recursive(&src, &dst).unwrap(); + + assert!(dst.join("file.sol").exists()); + assert!(!dst.join("external_link").exists()); + } + + #[test] + fn test_copy_dir_recursive_nonexistent_src() { + let temp = TempDir::new().unwrap(); + + let src = temp.path().join("nonexistent"); + let dst = temp.path().join("dst"); + + copy_dir_recursive(&src, &dst).unwrap(); + assert!(!dst.exists()); + } + + #[test] + fn test_copy_project_copies_include_paths_under_root() { + let temp = TempDir::new().unwrap(); + let root = temp.path().join("project"); + let out = temp.path().join("workspace"); + create_test_dir_structure( + &root, + &["src/Counter.sol", "test/Counter.t.sol", "include/Shared.sol"], + ); + + let config = Config { + root: root.clone(), + src: root.join("src"), + test: root.join("test"), + script: root.join("script"), + include_paths: vec![root.join("include")], + ..Default::default() + }; + + copy_project(&config, &out).unwrap(); + + assert!(out.join("include/Shared.sol").exists()); + } + + #[test] + fn test_copy_project_skips_include_paths_covered_by_libs() { + let temp = TempDir::new().unwrap(); + let root = temp.path().join("project"); + let out = temp.path().join("workspace"); + create_test_dir_structure( + &root, + &["src/Counter.sol", "test/Counter.t.sol", "lib/foo/Foo.sol", "lib/bar/Bar.sol"], + ); + + let config = Config { + root: root.clone(), + src: root.join("src"), + test: root.join("test"), + script: root.join("script"), + libs: vec![root.join("lib")], + include_paths: vec![root.join("lib/foo")], + ..Default::default() + }; + + copy_project(&config, &out).unwrap(); + + assert!(out.join("lib/foo/Foo.sol").exists()); + assert!(out.join("lib/bar/Bar.sol").exists()); + } + + #[test] + fn test_copy_project_rejects_external_include_paths() { + let temp = TempDir::new().unwrap(); + let root = temp.path().join("project"); + let outside = temp.path().join("outside"); + let out = temp.path().join("workspace"); + create_test_dir_structure(&root, &["src/Counter.sol", "test/Counter.t.sol"]); + create_test_dir_structure(&outside, &["Shared.sol"]); + + let config = Config { + root: root.clone(), + src: root.join("src"), + test: root.join("test"), + script: root.join("script"), + include_paths: vec![outside], + ..Default::default() + }; + + let err = copy_project(&config, &out).unwrap_err(); + + assert!( + err.to_string().contains("requires include/allow directory under project root"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_relative_to_root_basic() { + let root = PathBuf::from("/project"); + let path = PathBuf::from("/project/src/contracts"); + + let rel = relative_to_root(&root, &path); + assert_eq!(rel, PathBuf::from("src/contracts")); + } + + #[test] + fn test_relative_to_root_same_path() { + let root = PathBuf::from("/project"); + let path = PathBuf::from("/project"); + + let rel = relative_to_root(&root, &path); + assert_eq!(rel, PathBuf::from("")); + } + + #[test] + fn test_relative_to_root_outside_root() { + let root = PathBuf::from("/project"); + let path = PathBuf::from("/other/location"); + + let rel = relative_to_root(&root, &path); + assert_eq!(rel, path); + } + + #[test] + fn test_ensure_within_root_rejects_symlink_escape() { + let temp = TempDir::new().unwrap(); + let root = temp.path().join("project"); + let outside = temp.path().join("outside"); + fs::create_dir_all(&root).unwrap(); + fs::create_dir_all(&outside).unwrap(); + fs::write(outside.join("secret.txt"), "shhh").unwrap(); + + // src is a symlink that points outside the project root. + let src = root.join("src"); + symlink_dir(&outside, &src).unwrap(); + + let err = ensure_within_root(&root, &src, "src", &src).unwrap_err(); + assert!(err.to_string().contains("escapes project root"), "unexpected error: {err}"); + } + + #[test] + fn test_ensure_within_root_accepts_in_root_symlink() { + let temp = TempDir::new().unwrap(); + let root = temp.path().join("project"); + let real_src = root.join("real_src"); + fs::create_dir_all(&real_src).unwrap(); + + // src -> real_src is fine: stays inside the project root. + let src_link = root.join("src"); + symlink_dir(&real_src, &src_link).unwrap(); + + ensure_within_root(&root, &src_link, "src", &src_link).unwrap(); + } + + #[test] + fn test_symlink_nested_libs_rejects_traversal_in_dependency_config() { + let temp = TempDir::new().unwrap(); + + // Pretend lib_src is a malicious dependency whose foundry.toml says + // libs = ["../../escape"]. We can't easily write foundry.toml here, so + // exercise the lexical guard directly via is_safe_relative_path: any + // path containing `..` must be rejected before being joined with + // `lib_src`. + let malicious: PathBuf = PathBuf::from("../../escape"); + assert!(!is_safe_relative_path(&malicious)); + + // Sanity check: a benign relative path is still accepted. + let benign: PathBuf = PathBuf::from("lib"); + assert!(is_safe_relative_path(&benign)); + + // And the function returns Ok when there is nothing to do. + let lib_src = temp.path().join("lib_src"); + let lib_dst = temp.path().join("lib_dst"); + fs::create_dir_all(&lib_src).unwrap(); + fs::create_dir_all(&lib_dst).unwrap(); + symlink_nested_libs(&lib_src, &lib_dst, 0).unwrap(); + } + + #[test] + fn test_process_nested_lib_dir_skips_symlinks() { + let temp = TempDir::new().unwrap(); + let outside = temp.path().join("outside"); + fs::create_dir_all(outside.join("secret_pkg/src")).unwrap(); + fs::write(outside.join("secret_pkg/src/Secret.sol"), "secret").unwrap(); + + let lib_src = temp.path().join("lib_src"); + let nested = lib_src.join("lib"); + fs::create_dir_all(&nested).unwrap(); + // A dep that is a symlink pointing outside the lib root. + symlink_dir(&outside.join("secret_pkg"), &nested.join("evil")).unwrap(); + + let lib_dst = temp.path().join("lib_dst"); + fs::create_dir_all(&lib_dst).unwrap(); + + process_nested_lib_dir(&nested, &lib_dst, Path::new("lib"), 0).unwrap(); + + // The symlinked entry must not have been followed into the destination. + assert!(!lib_dst.join("lib/evil").exists(), "symlinked dep was followed"); + } +} diff --git a/crates/forge/tests/cli/config.rs b/crates/forge/tests/cli/config.rs index eb7e8eac51a2b..ceb31a487ff07 100644 --- a/crates/forge/tests/cli/config.rs +++ b/crates/forge/tests/cli/config.rs @@ -73,6 +73,7 @@ ignored_error_codes_from = [] ignored_warnings_from = [] deny = "never" test_failures_file = "cache/test-failures" +mutation_dir = "cache/mutation" show_progress = false ffi = false live_logs = false @@ -230,6 +231,10 @@ show_metrics = true show_solidity = false check_interval = 1 +[mutation] +include_operators = [] +exclude_operators = [] + [labels] [vyper] @@ -292,6 +297,7 @@ forgetest!(can_extract_config_values, |prj, cmd| { path_pattern_inverse: None, coverage_pattern_inverse: None, test_failures_file: "test-cache/test-failures".into(), + mutation_dir: "test-cache/mutation".into(), threads: None, show_progress: false, fuzz: FuzzConfig { @@ -311,6 +317,7 @@ forgetest!(can_extract_config_values, |prj, cmd| { }, ..Default::default() }, + mutation: Default::default(), ffi: true, live_logs: true, allow_internal_expect_revert: false, @@ -1355,6 +1362,7 @@ forgetest_init!(test_default_config, |prj, cmd| { "no_match_path": null, "no_match_coverage": null, "test_failures_file": "cache/test-failures", + "mutation_dir": "cache/mutation", "threads": null, "show_progress": false, "fuzz": { @@ -1416,6 +1424,11 @@ forgetest_init!(test_default_config, |prj, cmd| { "max_block_delay": null, "check_interval": 1 }, + "mutation": { + "include_operators": [], + "exclude_operators": [], + "timeout": null + }, "ffi": false, "live_logs": false, "allow_internal_expect_revert": false, diff --git a/crates/forge/tests/cli/test_cmd/core.rs b/crates/forge/tests/cli/test_cmd/core.rs index 22b4e3fe08a84..b8076c8c9276b 100644 --- a/crates/forge/tests/cli/test_cmd/core.rs +++ b/crates/forge/tests/cli/test_cmd/core.rs @@ -205,6 +205,23 @@ forgetest_init!(machine_mode_rejects_unsupported_flags, |_prj, cmd| { ); }); +forgetest_init!(machine_mode_rejects_mutation_testing, |_prj, cmd| { + let assert = cmd.args(["--machine", "test", "--mutate"]).assert_failure(); + let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); + let envelope: Value = serde_json::from_str(stdout.trim()) + .unwrap_or_else(|e| panic!("expected single-envelope error stdout: {stdout}: {e}")); + assert_eq!(envelope["success"], false); + assert_eq!(envelope["errors"][0]["code"], "cli.usage.invalid"); + assert_eq!(assert.get_output().status.code(), Some(2)); + let msg = envelope["errors"][0]["message"].as_str().unwrap_or(""); + assert!(msg.contains("--mutate"), "missing --mutate mention: {envelope}"); + assert_eq!( + envelope["errors"][0]["details"]["unsupported_flags"], + serde_json::json!(["--mutate"]), + "missing structured unsupported_flags details: {envelope}" + ); +}); + // `--allow-failure`: success envelope with `data.failed > 0` and exit 0. forgetest_init!(machine_mode_allow_failure_emits_success_envelope, |prj, cmd| { prj.add_test( diff --git a/crates/forge/tests/cli/test_cmd/mod.rs b/crates/forge/tests/cli/test_cmd/mod.rs index 4bcf71a52d222..5e024d6fec98a 100644 --- a/crates/forge/tests/cli/test_cmd/mod.rs +++ b/crates/forge/tests/cli/test_cmd/mod.rs @@ -15,6 +15,7 @@ mod core; mod fuzz; mod invariant; mod logs; +mod mutation; mod repros; mod showmap; mod spec; diff --git a/crates/forge/tests/cli/test_cmd/mutation.rs b/crates/forge/tests/cli/test_cmd/mutation.rs new file mode 100644 index 0000000000000..5708dcbdbfbe4 --- /dev/null +++ b/crates/forge/tests/cli/test_cmd/mutation.rs @@ -0,0 +1,1068 @@ +// CLI integration tests for mutation testing + +use foundry_test_utils::{str, util::OutputExt}; +use std::fs; + +fn mutation_summary(stdout: &str) -> serde_json::Value { + serde_json::from_str::(stdout.trim()).unwrap()["summary"].clone() +} + +forgetest_init!(can_run_mutation_testing, |prj, cmd| { + prj.add_source( + "Counter.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +contract Counter { + uint256 public number; + + function setNumber(uint256 newNumber) public { + number = newNumber; + } + + function increment() public { + number++; + } +} +"#, + ); + + prj.add_test( + "Counter.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/Counter.sol"; + +contract CounterTest { + Counter public counter; + + function setUp() public { + counter = new Counter(); + } + + function test_Increment() public { + counter.increment(); + assert(counter.number() == 1); + } + + function test_SetNumber() public { + counter.setNumber(42); + assert(counter.number() == 42); + } +} +"#, + ); + + // Run mutation testing + cmd.args(["test", "--mutate", "src/Counter.sol", "--mutation-jobs", "1"]); + cmd.assert_success().stdout_eq(str![[r#" +... +Running mutation tests with 1 parallel workers... +... +════════════════════════════════════════════════════════════ +MUTATION TESTING RESULTS +════════════════════════════════════════════════════════════ + +╭──────────┬───────────┬────────────╮ +│ Status ┆ # Mutants ┆ % of Total │ +╞══════════╪═══════════╪════════════╡ +│ Survived ┆ 1 ┆ 14.3% │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Killed ┆ 4 ┆ 57.1% │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Invalid ┆ 2 ┆ 28.6% │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Skipped ┆ 0 ┆ 0.0% │ +╰──────────┴───────────┴────────────╯ + +Legend: + Survived - tests did not catch the mutation + Killed - tests caught the mutation + Invalid - mutation produced a compilation error + Skipped - redundant mutation on the same expression + Timed out - compile/test exceeded the configured timeout + +Mutation Score: 80.0% (4/5 mutants killed); [ELAPSED] + +──────────────────────────────────────────────────────────── +Survived mutants +──────────────────────────────────────────────────────────── + +... + number++; + Mutation: + - number++ + + ++number +... +──────────────────────────────────────────────────────────── +4 mutants killed + +──────────────────────────────────────────────────────────── +2 mutants invalid + +════════════════════════════════════════════════════════════ + +"#]]); + + // Run mutation testing with --json - verify the output contains valid mutation JSON + cmd.forge_fuse().args(["test", "--mutate", "src/Counter.sol", "--mutation-jobs", "1", "--json"]).assert_success().stdout_eq(str![[r#" +{"summary":{"total":7,"killed":4,"survived":1,"invalid":2,"skipped":0,"timed_out":0,"mutation_score":80.0,"duration_secs":[..]},"survived_mutants":{"src/Counter.sol":[{"line":13,"column":9,"original":"number++","mutant":"++number"}]}} + +"#]]); +}); + +forgetest_init!(mutation_testing_rejects_all_skipped_baseline, |prj, cmd| { + prj.add_source( + "Counter.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +contract Counter { + uint256 public number; + + function increment() public { + number++; + } +} +"#, + ); + + prj.add_test( + "Counter.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "forge-std/Test.sol"; +import "../src/Counter.sol"; + +contract CounterTest is Test { + Counter public counter; + + function setUp() public { + vm.skip(true); + counter = new Counter(); + } + + function test_Increment() public { + counter.increment(); + assertEq(counter.number(), 1); + } +} +"#, + ); + + let output = cmd.args(["test", "--mutate", "src/Counter.sol"]).assert_failure(); + let stderr = output.get_output().stderr_lossy(); + + assert!( + stderr.contains("Mutation testing requires at least one passing baseline test"), + "unexpected stderr:\n{stderr}" + ); +}); + +forgetest_init!(mutation_testing_rejects_empty_mutate_path_selection, |prj, cmd| { + prj.add_source( + "Counter.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +contract Counter { + uint256 public number; + + function increment() public { + number++; + } +} +"#, + ); + + prj.add_test( + "Counter.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/Counter.sol"; + +contract CounterTest { + function test_Increment() public { + Counter counter = new Counter(); + counter.increment(); + assert(counter.number() == 1); + } +} +"#, + ); + + let output = + cmd.args(["test", "--mutate", "--mutate-path", "src/Missing*.sol"]).assert_failure(); + let stderr = output.get_output().stderr_lossy(); + + assert!( + stderr.contains("no source matched --mutate-path pattern"), + "unexpected stderr:\n{stderr}" + ); +}); + +forgetest_init!(mutation_testing_rejects_empty_mutate_contract_selection, |prj, cmd| { + prj.add_source( + "Counter.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +contract Counter { + uint256 public number; + + function increment() public { + number++; + } +} +"#, + ); + + prj.add_test( + "Counter.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/Counter.sol"; + +contract CounterTest { + function test_Increment() public { + Counter counter = new Counter(); + counter.increment(); + assert(counter.number() == 1); + } +} +"#, + ); + + let output = cmd.args(["test", "--mutate", "--mutate-contract", "Missing"]).assert_failure(); + let stderr = output.get_output().stderr_lossy(); + + assert!( + stderr.contains("no source matched --mutate-contract pattern"), + "unexpected stderr:\n{stderr}" + ); +}); + +forgetest_init!(mutation_testing_with_parallel_workers, |prj, cmd| { + prj.add_source( + "Simple.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +contract Simple { + function add(uint256 a, uint256 b) public pure returns (uint256) { + return a + b; + } +} +"#, + ); + + prj.add_test( + "Simple.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/Simple.sol"; + +contract SimpleTest { + Simple public simple; + + function setUp() public { + simple = new Simple(); + } + + function test_Add() public { + assert(simple.add(1, 2) == 3); + } +} +"#, + ); + + // Run mutation testing with 4 workers + cmd.args(["test", "--mutate", "src/Simple.sol", "--mutation-jobs", "4"]); + cmd.assert_success().stdout_eq(str![[r#" +... +Running mutation tests with 4 parallel workers... +... +════════════════════════════════════════════════════════════ +MUTATION TESTING RESULTS +════════════════════════════════════════════════════════════ + +╭──────────┬───────────┬────────────╮ +│ Status ┆ # Mutants ┆ % of Total │ +╞══════════╪═══════════╪════════════╡ +│ Survived ┆ 2 ┆ 18.2% │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Killed ┆ 8 ┆ 72.7% │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Invalid ┆ 1 ┆ 9.1% │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Skipped ┆ 0 ┆ 0.0% │ +╰──────────┴───────────┴────────────╯ +... +Mutation Score: 80.0% (8/10 mutants killed); [ELAPSED] +... +"#]]); +}); + +forgetest_init!(mutation_testing_with_show_progress, |prj, cmd| { + prj.add_source( + "Simple.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +contract Simple { + function add(uint256 a, uint256 b) public pure returns (uint256) { + return a + b; + } +} +"#, + ); + + prj.add_test( + "Simple.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/Simple.sol"; + +contract SimpleTest { + Simple public simple; + + function setUp() public { + simple = new Simple(); + } + + function test_Add() public { + assert(simple.add(1, 2) == 3); + } +} +"#, + ); + + // Run mutation testing with progress display (use 4 workers like parallel test for consistency) + cmd.args(["test", "--mutate", "src/Simple.sol", "--show-progress", "--mutation-jobs", "4"]); + cmd.assert_success().stdout_eq(str![[r#" +... +════════════════════════════════════════════════════════════ +MUTATION TESTING RESULTS +════════════════════════════════════════════════════════════ + +╭──────────┬───────────┬────────────╮ +│ Status ┆ # Mutants ┆ % of Total │ +╞══════════╪═══════════╪════════════╡ +│ Survived ┆ 2 ┆ 18.2% │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Killed ┆ 8 ┆ 72.7% │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Invalid ┆ 1 ┆ 9.1% │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Skipped ┆ 0 ┆ 0.0% │ +╰──────────┴───────────┴────────────╯ + +Legend: + Survived - tests did not catch the mutation + Killed - tests caught the mutation + Invalid - mutation produced a compilation error + Skipped - redundant mutation on the same expression + Timed out - compile/test exceeded the configured timeout + +Mutation Score: 80.0% (8/10 mutants killed); [ELAPSED] + +──────────────────────────────────────────────────────────── +Survived mutants +──────────────────────────────────────────────────────────── + +... + return a + b; + Mutation: + - a + b +... + return a + b; + Mutation: + - a + b +... +──────────────────────────────────────────────────────────── +8 mutants killed + +──────────────────────────────────────────────────────────── +1 mutants invalid + +════════════════════════════════════════════════════════════ + +"#]]); +}); + +forgetest_init!(mutation_result_cache_invalidates_when_tests_change, |prj, _cmd| { + prj.add_source( + "Calculator.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +contract Calculator { + function add(uint256 a, uint256 b) public pure returns (uint256) { + return a + b; + } +} +"#, + ); + + prj.add_test( + "Calculator.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/Calculator.sol"; + +contract CalculatorTest { + Calculator public calculator; + + function setUp() public { + calculator = new Calculator(); + } + + function test_Add() public view { + calculator.add(1, 2); + } +} +"#, + ); + + let mut weak_cmd = prj.forge_command(); + let weak_stdout = weak_cmd + .args(["test", "--mutate", "src/Calculator.sol", "--mutation-jobs", "1", "--json"]) + .assert_success() + .get_output() + .stdout_lossy(); + let weak_summary = mutation_summary(&weak_stdout); + + prj.add_test( + "Calculator.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/Calculator.sol"; + +contract CalculatorTest { + Calculator public calculator; + + function setUp() public { + calculator = new Calculator(); + } + + function test_Add() public view { + assert(calculator.add(1, 2) == 3); + } +} +"#, + ); + + let mut strong_cmd = prj.forge_command(); + let strong_stdout = strong_cmd + .args(["test", "--mutate", "src/Calculator.sol", "--mutation-jobs", "1", "--json"]) + .assert_success() + .get_output() + .stdout_lossy(); + let strong_summary = mutation_summary(&strong_stdout); + + assert_eq!(weak_summary["total"], strong_summary["total"]); + assert!( + strong_summary["killed"].as_u64().unwrap() > weak_summary["killed"].as_u64().unwrap(), + "expected changed tests to invalidate cached mutation results: weak={weak_summary}, strong={strong_summary}", + ); +}); + +forgetest_init!(mutation_result_cache_invalidates_when_match_test_changes, |prj, _cmd| { + prj.add_source( + "Calculator.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +contract Calculator { + function add(uint256 a, uint256 b) public pure returns (uint256) { + return a + b; + } +} +"#, + ); + + prj.add_test( + "Calculator.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/Calculator.sol"; + +contract CalculatorTest { + Calculator public calculator; + + function setUp() public { + calculator = new Calculator(); + } + + function test_Weak() public view { + calculator.add(1, 2); + } + + function test_Strong() public view { + assert(calculator.add(1, 2) == 3); + } +} +"#, + ); + + let mut weak_cmd = prj.forge_command(); + let weak_stdout = weak_cmd + .args([ + "test", + "--mutate", + "src/Calculator.sol", + "--mutation-jobs", + "1", + "--match-test", + "test_Weak", + "--json", + ]) + .assert_success() + .get_output() + .stdout_lossy(); + let weak_summary = mutation_summary(&weak_stdout); + + let mut strong_cmd = prj.forge_command(); + let strong_stdout = strong_cmd + .args([ + "test", + "--mutate", + "src/Calculator.sol", + "--mutation-jobs", + "1", + "--match-test", + "test_Strong", + "--json", + ]) + .assert_success() + .get_output() + .stdout_lossy(); + let strong_summary = mutation_summary(&strong_stdout); + + assert_eq!(weak_summary["total"], strong_summary["total"]); + assert!( + strong_summary["killed"].as_u64().unwrap() > weak_summary["killed"].as_u64().unwrap(), + "expected --match-test to invalidate cached mutation results: weak={weak_summary}, strong={strong_summary}", + ); +}); + +forgetest_init!(mutation_result_cache_invalidates_when_match_path_changes, |prj, _cmd| { + prj.add_source( + "Calculator.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +contract Calculator { + function add(uint256 a, uint256 b) public pure returns (uint256) { + return a + b; + } +} +"#, + ); + + prj.add_test( + "Weak.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/Calculator.sol"; + +contract WeakTest { + Calculator public calculator; + + function setUp() public { + calculator = new Calculator(); + } + + function test_Weak() public view { + calculator.add(1, 2); + } +} +"#, + ); + + prj.add_test( + "Strong.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/Calculator.sol"; + +contract StrongTest { + Calculator public calculator; + + function setUp() public { + calculator = new Calculator(); + } + + function test_Strong() public view { + assert(calculator.add(1, 2) == 3); + } +} +"#, + ); + + let mut weak_cmd = prj.forge_command(); + let weak_stdout = weak_cmd + .args([ + "test", + "--mutate", + "src/Calculator.sol", + "--mutation-jobs", + "1", + "--match-path", + "test/Weak.t.sol", + "--json", + ]) + .assert_success() + .get_output() + .stdout_lossy(); + let weak_summary = mutation_summary(&weak_stdout); + + let mut strong_cmd = prj.forge_command(); + let strong_stdout = strong_cmd + .args([ + "test", + "--mutate", + "src/Calculator.sol", + "--mutation-jobs", + "1", + "--match-path", + "test/Strong.t.sol", + "--json", + ]) + .assert_success() + .get_output() + .stdout_lossy(); + let strong_summary = mutation_summary(&strong_stdout); + + assert_eq!(weak_summary["total"], strong_summary["total"]); + assert!( + strong_summary["killed"].as_u64().unwrap() > weak_summary["killed"].as_u64().unwrap(), + "expected --match-path to invalidate cached mutation results: weak={weak_summary}, strong={strong_summary}", + ); +}); + +forgetest_init!(mutation_honors_match_path_at_compile_time, |prj, cmd| { + prj.add_source( + "Foo.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +contract Foo { + function add(uint256 a, uint256 b) public pure returns (uint256) { + return a + b; + } +} +"#, + ); + + prj.add_test( + "FooSelected.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/Foo.sol"; + +contract FooSelectedTest { + Foo internal foo; + + function setUp() public { + foo = new Foo(); + } + + function test_Add() public view { + assert(foo.add(2, 3) == 5); + } +} +"#, + ); + + prj.add_test( + "FooBroken.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/Foo.sol"; + +contract FooBrokenTest { + function test_Broken() public pure { + NonExistent x = NonExistent(0); + x.doSomething(); + } +} +"#, + ); + + cmd.forge_fuse().args(["test", "--match-path", "test/FooSelected.t.sol"]).assert_success(); + + cmd.forge_fuse().args([ + "test", + "--mutate", + "src/Foo.sol", + "--match-path", + "test/FooSelected.t.sol", + "--mutation-jobs", + "1", + "--json", + ]); + + let out = cmd.assert_success().get_output().stdout_lossy(); + let summary = mutation_summary(&out); + + let total = summary["total"].as_u64().unwrap_or(0); + let invalid = summary["invalid"].as_u64().unwrap_or(u64::MAX); + let killed = summary["killed"].as_u64().unwrap_or(0); + let survived = summary["survived"].as_u64().unwrap_or(0); + + assert!( + invalid < total, + "filtered-out FooBroken.t.sol must not make every mutant Invalid; summary={summary}" + ); + assert!( + killed + survived >= 1, + "expected at least one Killed/Survived mutant from arithmetic ops; summary={summary}" + ); +}); + +forgetest_init!(mutation_workspace_copies_include_paths, |prj, cmd| { + let include_dir = prj.root().join("include"); + fs::create_dir_all(&include_dir).unwrap(); + fs::write( + include_dir.join("Shared.sol"), + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +library Shared { + function value() internal pure returns (uint256) { + return 1; + } +} +"#, + ) + .unwrap(); + + prj.update_config(|config| { + config.include_paths = vec![include_dir.clone()]; + }); + + prj.add_source( + "UsesShared.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "Shared.sol"; + +contract UsesShared { + function value() public pure returns (uint256) { + return Shared.value() + 1; + } +} +"#, + ); + + prj.add_test( + "UsesShared.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/UsesShared.sol"; + +contract UsesSharedTest { + function test_Value() public { + UsesShared usesShared = new UsesShared(); + assert(usesShared.value() == 2); + } +} +"#, + ); + + let out = cmd + .args(["test", "--mutate", "src/UsesShared.sol", "--mutation-jobs", "1", "--json"]) + .assert_success() + .get_output() + .stdout_lossy(); + let summary = mutation_summary(&out); + let total = summary["total"].as_u64().unwrap_or(0); + let invalid = summary["invalid"].as_u64().unwrap_or(u64::MAX); + + assert!(total > 0, "expected mutation testing to generate mutants: summary={summary}"); + assert!( + invalid < total, + "include_paths imports should compile inside mutant workspaces: summary={summary}" + ); +}); + +// Test require/assert mutation for security-critical patterns +forgetest_init!(mutation_testing_require_mutator, |prj, cmd| { + // A contract with security-critical require checks (access control, input validation) + prj.add_source( + "Vault.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +contract Vault { + address public owner; + mapping(address => uint256) public balances; + bool public paused; + + constructor() { + owner = msg.sender; + } + + modifier onlyOwner() { + require(msg.sender == owner, "Not owner"); + _; + } + + function deposit() public payable { + require(!paused, "Contract paused"); + require(msg.value > 0, "Must send ETH"); + balances[msg.sender] += msg.value; + } + + function withdraw(uint256 amount) public { + require(!paused, "Contract paused"); + require(balances[msg.sender] >= amount, "Insufficient balance"); + balances[msg.sender] -= amount; + payable(msg.sender).transfer(amount); + } + + function pause() public onlyOwner { + paused = true; + } + + function unpause() public onlyOwner { + paused = false; + } +} +"#, + ); + + prj.add_test( + "Vault.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "forge-std/Test.sol"; +import "../src/Vault.sol"; + +contract VaultTest is Test { + Vault public vault; + + function setUp() public { + vault = new Vault(); + } + + // === DEPOSIT TESTS === + + function test_DepositRequiresValue() public { + vm.expectRevert("Must send ETH"); + vault.deposit(); + } + + function test_DepositWithValue() public { + vault.deposit{value: 1 ether}(); + assertEq(vault.balances(address(this)), 1 ether); + } + + function test_DepositWhenPausedReverts() public { + vault.pause(); + vm.expectRevert("Contract paused"); + vault.deposit{value: 1 ether}(); + } + + // === WITHDRAW TESTS === + + function test_WithdrawRequiresBalance() public { + vm.expectRevert("Insufficient balance"); + vault.withdraw(1 ether); + } + + function test_WithdrawExactBalance() public { + // Kills >= -> > and >= -> != mutations + vault.deposit{value: 1 ether}(); + vault.withdraw(1 ether); + assertEq(vault.balances(address(this)), 0); + } + + function test_WithdrawPartialBalance() public { + vault.deposit{value: 1 ether}(); + vault.withdraw(0.5 ether); + assertEq(vault.balances(address(this)), 0.5 ether); + } + + function test_WithdrawWhenPausedReverts() public { + vault.deposit{value: 1 ether}(); + vault.pause(); + vm.expectRevert("Contract paused"); + vault.withdraw(0.5 ether); + } + + // === PAUSE/UNPAUSE TESTS === + + function test_OnlyOwnerCanPause() public { + vault.pause(); + assertTrue(vault.paused()); + } + + function test_NonOwnerCannotPause() public { + vm.prank(address(1)); + vm.expectRevert("Not owner"); + vault.pause(); + } + + function test_OnlyOwnerCanUnpause() public { + vault.pause(); + vault.unpause(); + assertFalse(vault.paused()); + } + + function test_NonOwnerCannotUnpause() public { + vault.pause(); + vm.prank(address(1)); + vm.expectRevert("Not owner"); + vault.unpause(); + } + + function test_UnpauseAllowsDeposit() public { + vault.pause(); + vault.unpause(); + vault.deposit{value: 1 ether}(); + assertEq(vault.balances(address(this)), 1 ether); + } + + // Kills >= mutation on address comparison + function test_HigherAddressCannotPause() public { + vm.prank(address(type(uint160).max)); + vm.expectRevert("Not owner"); + vault.pause(); + } + + receive() external payable {} +} +"#, + ); + + // The surviving mutant (msg.value > 0 -> msg.value != 0) is equivalent for uint256 + let mut cmd2 = prj.forge_command(); + cmd2.args(["test", "--mutate", "src/Vault.sol", "--mutation-jobs", "2"]); + cmd2.assert_success().stdout_eq(str![[r#" +... +Running mutation tests with 2 parallel workers... +... +════════════════════════════════════════════════════════════ +MUTATION TESTING RESULTS +════════════════════════════════════════════════════════════ + +╭──────────┬───────────┬────────────╮ +│ Status ┆ # Mutants ┆ % of Total │ +╞══════════╪═══════════╪════════════╡ +│ Survived ┆ 3 ┆ 5.0% │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Killed ┆ 48 ┆ 80.0% │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Invalid ┆ 7 ┆ 11.7% │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Skipped ┆ 2 ┆ 3.3% │ +╰──────────┴───────────┴────────────╯ +... +Mutation Score: 94.1% (48/51 mutants killed); [ELAPSED] +... +"#]]); +}); + +forgetest_init!(mutation_testing_assembly_code, |prj, cmd| { + prj.add_source( + "AsmMath.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +library AsmMath { + function add(uint256 a, uint256 b) internal pure returns (uint256 result) { + assembly { + result := add(a, b) + } + } + + function sub(uint256 a, uint256 b) internal pure returns (uint256 result) { + assembly { + result := sub(a, b) + } + } +} +"#, + ); + + prj.add_test( + "AsmMath.t.sol", + r#" +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.13; + +import "../src/AsmMath.sol"; + +contract AsmMathTest { + using AsmMath for uint256; + + function test_Add() public pure { + assert(uint256(2).add(3) == 5); + assert(uint256(0).add(0) == 0); + } + + function test_Sub() public pure { + assert(uint256(5).sub(3) == 2); + } +} +"#, + ); + + cmd.args(["test", "--mutate", "src/AsmMath.sol", "--mutation-jobs", "1"]); + cmd.assert_success().stdout_eq(str![[r#" +... +Running mutation tests with 1 parallel workers... +... +════════════════════════════════════════════════════════════ +MUTATION TESTING RESULTS +════════════════════════════════════════════════════════════ +... +"#]]); +});