From 2d91c6e126a5655695ebb9dffbe0023328a1e342 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 24 Jun 2026 20:42:58 +0200 Subject: [PATCH 1/2] perf(dedup): run duplicate check only on the outermost runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A task inside a workflow, and a workflow inside a scan, each ran their own duplicate check. On scans with 10,000s of findings the dedup ran redundantly at every nesting level — a real runtime toll for no benefit. mark_duplicates() operates on self.results, and descendant results aggregate up into the outermost runner's self.results via the yielder, so its single dedup pass already covers every descendant's findings. Gate the check so nested runners (has_parent=True) skip it by default, while still respecting an explicit enable_duplicate_check override if the caller set one. Standalone tasks/workflows (no parent) still dedup. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01P5vSjfkBuGAAHdKxHS3ySm --- secator/runners/_base.py | 10 ++++++++++ tests/unit/test_runners.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/secator/runners/_base.py b/secator/runners/_base.py index 38454a0df..d08fb24de 100644 --- a/secator/runners/_base.py +++ b/secator/runners/_base.py @@ -138,7 +138,17 @@ def __init__(self, config, inputs=[], results=[], run_opts={}, hooks={}, validat self.raise_on_error = self.run_opts.get('raise_on_error', False) # Runner toggles + # Duplicate check should run ONCE, on the outermost runner only. A nested runner + # (has_parent=True) — e.g. a task inside a workflow, or a workflow inside a scan — + # would otherwise re-run the dedup at every nesting level, which on scans with + # 10,000s of findings is a real runtime toll for no benefit: results aggregate up to + # the outermost runner (descendant findings flow into its self.results via the + # yielder), so its single mark_duplicates() pass already covers every descendant's + # findings. We therefore disable the check for nested runners by default, while still + # respecting an explicit `enable_duplicate_check` override if the caller set one. self.enable_duplicate_check = self.run_opts.get('enable_duplicate_check', self.enable_duplicate_check) + if self.has_parent and 'enable_duplicate_check' not in self.run_opts: + self.enable_duplicate_check = False self.enable_profiles = self.run_opts.get('enable_profiles', True) self.enable_reports = self.run_opts.get('enable_reports', not self.sync) and not self.dry_run and not self.no_process and not self.no_poll # noqa: E501 self.enable_hooks = self.run_opts.get('enable_hooks', True) and not self.dry_run and not self.no_process # noqa: E501 diff --git a/tests/unit/test_runners.py b/tests/unit/test_runners.py index 314f8d134..16f6be2d3 100644 --- a/tests/unit/test_runners.py +++ b/tests/unit/test_runners.py @@ -918,3 +918,36 @@ def test_self_errors_includes_chunk_errors(self): cmd.unique_name = 'nmap' cmd.results.append(chunk_error) self.assertEqual(cmd.self_errors, [chunk_error]) + + +class TestDuplicateCheckGating(unittest.TestCase): + """Duplicate check must run only on the outermost runner. + + A nested runner (has_parent=True) — a task inside a workflow, or a workflow inside + a scan — must NOT run its own dedup pass; the outermost runner does it once over its + fully-aggregated results. See Runner.__init__ in secator/runners/_base.py. + """ + + def test_outermost_runner_dedups(self): + """An outermost runner (has_parent defaults to False) keeps the duplicate check on.""" + cmd = MyCommand(TARGETS) + self.assertFalse(cmd.has_parent) + self.assertTrue(cmd.enable_duplicate_check) + + def test_nested_runner_does_not_dedup(self): + """A nested runner (has_parent=True) has its duplicate check disabled by default.""" + cmd = MyCommand(TARGETS, has_parent=True) + self.assertTrue(cmd.has_parent) + self.assertFalse(cmd.enable_duplicate_check) + + def test_nested_runner_respects_explicit_enable_override(self): + """An explicit enable_duplicate_check=True override wins even when nested.""" + cmd = MyCommand(TARGETS, has_parent=True, enable_duplicate_check=True) + self.assertTrue(cmd.has_parent) + self.assertTrue(cmd.enable_duplicate_check) + + def test_outermost_runner_respects_explicit_disable_override(self): + """An explicit enable_duplicate_check=False override wins on an outermost runner.""" + cmd = MyCommand(TARGETS, enable_duplicate_check=False) + self.assertFalse(cmd.has_parent) + self.assertFalse(cmd.enable_duplicate_check) From f2dd0b6fac29533230da8180fe3a3ca50781252f Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Mon, 29 Jun 2026 11:16:32 +0200 Subject: [PATCH 2/2] Update _base.py --- secator/runners/_base.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/secator/runners/_base.py b/secator/runners/_base.py index d08fb24de..9c24f91e7 100644 --- a/secator/runners/_base.py +++ b/secator/runners/_base.py @@ -138,16 +138,8 @@ def __init__(self, config, inputs=[], results=[], run_opts={}, hooks={}, validat self.raise_on_error = self.run_opts.get('raise_on_error', False) # Runner toggles - # Duplicate check should run ONCE, on the outermost runner only. A nested runner - # (has_parent=True) — e.g. a task inside a workflow, or a workflow inside a scan — - # would otherwise re-run the dedup at every nesting level, which on scans with - # 10,000s of findings is a real runtime toll for no benefit: results aggregate up to - # the outermost runner (descendant findings flow into its self.results via the - # yielder), so its single mark_duplicates() pass already covers every descendant's - # findings. We therefore disable the check for nested runners by default, while still - # respecting an explicit `enable_duplicate_check` override if the caller set one. self.enable_duplicate_check = self.run_opts.get('enable_duplicate_check', self.enable_duplicate_check) - if self.has_parent and 'enable_duplicate_check' not in self.run_opts: + if self.has_parent and 'enable_duplicate_check' not in self.run_opts: # disable duplicate check on children runners self.enable_duplicate_check = False self.enable_profiles = self.run_opts.get('enable_profiles', True) self.enable_reports = self.run_opts.get('enable_reports', not self.sync) and not self.dry_run and not self.no_process and not self.no_poll # noqa: E501