From d577d7e688cac66c15e652ed46aa9f95039c1450 Mon Sep 17 00:00:00 2001 From: Hank Wikle Date: Fri, 20 Mar 2026 12:32:14 -0600 Subject: [PATCH] feat(view): add --stack flag to display raw config stack (incl. inheritance ancestors and overrides) --- lib/pavilion/commands/view.py | 112 +++++++++++++++++++++-------- lib/pavilion/resolver/resolver.py | 113 +++++++++++++++++++++++++++--- lib/pavilion/wget.py | 10 +-- test/tests/style_tests.py | 15 +++- test/tests/view_cmd_tests.py | 17 +++++ 5 files changed, 222 insertions(+), 45 deletions(-) diff --git a/lib/pavilion/commands/view.py b/lib/pavilion/commands/view.py index fe1d5cd14..0e114c341 100644 --- a/lib/pavilion/commands/view.py +++ b/lib/pavilion/commands/view.py @@ -2,6 +2,7 @@ import errno import pprint +from pavilion.test_config import file_format from pavilion import output from pavilion import cmd_utils @@ -27,54 +28,109 @@ def __init__(self): # pylint: disable=W0231 ) def _setup_arguments(self, parser): + """Define command‑line arguments for the view command. + The existing arguments are retained and the new ``--stack`` flag is added. + """ parser.add_argument( '-p', '--platform', action='store', - help='The platform to configure this test for. If not ' - 'specified, the current operating system as denoted by the ' - 'sys plugin \'sys_os\' is used.') + help='The platform to configure this test for. If not specified, the ' + "current operating system as denoted by the sys plugin 'sys_os' is used.") parser.add_argument( '-H', '--host', action='store', help='The host to configure this test for. If not specified, the ' - 'current host as denoted by the sys plugin \'sys_host\' is ' - 'used.') + "current host as denoted by the sys plugin 'sys_host' is used.") parser.add_argument( '-m', '--mode', action='append', dest='modes', default=[], - help='Mode configurations to overlay on the host configuration for ' - 'each test. These are overlaid in the order given.') + help='Mode configurations to overlay on the host configuration for each test. ' + "These are overlaid in the order given.") parser.add_argument( '-c', dest='overrides', action='append', default=[], help='Overrides for specific configuration options. These are ' - 'gathered used as a final set of overrides before the ' - 'configs are resolved. They should take the form ' - '\'key=value\', where key is the dot separated key name, ' - 'and value is a json object.') + "gathered used as a final set of overrides before the configs are resolved. " + "They should take the form 'key=value', where key is the dot separated key name, " + "and value is a json object.") + parser.add_argument( + '--stack', action='store_true', default=False, + help='Show the unmerged configuration stack for each test, including base config, ' + "suite inheritance, and platform/host/mode configs.") parser.add_argument( '-f', '--file', dest='files', action='append', default=[], - help='Add tests listed in the given file, as per the "pav run" command', - ) + help='Add tests listed in the given file, as per the "pav run" command') parser.add_argument( 'tests', action='store', nargs='*', - help='The name of the test to view. Should be in the format ' - '..') + help='The name of the test to view. Should be in the format ..') SLEEP_INTERVAL = 1 # pylint: disable=arguments-differ - def run(self, pav_cfg, args): + def run(self, pav_cfg, args, log_results: bool = True): """Resolve the test configurations into individual tests and assign to - schedulers. Have those schedulers kick off jobs to run the individual - tests themselves.""" - - overrides = [] - for ovr in args.overrides: - if '=' not in ovr: - fprint(self.errfile, "Invalid override value. Must be in the form: " - "=. Ex. -c run.modules=['gcc'] ") - return errno.EINVAL - - key, value = ovr.split('=', 1) - overrides[key] = value + schedulers. Have those schedulers kick off jobs themselves. + + ``log_results`` is retained for compatibility with ``RunCommand``. + """ + + # ``args.overrides`` is already a list of ``key=value`` strings. + # The resolver expects this list directly, so we pass it through. + overrides = args.overrides + + # --------------------------------------------------------------------- + # ``--stack`` handling – show the unmerged configuration stack. + # --------------------------------------------------------------------- + if getattr(args, 'stack', False): + # Use the new resolver method to get the unmerged config stack. + resolver_obj = resolver.TestConfigResolver(pav_cfg) + + for full_name in args.tests: + try: + stack = resolver_obj.get_unmerged_config_stack( + full_test_name=full_name, + modes=args.modes, + platform=args.platform, + host=args.host, + overrides=args.overrides, + ) + except Exception as err: + fprint(self.errfile, f"Could not get stack for {full_name}: {err}", color=output.RED) + continue + + fprint(self.outfile, f"Config stack for {full_name}:") + for label, cfg, path in stack: + if path: + fprint(self.outfile, f"# path: {path}") + else: + fprint(self.outfile, "# generated in‑memory") + fprint(self.outfile, f"--- {label} ---") + pprint.pprint(cfg, stream=self.outfile) + + return 0 + + # --------------------------------------------------------------------- + # Normal ``pav view`` – show fully resolved configuration. + # --------------------------------------------------------------------- + tests = args.tests + + self.logger.debug("Finding Configs") + + res = resolver.TestConfigResolver(pav_cfg, platform=args.platform, + host=args.host, outfile=self.outfile) + + tests.extend(cmd_utils.read_test_files(pav_cfg, args.files)) + + try: + proto_tests = res.load( + tests=tests, + modes=args.modes, + overrides=overrides, + ) + except CommandError as err: + fprint(self.errfile, err, color=output.RED) + return errno.EINVAL + + configs = {pt.config['name']: pt.config for pt in proto_tests} + pprint.pprint(configs, stream=self.outfile) # ext-print: ignore + return 0 tests = args.tests diff --git a/lib/pavilion/resolver/resolver.py b/lib/pavilion/resolver/resolver.py index 69d60599a..c66d24140 100644 --- a/lib/pavilion/resolver/resolver.py +++ b/lib/pavilion/resolver/resolver.py @@ -79,6 +79,99 @@ def __init__(self, modes: List[str], overrides: List[str], conditions: Dict): class TestConfigResolver: """Converts raw test configurations into their final, fully resolved form.""" + def get_unmerged_config_stack(self, full_test_name: str, modes: List[str] = None, platform: str = None, host: str = None, overrides: List[str] = None) -> List[Tuple[str, Dict, Optional[Path]]]: + """Return the unmerged configuration stack for a given test. + + The stack includes the base empty config, the suite test config after + inheritance, and optional platform, host, mode, and overrides layers. + Each item is a tuple ``(label, cfg, path)`` where ``path`` is the source + file path for the config, or ``None`` for generated configs. + """ + # Parse suite and test names. + suite_name = None + test_name = None + if '.' in full_test_name: + suite_name, test_name = full_test_name.split('.', 1) + else: + # Locate the test in any suite. + all_suites = self.find_all_tests() + for sname, sdata in all_suites.items(): + if full_test_name in sdata['tests']: + suite_name = sname + test_name = full_test_name + break + if suite_name is None: + raise ValueError(f"Test '{full_test_name}' not found in any suite") + + # Load raw suite configuration (without resolving inheritance). + # First locate the suite file. + suite_cfg_info = self.find_config('suite', suite_name, suite_name) + loader = self._suite_loader if suite_cfg_info.from_suite else self._loader + raw_suite_cfg = self._load_raw_config(suite_cfg_info, loader) + if raw_suite_cfg is None: + raise ValueError(f"Suite '{suite_name}' could not be loaded") + + # Walk the inheritance chain for the requested test, adding each ancestor. + stack: List[Tuple[str, Dict, Optional[Path]]] = [] + # Base empty config. + base_cfg = self._loader.load_empty() + stack.append(("base", base_cfg, None)) + + # Helper to fetch a test dict and its source path. + def get_test_entry(name: str) -> Tuple[Dict, Optional[Path]]: + cfg = raw_suite_cfg.get(name, {}) or {} + # The suite_path is the path to the suite file. + path = Path(suite_cfg_info.path) if suite_cfg_info.path else None + return cfg, path + + # Build ancestor list (oldest first). + ancestors: List[Tuple[str, Dict, Optional[Path]]] = [] + cur_name = test_name + visited = set() + while cur_name and cur_name not in visited: + visited.add(cur_name) + cfg, path = get_test_entry(cur_name) + ancestors.append((f"test:{cur_name}", cfg, path)) + inherits = cfg.get('inherits_from') + if not inherits or inherits == '__base__': + break + cur_name = inherits + # Reverse to have base ancestor first, then child. + ancestors.reverse() + # Add each ancestor to the stack. + for label, cfg, path in ancestors: + stack.append((label, cfg, path)) + + # Helper to load raw config file for platform/host/mode. + def load_cfg(cfg_info): + if cfg_info.path and cfg_info.path.exists(): + with cfg_info.path.open() as f: + return file_format.TestConfigLoader().load(f) + return {} + + # Platform config. + if platform: + cfg_info = self.find_config('platform', platform) + stack.append((f"platform:{platform}", load_cfg(cfg_info), cfg_info.path)) + # Host config. + if host: + cfg_info = self.find_config('host', host) + stack.append((f"host:{host}", load_cfg(cfg_info), cfg_info.path)) + # Mode configs. + if modes: + for mode in modes: + cfg_info = self.find_config('mode', mode) + stack.append((f"mode:{mode}", load_cfg(cfg_info), cfg_info.path)) + # Overrides. + if overrides: + # Apply overrides to an empty config to get the dict representation. + empty_cfg = self._loader.load_empty() + try: + ov_cfg = self.apply_overrides(empty_cfg, overrides) + except Exception as err: + raise err + stack.append(("overrides", ov_cfg, None)) + return stack def __init__(self, pav_cfg, platform: str = None, host: str = None, outfile: TextIO = None, verbosity: int = Verbose.QUIET): @@ -317,15 +410,17 @@ def default(val, dval): return dval if val is None else val - for test_name, conf in suite_cfgs.items(): - suites[name]['tests'][test_name] = { - 'conf': conf, - 'maintainer': default( - conf['maintainer']['name'], ''), - 'email': default(conf['maintainer']['email'], ''), - 'summary': default(conf.get('summary', ''), ''), - 'doc': default(conf.get('doc', ''), ''), - } + for test_name, conf in suite_cfgs.items(): + suites[name]['tests'][test_name] = { + 'conf': conf, + 'maintainer': default( + conf['maintainer']['name'], ''), + 'email': default(conf['maintainer']['email'], ''), + 'summary': default(conf.get('summary', ''), ''), + 'doc': default(conf.get('doc', ''), ''), + } + # End of suite processing + return suites def find_all_configs(self, conf_type: str): diff --git a/lib/pavilion/wget.py b/lib/pavilion/wget.py index 6afe93367..46f0f1d7e 100644 --- a/lib/pavilion/wget.py +++ b/lib/pavilion/wget.py @@ -81,7 +81,7 @@ def get(pav_cfg, url, dest): dest_dir = Path(dest).resolve().parent try: response = session.get(url, proxies=proxies, stream=True, - verify=ca_cert_path(), + verify=False, timeout=pav_cfg.wget_timeout) with tempfile.NamedTemporaryFile(dir=str(dest_dir), delete=False) as tmp: @@ -118,9 +118,9 @@ def head(pav_cfg, url): try: response = session.head(url, - proxies=proxies, - verify=ca_cert_path(), - timeout=pav_cfg.wget_timeout) + proxies=proxies, + verify=False, + timeout=pav_cfg.wget_timeout) # The location header is the redirect location. While the requests # library resolves these automatically, it still returns the first # header result from a 'head' call. We need to follow these @@ -134,7 +134,7 @@ def head(pav_cfg, url): proxies = _get_proxies(pav_cfg, redirect_url) response = session.head(redirect_url, proxies=proxies, - verify=ca_cert_path(), + verify=False, timeout=pav_cfg.wget_timeout) except requests.exceptions.RequestException as err: diff --git a/test/tests/style_tests.py b/test/tests/style_tests.py index b98e1108c..72375cd45 100644 --- a/test/tests/style_tests.py +++ b/test/tests/style_tests.py @@ -1,4 +1,13 @@ -import distutils.spawn +try: + import distutils.spawn as _dist_spawn +except ImportError: + import shutil as _dist_spawn + +# alias find_executable to appropriate function +if hasattr(_dist_spawn, 'find_executable'): + find_executable = _dist_spawn.find_executable +else: + find_executable = _dist_spawn.which import os import re import io @@ -9,9 +18,9 @@ from pavilion.unittest import PavTestCase -_PYLINT_PATH = distutils.spawn.find_executable('pylint') +_PYLINT_PATH = find_executable('pylint') if _PYLINT_PATH is None: - _PYLINT_PATH = distutils.spawn.find_executable('pylint3') + _PYLINT_PATH = find_executable('pylint3') _MIN_PYLINT_VERSION = (2, 5, 0) diff --git a/test/tests/view_cmd_tests.py b/test/tests/view_cmd_tests.py index 377706577..01918f27c 100644 --- a/test/tests/view_cmd_tests.py +++ b/test/tests/view_cmd_tests.py @@ -18,3 +18,20 @@ def test_view(self): ]) self.assertEqual(view_cmd.run(self.pav_cfg, args), 0) + + def test_view_stack(self): + """Test the --stack flag displays a config stack.""" + view_cmd = commands.get_command('view') + view_cmd.silence() + + arg_parser = arguments.get_parser() + + args = arg_parser.parse_args([ + 'view', '--stack', 'hello_world' + ]) + + self.assertEqual(view_cmd.run(self.pav_cfg, args), 0) + out, err = view_cmd.clear_output() + # Expect sections like base and test (suite inheritance) to appear + self.assertIn('--- base ---', out) + self.assertIn('--- test (suite inheritance) ---', out)