From 48a288d64ea4575f9a5e37b1a55a022d36260f17 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 5 May 2026 14:53:51 -0400 Subject: [PATCH 1/3] Add processing and comparison of ctests in JEDI builds --- src/swell/suites/build_jedi/flow.cylc | 13 ++ src/swell/suites/compare_jedi/flow.cylc | 42 ++++++ src/swell/suites/compare_jedi/suite_config.py | 33 +++++ src/swell/swell.py | 3 + src/swell/tasks/compare_jedi_ctests.py | 140 ++++++++++++++++++ src/swell/tasks/run_jedi_ctests.py | 55 +++++++ src/swell/tasks/task_questions.py | 19 +++ src/swell/test/test_driver.py | 3 + src/swell/utilities/question_defaults.py | 21 +++ src/swell/utilities/slurm.py | 1 + 10 files changed, 330 insertions(+) create mode 100644 src/swell/suites/compare_jedi/flow.cylc create mode 100644 src/swell/suites/compare_jedi/suite_config.py create mode 100644 src/swell/tasks/compare_jedi_ctests.py create mode 100644 src/swell/tasks/run_jedi_ctests.py diff --git a/src/swell/suites/build_jedi/flow.cylc b/src/swell/suites/build_jedi/flow.cylc index b7e347dee..068f4fdb1 100644 --- a/src/swell/suites/build_jedi/flow.cylc +++ b/src/swell/suites/build_jedi/flow.cylc @@ -22,6 +22,10 @@ CloneJedi => BuildJediByLinking? BuildJediByLinking:fail? => BuildJedi + + {% if bundles_to_run_ctests | length > 0 %} + BuildJedi => RunJediCtests + {% endif %} """ # -------------------------------------------------------------------------------------------------- @@ -53,4 +57,13 @@ --{{key}} = {{value}} {%- endfor %} + [[RunJediCtests]] + script = "swell task RunJediCtests $config" + platform = {{platform}} + execution time limit = {{scheduling["RunJediCtests"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediCtests"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_jedi/flow.cylc b/src/swell/suites/compare_jedi/flow.cylc new file mode 100644 index 000000000..0e4bade4a --- /dev/null +++ b/src/swell/suites/compare_jedi/flow.cylc @@ -0,0 +1,42 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for comparing two builds of JEDI directly + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + [[graph]] + R1 = """ + CompareJediCtests + """ + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CompareJediCtests]] + script = "swell task CompareJediCtests $config" + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_jedi/suite_config.py b/src/swell/suites/compare_jedi/suite_config.py new file mode 100644 index 000000000..a037744b5 --- /dev/null +++ b/src/swell/suites/compare_jedi/suite_config.py @@ -0,0 +1,33 @@ +# -------------------------------------------------------------------------------------------------- +# @package configuration +# +# Class containing the configuration. This is a dictionary that is converted from +# an input yaml configuration file. Various function are included for interacting with the +# dictionary. +# +# -------------------------------------------------------------------------------------------------- + + +from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.suites.suite_questions import SuiteQuestions as sq + +from enum import Enum + +from swell.utilities.question_defaults import QuestionDefaults as qd + +# -------------------------------------------------------------------------------------------------- + + +class SuiteConfig(QuestionContainer, Enum): + + # -------------------------------------------------------------------------------------------------- + + compare_jedi = QuestionList( + list_name="compare_jedi", + questions=[ + sq.all_suites, + qd.comparison_experiment_paths() + ] + ) + + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index 89f55a3ad..e8e7e7606 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -303,3 +303,6 @@ def main() -> None: # -------------------------------------------------------------------------------------------------- + +if __name__ == '__main__': + test('code_tests') diff --git a/src/swell/tasks/compare_jedi_ctests.py b/src/swell/tasks/compare_jedi_ctests.py new file mode 100644 index 000000000..17f8e08fd --- /dev/null +++ b/src/swell/tasks/compare_jedi_ctests.py @@ -0,0 +1,140 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +import re + +from swell.utilities.comparisons import comparison_tags +from swell.tasks.base.task_base import taskBase + +# -------------------------------------------------------------------------------------------------- + + +class CompareJediCtests(taskBase): + + def parse_results(self, results_file) -> list: + ''' + Read results from a file containing output from JEDI ctests, + and parse to a list of failed tests + + Parameters: + results_file: path to a file containing output from bundle ctests + (e.g. one generated by the RunJediCtests task) + + Returns: + failed_tests: list of tests failed for the build + ''' + + with open(results_file, 'r') as f: + lines = f.readlines() + + if len(lines) == 0: + raise Exception(f'File {results_file} does not contain results') + + failed_tests = [] + + for line in lines: + if re.search('- .* \(Failed\)', line): # noqa + failed_tests.append(line.split('-')[1].split('(Failed)')[0].strip()) + + failed_tests = list(set(failed_tests)) + + return failed_tests + + def execute(self) -> None: + + # Paths to experiments to compare + experiment_paths = self.config.comparison_experiment_paths() + + # Bundles to consider ctest results for + bundles = self.config.bundles_to_run_ctests() + + # Attach tags to paths, if not present + experiment_tag_paths = comparison_tags(experiment_paths, self.logger) + + experiment_tag_1 = list(experiment_tag_paths.keys())[0] + experiment_tag_2 = list(experiment_tag_paths.keys())[1] + + experiment_path_1 = list(experiment_tag_paths.values())[0] + experiment_path_2 = list(experiment_tag_paths.values())[1] + + # Paths to the ctest results rendered by the RunJediCtests task + ctest_path_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'ctests') + ctest_path_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'ctests') + + # Dict tracking all test results + results_dict = {} + + for bundle in bundles: + ctest_file_1 = os.path.join(ctest_path_1, f'ctest_results-{bundle}.txt') + ctest_file_2 = os.path.join(ctest_path_2, f'ctest_results-{bundle}.txt') + + # Parse for failed tests + failed_results_1 = self.parse_results(ctest_file_1) + failed_results_2 = self.parse_results(ctest_file_2) + + results_dict[bundle] = {} + + # Track which tests have failed for both builds + for test in failed_results_1: + if test not in results_dict[bundle].keys(): + results_dict[bundle][test] = {experiment_tag_2: 'Pass'} + results_dict[bundle][test][experiment_tag_1] = 'Fail' + results_dict[bundle][test]['width'] = len(test) + + for test in failed_results_2: + if test not in results_dict[bundle].keys(): + results_dict[bundle][test] = {experiment_tag_1: 'Pass'} + results_dict[bundle][test][experiment_tag_2] = 'Fail' + results_dict[bundle][test]['width'] = len(test) + + # Whether the same number of tests pass + passed = True + + # Format the string for readable output + results_str = 'JEDI CTest Results Comparison\n' + results_str += f'{experiment_tag_1}: {experiment_path_1}\n' + results_str += f'{experiment_tag_2}: {experiment_path_2}\n\n' + + width_col_1 = max(len('Fail'), len(experiment_tag_1)) + 2 + width_col_2 = max(len('Fail'), len(experiment_tag_2)) + 2 + + for bundle in bundles: + max_width = max(len(bundle), max([results_dict[bundle][test]['width'] + for test in results_dict[bundle]])) + 2 + results_str += bundle + ' ' * (max_width - len(bundle)) + results_str += experiment_tag_1 + ' ' * (width_col_1 - len(experiment_tag_1)) + results_str += experiment_tag_2 + ' ' * (width_col_2 - len(experiment_tag_2)) + results_str += '\n' + for test in results_dict[bundle].keys(): + results_str += test + ' ' * (max_width - len(test)) + result_1 = results_dict[bundle][test][experiment_tag_1] + result_2 = results_dict[bundle][test][experiment_tag_2] + if result_1 != result_2: + passed = False + results_str += result_1 + ' ' * (width_col_1 - len(result_1)) + results_str += result_2 + ' ' * (width_col_2 - len(result_2)) + results_str += '\n' + + results_str += '\n' + + self.logger.info(results_str) + + out_path = os.path.join(self.experiment_path(), 'ctests') + os.makedirs(out_path, exist_ok=True) + with open(os.path.join(out_path, 'ctest_comparison.txt'), 'w') as f: + f.write(results_str) + + if not passed: + # Send the result to job.err as well + self.logger.error(results_str) + raise Exception(f'Differing tests passed between experiments') + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_ctests.py b/src/swell/tasks/run_jedi_ctests.py new file mode 100644 index 000000000..87876c387 --- /dev/null +++ b/src/swell/tasks/run_jedi_ctests.py @@ -0,0 +1,55 @@ + +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import subprocess + +from swell.tasks.base.task_base import taskBase + +# -------------------------------------------------------------------------------------------------- + + +class RunJediCtests(taskBase): + + def execute(self) -> None: + + # Locate the experiment's jedi build dir, must be built or linked first + build_dir = os.path.join(self.experiment_path(), 'jedi_bundle', 'build') + + # Identify the bundles to run ctests on + bundles = self.config.bundles_to_run_ctests() + + for bundle in bundles: + bundle_dir = os.path.join(build_dir, bundle) + + # Run the ctests + cwd = os.getcwd() + os.chdir(bundle_dir) + command = ['ctest', '-V', '-I'] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # Record the output + results, error = process.communicate() + os.chdir(cwd) + + # Get the output file name + out_name = f'ctest_results-{bundle}.txt' + + # Make the output directory + out_path = os.path.join(self.experiment_path(), 'ctests') + os.makedirs(out_path, exist_ok=True) + + # Write the results + with open(os.path.join(out_path, out_name), 'w') as f: + f.write(f'CTest results for {bundle} bundle located at: {bundle_dir}\n') + f.write(results.decode('utf-8')) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index df9b815cd..60305f031 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -197,6 +197,16 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + CompareJediCtests = QuestionList( + list_name="CompareJediCtests", + questions=[ + qd.bundles_to_run_ctests(), + qd.comparison_experiment_paths() + ] + ) + + # -------------------------------------------------------------------------------------------------- + EvaComparisonJediLog = QuestionList( list_name="EvaJediLog", questions=[ @@ -598,6 +608,15 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + RunJediCtests = QuestionList( + list_name="RunJediCtests", + questions=[ + qd.bundles_to_run_ctests() + ] + ) + + # -------------------------------------------------------------------------------------------------- + RunJediEnsembleMeanVariance = QuestionList( list_name="RunJediEnsembleMeanVariance", questions=[ diff --git a/src/swell/test/test_driver.py b/src/swell/test/test_driver.py index 16060acaa..98d0eeef1 100644 --- a/src/swell/test/test_driver.py +++ b/src/swell/test/test_driver.py @@ -32,3 +32,6 @@ def test_wrapper(test: str) -> None: # -------------------------------------------------------------------------------------------------- + +if __name__ == '__main__': + test_wrapper('code_tests') diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 130c8ccc5..6fa59399c 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -357,6 +357,27 @@ class bundles(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class bundles_to_run_ctests(TaskQuestion): + default_value: list[str] = mutable_field([ + "fv3-jedi" + ]) + question_name: str = "bundles_to_run_ctests" + ask_question: bool = True + options: list[str] = mutable_field([ + "fv3-jedi", + "soca", + "iodaconv", + "ufo", + "ioda", + "oops", + "saber" + ]) + prompt: str = "Which JEDI bundles to you wish to run ctests on?" + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + @dataclass class check_for_obs(TaskQuestion): default_value: bool = True diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 787650d81..4ee855217 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -88,6 +88,7 @@ def prepare_scheduling_dict( 'RunJediObsfiltersExecutable', 'RunJediUfoTestsExecutable', 'RunJediVariationalExecutable', + 'RunJediCtests' } # Throw an error if a user tries to set SLURM directives for a task that From 091a87201f4e6d7a5bde2035544a7f6e2263bfa9 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 8 May 2026 14:11:18 -0400 Subject: [PATCH 2/3] Add to docs --- docs/examples/comparison_workflows.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/examples/comparison_workflows.md b/docs/examples/comparison_workflows.md index 0ec9b2a4e..cebffec54 100644 --- a/docs/examples/comparison_workflows.md +++ b/docs/examples/comparison_workflows.md @@ -21,3 +21,24 @@ comparison_experiment_paths: These experiments should have matching assimilation window parameters. By default in this suite, start and end cycle points are not specified, in which case Swell will parse the two experiments to find the matching cycle times between the two. Alternatively, start and end cycle points can be set manually. The experiment can then be created using `swell create compare_variational_marine -o override.yaml` or `swell create compare_variational_atmosphere -o override`, depending on the type of experiments being compared. Launching the experiment will run tasks analyzing the jedi log and generating plots using Eva for increments. Comparison of the log analysis will be placed under the comparison suite's directory in a file named `jedi_log_comparison.txt`, while the eva plots will be located under the cycle directory for each cycle. + +## Comparing JEDI builds + +This section describes how to run `ctests` on JEDI builds. The task `RunJediCtests` can be run in `build_jedi` experiments run by swell to output the results to a text file. `ctests` will be run for bundles specified in `bundles_to_run_ctests`. This field defaults to `fv3-jedi`, but any bundle with ctests can be run by this task. The output of the `ctest` execution is sent to `/ctests/ctest_results-.txt` + + +The `compare_jedi` suite checks the ctest results to ensure the two `build_jedi` experiments listed under `comparison_experiment_paths` pass the same ctests. The `bundles_to_run_ctests` key is also used by the `compare_jedi` suite to specify which bundles should be compared. The `compare_jedi` suite assumes that the task `RunJediCtests` has been run in the `build_jedi` experiments listed under `comparison_experiment_paths`. The task `CompareJediCtests` parses the output to figure out which tests fail for both experiments (the assumption is that some tests will always fail for most bundles, so the condition for zero-diff is ensuring the same tasks fail for both builds). If a mismatch in passed tests is detected, this task generates an error. The log output of this task lists the failed tasks for the two suites, and displays if any pass for one that is not passed for the other. For example, the output for comparing `fv3-jedi`: + +``` +fv3-jedi CTL EXP +fv3jedi_staticb_nicas_gfs Fail Fail +fv3jedi_hofx_nomodel_abi_radii Fail Fail +fv3jedi_staticb_split_nicas_gfs Fail Fail +fv3jedi_hyb Fail Fail +fv3jedi_staticb_dirac_local_gfs_12pe Fail Fail +fv3jedi_staticb_cor_geos Fail Fail +fv3jedi_staticb_dirac_local_gfs_6pe Fail Fail +fv3jedi_staticb_nicas_geos Fail Fail +fv3jedi_staticb_dirac_global_gfs_6pe Fail Fail +fv3jedi_staticb_dirac_global_gfs_12pe Fail Fail +``` From f84ca6914f88e607df6ee287dc36f583fa561d7b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 8 May 2026 16:11:51 -0400 Subject: [PATCH 3/3] address comments --- src/swell/tasks/compare_jedi_ctests.py | 5 +++++ src/swell/tasks/run_jedi_ctests.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/compare_jedi_ctests.py b/src/swell/tasks/compare_jedi_ctests.py index 17f8e08fd..6c9c271ec 100644 --- a/src/swell/tasks/compare_jedi_ctests.py +++ b/src/swell/tasks/compare_jedi_ctests.py @@ -112,6 +112,11 @@ def execute(self) -> None: results_str += experiment_tag_1 + ' ' * (width_col_1 - len(experiment_tag_1)) results_str += experiment_tag_2 + ' ' * (width_col_2 - len(experiment_tag_2)) results_str += '\n' + + # Specify if all tests have passed + if len(results_dict[bundle].keys()) == 0: + results_str += 'All tests passed.\n' + for test in results_dict[bundle].keys(): results_str += test + ' ' * (max_width - len(test)) result_1 = results_dict[bundle][test][experiment_tag_1] diff --git a/src/swell/tasks/run_jedi_ctests.py b/src/swell/tasks/run_jedi_ctests.py index 87876c387..9bd5c6462 100644 --- a/src/swell/tasks/run_jedi_ctests.py +++ b/src/swell/tasks/run_jedi_ctests.py @@ -33,7 +33,7 @@ def execute(self) -> None: # Run the ctests cwd = os.getcwd() os.chdir(bundle_dir) - command = ['ctest', '-V', '-I'] + command = ['ctest', '-V'] process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Record the output