From 933b1422478c535c4fc84bcc05273f1d63bbc334 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 1 May 2025 18:36:17 -0400 Subject: [PATCH 001/299] Initial commit --- src/swell/deployment/create_experiment.py | 74 +-- .../prepare_config_and_suite.py | 529 ++++++------------ src/swell/suites/3dvar/workflow.py | 95 ++++ src/swell/suites/all_suites.py | 59 +- src/swell/swell.py | 4 +- src/swell/tasks/task_runtimes.py | 81 +++ src/swell/utilities/cylc_formatting.py | 70 +++ src/swell/utilities/cylc_runtime.py | 164 ++++++ src/swell/utilities/cylc_workflow.py | 198 +++++++ src/swell/utilities/dictionary.py | 18 +- src/swell/utilities/slurm.py | 191 ++----- src/swell/utilities/swell_questions.py | 11 +- 12 files changed, 926 insertions(+), 568 deletions(-) create mode 100644 src/swell/suites/3dvar/workflow.py create mode 100644 src/swell/tasks/task_runtimes.py create mode 100644 src/swell/utilities/cylc_formatting.py create mode 100644 src/swell/utilities/cylc_runtime.py create mode 100644 src/swell/utilities/cylc_workflow.py diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 8d5d4baf4..4a3f5d575 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -16,14 +16,15 @@ import yaml from typing import Union, Optional -from swell.suites.all_suites import AllSuites +from swell.suites.all_suites import SuiteConfigs from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import \ PrepareExperimentConfigAndSuite from swell.swell_path import get_swell_path from swell.utilities.dictionary import add_comments_to_dictionary, dict_get from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.logger import Logger, get_logger -from swell.utilities.slurm import prepare_scheduling_dict +from swell.utilities.slurm import prepare_slurm_defaults_and_overrides +from swell.suites.all_suites import SuiteConfigs, Workflows # -------------------------------------------------------------------------------------------------- @@ -91,50 +92,27 @@ def prepare_config( # --------------------------------------------------------------- prepare_config_and_suite = PrepareExperimentConfigAndSuite(logger, suite, suite_config, platform, method, override) + + suite_dict = prepare_config_and_suite.get_experiment_dict() - # Ask questions as the suite gets configured - # ------------------------------------------ - experiment_dict, comment_dict = prepare_config_and_suite.ask_questions_and_configure_suite() - # Add the datetime to the dictionary - # ---------------------------------- - experiment_dict['datetime_created'] = datetime.datetime.today().strftime("%Y%m%d_%H%M%SZ") - comment_dict['datetime_created'] = 'Datetime this file was created (auto added)' + slurm_dict = prepare_slurm_defaults_and_overrides(logger, slurm, platform) + + workflow = Workflows[suite](suite_dict, slurm_dict) + + model_ind_tasks, model_dep_tasks = workflow.get_independent_and_model_tasks() + + prepare_config_and_suite.set_model_ind_tasks(model_ind_tasks) + prepare_config_and_suite.set_model_dep_tasks(model_dep_tasks) - # Add the platform the dictionary - # ------------------------------- - experiment_dict['platform'] = platform - comment_dict['platform'] = 'Computing platform to run the experiment' + experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() - # Add the suite_to_run to the dictionary - # -------------------------------------- - experiment_dict['suite_to_run'] = suite - comment_dict['suite_to_run'] = 'Record of the suite being executed' + workflow.set_experiment_dict(experiment_dict) - # Add the model components to the dictionary + workflow_string = workflow.get_workflow_str() + + # Ask questions as the suite gets configured # ------------------------------------------ - if 'models' in experiment_dict: - experiment_dict['model_components'] = list(experiment_dict['models'].keys()) - comment_dict['model_components'] = 'List of models in this experiment' - - # Expand experiment dict with SLURM overrides. - # NOTE: This is a bit of a hack. We should really either commit to using a - # separate file and pass it around everywhere, or commit fully to keeping - # everything in `experiment.yaml` and support it through the Questionary - # infrastructure. - # ---------------------------------- - if slurm is not None: - logger.info(f"Reading SLURM directives from {slurm}.") - assert os.path.exists(slurm) - with open(slurm, "r") as slurmfile: - slurm_dict = yaml.safe_load(slurmfile) - # Ensure that SLURM dict is _only_ used for SLURM directives. - slurm_invalid_keys = set(slurm_dict.keys()).difference({ - "slurm_directives_global", - "slurm_directives_tasks" - }) - if slurm_invalid_keys: - logger.abort(f'SLURM file contains invalid keys: {slurm_invalid_keys}') - experiment_dict = {**experiment_dict, **slurm_dict} + experiment_dict, comment_dict = prepare_config_and_suite.ask_questions_and_configure_suite() # Expand all environment vars in the dictionary # --------------------------------------------- @@ -152,7 +130,7 @@ def prepare_config( # Return path to dictionary file # ------------------------------ - return experiment_dict_string_comments + return experiment_dict_string_comments, workflow_string # -------------------------------------------------------------------------------------------------- @@ -169,7 +147,7 @@ def create_experiment_directory( # Get the base name of the suite # ------------------------------ - suite = AllSuites.base_suite(suite_config) + suite = SuiteConfigs.base_suite(suite_config) # Create a logger # --------------- @@ -177,7 +155,7 @@ def create_experiment_directory( # Call the experiment config and suite generation # ------------------------------------------------ - experiment_dict_str = prepare_config(suite, suite_config, method, platform, + experiment_dict_str, workflow_str = prepare_config(suite, suite_config, method, platform, override, advanced, slurm) # Load the string using yaml @@ -205,12 +183,8 @@ def create_experiment_directory( with open(os.path.join(exp_suite_path, 'experiment.yaml'), 'w') as file: file.write(experiment_dict_str) - # At this point we need to write the complete suite file with all templates resolved. Call the - # function to build the scheduling dictionary, combine with the experiment dictionary, - # resolve the templates and write the suite file to the experiment suite directory. - # -------------------------------------------------------------------------------------------- - swell_suite_path = os.path.join(get_swell_path(), 'suites', suite) - prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experiment_dict, platform) + with open(os.path.join(exp_suite_path, 'flow.cylc'), 'w') as file: + file.write(workflow_str) # Copy suite and platform files to experiment suite directory # ----------------------------------------------------------- diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 3e490984c..6774b7b3c 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -13,16 +13,20 @@ import yaml from collections.abc import Mapping from typing import Union, Tuple, Optional +from enum import StrEnum +import datetime from swell.swell_path import get_swell_path +from swell.utilities.suite_utils import get_model_components from swell.deployment.prepare_config_and_suite.question_and_answer_cli import GetAnswerCli from swell.deployment.prepare_config_and_suite.question_and_answer_defaults import GetAnswerDefaults from swell.utilities.dictionary import dict_get from swell.utilities.logger import Logger from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.dictionary import update_dict +from swell.utilities.dictionary import update_dict, add_dict from swell.tasks.task_questions import TaskQuestions as task_questions -from swell.suites.all_suites import AllSuites +from swell.suites.all_suites import SuiteConfigs, Workflows +from swell.utilities.swell_questions import QuestionType # -------------------------------------------------------------------------------------------------- @@ -78,213 +82,153 @@ def __init__( # Big dictionary that contains all user responses as well a dictionary containing the # questions that were asked self.experiment_dict = {} - self.questions_dict = {} + self.comment_dict = {} + + # Add the datetime to the dictionary + # ---------------------------------- + self.experiment_dict['datetime_created'] = datetime.datetime.today().strftime("%Y%m%d_%H%M%SZ") + self.comment_dict['datetime_created'] = 'Datetime this file was created (auto added)' + + # Add the platform the dictionary + # ------------------------------- + self.experiment_dict['platform'] = platform + self.comment_dict['platform'] = 'Computing platform to run the experiment' + + # Add the suite_to_run to the dictionary + # -------------------------------------- + self.experiment_dict['suite_to_run'] = suite + self.comment_dict['suite_to_run'] = 'Record of the suite being executed' # Get list of all possible models - self.possible_model_components = os.listdir(os.path.join(get_swell_path(), 'configuration', - 'jedi', 'interfaces')) + self.possible_model_components = get_model_components() - # Read suite file into a string - suite_file = os.path.join(get_swell_path(), 'suites', self.suite, 'flow.cylc') - with open(suite_file, 'r') as suite_file_open: - self.suite_str = suite_file_open.read() + self.prepare_suite_question_dictionary() + self.override_with_defaults(QuestionType.SUITE) + self.override_with_external(QuestionType.SUITE) + self.ask_questions_and_configure(QuestionType.SUITE) - # Get a list of model-independent and dependent questions - self.model_ind_tasks = self.get_suite_task_list_model_ind(self.suite_str) - self.all_model_dep_tasks = self.get_all_model_dep_tasks(self.suite_str) + # ---------------------------------------------------------------------------------------------- - # Perform the assembly of the dictionaries that contain all the questions that can possibly - # be asked. This + def configure_and_ask_task_questions(self) -> None: + self.prepare_task_question_dictionary() + self.override_with_defaults(QuestionType.TASK) + self.override_with_external(QuestionType.TASK) + self.ask_questions_and_configure(QuestionType.TASK) - self.prepare_question_dictionaries() - self.override_with_defaults() - self.override_with_external() + return self.experiment_dict, self.comment_dict # ---------------------------------------------------------------------------------------------- - def prepare_question_dictionaries(self) -> None: + def set_model_ind_tasks(self, tasks: list) -> None: + self.model_ind_tasks = tasks - # Create a dictionary of all suite questions - question_dictionary = {} + # ---------------------------------------------------------------------------------------------- - # Create a dictionary of all task questions - question_dictionary_tasks = {} + def set_model_dep_tasks(self, model_task_dict: Mapping) -> None: + self.model_dep_tasks = model_task_dict - # Create a dictionary associating each task with its list of questions - self.questions_per_task = {} + # ---------------------------------------------------------------------------------------------- - # Create an override dictionary for model-dependent questions - # This will later be used to set defaults - model_dep_questions_override = {} + def get_experiment_dict(self) -> Mapping: + return self.experiment_dict + + # ---------------------------------------------------------------------------------------------- - # Get a list of all questions associated with the suite, except for those specified - # seperately for models + def get_workflow_file_str(self, slurm_dict): + self.workflow.set_experiment_dict(self.experiment_dict, slurm_dict) + self.workflow.set_runtime_str() - suite_config_obj = AllSuites.get_config(self.suite_config) - suite_question_list = suite_config_obj.expand_question_list() + self.workflow_str = self.workflow.get_workflow_str() - # Allow for adding extra tasks manually from configuration - # For dynamic suite creation (e.g. comparison tests) + return self.workflow_str + + # ---------------------------------------------------------------------------------------------- - dynamic_tasks = self.get_dynamic_tasks(suite_question_list) - # Loop through all tasks and get their associated tasks - for task in self.model_ind_tasks + self.all_model_dep_tasks + dynamic_tasks: + def prepare_task_question_dictionary(self): + for task in self.model_independent_tasks: if task in task_questions.get_all(): - question_list = task_questions[task].value.expand_question_list() + if self.suite_needs_model_components: + for model in self.experiment_dict['model_components'].keys(): + model_dict = {model: {}} + for question in task_questions[task].expand_question_list(model): + model_dict[model][question['question_name']] = question + + self.question_dictionary_model_dep = add_dict(self.question_dictionary_model_dep, model_dict) + + for question in task_questions[task].expand_question_list(): + if question['models'] is not None: + for model in self.experiment_dict['model_components'].keys(): + model_dict = {model: {}} + for question in task_questions[task].expand_question_list(model): + if model in question['models'] or 'all_models' in question['models']: + model_dict[model][question['question_name']] = question + + self.question_dictionary_model_dep = add_dict(self.question_dictionary_model_dep, model_dict) + + else: + question_dict = {} + for question in task_questions[task].expand_question_list(): + if question['models'] is not None: + self.logger.abort('The model components question has not been answered.') + else: + question_dict[question['question_name']] = question - for question in question_list: - question_dictionary_tasks[question['question_name']] = question + self.question_dictionary_model_ind = add_dict(self.question_dictionary_model_ind, question_dict) - for model in self.possible_model_components: - for question in task_questions[task].value.expand_question_list(model): - model_dep_questions_override[model][question['question_name']] = question + def prepare_suite_question_dictionary(self) -> None: - self.questions_per_task[task] = [question['question_name'] - for question in question_list] - else: - self.questions_per_task[task] = [] + question_dictionary_model_ind = {} + question_dictionary_model_dep = {} - # Convert the list of questions into a dictionary indexed by the question name - for question in suite_question_list: - question_dictionary[question['question_name']] = question + suite_config_obj = SuiteConfigs.get_config(self.suite_config) + suite_question_list = suite_config_obj.expand_question_list() - # Update model dependent overrides with suite questions for model in self.possible_model_components: - model_dep_questions_override[model] = {} - for question in suite_config_obj.expand_question_list(model): - model_dep_questions_override[model][question['question_name']] = question - - # Merge the dictionaries for task questions into the suite question - # list, but keep suite questions at the top of the order - # Override priority is given to questions defined in the suite_question_list + question_dictionary_model_dep[model] = {} - # Iterate through the task questions - # ---------------------------------- - for key, value in question_dictionary_tasks.items(): + for question in suite_config_obj.expand_question_list(model): + question_dictionary_model_dep[model][question['question_name']] = question - # If a question has no counterpart specified in suite questions, merge it - # ----------------------------------------------------------------------- - if key not in question_dictionary: - question_dictionary[key] = value + for question in suite_question_list: + if question['models'] is None: + question_dictionary_model_ind[question['question_name']] = question else: + if 'all_models' in question['models']: + question_models = self.possible_model_components + else: + question_models = question['models'] + + for model in question_models: + question_dictionary_model_dep = add_dict(question_dictionary_model_dep, {model: {question['question_name']: question}}) - # Otherwise, we need to check the suite question to - # see if there are any model-dependent fields which are not specified - # ---------------------------------------------------------------------------------------------------------------------- - for sub_key, sub_val in question_dictionary[key].items(): - if isinstance(sub_val, Mapping) and 'depends_on_model' in sub_val.keys(): - for model in self.possible_model_components: - if model not in sub_val[ - 'depends_on_model'].keys() and sub_key in value.keys(): - - # If the value is a model-dependent specification, - # grab the value associated with each model, if present - # ------------------------------------------------------------------------------------------------------ - if isinstance(value[sub_key], Mapping) and ( - 'depends_on_model' in value[sub_key].keys()): - if model in value[sub_key]['depends_on_model'].keys(): - question_dictionary[key][sub_key][ - 'depends_on_model'][model] = \ - value[sub_key]['depends_on_model'][model] - else: - question_dictionary[key][sub_key][ - 'depends_on_model'][model] = 'defer_to_model' - - # If the value is not a model-dependent dictionary, - # set the missing model to its value - # ----------------------------------------------------------------------------------- - else: - question_dictionary[key][sub_key][ - 'depends_on_model'][model] = value[sub_key] - - # At this point we can check to see if this is a suite that requires model components self.suite_needs_model_components = True - if 'model_components' not in question_dictionary.keys(): + if 'model_components' not in question_dictionary_model_ind.keys(): self.suite_needs_model_components = False - # Create copy of the question_dictionary for model independent questions - question_dictionary_model_ind = copy.deepcopy(question_dictionary) - - # Iterate through the model_ind dictionary and remove questions associated with models - # and questions not required by the suite - keys_to_remove = [] - for key, val in question_dictionary_model_ind.items(): - if dict_get(self.logger, val, 'models', None) is not None: - keys_to_remove.append(key) - - # Cycle times can be a special case that is needed even when models are not. Though if they - # are then the cycle times are needed for each model component. So we need to check if the - # suite needs cycle_times - - # If there are no models and the cycle_times is in the keys to remove then remove it - if not self.suite_needs_model_components and 'cycle_times' in keys_to_remove: - keys_to_remove.remove('cycle_times') - - # Now remove the keys - for key in keys_to_remove: - del question_dictionary_model_ind[key] - self.question_dictionary_model_ind = copy.deepcopy(question_dictionary_model_ind) - - # If there are no models and the cycle_times is in the keys then remove the models key from - # the cycle_times question dictionary - if 'cycle_times' in self.question_dictionary_model_ind.keys(): - if not self.suite_needs_model_components: - self.question_dictionary_model_ind['cycle_times'].pop('models') - self.question_dictionary_model_ind['cycle_times']['default_value'] = 'T00' - - # At this point we can return if there are no model components - if not self.suite_needs_model_components: - return - - # Create copy of the question_dictionary for model dependent questions - question_dictionary_model_dep = copy.deepcopy(question_dictionary) - - # Iterate through the model_dep dictionary and remove questions not associated with models - # and questions not required by the suite - keys_to_remove = [] - for key, val in question_dictionary_model_dep.items(): - if dict_get(self.logger, val, 'models', None) is None: - keys_to_remove.append(key) - for key in keys_to_remove: - del question_dictionary_model_dep[key] - - # Create new questions dictionary for each model component - self.question_dictionary_model_dep = {} - for model in self.possible_model_components: - self.question_dictionary_model_dep[model] = update_dict( - copy.deepcopy(question_dictionary_model_dep), - model_dep_questions_override[model]) - - # Remove any questions that are not associated with the model component - for model in self.possible_model_components: - keys_to_remove = [] - for key, val in self.question_dictionary_model_dep[model].items(): - if val['models'] != ['all_models'] and model not in val['models']: - keys_to_remove.append(key) # Remove if not needed by this model + self.question_dictionary_model_ind = question_dictionary_model_ind + self.question_dictionary_model_dep = question_dictionary_model_dep - for key in keys_to_remove: - del self.question_dictionary_model_dep[model][key] - - # ---------------------------------------------------------------------------------------------- - - def override_with_defaults(self) -> None: + def override_with_defaults(self, suite_task: QuestionType) -> None: # Perform a platform override on the model_ind dictionary # ------------------------------------------------------- platform_defaults = {} - for suite_task in ['suite', 'task']: - platform_dict_file = os.path.join(get_swell_path(), 'deployment', 'platforms', - self.platform, f'{suite_task}_questions.yaml') - with open(platform_dict_file, 'r') as ymlfile: - platform_defaults.update(yaml.safe_load(ymlfile)) + + platform_dict_file = os.path.join(get_swell_path(), 'deployment', 'platforms', + self.platform, f'{suite_task.value}_questions.yaml') + with open(platform_dict_file, 'r') as ymlfile: + platform_defaults.update(yaml.safe_load(ymlfile)) # Loop over the keys in self.question_dictionary_model_ind and update with platform_defaults # if that dictionary shares the key - for key, val in self.question_dictionary_model_ind.items(): - if key in platform_defaults.keys(): - self.question_dictionary_model_ind[key].update(platform_defaults[key]) - + for question_name, question in self.question_dictionary_model_ind.items(): + if question['question_type'] == suite_task: + if question_name in platform_defaults.keys(): + for key, val in platform_defaults[question_name].items(): + if key not in question.keys() or question[key] == 'defer_to_platform': + question[key] = val + # Perform a model override on the model_dep dictionary # ---------------------------------------------------- if self.suite_needs_model_components: @@ -292,51 +236,52 @@ def override_with_defaults(self) -> None: # Open the suite and task default dictionaries model_defaults = {} - for suite_task in ['suite', 'task']: - model_dict_file = os.path.join(get_swell_path(), 'configuration', 'jedi', - 'interfaces', model, - f'{suite_task}_questions.yaml') - with open(model_dict_file, 'r') as ymlfile: - model_defaults.update(yaml.safe_load(ymlfile)) + + model_dict_file = os.path.join(get_swell_path(), 'configuration', 'jedi', + 'interfaces', model, f'{suite_task.value}_questions.yaml') + with open(model_dict_file, 'r') as ymlfile: + model_defaults.update(yaml.safe_load(ymlfile)) # Loop over the keys in self.question_dictionary_model_ind and update with # model_defaults or platform_defaults if that dictionary shares the key for question_name, question in model_dict.items(): - if question_name in model_defaults.keys(): - for key, val in question.items(): - # If the value of the question is still set as model-dependent, - # set the value for that model - if isinstance(val, Mapping) and \ + if question['question_type'] == suite_task: + if question_name in model_defaults.keys(): + for key, val in question.items(): + # If the value of the question is still set as model-dependent, + # set the value for that model + if isinstance(val, Mapping) and \ 'depends_on_model' in val.keys() and \ model in val['depends_on_model'].keys() and \ val['depends_on_model'][model] != 'defer_to_model': - model_dict[question_name][key] = val['depends_on_model'][model] - elif key in model_defaults[question_name].keys() and ( - val == 'defer_to_model' or val is None): - model_dict[question_name][key] = model_defaults[question_name][key] + model_dict[question_name][key] = val['depends_on_model'][model] + elif key in model_defaults[question_name].keys() and ( + val == 'defer_to_model' or val is None): + model_dict[question_name][key] = model_defaults[question_name][key] - if question_name in platform_defaults.keys(): - for key, val in question.items(): - if val == 'defer_to_platform': - model_dict[question_name][key] = platform_defaults[ + if question_name in platform_defaults.keys(): + for key, val in platform_defaults[question_name].items(): + if val == 'defer_to_platform': + model_dict[question_name][key] = platform_defaults[ question_name][key] # Look for defer_to_code in the model_ind dictionary # -------------------------------------------------- - for key, val in self.question_dictionary_model_ind.items(): - if key == 'model_components': - if val['default_value'] == 'defer_to_code': - val['default_value'] = self.possible_model_components - if val['options'] == 'defer_to_code': - val['options'] = self.possible_model_components + for question_name, question in self.question_dictionary_model_ind.items(): + if question['question_type'] == suite_task: + if question_name == 'model_components': + if question['default_value'] == 'defer_to_code': + question['default_value'] = self.possible_model_components + if question['options'] == 'defer_to_code': + question['options'] = self.possible_model_components - if key == 'experiment_id' and val['default_value'] == 'defer_to_code': - val['default_value'] = f'swell-{self.suite}' + if question_name == 'experiment_id' and question['default_value'] == 'defer_to_code': + question['default_value'] = f'swell-{self.suite}' # ---------------------------------------------------------------------------------------------- - def override_with_external(self) -> None: + def override_with_external(self, suite_task: QuestionType) -> None: # Append with any user provide overrides if self.override is not None: @@ -359,171 +304,57 @@ def override_with_external(self) -> None: # Iterate over the model_ind dictionary and override # -------------------------------------------------- - for key, val in self.question_dictionary_model_ind.items(): - if key in override_dict: - val['default_value'] = override_dict[key] + for question_name, question in self.question_dictionary_model_ind.items(): + if question['question_type'] == suite_task: + if question_name in override_dict: + question['default_value'] = override_dict[question_name] # Iterate over the model_dep dictionary and override # -------------------------------------------------- - if self.suite_needs_model_components and 'models' in override_dict.keys(): + if self.suite_needs_model_components and override_dict['models'] is not None: for model, model_dict in self.question_dictionary_model_dep.items(): - for key, val in model_dict.items(): - if model in override_dict['models']: - if key in override_dict['models'][model]: - val['default_value'] = override_dict['models'][model][key] + for question_name, question in model_dict.items(): + if question['question_type'] == suite_task: + if model in override_dict['models']: + if question_name in override_dict['models'][model]: + question['default_value'] = override_dict['models'][model][question_name] # ---------------------------------------------------------------------------------------------- - def ask_questions_and_configure_suite(self) -> Tuple[dict, dict]: - - """ - This is where we ask all the questions and as we go configure the suite file. The process - is rather complex and proceeds as described below. The order is determined by what makes - sense to a user that is going through answering questions. For example we want them to be - able to answer all the questions associated with a certain model together. While there is - work going on behind the scenes to configure the suite file the user should not see a break - in the questioning or a back and forth that causes confusion. - - 1. Ask the model independent suite questions. - - 2. Perform a non-exhaustive resolving of suite file templates. Non-exhaustive because at - this point we have not asked the model dependent suite questions so there may be more - templates to resolve. - - 3. Get a list of tasks that do not depend on the model component. + def get_questions_of_type(self, suite_task: QuestionType, question_dictionary: Mapping) -> Mapping: + out_dict = {} + + if 'models' in question_dictionary.keys(): + for model in self.possible_model_components: + if model in question_dictionary.keys(): + out_dict[model] = self.get_questions_of_type(suite_task, question_dictionary[model]) + + else: + for question_name, question in question_dictionary.items(): + if question['question_type'] == suite_task: + out_dict[question['question_name']] = question + + return out_dict - 4. Ask the model independent task questions. - - 5. Check that the suite in question has model_components - - 6. Ask the model dependent suite questions. - - 7. Perform an exhaustive resolving of suite file templates. Now it is exhaustive because at - this point we should have all the required information to resolve all the templates. - - 8. Ask the new task questions that do not actually depend on the model.. - - 9.1 Build a list of tasks for each model component. + # ---------------------------------------------------------------------------------------------- - 9.2 Iterate over the model_dep dictionary and ask task questions. - """ + def ask_questions_and_configure(self, suite_task: QuestionType) -> Tuple[dict, dict]: - # If the client is CLI put out some information about what is due to happen next - if self.config_client.__class__.__name__ == 'GetAnswerCli': + if self.config_client.__class__.__name__ == 'GetAnswerCli' and suite_task == QuestionType.SUITE: self.logger.info("Please answer the following questions to configure your experiment ") - # 1. Iterate over the model_ind dictionary and ask questions - # ---------------------------------------------------------- - for question_key in self.question_dictionary_model_ind: - - # Ask only the suite questions first - # ---------------------------------- - if self.question_dictionary_model_ind[question_key]['question_type'] == 'suite': - - # Ask the question - self.ask_a_question(self.question_dictionary_model_ind, question_key) - - # 2. Perform a non-exhaustive resolving of suite file templates - # ------------------------------------------------------------- - suite_str = template_string_jinja2(self.logger, self.suite_str, self.experiment_dict, True) - - # 3. Get a list of tasks that do not depend on the model component - # ---------------------------------------------------------------- - model_ind_tasks = self.get_suite_task_list_model_ind(suite_str) - - # 4.1 Iterate over the model_ind dictionary and ask task questions - # ---------------------------------------------------------------- - for question_key in self.question_dictionary_model_ind: - - # Ask the task questions - # ---------------------- - if self.question_dictionary_model_ind[question_key]['question_type'] != 'suite': - - # Check if the question is associated with any model independent tasks - if any(question_key in self.questions_per_task[task] for task in model_ind_tasks - if task in self.questions_per_task.keys()): - - # Ask the question - self.ask_a_question(self.question_dictionary_model_ind, question_key) - - # 5. Check that the suite in question has model_components - # -------------------------------------------------------- - if not self.suite_needs_model_components: - return self.experiment_dict, self.questions_dict - - # 6. Iterate over the model_dep dictionary and ask suite questions - # ---------------------------------------------------------------- - - # At this point the user should have provided the model components answer. Check that it is - # in the experiment dictionary and retrieve the response + for question_name, question in self.get_questions_of_type(suite_task, self.question_dictionary_model_ind).items(): + self.ask_a_question(self.question_dictionary_model_ind, question_name) - if 'model_components' not in self.experiment_dict: - self.logger.abort('The model components question has not been answered.') - - for model in self.experiment_dict['model_components']: - - model_dict = self.question_dictionary_model_dep[model] - - # Loop over keys of each model - for question_key in model_dict: - - # Ask only the suite questions first - if model_dict[question_key]['question_type'] == 'suite': - - # Ask the question - self.ask_a_question(model_dict, question_key, model) - - # 7. Perform a more exhaustive resolving of suite file templates - # -------------------------------------------------------------- - # Note that we reset the suite file to avoid templates having been left unresolved - # (removed) from the previous attempt. We still do not ask for an exhaustive resolving - # of templates because there are things related to scheduling that are not yet able to be - # resolved. In the future it might be good to bring some of that information into the - # sphere of suite questions but that requires some careful thought so as not to overload - # the user with questions. - suite_str = template_string_jinja2(self.logger, self.suite_str, self.experiment_dict, - True) - - # 8. Ask the new task questions that do not actually depend on the model - # ----------------------------------------------------------------------- - for question_key in self.question_dictionary_model_ind: - - if self.question_dictionary_model_ind[question_key]['question_type'] == 'task': - - # Check whether the question is associated with any model dependent tasks - if any(question_key in self.questions_per_task[task] - for task in self.all_model_dep_tasks): - - # Ask the question - self.ask_a_question(self.question_dictionary_model_ind, question_key) - - # 9.1 Build a list of tasks for each model component - # ------------------------------------------------- - model_dep_tasks = self.get_suite_task_list_model_dep(suite_str) - - # 9.2 Iterate over the model_dep dictionary and ask task questions - # ---------------------------------------------------------------- - for model in self.experiment_dict['model_components']: - - # Iterate over the model_dep dictionary and ask questions - # ------------------------------------------------------- - for question_key in self.question_dictionary_model_dep[model]: - - # Ask only the task questions first - # ---------------------------------- - if self.question_dictionary_model_dep[model][ - question_key]['question_type'] == 'task': - - # Check whether any of tasks_dep_per_model are in question_tasks - if any(question_key in self.questions_per_task[task] - for task in model_dep_tasks[model]): + if self.suite_needs_model_components: + if 'model_components' not in self.experiment_dict: + self.logger.abort('The model components question has not been answered.') - # Ask the question - self.ask_a_question(self.question_dictionary_model_dep[model], question_key, - model) + for model in self.experiment_dict['model_components']: + model_dict = self.question_dictionary_model_dep[model] - # Return the main experiment dictionary - return self.experiment_dict, self.questions_dict + for question_name, question in self.get_questions_of_type(suite_task, model_dict).items(): + self.ask_a_question(model_dict, question_name, model) # ---------------------------------------------------------------------------------------------- def ask_a_question( @@ -542,10 +373,10 @@ def ask_a_question( if model is not None: if 'models' not in self.experiment_dict: self.experiment_dict['models'] = {} - self.questions_dict['models'] = f"Configurations for the model components." + self.comment_dict['models'] = f"Configurations for the model components." if model not in self.experiment_dict['models']: self.experiment_dict['models'][model] = {} - self.questions_dict[f'models.{model}'] = \ + self.comment_dict[f'models.{model}'] = \ f"Configuration for the {model} model component." # Check the dependency chain for the question diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py new file mode 100644 index 000000000..72888b08d --- /dev/null +++ b/src/swell/suites/3dvar/workflow.py @@ -0,0 +1,95 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.cylc_workflow import CylcWorkflow + +# -------------------------------------------------------------------------------------------------- + +class Workflow_3dvar(CylcWorkflow): + def define_description(self): + description = """ + # Cylc suite for executing JEDI-based non-cycling variational data assimilation + """ + + return description + + def define_graph_section(self): + r1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + """ + + for model_component in self.experiment_dict['model_components']: + r1 += """ + + # Stage JEDI static files + CloneJedi => StageJedi-{model_component} + """.format(model_component=model_component) + + graph_str += self.format_cycle('R1', r1) + + for model_component in self.experiment_dict['model_components']: + if 'cycle_times' in self.experiment_dict[model_component]: + for cycle_time in self.experiment_dict[model_component]['cycle_times']: + cycle_str = """ + # Task triggers for: {model_component} + # ------------------ + # Get background + GetBackground-{model_component} + + # Get observations + GetObservations-{model_component} + + # GenerateBClimatology, for ocean it is cycle dependent + GenerateBClimatologyByLinking-{model_component} :fail? => GenerateBClimatology-{model_component} + GetBackground-{model_component} => GenerateBClimatology-{model_component} + + # Perform staging that is cycle dependent + StageJediCycle-{model_component} + + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} + StageJedi-{model_component}[^] => RunJediVariationalExecutable-{model_component} + StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} + GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} + GenerateBClimatologyByLinking-{model_component}? | GenerateBClimatology-{model_component} => RunJediVariationalExecutable-{model_component} + GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} + + # EvaObservations + RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} + + # EvaJediLog + RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} + + # EvaIncrement + RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} + + # Save observations + RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} + + # Clean up large files + EvaObservations-{model_component} & SaveObsDiags-{model_component} => + CleanCycle-{model_component} + """ + + cycle_str = cycle_str.format(model_component=model_component) + + graph_str += self.format_cycle(cycle_time, cycle_str) + + return self.create_new_section('graph', graph_str) + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/suites/all_suites.py b/src/swell/suites/all_suites.py index a5bd9983b..277c95f30 100644 --- a/src/swell/suites/all_suites.py +++ b/src/swell/suites/all_suites.py @@ -14,6 +14,7 @@ from swell.swell_path import get_swell_path from swell.utilities.suite_utils import get_suites +from swell.suites.suite_questions import SuiteQuestions # -------------------------------------------------------------------------------------------------- @@ -36,12 +37,13 @@ def base_suite(cls, config: str) -> str: # -------------------------------------------------------------------------------------------------- +def format_suite_name(suite_name): + return suite_name[1:] if suite_name[0] == '_' else suite_name -def construct_suite_enum(): - # Automatically construct enum of all suite configs +# -------------------------------------------------------------------------------------------------- - def format_config_name(config_name): - return config_name[1:] if config_name[0] == '_' else config_name +def construct_suite_config_enum(): + # Automatically construct enum of all suite configs def wrapper(suite_config_enum): # Dictionary used to create the enum @@ -58,10 +60,10 @@ def wrapper(suite_config_enum): suite_configs = suite_container.get_all() for config in suite_configs: - enum_dict[format_config_name(config)] = getattr(suite_container, config) - config_suite_map[format_config_name(config)] = suite + enum_dict[format_suite_name(config)] = getattr(suite_container, config) + config_suite_map[format_suite_name(config)] = suite else: - enum_dict[suite] = SuiteQuestions.all_suites + enum_dict[suite] = SuiteQuestions.all_suites.value config_suite_map[suite] = suite # Set the map dictionary to a hidden attribute @@ -85,9 +87,48 @@ def wrapper(suite_config_enum): # -------------------------------------------------------------------------------------------------- -@construct_suite_enum() -class AllSuites(Enum): +def construct_workflow_enum(): + # Automatically construct enum of all suite configs + + def wrapper(workflow_enum): + # Dictionary used to create the enum + enum_dict = {} + # Map of config names to their parent suites + config_suite_map = {} + + # Find all of the suite configs + for suite in get_suites(): + config_path = os.path.join(get_swell_path(), 'suites', suite, 'workflow.py') + if os.path.exists(config_path): + workflow = getattr( + import_module(f'swell.suites.{suite}.workflow'), f'Workflow_{suite}') + + enum_dict[suite] = workflow + + # Set the map dictionary to a hidden attribute + enum_dict['__config_suite_map__'] = config_suite_map + + # Override with manually specified keys in enum + for item in workflow_enum: + enum_dict[item.name] = item.value + + # Build the enum + enum_cls = Enum(workflow_enum.__name__, enum_dict) + + return enum_cls + return wrapper + +# -------------------------------------------------------------------------------------------------- + +@construct_suite_config_enum() +class SuiteConfigs(Enum): pass # -------------------------------------------------------------------------------------------------- + +@construct_workflow_enum() +class Workflows(Enum): + pass + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index c4b9105a2..2d8da6e85 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import task_wrapper, get_tasks from swell.test.test_driver import test_wrapper, valid_tests from swell.test.suite_tests.suite_tests import run_suite, TestSuite -from swell.suites.all_suites import AllSuites +from swell.suites.all_suites import SuiteConfigs from swell.utilities.welcome_message import write_welcome_message from swell.utilities.scripts.utility_driver import get_utilities, utility_wrapper @@ -87,7 +87,7 @@ def swell_driver() -> None: @swell_driver.command() -@click.argument('suite', type=click.Choice(AllSuites.config_names())) +@click.argument('suite', type=click.Choice(SuiteConfigs.config_names())) @click.option('-m', '--input_method', 'input_method', default='defaults', type=click.Choice(['defaults', 'cli']), help=input_method_help) @click.option('-p', '--platform', 'platform', default='nccs_discover_sles15', diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py new file mode 100644 index 000000000..216a9dd00 --- /dev/null +++ b/src/swell/tasks/task_runtimes.py @@ -0,0 +1,81 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from typing import Union, Optional, Self +from collections.abc import Mapping +from enum import Enum, member + +from swell.utilities.cylc_runtime import Task, Model, Cycling, Slurm + +# -------------------------------------------------------------------------------------------------- + +class TaskRuntimes(Enum): + + @member + class CloneJedi(Task): + pass + + @member + class BuildJediByLinking(Task): + pass + + @member + class BuildJedi(Slurm): + time_limit: bool = True + + @member + class StageJedi(Model): + pass + + @member + class StageJediCycle(Cycling, Model): + pass + + @member + class GetBackground(Cycling, Model): + pass + + @member + class GetObservations(Cycling, Model): + pass + + @member + class GenerateBClimatology(Cycling, Model, Slurm): + time_limit: bool = True + + @member + class GenerateBClimatologyByLinking(Cycling, Model): + pass + + @member + class RunJediVariationalExecutable(Cycling, Model, Slurm): + time_limit: bool = True + + @member + class EvaJediLog(Cycling, Model): + pass + + @member + class EvaIncrement(Cycling, Model): + pass + + @member + class EvaObservations(Cycling, Model, Slurm): + time_limit: bool = True + + @member + class SaveObsDiags(Cycling, Model): + pass + + @member + class CleanCycle(Cycling, Model): + pass + + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/utilities/cylc_formatting.py b/src/swell/utilities/cylc_formatting.py new file mode 100644 index 000000000..3c6ae8b2c --- /dev/null +++ b/src/swell/utilities/cylc_formatting.py @@ -0,0 +1,70 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from typing import Union, Optional, Self +from collections.abc import Mapping + +# -------------------------------------------------------------------------------------------------- + +indent = ' ' + +def format_dict(dictionary: Mapping): + dict_str = '' + + for key, value in dictionary.items(): + dict_str += f'{key} = {value}\n' + + return dict_str + +def indent_lines(string: str, level: int = 0): + out_string = '' + + for line in string.split('\n'): + line = line.strip() + + if len(line) > 0: + line = f'{indent*level}{line}' + + out_string += f'{line}\n' + + return out_string + +class CylcSection(): + def __init__(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int = 0) -> None: + self.name = name + self.content = content + self.level = level + + self.section_str = self.__format_section__(self.name, self.content, self.level) + + def __format_section__(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int = 0) -> str: + section_str = '' + + if name is not None: + section_str += f'{level*"["}{name}{"]"*level}\n' + else: + level -= 1 + + if isinstance(content, Mapping): + content = format_dict(content) + + section_str += indent_lines(content, level+1) + + if level == 0: + section_str += f'# {"-"*98}\n' + + return section_str + + def add_subsection(self, subsection: Self): + self.section_str += subsection.__format_section__(subsection.name, subsection.content, self.level+1) + + def get_section_str(self): + return self.section_str + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py new file mode 100644 index 000000000..11c65b013 --- /dev/null +++ b/src/swell/utilities/cylc_runtime.py @@ -0,0 +1,164 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from typing import Union, Optional, Self +from collections.abc import Mapping + +from swell.utilities.cylc_formatting import Section, indent_lines + +# -------------------------------------------------------------------------------------------------- + +indent = ' ' + +@dataclass +class Task: + base_name: Optional[str] = None + scheduling_name: Optional[str] = None + + pre_script: Union[str, bool, None] = False + script: Union[str, bool, None] = None + + retry: Union[str, Mapping, None] = None + time_limit: Union[str, Mapping, None] = None + environment: Optional[Mapping] = None + slurm: Union[bool, Mapping, None] = None + + is_cycling: bool = False + is_model: bool = False + + def __post_init__(self): + + if self.base_name is None: + self.base_name = self.__class__.__name__ + + if self.scheduling_name is None: + self.scheduling_name = self.base_name + + if self.is_model: + self.scheduling_name += '-{model_component}' + + if self.script is None: + self.script = f'swell task {self.base_name} $config' + + if self.is_cycling: + self.script += ' -d $datetime' + + if self.is_model: + self.script += ' -m {model_component}' + + def format_string_block(self, string: str) -> str: + out_string = '"""\n' + out_string += indent_lines(string, 1) + out_string += '"""\n' + + return out_string + + def match_platform(self, content: Union[str, dict], platform: str): + if isinstance(content, Mapping): + if platform in content.keys(): + content = content[platform] + elif 'all' in content.keys(): + content = content['all'] + + return content + + def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int =0) -> Section: + return Section(name, content, level) + + def get_runtime_section(self, model_component: Optional[str], experiment_dict: Mapping, slurm_external: Mapping): + platform = experiment_dict['platform'] + + runtime_dict = {} + + if self.pre_script: + runtime_dict['pre_script'] = self.format_string_block(self.pre_script) + + if self.script: + runtime_dict['script'] = self.format_string_block(self.script) + + if self.slurm: + runtime_dict['platform'] = platform + + if self.time_limit is True: + runtime_dict['execution time limit'] = 'PT1H' + elif self.time_limit: + time_limit = self.match_platform(self.time_limit, platform) + runtime_dict['execution time limit'] = time_limit + + if self.retry is True: + runtime_dict['execution retry delays'] = '2*PT1M' + if self.retry: + retry = self.match_platform(self.retry, platform) + runtime_dict['execution retry delays'] = retry + + runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) + + if self.slurm: + if isinstance(self.slurm, Mapping): + slurm_dict = {} + for key, value in self.slurm.items(): + slurm_dict[key] = self.match_platform(value, platform) + else: + slurm_dict = {} + + slurm_globals = slurm_external['slurm_directives_global'] + slurm_task = {} + + if 'slurm_directives_tasks' in slurm_external.keys(): + task_directives = slurm_external['slurm_directives_tasks'] + + if self.base_name in task_directives: + slurm_task = task_directives[self.base_name] + if self.scheduling_name in task_directives: + slurm_task = task_directives[self.scheduling_name] + else: + slurm_task = {} + + slurm_dict = {'job-name': self.scheduling_name, + **slurm_globals, + **slurm_dict + **slurm_task + } + + slurm_section_dict = {} + + for key, value in slurm_dict.items(): + slurm_section_dict[f'--{key}'] = value + + runtime_section.add_section(self.create_new_section('directives', slurm_section_dict)) + + return runtime_section + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class Model(Task): + def __post_init(self): + self.is_model = True + super().__post_init__() + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class Cycling(Task): + def __post_init(self): + self.is_cycling = True + super().__post_init__() + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class Slurm(Task): + def __post_init__(self): + if not self.slurm: + self.slurm = {} + + super().__post_init__() + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py new file mode 100644 index 000000000..811721287 --- /dev/null +++ b/src/swell/utilities/cylc_workflow.py @@ -0,0 +1,198 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from typing import Union, Optional, Self +from collections.abc import Mapping + +from swell.utilities.cylc_formatting import Section, indent_lines + +# -------------------------------------------------------------------------------------------------- + +class CylcWorkflow(): + def __init__(self, experiment_dict, slurm_external) -> None: + self.experiment_dict = experiment_dict + self.slurm_external = slurm_external + + self.setup_workflow() + + def set_experiment_dict(self, experiment_dict) -> None: + self.experiment_dict = experiment_dict + + def format_string_block(self, string) -> str: + out_string = '"""\n' + out_string += indent_lines(string, 1) + out_string += '"""\n' + + return out_string + + def format_cycle(self, name: str, cycle: str) -> str: + cycle_string = f'{name} = ' + cycle_string += self.format_string_block(cycle) + return cycle_string + + def reset_indentation(self, string: str) -> str: + out_string = '' + + start = False + for line in string.split('\n'): + line = line.strip() + + if len(line) > 0: + start = True + + if start: + out_string += line + + return out_string + + def setup_workflow(self) -> None: + self.header = self.define_header() + self.description = self.define_description() + self.scheduler = self.define_scheduler() + + self.scheduling_section = self.define_scheduling_section() + self.graph_section = self.define_graph_section() + + self.scheduling = self.define_scheduling() + + self.tasks = self.parse_graph_for_tasks() + + + def define_header(self) -> str: + return self.reset_indentation(""" + # (C) Copyright 2021- United States Government as represented by the Administrator of the + # National Aeronautics and Space Administration. All Rights Reserved. + # + # This software is licensed under the terms of the Apache Licence Version 2.0 + # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.""") + + def define_description(self) -> str: + return self.reset_indentation("""# Cylc workflow auto-generated for suite + {suite_to_run} by Swell.""".format(**self.experiment_dict)) + + def define_scheduler(self) -> str: + scheduler_dict = {'UTC mode': True, 'allow implicit tasks': False} + scheduler = self.create_new_section('scheduler', scheduler_dict) + + return scheduler.get_section_str() + + def define_scheduling(self) -> str: + scheduling = self.scheduling_section.add_subsection(self.graph_section) + + return scheduling.get_section_str() + + def define_scheduling_section(self) -> Section: + scheduling_dict = {'initial cycle point': self.experiment_dict['start_cycle_point'], + 'final cycle point': self.experiment_dict['final_cycle_point'], + 'runahead limit': self.experiment_dict['runahead_limit']} + + scheduling_section = Section('scheduling', scheduling_dict) + + return scheduling_section + + def define_graph_section(self) -> Section: + return self.create_new_section('graph') + + def parse_graph_for_tasks(self) -> Section: + tasks = [] + + cylc_characters = [':', '[', ']'] + + in_graph = False + in_cycle = False + + for line in self.scheduling: + comment = False + sub_strings = line.split(' ') + + for sub_string in sub_strings: + sub_string = sub_string.strip() + + if '#' in sub_string: + comment = True + + if '"""' in sub_string: + in_cycle = not in_cycle + + if not comment and in_graph and in_cycle: + if len(sub_string) > 0 and sub_string not in ['=>', '&', '|']: + task = sub_string + for i, char in enumerate(task): + if char in cylc_characters: + task = task.split(char)[0] + + if task not in tasks: + tasks.append(task) + + if '[graph]' in line: + in_graph = True + + return tasks + + def get_independent_and_model_tasks(self) -> Tuple[list, dict]: + ind_tasks = [] + model_tasks = {} + + models = [] + if 'model_components' in self.experiment_dict: + models = self.experiment_dict['model_components'] + + for model in models: + model_tasks[model] = [] + + for task in self.tasks: + if '-' in task: + task_name = task.split('-')[0] + model = task.split('-')[1] + + if model in models: + model_tasks[model].append(task_name) + else: + ind_tasks.append(task) + + return ind_tasks, model_tasks + + def define_runtime_task_overrides(self) -> dict: + return {} + + def create_new_section(name: Optional[str], content: Union[str, dict], level: int = 0): + return Section(name, content, level) + + def define_runtime(self) -> str: + runtime_section = Section('runtime', '# Task defaults\n# -------------\n') + + for task in ['root'] + self.tasks: + if task in self.runtime_task_overrides.keys(): + task_section = self.define_runtime_task_overrides[task] + + runtime_section.add_subsection(task_section) + + else: + if '-' in task: + task_name = task.split('-')[0] + model = task.split('-')[1] + if model not in self.experiment_dict['model_components']: + task_name = task + model = None + else: + task_name = task + model = None + + task_class = TaskRuntime[task_name] + task_section = task_class().get_section(model, self.experiment_dict, self.slurm_external) + + + return runtime_str + + + + + + + diff --git a/src/swell/utilities/dictionary.py b/src/swell/utilities/dictionary.py index 82f0a73b2..ae7f2324c 100644 --- a/src/swell/utilities/dictionary.py +++ b/src/swell/utilities/dictionary.py @@ -8,7 +8,7 @@ import yaml -from collections.abc import Hashable +from collections.abc import Hashable, Mapping from typing import Union from swell.utilities.logger import Logger @@ -176,3 +176,19 @@ def dictionary_override(logger: Logger, orig_dict: dict, override_dict: dict) -> # -------------------------------------------------------------------------------------------------- + +def add_dict(priority_dict: Mapping, additional_dict: Mapping) -> Mapping: + # Return version of dictionary 1 updated with additional keys from dictionary 2 without + # overwriting entries in dictionary 1 + + for key, value in additional_dict.items(): + if key in priority_dict.keys(): + priority_value = priority_dict[key] + if isinstance(value, Mapping) and isinstance(priority_value, Mapping): + priority_dict[key] = add_dict(priority_value, value) + else: + priority_dict[key] = value + + return priority_dict + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 6752bbf6a..8796d4835 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -10,18 +10,18 @@ import os import re import yaml +from typing import Optional from importlib import resources from swell.utilities.logger import Logger - -def prepare_scheduling_dict( +def prepare_slurm_defaults_and_overrides( logger: Logger, - experiment_dict: dict, platform: str, + slurm_file: Optional[str], ) -> dict: - + # Obtain platform-specific SLURM directives and set them as global defaults # Start by constructing the full platforms path # ------------------------------------------- @@ -35,18 +35,13 @@ def prepare_scheduling_dict( raise Exception(f"Platform '{platform}' has not been configured in SWELL") except Exception as err: raise err + + global_defaults = {} + global_defaults['slurm_directives_global'] = {} logger.info(f'Loading SLURM user configuration for the "{platform}" platform') with resources.open_text(path_import, 'slurm.yaml') as yaml_file: - global_defaults = yaml.safe_load(yaml_file) - - # Hard-coded SLURM defaults for certain tasks - # ------------------------------------------- - task_defaults = { - "RunJediVariationalExecutable": {"all": {"nodes": 3}}, - "RunJediUfoTestsExecutable": {"all": {"ntasks-per-node": 1}}, - "RunJediConvertStateSoca2ciceExecutable": {"all": {"nodes": 1}} - } + global_defaults['slurm_directives_global'] = yaml.safe_load(yaml_file) # Global SLURM settings stored in $HOME/.swell/swell-slurm.yaml # ---------------------------------------------- @@ -54,146 +49,32 @@ def prepare_scheduling_dict( # See https://github.com/GEOS-ESM/swell/issues/351 user_globals = slurm_global_defaults(logger) - # Global SLURM settings from experiment dict (questionary / overrides YAML) - # ---------------------------------------------- - experiment_globals = {} - if "slurm_directives_global" in experiment_dict: - logger.info(f"Loading additional SLURM globals from experiment dict") - experiment_globals = experiment_dict["slurm_directives_global"] - - # Task-specific SLURM settings from experiment dict (questionary / overrides YAML) - # ---------------------------------------------- - experiment_task_directives = {} - if "slurm_directives_tasks" in experiment_dict: - logger.info(f"Loading experiment-specific SLURM configs from experiment dict") - experiment_task_directives = experiment_dict["slurm_directives_tasks"] - - # List of tasks using slurm - # ------------------------- - slurm_tasks = { - 'BuildJedi', - 'BuildGeos', - 'EvaObservations', - 'GenerateBClimatology', - 'RunJediEnsembleMeanVariance', - 'RunJediConvertStateSoca2ciceExecutable', - 'RunJediFgatExecutable', - 'RunJediHofxEnsembleExecutable', - 'RunJediHofxExecutable', - 'RunJediLocalEnsembleDaExecutable', - 'RunJediObsfiltersExecutable', - 'RunJediUfoTestsExecutable', - 'RunJediVariationalExecutable', - 'RunGeosExecutable' - } - - # Throw an error if a user tries to set SLURM directives for a task that - # doesn't use SLURM. - experiment_slurm_tasks = set(experiment_task_directives.keys()) - non_slurm_tasks = experiment_slurm_tasks.difference(slurm_tasks) - assert len(non_slurm_tasks) == 0, \ - f"The following tasks cannot use SLURM: {non_slurm_tasks}" - - model_components = experiment_dict["model_components"] \ - if "model_components" in experiment_dict \ - else [] - - scheduling_dict = {} - for slurm_task in slurm_tasks: - # Priority order (first = highest priority) - # 1. Task-specific directives from experiment - # (experiment_task_directives[slurm_task]["all"]) - # 2. Global directives from experiment (experiment_globals) - # 3. Directives from user config (user_globals) - # 4. Hard-coded task-specific defaults (task_defaults) - # 5. Hard-coded global defaults (global_defaults) - # NOTE: Hard-code "job-name" to SWELL task here but it can be - # overwritten in task-specific directives. - directives = { - "job-name": slurm_task, - **global_defaults, - **user_globals, - **experiment_globals - } - if slurm_task in task_defaults: - if "all" in task_defaults[slurm_task]: - directives = { - **directives, - **task_defaults[slurm_task]["all"] - } - if slurm_task in experiment_task_directives: - if "all" in experiment_task_directives[slurm_task]: - directives = { - **directives, - **experiment_task_directives[slurm_task]["all"] - } - # Set model_agnostic directives - validate_directives(directives) - scheduling_dict[slurm_task] = {"directives": {"all": directives}} - - # Now, add model component-specific logic. The inheritance here is more - # complicated: - # - Experiment global defaults (`experiment_globals`) - # - User global defaults (`user_globals`) - # - Task- and model-specific hard-coded defaults - # - Task-specific, model-generic hard-coded defaults - # - Global hard-coded defaults - # Now, for every model component, set the model-generic directives - # (`directives`) but overwrite with model-specific directives if - # present. - for model_component in model_components: - model_directives = { - "job-name": f"{slurm_task}-{model_component}", - **global_defaults - } - if slurm_task in task_defaults: - model_directives = add_directives( - model_directives, - task_defaults[slurm_task], - "all" - ) - model_directives = add_directives( - model_directives, - task_defaults[slurm_task], - model_component - ) - model_directives = { - **model_directives, - **user_globals, - **experiment_globals - } - if slurm_task in experiment_task_directives: - model_directives = add_directives( - model_directives, - experiment_task_directives[slurm_task], - "all" - ) - model_directives = add_directives( - model_directives, - experiment_task_directives[slurm_task], - model_component - ) - validate_directives(model_directives) - scheduling_dict[slurm_task]["directives"][model_component] = model_directives - - # Default execution time limit for everthing is PT1H - x = 'PT1H' - if slurm_task in experiment_task_directives.keys(): - x = experiment_task_directives[slurm_task].get('execution_time_limit', x) - scheduling_dict[slurm_task]['execution_time_limit'] = x - - return scheduling_dict - - -def add_directives(target_dict: dict, input_dict: dict, key: str) -> dict: - if key in input_dict: - return { - **target_dict, - **input_dict[key] - } - else: - return target_dict - + # Expand experiment dict with SLURM overrides. + # NOTE: This is a bit of a hack. We should really either commit to using a + # separate file and pass it around everywhere, or commit fully to keeping + # everything in `experiment.yaml` and support it through the Questionary + # infrastructure. + # ---------------------------------- + if slurm_file is not None: + logger.info(f"Reading SLURM directives from {slurm_file}.") + assert os.path.exists(slurm_file) + with open(slurm_file, "r") as slurmfile: + slurm_overrides = yaml.safe_load(slurmfile) + # Ensure that SLURM dict is _only_ used for SLURM directives. + slurm_invalid_keys = set(slurm_overrides.keys()).difference({ + "slurm_directives_global", + "slurm_directives_tasks" + }) + if slurm_invalid_keys: + logger.abort(f'SLURM file contains invalid keys: {slurm_invalid_keys}') + + slurm_dict = {**global_defaults, + **user_globals, + **slurm_overrides} + + validate_directives(slurm_dict) + + return slurm_dict def validate_directives(directive_dict: dict) -> None: directive_pattern = r'(?<=--)[a-zA-Z-]+' @@ -217,10 +98,12 @@ def slurm_global_defaults( ) -> dict: yaml_path = os.path.expanduser(yaml_path) user_globals = {} + user_globals['slurm_directives_global'] = {} + if os.path.exists(yaml_path): logger.info(f"Loading SLURM user configuration from {yaml_path}") with open(yaml_path, "r") as yaml_file: - user_globals = yaml.safe_load(yaml_file) + user_globals['slurm_directives_global'] = yaml.safe_load(yaml_file) return user_globals diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index 9d4615c58..fb0d037ed 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -11,13 +11,18 @@ import os from dataclasses import dataclass, asdict, field from typing import List, Optional, Self, Union, Literal -from enum import Enum +from enum import Enum, StrEnum from isodate import parse_datetime, parse_duration, ISO8601Error from swell.swell_path import get_swell_path # -------------------------------------------------------------------------------------------------- +class QuestionType(StrEnum): + SUITE = 'suite' + TASK = 'task' + +# -------------------------------------------------------------------------------------------------- class WidgetType(Enum): STRING = "string" @@ -189,14 +194,14 @@ def expand_question_list(self, model: Optional[str] = None): @dataclass class SuiteQuestion(SwellQuestion): - question_type: str = "suite" + question_type: QuestionType = QuestionType.SUITE # -------------------------------------------------------------------------------------------------- @dataclass class TaskQuestion(SwellQuestion): - question_type: str = "task" + question_type: QuestionType = QuestionType.TASK # -------------------------------------------------------------------------------------------------- From d46b22fffa714ca7464c869d940d9c279714ade5 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 2 May 2025 10:17:01 -0400 Subject: [PATCH 002/299] adding workflow string --- src/swell/deployment/create_experiment.py | 10 ++-- .../prepare_config_and_suite.py | 12 ++-- src/swell/suites/3dvar/workflow.py | 8 ++- src/swell/suites/all_suites.py | 15 +++-- src/swell/utilities/config.py | 4 +- src/swell/utilities/cylc_formatting.py | 7 ++- src/swell/utilities/cylc_workflow.py | 26 ++++----- src/swell/utilities/slurm.py | 11 +++- src/swell/utilities/suite_utils.py | 56 ++++--------------- src/swell/utilities/swell_questions.py | 1 + 10 files changed, 64 insertions(+), 86 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 4a3f5d575..ba5e7d686 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -94,15 +94,15 @@ def prepare_config( platform, method, override) suite_dict = prepare_config_and_suite.get_experiment_dict() + print(suite_dict) + slurm_dict = prepare_slurm_defaults_and_overrides(logger, platform, slurm) - slurm_dict = prepare_slurm_defaults_and_overrides(logger, slurm, platform) - - workflow = Workflows[suite](suite_dict, slurm_dict) + workflow = Workflows.get_workflow(suite)(suite_dict, slurm_dict) model_ind_tasks, model_dep_tasks = workflow.get_independent_and_model_tasks() - prepare_config_and_suite.set_model_ind_tasks(model_ind_tasks) - prepare_config_and_suite.set_model_dep_tasks(model_dep_tasks) + prepare_config_and_suite.set_model_independent_tasks(model_ind_tasks) + prepare_config_and_suite.set_model_dependent_tasks(model_dep_tasks) experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 6774b7b3c..8f597810f 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -119,13 +119,13 @@ def configure_and_ask_task_questions(self) -> None: # ---------------------------------------------------------------------------------------------- - def set_model_ind_tasks(self, tasks: list) -> None: - self.model_ind_tasks = tasks + def set_model_independent_tasks(self, tasks: list) -> None: + self.model_independent_tasks = tasks # ---------------------------------------------------------------------------------------------- - def set_model_dep_tasks(self, model_task_dict: Mapping) -> None: - self.model_dep_tasks = model_task_dict + def set_model_dependent_tasks(self, model_task_dict: Mapping) -> None: + self.model_dependent_tasks = model_task_dict # ---------------------------------------------------------------------------------------------- @@ -403,11 +403,11 @@ def ask_a_question( if model is None: self.experiment_dict[question_key] = self.config_client.get_answer( self.logger, question_key, qd) - self.questions_dict[question_key] = qd['prompt'] + self.comment_dict[question_key] = qd['prompt'] else: self.experiment_dict['models'][model][question_key] = \ self.config_client.get_answer(self.logger, question_key, qd, model) - self.questions_dict[f'models.{model}.{question_key}'] = qd['prompt'] + self.comment_dict[f'models.{model}.{question_key}'] = qd['prompt'] # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 72888b08d..a925551ca 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -20,6 +20,8 @@ def define_description(self): return description def define_graph_section(self): + graph_str = '' + r1 = """ # Triggers for non cycle time dependent tasks # ------------------------------------------- @@ -43,8 +45,8 @@ def define_graph_section(self): graph_str += self.format_cycle('R1', r1) for model_component in self.experiment_dict['model_components']: - if 'cycle_times' in self.experiment_dict[model_component]: - for cycle_time in self.experiment_dict[model_component]['cycle_times']: + if 'cycle_times' in self.experiment_dict['models'][model_component]: + for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: cycle_str = """ # Task triggers for: {model_component} # ------------------ @@ -92,4 +94,4 @@ def define_graph_section(self): return self.create_new_section('graph', graph_str) -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/all_suites.py b/src/swell/suites/all_suites.py index 277c95f30..10c47bd71 100644 --- a/src/swell/suites/all_suites.py +++ b/src/swell/suites/all_suites.py @@ -19,11 +19,11 @@ # -------------------------------------------------------------------------------------------------- -# Class methods for AllSuites enum +# Class methods for SuiteConfigs enum @classmethod -def get_config(cls, config_name): - return getattr(cls, config_name).value.value +def get(cls, name): + return getattr(cls, name).value.value @classmethod @@ -77,7 +77,7 @@ def wrapper(suite_config_enum): enum_cls = Enum(suite_config_enum.__name__, enum_dict) # Set classmethods for the enum - setattr(enum_cls, 'get_config', get_config) + setattr(enum_cls, 'get_config', get) setattr(enum_cls, 'config_names', config_names) setattr(enum_cls, 'base_suite', base_suite) @@ -86,6 +86,10 @@ def wrapper(suite_config_enum): # -------------------------------------------------------------------------------------------------- +@classmethod +def get_workflow(cls, name): + return getattr(cls, name).value + def construct_workflow_enum(): # Automatically construct enum of all suite configs @@ -115,6 +119,9 @@ def wrapper(workflow_enum): # Build the enum enum_cls = Enum(workflow_enum.__name__, enum_dict) + # Set classmethods + setattr(enum_cls, 'get_workflow', get_workflow) + return enum_cls return wrapper diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index a9b7c1715..663709753 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -12,7 +12,7 @@ from swell.tasks.task_questions import TaskQuestions as task_questions from swell.utilities.logger import Logger -from swell.suites.all_suites import AllSuites +from swell.suites.all_suites import SuiteConfigs # -------------------------------------------------------------------------------------------------- @@ -103,7 +103,7 @@ def __init__(self, input_file: str, logger: Logger, task_name: str, model: str) # ------------------------------------------------------------------------- # Check for suite questions - suite_questions = AllSuites.get_config( + suite_questions = SuiteConfigs.get_config( self.__suite_to_run__).get_all_question_names('suite') question_list = [] diff --git a/src/swell/utilities/cylc_formatting.py b/src/swell/utilities/cylc_formatting.py index 3c6ae8b2c..053ea5bde 100644 --- a/src/swell/utilities/cylc_formatting.py +++ b/src/swell/utilities/cylc_formatting.py @@ -40,12 +40,13 @@ def __init__(self, name: Optional[str] = None, content: Union[str, dict] = '', l self.name = name self.content = content self.level = level - + print(name) + print(content) self.section_str = self.__format_section__(self.name, self.content, self.level) def __format_section__(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int = 0) -> str: section_str = '' - + print(level) if name is not None: section_str += f'{level*"["}{name}{"]"*level}\n' else: @@ -67,4 +68,4 @@ def add_subsection(self, subsection: Self): def get_section_str(self): return self.section_str -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 811721287..90ed93c39 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -7,10 +7,10 @@ # -------------------------------------------------------------------------------------------------- -from typing import Union, Optional, Self +from typing import Union, Optional, Self, Tuple from collections.abc import Mapping -from swell.utilities.cylc_formatting import Section, indent_lines +from swell.utilities.cylc_formatting import CylcSection, indent_lines # -------------------------------------------------------------------------------------------------- @@ -83,23 +83,26 @@ def define_scheduler(self) -> str: return scheduler.get_section_str() def define_scheduling(self) -> str: - scheduling = self.scheduling_section.add_subsection(self.graph_section) + scheduling = self.define_scheduling_section() + graph = self.define_graph_section() + + scheduling.add_subsection(graph) return scheduling.get_section_str() - def define_scheduling_section(self) -> Section: + def define_scheduling_section(self) -> CylcSection: scheduling_dict = {'initial cycle point': self.experiment_dict['start_cycle_point'], 'final cycle point': self.experiment_dict['final_cycle_point'], 'runahead limit': self.experiment_dict['runahead_limit']} - scheduling_section = Section('scheduling', scheduling_dict) + scheduling_section = self.create_new_section('scheduling', scheduling_dict) return scheduling_section - def define_graph_section(self) -> Section: + def define_graph_section(self) -> CylcSection: return self.create_new_section('graph') - def parse_graph_for_tasks(self) -> Section: + def parse_graph_for_tasks(self) -> CylcSection: tasks = [] cylc_characters = [':', '[', ']'] @@ -161,8 +164,8 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: def define_runtime_task_overrides(self) -> dict: return {} - def create_new_section(name: Optional[str], content: Union[str, dict], level: int = 0): - return Section(name, content, level) + def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int = 0): + return CylcSection(name, content, level) def define_runtime(self) -> str: runtime_section = Section('runtime', '# Task defaults\n# -------------\n') @@ -191,8 +194,3 @@ def define_runtime(self) -> str: return runtime_str - - - - - diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 8796d4835..89700ce7e 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -67,13 +67,18 @@ def prepare_slurm_defaults_and_overrides( }) if slurm_invalid_keys: logger.abort(f'SLURM file contains invalid keys: {slurm_invalid_keys}') + else: + slurm_overrides = {} slurm_dict = {**global_defaults, **user_globals, **slurm_overrides} - - validate_directives(slurm_dict) - + print('huh') + validate_directives(slurm_dict["slurm_directives_global"]) + if 'slurm_directives_tasks' in slurm_dict: + for task in slurm_dict["slurm_directives_tasks"].keys(): + validate_directives(slurm_dict["slurm_directives_tasks"][task]) + print('huh') return slurm_dict def validate_directives(directive_dict: dict) -> None: diff --git a/src/swell/utilities/suite_utils.py b/src/swell/utilities/suite_utils.py index 4e9c8a144..5762d5ad9 100644 --- a/src/swell/utilities/suite_utils.py +++ b/src/swell/utilities/suite_utils.py @@ -15,6 +15,16 @@ from swell.swell_path import get_swell_path +# -------------------------------------------------------------------------------------------------- + +def get_model_components() -> list: + + # Path to model interfaces + interface_directory = os.path.join(get_swell_path(), 'configuration', 'jedi', 'interfaces') + + # Get models + return os.listdir(interface_directory) + # -------------------------------------------------------------------------------------------------- def get_suites() -> list: @@ -31,49 +41,3 @@ def get_suites() -> list: # -------------------------------------------------------------------------------------------------- - -def get_suite_configs() -> list: - - suites = get_suites() - - # List of suites and associated sub-suites - suite_config_list = [] - - for suite in suites: - suite_sub_list = [] - suite_module = importlib.import_module(f'swell.suites.{suite}.suite_config') - suite_configs = getattr(suite_module, 'SuiteConfig') - - [suite_sub_list.append(suite_config[1:] if suite_config[0] == '_' else suite_config) - for suite_config in suite_configs.get_all()] - - suite_config_list.extend(sorted(suite_sub_list)) - - # List all directories in platform_directory - return suite_config_list - - -# -------------------------------------------------------------------------------------------------- - - -def get_suite_tests() -> list: - - # Path to platforms - suite_tests_directory = os.path.join(get_swell_path(), 'test', 'suite_tests', '*.yaml') - - # List of tasks - suite_test_files = sorted(glob.glob(suite_tests_directory)) - - # Get just the task name - suite_tests = [] - for suite_test_file in suite_test_files: - suite_tests.append(os.path.basename(suite_test_file)[0:-5]) - - # Sort list alphabetically - suite_tests = sorted(suite_tests) - - # Return list of valid task choices - return suite_tests - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index fb0d037ed..13c4ea7ed 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -105,6 +105,7 @@ class SwellQuestion: question_name: str widget_type: WidgetType prompt: str + models: Optional[list] = None question_type: str = None ask_question: bool = False options: Optional[str] = None From e1c3816d8dce57682cb06ae13872a41fa22beb90 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 2 May 2025 12:03:33 -0400 Subject: [PATCH 003/299] working --- src/swell/deployment/create_experiment.py | 5 --- src/swell/tasks/task_runtimes.py | 10 ++++- src/swell/utilities/cylc_formatting.py | 19 ++++---- src/swell/utilities/cylc_runtime.py | 11 ++--- src/swell/utilities/cylc_workflow.py | 53 +++++++++++++++++++---- src/swell/utilities/slurm.py | 2 - 6 files changed, 71 insertions(+), 29 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index ba5e7d686..1f3f01508 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -94,7 +94,6 @@ def prepare_config( platform, method, override) suite_dict = prepare_config_and_suite.get_experiment_dict() - print(suite_dict) slurm_dict = prepare_slurm_defaults_and_overrides(logger, platform, slurm) workflow = Workflows.get_workflow(suite)(suite_dict, slurm_dict) @@ -110,10 +109,6 @@ def prepare_config( workflow_string = workflow.get_workflow_str() - # Ask questions as the suite gets configured - # ------------------------------------------ - experiment_dict, comment_dict = prepare_config_and_suite.ask_questions_and_configure_suite() - # Expand all environment vars in the dictionary # --------------------------------------------- experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 216a9dd00..45925ee46 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -17,6 +17,14 @@ class TaskRuntimes(Enum): + @member + class root(Task): + script: bool = False + pre_script: str = "source $CYLC_SUITE_DEF_PATH/modules" + environment: dict = {'datetime': '$CYLC_TASK_CYCLE_POINT', + 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'} + + @member class CloneJedi(Task): pass @@ -78,4 +86,4 @@ class CleanCycle(Cycling, Model): pass -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_formatting.py b/src/swell/utilities/cylc_formatting.py index 053ea5bde..f2ab38839 100644 --- a/src/swell/utilities/cylc_formatting.py +++ b/src/swell/utilities/cylc_formatting.py @@ -22,11 +22,12 @@ def format_dict(dictionary: Mapping): return dict_str -def indent_lines(string: str, level: int = 0): +def indent_lines(string: str, level: int = 0, reset: bool = False): out_string = '' for line in string.split('\n'): - line = line.strip() + if reset: + line = line.strip() if len(line) > 0: line = f'{indent*level}{line}' @@ -40,22 +41,19 @@ def __init__(self, name: Optional[str] = None, content: Union[str, dict] = '', l self.name = name self.content = content self.level = level - print(name) - print(content) self.section_str = self.__format_section__(self.name, self.content, self.level) def __format_section__(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int = 0) -> str: section_str = '' - print(level) if name is not None: - section_str += f'{level*"["}{name}{"]"*level}\n' + section_str += f'{indent*level}{(level+1)*"["}{name}{"]"*(level+1)}\n' else: level -= 1 if isinstance(content, Mapping): content = format_dict(content) - section_str += indent_lines(content, level+1) + section_str += indent_lines(content, level+1, False) if level == 0: section_str += f'# {"-"*98}\n' @@ -66,6 +64,11 @@ def add_subsection(self, subsection: Self): self.section_str += subsection.__format_section__(subsection.name, subsection.content, self.level+1) def get_section_str(self): - return self.section_str + section_str = self.section_str + + if self.level == 0: + section_str += f'# {"-"*98}\n' + + return section_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 11c65b013..8caf5beca 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -9,8 +9,9 @@ from typing import Union, Optional, Self from collections.abc import Mapping +from dataclasses import dataclass -from swell.utilities.cylc_formatting import Section, indent_lines +from swell.utilities.cylc_formatting import CylcSection, indent_lines # -------------------------------------------------------------------------------------------------- @@ -68,10 +69,10 @@ def match_platform(self, content: Union[str, dict], platform: str): return content - def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int =0) -> Section: - return Section(name, content, level) + def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int =0) -> CylcSection: + return CylcSection(name, content, level) - def get_runtime_section(self, model_component: Optional[str], experiment_dict: Mapping, slurm_external: Mapping): + def get_section(self, model_component: Optional[str], experiment_dict: Mapping, slurm_external: Mapping): platform = experiment_dict['platform'] runtime_dict = {} @@ -161,4 +162,4 @@ def __post_init__(self): super().__post_init__() -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 90ed93c39..a9779f9a5 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -11,6 +11,7 @@ from collections.abc import Mapping from swell.utilities.cylc_formatting import CylcSection, indent_lines +from swell.tasks.task_runtimes import TaskRuntimes # -------------------------------------------------------------------------------------------------- @@ -26,7 +27,7 @@ def set_experiment_dict(self, experiment_dict) -> None: def format_string_block(self, string) -> str: out_string = '"""\n' - out_string += indent_lines(string, 1) + out_string += indent_lines(string, 1, True) out_string += '"""\n' return out_string @@ -65,17 +66,34 @@ def setup_workflow(self) -> None: def define_header(self) -> str: - return self.reset_indentation(""" + header = self.comment_block(string = """ # (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.""") + + return header def define_description(self) -> str: - return self.reset_indentation("""# Cylc workflow auto-generated for suite - {suite_to_run} by Swell.""".format(**self.experiment_dict)) - + description = self.comment_block("""# Cylc workflow auto-generated for suite {suite_to_run} by Swell.""".format(**self.experiment_dict)) + return description + + def comment_block(self, string, level: int = 0, section_break: bool = True): + out_string = '' + + string = indent_lines(string, level, reset=True) + + for line in string.split('\n'): + if len(line.strip()) > 0: + out_string += line + + if section_break: + out_string += f'\n# {"-" *98}' + + return out_string + + def define_scheduler(self) -> str: scheduler_dict = {'UTC mode': True, 'allow implicit tasks': False} scheduler = self.create_new_section('scheduler', scheduler_dict) @@ -168,10 +186,12 @@ def create_new_section(self, name: Optional[str] = None, content: Union[str, dic return CylcSection(name, content, level) def define_runtime(self) -> str: - runtime_section = Section('runtime', '# Task defaults\n# -------------\n') + runtime_section = self.create_new_section('runtime', '# Task defaults\n# -------------\n') + + runtime_overrides = self.define_runtime_task_overrides() for task in ['root'] + self.tasks: - if task in self.runtime_task_overrides.keys(): + if task in runtime_overrides.keys(): task_section = self.define_runtime_task_overrides[task] runtime_section.add_subsection(task_section) @@ -187,10 +207,27 @@ def define_runtime(self) -> str: task_name = task model = None - task_class = TaskRuntime[task_name] + task_class = TaskRuntimes[task_name].value task_section = task_class().get_section(model, self.experiment_dict, self.slurm_external) + runtime_section.add_subsection(task_section) + + runtime_str = runtime_section.get_section_str() return runtime_str + def get_workflow_str(self) -> str: + + workflow_str = '' + + workflow_str += self.header + print('DESCRIPTION', self.description) + workflow_str += self.description + workflow_str += self.scheduler + workflow_str += self.scheduling + + runtime = self.define_runtime() + workflow_str += runtime + print(workflow_str) + return workflow_str diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 89700ce7e..ebd5d7dcd 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -73,12 +73,10 @@ def prepare_slurm_defaults_and_overrides( slurm_dict = {**global_defaults, **user_globals, **slurm_overrides} - print('huh') validate_directives(slurm_dict["slurm_directives_global"]) if 'slurm_directives_tasks' in slurm_dict: for task in slurm_dict["slurm_directives_tasks"].keys(): validate_directives(slurm_dict["slurm_directives_tasks"][task]) - print('huh') return slurm_dict def validate_directives(directive_dict: dict) -> None: From e3e7d747e350317d2f74a9fd007ea61351ae5cba Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 2 May 2025 15:43:06 -0400 Subject: [PATCH 004/299] fixing section formatting --- .../prepare_config_and_suite.py | 12 ++--- src/swell/suites/3dvar/workflow.py | 6 +-- src/swell/tasks/task_runtimes.py | 49 ++++++++++++------- src/swell/utilities/cylc_formatting.py | 5 +- src/swell/utilities/cylc_runtime.py | 38 +++++++------- src/swell/utilities/cylc_workflow.py | 31 +++++++----- src/swell/utilities/slurm.py | 17 +++++-- 7 files changed, 92 insertions(+), 66 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 8f597810f..54ebbd596 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -149,18 +149,18 @@ def prepare_task_question_dictionary(self): for task in self.model_independent_tasks: if task in task_questions.get_all(): if self.suite_needs_model_components: - for model in self.experiment_dict['model_components'].keys(): + for model in self.experiment_dict['model_components']: model_dict = {model: {}} - for question in task_questions[task].expand_question_list(model): + for question in task_questions[task].value.expand_question_list(model): model_dict[model][question['question_name']] = question self.question_dictionary_model_dep = add_dict(self.question_dictionary_model_dep, model_dict) - for question in task_questions[task].expand_question_list(): + for question in task_questions[task].value.expand_question_list(): if question['models'] is not None: - for model in self.experiment_dict['model_components'].keys(): + for model in self.experiment_dict['model_components']: model_dict = {model: {}} - for question in task_questions[task].expand_question_list(model): + for question in task_questions[task].value.expand_question_list(model): if model in question['models'] or 'all_models' in question['models']: model_dict[model][question['question_name']] = question @@ -168,7 +168,7 @@ def prepare_task_question_dictionary(self): else: question_dict = {} - for question in task_questions[task].expand_question_list(): + for question in task_questions[task].value.expand_question_list(): if question['models'] is not None: self.logger.abort('The model components question has not been answered.') else: diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index a925551ca..3ada33b0b 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -13,9 +13,9 @@ class Workflow_3dvar(CylcWorkflow): def define_description(self): - description = """ + description = self.comment_block(""" # Cylc suite for executing JEDI-based non-cycling variational data assimilation - """ + """) return description @@ -57,7 +57,7 @@ def define_graph_section(self): GetObservations-{model_component} # GenerateBClimatology, for ocean it is cycle dependent - GenerateBClimatologyByLinking-{model_component} :fail? => GenerateBClimatology-{model_component} + GenerateBClimatologyByLinking-{model_component}:fail? => GenerateBClimatology-{model_component} GetBackground-{model_component} => GenerateBClimatology-{model_component} # Perform staging that is cycle dependent diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 45925ee46..c6c7ee4fc 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -10,80 +10,91 @@ from typing import Union, Optional, Self from collections.abc import Mapping from enum import Enum, member +from dataclasses import dataclass +from swell.utilities.dataclass_utils import mutable_field from swell.utilities.cylc_runtime import Task, Model, Cycling, Slurm # -------------------------------------------------------------------------------------------------- -class TaskRuntimes(Enum): +class TaskRuntimes(): - @member + @dataclass class root(Task): script: bool = False pre_script: str = "source $CYLC_SUITE_DEF_PATH/modules" - environment: dict = {'datetime': '$CYLC_TASK_CYCLE_POINT', - 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'} + environment: dict = mutable_field({'datetime': '$CYLC_TASK_CYCLE_POINT', + 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'}) - @member + @dataclass class CloneJedi(Task): pass - @member + @dataclass class BuildJediByLinking(Task): pass - @member + @dataclass class BuildJedi(Slurm): time_limit: bool = True - @member + @dataclass class StageJedi(Model): pass - @member + @dataclass class StageJediCycle(Cycling, Model): pass - @member + @dataclass class GetBackground(Cycling, Model): pass - @member + @dataclass class GetObservations(Cycling, Model): pass - @member + @dataclass class GenerateBClimatology(Cycling, Model, Slurm): time_limit: bool = True - @member + @dataclass class GenerateBClimatologyByLinking(Cycling, Model): pass - @member + @dataclass class RunJediVariationalExecutable(Cycling, Model, Slurm): time_limit: bool = True - @member + @dataclass class EvaJediLog(Cycling, Model): pass - @member + @dataclass class EvaIncrement(Cycling, Model): pass - @member + @dataclass class EvaObservations(Cycling, Model, Slurm): time_limit: bool = True - @member + @dataclass class SaveObsDiags(Cycling, Model): pass - @member + @dataclass class CleanCycle(Cycling, Model): pass + @classmethod + def get(cls, name: str) -> Task: + return getattr(cls, name) + # -------------------------------------------------------------------------------------------------- + +if __name__ == "__main__": + print(TaskRuntimes.get('RunJediVariationalExecutable')()) + r = TaskRuntimes.get('root')() + print(r.pre_script) diff --git a/src/swell/utilities/cylc_formatting.py b/src/swell/utilities/cylc_formatting.py index f2ab38839..ff4da1381 100644 --- a/src/swell/utilities/cylc_formatting.py +++ b/src/swell/utilities/cylc_formatting.py @@ -55,9 +55,6 @@ def __format_section__(self, name: Optional[str] = None, content: Union[str, dic section_str += indent_lines(content, level+1, False) - if level == 0: - section_str += f'# {"-"*98}\n' - return section_str def add_subsection(self, subsection: Self): @@ -67,7 +64,7 @@ def get_section_str(self): section_str = self.section_str if self.level == 0: - section_str += f'# {"-"*98}\n' + section_str += f'# {"-"*98}\n\n' return section_str diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 8caf5beca..b060b8294 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -22,13 +22,15 @@ class Task: base_name: Optional[str] = None scheduling_name: Optional[str] = None + model: Optional[str] = None + pre_script: Union[str, bool, None] = False script: Union[str, bool, None] = None retry: Union[str, Mapping, None] = None time_limit: Union[str, Mapping, None] = None environment: Optional[Mapping] = None - slurm: Union[bool, Mapping, None] = None + slurm: Optional[Mapping] = None is_cycling: bool = False is_model: bool = False @@ -42,7 +44,7 @@ def __post_init__(self): self.scheduling_name = self.base_name if self.is_model: - self.scheduling_name += '-{model_component}' + self.scheduling_name += f'-{self.model}' if self.script is None: self.script = f'swell task {self.base_name} $config' @@ -51,12 +53,12 @@ def __post_init__(self): self.script += ' -d $datetime' if self.is_model: - self.script += ' -m {model_component}' + self.script += f' -m {self.model}' def format_string_block(self, string: str) -> str: out_string = '"""\n' out_string += indent_lines(string, 1) - out_string += '"""\n' + out_string += '"""' return out_string @@ -72,9 +74,8 @@ def match_platform(self, content: Union[str, dict], platform: str): def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int =0) -> CylcSection: return CylcSection(name, content, level) - def get_section(self, model_component: Optional[str], experiment_dict: Mapping, slurm_external: Mapping): + def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): platform = experiment_dict['platform'] - runtime_dict = {} if self.pre_script: @@ -99,12 +100,10 @@ def get_section(self, model_component: Optional[str], experiment_dict: Mapping, runtime_dict['execution retry delays'] = retry runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) - - if self.slurm: - if isinstance(self.slurm, Mapping): - slurm_dict = {} - for key, value in self.slurm.items(): - slurm_dict[key] = self.match_platform(value, platform) + if self.slurm is not None: + slurm_dict = {} + for key, value in self.slurm.items(): + slurm_dict[key] = self.match_platform(value, platform) else: slurm_dict = {} @@ -123,16 +122,19 @@ def get_section(self, model_component: Optional[str], experiment_dict: Mapping, slurm_dict = {'job-name': self.scheduling_name, **slurm_globals, - **slurm_dict + **slurm_dict, **slurm_task } slurm_section_dict = {} - + print('slurm', slurm_section_dict) for key, value in slurm_dict.items(): slurm_section_dict[f'--{key}'] = value - - runtime_section.add_section(self.create_new_section('directives', slurm_section_dict)) + print('slurm', slurm_section_dict) + directive_section = self.create_new_section('directives', slurm_section_dict) + + runtime_section.add_subsection(directive_section) + print(runtime_section.get_section_str()) return runtime_section @@ -140,7 +142,7 @@ def get_section(self, model_component: Optional[str], experiment_dict: Mapping, @dataclass class Model(Task): - def __post_init(self): + def __post_init__(self): self.is_model = True super().__post_init__() @@ -148,7 +150,7 @@ def __post_init(self): @dataclass class Cycling(Task): - def __post_init(self): + def __post_init__(self): self.is_cycling = True super().__post_init__() diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index a9779f9a5..640d5476f 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -63,6 +63,7 @@ def setup_workflow(self) -> None: self.scheduling = self.define_scheduling() self.tasks = self.parse_graph_for_tasks() + print(self.tasks) def define_header(self) -> str: @@ -84,12 +85,16 @@ def comment_block(self, string, level: int = 0, section_break: bool = True): string = indent_lines(string, level, reset=True) + start = False for line in string.split('\n'): if len(line.strip()) > 0: - out_string += line + start = True + + if start: + out_string += f'{line}\n' if section_break: - out_string += f'\n# {"-" *98}' + out_string += f'\n# {"-" *98}\n\n' return out_string @@ -123,12 +128,12 @@ def define_graph_section(self) -> CylcSection: def parse_graph_for_tasks(self) -> CylcSection: tasks = [] - cylc_characters = [':', '[', ']'] + cylc_characters = [':', '[', ']', '?'] in_graph = False in_cycle = False - for line in self.scheduling: + for line in self.scheduling.split('\n'): comment = False sub_strings = line.split(' ') @@ -138,19 +143,19 @@ def parse_graph_for_tasks(self) -> CylcSection: if '#' in sub_string: comment = True - if '"""' in sub_string: - in_cycle = not in_cycle - if not comment and in_graph and in_cycle: - if len(sub_string) > 0 and sub_string not in ['=>', '&', '|']: + if len(sub_string) > 0 and sub_string not in ['=>', '&', '|', '"""']: task = sub_string for i, char in enumerate(task): if char in cylc_characters: task = task.split(char)[0] - if task not in tasks: + if len(task) > 0 and task not in tasks: tasks.append(task) + if '"""' in sub_string: + in_cycle = not in_cycle + if '[graph]' in line: in_graph = True @@ -186,7 +191,7 @@ def create_new_section(self, name: Optional[str] = None, content: Union[str, dic return CylcSection(name, content, level) def define_runtime(self) -> str: - runtime_section = self.create_new_section('runtime', '# Task defaults\n# -------------\n') + runtime_section = self.create_new_section('runtime', '\n# Task defaults\n# -------------\n') runtime_overrides = self.define_runtime_task_overrides() @@ -207,8 +212,8 @@ def define_runtime(self) -> str: task_name = task model = None - task_class = TaskRuntimes[task_name].value - task_section = task_class().get_section(model, self.experiment_dict, self.slurm_external) + task_class = TaskRuntimes.get(task_name) + task_section = task_class(model=model).get_section(self.experiment_dict, self.slurm_external) runtime_section.add_subsection(task_section) @@ -222,12 +227,12 @@ def get_workflow_str(self) -> str: workflow_str = '' workflow_str += self.header - print('DESCRIPTION', self.description) workflow_str += self.description workflow_str += self.scheduler workflow_str += self.scheduling runtime = self.define_runtime() workflow_str += runtime + print(workflow_str) return workflow_str diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index ebd5d7dcd..448acba7b 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -70,10 +70,21 @@ def prepare_slurm_defaults_and_overrides( else: slurm_overrides = {} - slurm_dict = {**global_defaults, - **user_globals, - **slurm_overrides} + if 'slurm_directives_global' not in slurm_overrides.keys(): + slurm_overrides['slurm_directives_global'] = {} + + if 'slurm_directives_tasks' not in slurm_overrides.keys(): + slurm_overrides['slurm_directives_tasks'] = {} + + slurm_dict = {} + + slurm_dict['slurm_directives_global'] = { + **global_defaults['slurm_directives_global'], + **user_globals['slurm_directives_global'], + **slurm_overrides['slurm_directives_global']} + validate_directives(slurm_dict["slurm_directives_global"]) + if 'slurm_directives_tasks' in slurm_dict: for task in slurm_dict["slurm_directives_tasks"].keys(): validate_directives(slurm_dict["slurm_directives_tasks"][task]) From 3788c9c78da6f52d87e17db27948b89477ae707e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 2 May 2025 16:39:28 -0400 Subject: [PATCH 005/299] redo task questions --- .../prepare_config_and_suite.py | 2 + src/swell/utilities/cylc_formatting.py | 45 +++++++++++++++---- src/swell/utilities/cylc_runtime.py | 12 ++--- src/swell/utilities/cylc_workflow.py | 5 +-- src/swell/utilities/slurm.py | 2 + 5 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 54ebbd596..e98956458 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -147,6 +147,7 @@ def get_workflow_file_str(self, slurm_dict): def prepare_task_question_dictionary(self): for task in self.model_independent_tasks: + print(task) if task in task_questions.get_all(): if self.suite_needs_model_components: for model in self.experiment_dict['model_components']: @@ -357,6 +358,7 @@ def ask_questions_and_configure(self, suite_task: QuestionType) -> Tuple[dict, d self.ask_a_question(model_dict, question_name, model) # ---------------------------------------------------------------------------------------------- + def ask_a_question( self, full_question_dictionary: dict, diff --git a/src/swell/utilities/cylc_formatting.py b/src/swell/utilities/cylc_formatting.py index ff4da1381..5e2409d34 100644 --- a/src/swell/utilities/cylc_formatting.py +++ b/src/swell/utilities/cylc_formatting.py @@ -36,36 +36,63 @@ def indent_lines(string: str, level: int = 0, reset: bool = False): return out_string +def format_section(section: Self, level: int = 0) -> str: + section_str = '' + + name = section.name + if name is not None: + section_str += f'{indent*level}{(level+1)*"["}{name}{"]"*(level+1)}\n' + else: + level -= 1 + + content = section.content + if isinstance(content, Mapping): + content = format_dict(content) + + section_str += indent_lines(content, level+1) + + return section_str + class CylcSection(): - def __init__(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int = 0) -> None: + def __init__(self, name: Optional[str] = None, content: Union[str, dict] = '') -> None: self.name = name self.content = content - self.level = level - self.section_str = self.__format_section__(self.name, self.content, self.level) + + self.subsections = [] - def __format_section__(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int = 0) -> str: + def __format_section__(self, section: Self, level: int = 0) -> str: section_str = '' + + name = section.name if name is not None: section_str += f'{indent*level}{(level+1)*"["}{name}{"]"*(level+1)}\n' else: level -= 1 + content = section.content if isinstance(content, Mapping): content = format_dict(content) section_str += indent_lines(content, level+1, False) + if level == 0: + section_str += f'# {"-"*98}\n\n' + return section_str - def add_subsection(self, subsection: Self): - self.section_str += subsection.__format_section__(subsection.name, subsection.content, self.level+1) + def add_subsection(self, subsection: Self) -> None: + self.subsections.append(subsection) - def get_section_str(self): - section_str = self.section_str + def get_section_str(self, level: int = 0) -> str: + section_str = format_section(self, level) - if self.level == 0: + for subsection in self.subsections: + section_str += subsection.get_section_str(level+1) + + if level == 0: section_str += f'# {"-"*98}\n\n' return section_str + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index b060b8294..5ba5ef39f 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -71,8 +71,8 @@ def match_platform(self, content: Union[str, dict], platform: str): return content - def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int =0) -> CylcSection: - return CylcSection(name, content, level) + def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = '') -> CylcSection: + return CylcSection(name, content) def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): platform = experiment_dict['platform'] @@ -100,6 +100,11 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): runtime_dict['execution retry delays'] = retry runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) + + if self.environment is not None: + environment_section = self.create_new_section('environment', self.environment) + runtime_section.add_subsection(environment_section) + if self.slurm is not None: slurm_dict = {} for key, value in self.slurm.items(): @@ -127,14 +132,11 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): } slurm_section_dict = {} - print('slurm', slurm_section_dict) for key, value in slurm_dict.items(): slurm_section_dict[f'--{key}'] = value - print('slurm', slurm_section_dict) directive_section = self.create_new_section('directives', slurm_section_dict) runtime_section.add_subsection(directive_section) - print(runtime_section.get_section_str()) return runtime_section diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 640d5476f..5b31562f2 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -187,8 +187,8 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: def define_runtime_task_overrides(self) -> dict: return {} - def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = '', level: int = 0): - return CylcSection(name, content, level) + def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = ''): + return CylcSection(name, content) def define_runtime(self) -> str: runtime_section = self.create_new_section('runtime', '\n# Task defaults\n# -------------\n') @@ -234,5 +234,4 @@ def get_workflow_str(self) -> str: runtime = self.define_runtime() workflow_str += runtime - print(workflow_str) return workflow_str diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 448acba7b..9250efac0 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -85,6 +85,8 @@ def prepare_slurm_defaults_and_overrides( validate_directives(slurm_dict["slurm_directives_global"]) + slurm_dict['slurm_directives_tasks'] = slurm_overrides['slurm_directives_tasks'] + if 'slurm_directives_tasks' in slurm_dict: for task in slurm_dict["slurm_directives_tasks"].keys(): validate_directives(slurm_dict["slurm_directives_tasks"][task]) From f661549c5adedc3b0114e15e23430403c51f5ec8 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 2 May 2025 16:59:39 -0400 Subject: [PATCH 006/299] update task questions --- .../prepare_config_and_suite.py | 35 +++++++++++++++++++ src/swell/utilities/cylc_workflow.py | 2 ++ 2 files changed, 37 insertions(+) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index e98956458..a92f83b4e 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -144,6 +144,41 @@ def get_workflow_file_str(self, slurm_dict): # ---------------------------------------------------------------------------------------------- + def prepare_task_question_dictionary(self): + for task in self.model_independent_tasks: + if task in task_questions.get_all(): + question_list = task_questions[task].value.expand_question_list() + for question in question_list: + question_dict = {question['question_name']: question} + + if question['models'] is not None: + model_dict = {} + + for question_model in question['models']: + if question_model == 'all_models': + for model in self.experiment_dict['model_components']: + model_dict[model] = question_dict + elif question_model in self.experiment_dict['model_components']: + model_dict[question_model] = question_dict + + self.question_dictionary_model_dep = add_dict(self.question_dictionary_model_dep, model_dict) + + else: + self.question_dictionary_model_ind = add_dict(self.question_dictionary_model_ind, question_dict) + + for model, task_list in self.model_dependent_tasks.items(): + for task in task_list: + if task in task_questions.get_all(): + question_list = task_questions[task].value.expand_question_list() + + for question in question_list: + question_dict = {question['question_name']: question} + if question['models'] is None: + self.question_dictionary_model_ind = add_dict(self.question_dictionary_model_ind, question_dict) + elif model in question['models']: + self.question_dictionary_model_dep = add_dict(self.question_dictionary_model_dep, {model: question_dict}) + + def prepare_task_question_dictionary(self): for task in self.model_independent_tasks: diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 5b31562f2..bbd239e4f 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -168,6 +168,8 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: models = [] if 'model_components' in self.experiment_dict: models = self.experiment_dict['model_components'] + else: + models = [] for model in models: model_tasks[model] = [] From 4d94820685138c8adb6a80451f137139164e0303 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 2 May 2025 17:00:54 -0400 Subject: [PATCH 007/299] update task questions again --- .../prepare_config_and_suite.py | 66 ------------------- 1 file changed, 66 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index a92f83b4e..a8de49ac4 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -179,72 +179,6 @@ def prepare_task_question_dictionary(self): self.question_dictionary_model_dep = add_dict(self.question_dictionary_model_dep, {model: question_dict}) - - def prepare_task_question_dictionary(self): - for task in self.model_independent_tasks: - print(task) - if task in task_questions.get_all(): - if self.suite_needs_model_components: - for model in self.experiment_dict['model_components']: - model_dict = {model: {}} - for question in task_questions[task].value.expand_question_list(model): - model_dict[model][question['question_name']] = question - - self.question_dictionary_model_dep = add_dict(self.question_dictionary_model_dep, model_dict) - - for question in task_questions[task].value.expand_question_list(): - if question['models'] is not None: - for model in self.experiment_dict['model_components']: - model_dict = {model: {}} - for question in task_questions[task].value.expand_question_list(model): - if model in question['models'] or 'all_models' in question['models']: - model_dict[model][question['question_name']] = question - - self.question_dictionary_model_dep = add_dict(self.question_dictionary_model_dep, model_dict) - - else: - question_dict = {} - for question in task_questions[task].value.expand_question_list(): - if question['models'] is not None: - self.logger.abort('The model components question has not been answered.') - else: - question_dict[question['question_name']] = question - - self.question_dictionary_model_ind = add_dict(self.question_dictionary_model_ind, question_dict) - - def prepare_suite_question_dictionary(self) -> None: - - question_dictionary_model_ind = {} - question_dictionary_model_dep = {} - - suite_config_obj = SuiteConfigs.get_config(self.suite_config) - suite_question_list = suite_config_obj.expand_question_list() - - for model in self.possible_model_components: - question_dictionary_model_dep[model] = {} - - for question in suite_config_obj.expand_question_list(model): - question_dictionary_model_dep[model][question['question_name']] = question - - for question in suite_question_list: - if question['models'] is None: - question_dictionary_model_ind[question['question_name']] = question - else: - if 'all_models' in question['models']: - question_models = self.possible_model_components - else: - question_models = question['models'] - - for model in question_models: - question_dictionary_model_dep = add_dict(question_dictionary_model_dep, {model: {question['question_name']: question}}) - - self.suite_needs_model_components = True - if 'model_components' not in question_dictionary_model_ind.keys(): - self.suite_needs_model_components = False - - self.question_dictionary_model_ind = question_dictionary_model_ind - self.question_dictionary_model_dep = question_dictionary_model_dep - def override_with_defaults(self, suite_task: QuestionType) -> None: # Perform a platform override on the model_ind dictionary From 6e5fa8df5d4915340ac07a9fee395639d6863a3d Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 2 May 2025 17:44:18 -0400 Subject: [PATCH 008/299] mostly working --- .../prepare_config_and_suite.py | 35 ++++++++++++++++++- src/swell/tasks/task_runtimes.py | 3 +- src/swell/utilities/cylc_runtime.py | 5 ++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index a8de49ac4..b01b4366f 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -144,6 +144,39 @@ def get_workflow_file_str(self, slurm_dict): # ---------------------------------------------------------------------------------------------- + def prepare_suite_question_dictionary(self) -> None: + + question_dictionary_model_ind = {} + question_dictionary_model_dep = {} + + suite_config_obj = SuiteConfigs.get_config(self.suite_config) + suite_question_list = suite_config_obj.expand_question_list() + + for model in self.possible_model_components: + question_dictionary_model_dep[model] = {} + + for question in suite_config_obj.expand_question_list(model): + question_dictionary_model_dep[model][question['question_name']] = question + + for question in suite_question_list: + if question['models'] is None: + question_dictionary_model_ind[question['question_name']] = question + else: + if 'all_models' in question['models']: + question_models = self.possible_model_components + else: + question_models = question['models'] + + for model in question_models: + question_dictionary_model_dep = add_dict(question_dictionary_model_dep, {model: {question['question_name']: question}}) + + self.suite_needs_model_components = True + if 'model_components' not in question_dictionary_model_ind.keys(): + self.suite_needs_model_components = False + + self.question_dictionary_model_ind = question_dictionary_model_ind + self.question_dictionary_model_dep = question_dictionary_model_dep + def prepare_task_question_dictionary(self): for task in self.model_independent_tasks: if task in task_questions.get_all(): @@ -175,7 +208,7 @@ def prepare_task_question_dictionary(self): question_dict = {question['question_name']: question} if question['models'] is None: self.question_dictionary_model_ind = add_dict(self.question_dictionary_model_ind, question_dict) - elif model in question['models']: + elif model in question['models'] or 'all_models' in question['models']: self.question_dictionary_model_dep = add_dict(self.question_dictionary_model_dep, {model: question_dict}) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index c6c7ee4fc..34268919b 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -45,7 +45,8 @@ class StageJedi(Model): @dataclass class StageJediCycle(Cycling, Model): - pass + base_name: str = "StageJedi" + scheduling_name: str = "StageJediCycle-{model}" @dataclass class GetBackground(Cycling, Model): diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 5ba5ef39f..f61a1405a 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -46,6 +46,9 @@ def __post_init__(self): if self.is_model: self.scheduling_name += f'-{self.model}' + elif self.is_model: + self.scheduling_name = self.scheduling_name.format(model = self.model) + if self.script is None: self.script = f'swell task {self.base_name} $config' @@ -79,7 +82,7 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): runtime_dict = {} if self.pre_script: - runtime_dict['pre_script'] = self.format_string_block(self.pre_script) + runtime_dict['pre-script'] = self.format_string_block(self.pre_script) if self.script: runtime_dict['script'] = self.format_string_block(self.script) From b69e974a399915213cabd80b0170ece3f071fc3f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 5 May 2025 11:49:28 -0400 Subject: [PATCH 009/299] Functional, fixing indentation --- src/swell/utilities/cylc_formatting.py | 19 +++++++++++ src/swell/utilities/cylc_runtime.py | 7 +++- src/swell/utilities/cylc_workflow.py | 47 ++++++++++++++++++++++++-- 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/src/swell/utilities/cylc_formatting.py b/src/swell/utilities/cylc_formatting.py index 5e2409d34..819579f79 100644 --- a/src/swell/utilities/cylc_formatting.py +++ b/src/swell/utilities/cylc_formatting.py @@ -15,6 +15,8 @@ indent = ' ' def format_dict(dictionary: Mapping): + # Convert a dictionary into a string + dict_str = '' for key, value in dictionary.items(): @@ -22,7 +24,11 @@ def format_dict(dictionary: Mapping): return dict_str +# -------------------------------------------------------------------------------------------------- + def indent_lines(string: str, level: int = 0, reset: bool = False): + # Reset line indentation for string, and indent lines by level + out_string = '' for line in string.split('\n'): @@ -36,7 +42,12 @@ def indent_lines(string: str, level: int = 0, reset: bool = False): return out_string +# -------------------------------------------------------------------------------------------------- + def format_section(section: Self, level: int = 0) -> str: + # Format a string to match cylc's section syntax + # format the header with the appropriate amount of enclosing brackets and indents + section_str = '' name = section.name @@ -53,7 +64,15 @@ def format_section(section: Self, level: int = 0) -> str: return section_str +# -------------------------------------------------------------------------------------------------- + class CylcSection(): + ''' + Holds the information contained in a section, including the name and contents, which can be a + string or dictionary. Also tracks child subsections, automatically handling indentation and syntax + at the time when the string is called. + ''' + def __init__(self, name: Optional[str] = None, content: Union[str, dict] = '') -> None: self.name = name self.content = content diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index f61a1405a..d96345468 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -19,6 +19,11 @@ @dataclass class Task: + + ''' + Contains the basic properties and information needed to format the cylc [runtime] section. + ''' + base_name: Optional[str] = None scheduling_name: Optional[str] = None @@ -87,7 +92,7 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): if self.script: runtime_dict['script'] = self.format_string_block(self.script) - if self.slurm: + if self.slurm is not None: runtime_dict['platform'] = platform if self.time_limit is True: diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index bbd239e4f..1c7c26a16 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -16,15 +16,28 @@ # -------------------------------------------------------------------------------------------------- class CylcWorkflow(): + + ''' + Handles generating the flow.cylc file contents using the CylcSection syntax for each + necessary section in the cylc file. Since Swell workflows share a lot of common language, + this method has the convenience of automatically setting a lot of the contents. This means + that the graph section is the only part that will need to be adjusted in many cases, + and tasks may need to be altered in src/swell/tasks/task_runtimes.py. + ''' + def __init__(self, experiment_dict, slurm_external) -> None: self.experiment_dict = experiment_dict self.slurm_external = slurm_external self.setup_workflow() + # -------------------------------------------------------------------------------------------------- + def set_experiment_dict(self, experiment_dict) -> None: self.experiment_dict = experiment_dict + # -------------------------------------------------------------------------------------------------- + def format_string_block(self, string) -> str: out_string = '"""\n' out_string += indent_lines(string, 1, True) @@ -32,10 +45,14 @@ def format_string_block(self, string) -> str: return out_string + # -------------------------------------------------------------------------------------------------- + def format_cycle(self, name: str, cycle: str) -> str: cycle_string = f'{name} = ' cycle_string += self.format_string_block(cycle) return cycle_string + + # -------------------------------------------------------------------------------------------------- def reset_indentation(self, string: str) -> str: out_string = '' @@ -51,6 +68,8 @@ def reset_indentation(self, string: str) -> str: out_string += line return out_string + + # -------------------------------------------------------------------------------------------------- def setup_workflow(self) -> None: self.header = self.define_header() @@ -64,7 +83,8 @@ def setup_workflow(self) -> None: self.tasks = self.parse_graph_for_tasks() print(self.tasks) - + + # -------------------------------------------------------------------------------------------------- def define_header(self) -> str: header = self.comment_block(string = """ @@ -80,6 +100,8 @@ def define_description(self) -> str: description = self.comment_block("""# Cylc workflow auto-generated for suite {suite_to_run} by Swell.""".format(**self.experiment_dict)) return description + # -------------------------------------------------------------------------------------------------- + def comment_block(self, string, level: int = 0, section_break: bool = True): out_string = '' @@ -98,12 +120,16 @@ def comment_block(self, string, level: int = 0, section_break: bool = True): return out_string + # -------------------------------------------------------------------------------------------------- + def define_scheduler(self) -> str: scheduler_dict = {'UTC mode': True, 'allow implicit tasks': False} scheduler = self.create_new_section('scheduler', scheduler_dict) return scheduler.get_section_str() + + # -------------------------------------------------------------------------------------------------- def define_scheduling(self) -> str: scheduling = self.define_scheduling_section() @@ -112,6 +138,8 @@ def define_scheduling(self) -> str: scheduling.add_subsection(graph) return scheduling.get_section_str() + + # -------------------------------------------------------------------------------------------------- def define_scheduling_section(self) -> CylcSection: scheduling_dict = {'initial cycle point': self.experiment_dict['start_cycle_point'], @@ -121,9 +149,13 @@ def define_scheduling_section(self) -> CylcSection: scheduling_section = self.create_new_section('scheduling', scheduling_dict) return scheduling_section - + + # -------------------------------------------------------------------------------------------------- + def define_graph_section(self) -> CylcSection: return self.create_new_section('graph') + + # -------------------------------------------------------------------------------------------------- def parse_graph_for_tasks(self) -> CylcSection: tasks = [] @@ -160,6 +192,8 @@ def parse_graph_for_tasks(self) -> CylcSection: in_graph = True return tasks + + # -------------------------------------------------------------------------------------------------- def get_independent_and_model_tasks(self) -> Tuple[list, dict]: ind_tasks = [] @@ -185,12 +219,18 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: ind_tasks.append(task) return ind_tasks, model_tasks + + # -------------------------------------------------------------------------------------------------- def define_runtime_task_overrides(self) -> dict: return {} + + # -------------------------------------------------------------------------------------------------- def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = ''): return CylcSection(name, content) + + # -------------------------------------------------------------------------------------------------- def define_runtime(self) -> str: runtime_section = self.create_new_section('runtime', '\n# Task defaults\n# -------------\n') @@ -223,6 +263,7 @@ def define_runtime(self) -> str: return runtime_str + # -------------------------------------------------------------------------------------------------- def get_workflow_str(self) -> str: @@ -237,3 +278,5 @@ def get_workflow_str(self) -> str: workflow_str += runtime return workflow_str + + # -------------------------------------------------------------------------------------------------- From 71370dd6533720dd99dff6e66401be0dea79a696 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 5 May 2025 12:36:02 -0400 Subject: [PATCH 010/299] pycodestyle fixes --- src/swell/deployment/create_experiment.py | 21 +++++- .../prepare_config_and_suite.py | 72 ++++++++++++------- src/swell/suites/3dvar/workflow.py | 23 ++++-- src/swell/suites/all_suites.py | 7 ++ src/swell/tasks/task_runtimes.py | 4 +- src/swell/utilities/cylc_formatting.py | 16 +++-- src/swell/utilities/cylc_runtime.py | 21 ++++-- src/swell/utilities/cylc_workflow.py | 51 ++++++------- src/swell/utilities/slurm.py | 6 +- src/swell/utilities/suite_utils.py | 3 +- src/swell/utilities/swell_questions.py | 2 + 11 files changed, 148 insertions(+), 78 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 1f3f01508..a94860937 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -92,21 +92,38 @@ def prepare_config( # --------------------------------------------------------------- prepare_config_and_suite = PrepareExperimentConfigAndSuite(logger, suite, suite_config, platform, method, override) - + + # Retrieved the answered suite questions + # -------------------------------------- suite_dict = prepare_config_and_suite.get_experiment_dict() + + # Get the slurm defaults from the user and platform + # ------------------------------------------------- slurm_dict = prepare_slurm_defaults_and_overrides(logger, platform, slurm) + # Initialize the workflow + # ----------------------- workflow = Workflows.get_workflow(suite)(suite_dict, slurm_dict) + # Get the list of tasks from the workflow's graph + # ----------------------------------------------- model_ind_tasks, model_dep_tasks = workflow.get_independent_and_model_tasks() + # Set the tasks to be used in preparing the suite + # ----------------------------------------------- prepare_config_and_suite.set_model_independent_tasks(model_ind_tasks) prepare_config_and_suite.set_model_dependent_tasks(model_dep_tasks) + # Ask the task questions + # ---------------------- experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() + # Update the workflow with the answered task questions + # ---------------------------------------------------- workflow.set_experiment_dict(experiment_dict) + # Finalize the workflow by adding the runtime section, and get the contents + # ------------------------------------------------------------------------- workflow_string = workflow.get_workflow_str() # Expand all environment vars in the dictionary @@ -151,7 +168,7 @@ def create_experiment_directory( # Call the experiment config and suite generation # ------------------------------------------------ experiment_dict_str, workflow_str = prepare_config(suite, suite_config, method, platform, - override, advanced, slurm) + override, advanced, slurm) # Load the string using yaml # -------------------------- diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index b01b4366f..dfd973a13 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -86,7 +86,8 @@ def __init__( # Add the datetime to the dictionary # ---------------------------------- - self.experiment_dict['datetime_created'] = datetime.datetime.today().strftime("%Y%m%d_%H%M%SZ") + self.experiment_dict['datetime_created'] = datetime.datetime.today().strftime( + "%Y%m%d_%H%M%SZ") self.comment_dict['datetime_created'] = 'Datetime this file was created (auto added)' # Add the platform the dictionary @@ -131,7 +132,7 @@ def set_model_dependent_tasks(self, model_task_dict: Mapping) -> None: def get_experiment_dict(self) -> Mapping: return self.experiment_dict - + # ---------------------------------------------------------------------------------------------- def get_workflow_file_str(self, slurm_dict): @@ -168,7 +169,9 @@ def prepare_suite_question_dictionary(self) -> None: question_models = question['models'] for model in question_models: - question_dictionary_model_dep = add_dict(question_dictionary_model_dep, {model: {question['question_name']: question}}) + question_dictionary_model_dep = add_dict(question_dictionary_model_dep, + {model: {question['question_name']: + question}}) self.suite_needs_model_components = True if 'model_components' not in question_dictionary_model_ind.keys(): @@ -194,11 +197,13 @@ def prepare_task_question_dictionary(self): elif question_model in self.experiment_dict['model_components']: model_dict[question_model] = question_dict - self.question_dictionary_model_dep = add_dict(self.question_dictionary_model_dep, model_dict) - + self.question_dictionary_model_dep = add_dict( + self.question_dictionary_model_dep, model_dict) + else: - self.question_dictionary_model_ind = add_dict(self.question_dictionary_model_ind, question_dict) - + self.question_dictionary_model_ind = add_dict( + self.question_dictionary_model_ind, question_dict) + for model, task_list in self.model_dependent_tasks.items(): for task in task_list: if task in task_questions.get_all(): @@ -207,10 +212,11 @@ def prepare_task_question_dictionary(self): for question in question_list: question_dict = {question['question_name']: question} if question['models'] is None: - self.question_dictionary_model_ind = add_dict(self.question_dictionary_model_ind, question_dict) + self.question_dictionary_model_ind = add_dict( + self.question_dictionary_model_ind, question_dict) elif model in question['models'] or 'all_models' in question['models']: - self.question_dictionary_model_dep = add_dict(self.question_dictionary_model_dep, {model: question_dict}) - + self.question_dictionary_model_dep = add_dict( + self.question_dictionary_model_dep, {model: question_dict}) def override_with_defaults(self, suite_task: QuestionType) -> None: @@ -231,7 +237,7 @@ def override_with_defaults(self, suite_task: QuestionType) -> None: for key, val in platform_defaults[question_name].items(): if key not in question.keys() or question[key] == 'defer_to_platform': question[key] = val - + # Perform a model override on the model_dep dictionary # ---------------------------------------------------- if self.suite_needs_model_components: @@ -239,9 +245,11 @@ def override_with_defaults(self, suite_task: QuestionType) -> None: # Open the suite and task default dictionaries model_defaults = {} - + model_dict_file = os.path.join(get_swell_path(), 'configuration', 'jedi', - 'interfaces', model, f'{suite_task.value}_questions.yaml') + 'interfaces', model, + f'{suite_task.value}_questions.yaml') + with open(model_dict_file, 'r') as ymlfile: model_defaults.update(yaml.safe_load(ymlfile)) @@ -254,14 +262,15 @@ def override_with_defaults(self, suite_task: QuestionType) -> None: # If the value of the question is still set as model-dependent, # set the value for that model if isinstance(val, Mapping) and \ - 'depends_on_model' in val.keys() and \ - model in val['depends_on_model'].keys() and \ - val['depends_on_model'][model] != 'defer_to_model': + 'depends_on_model' in val.keys() and \ + model in val['depends_on_model'].keys() and \ + val['depends_on_model'][model] != 'defer_to_model': model_dict[question_name][key] = val['depends_on_model'][model] elif key in model_defaults[question_name].keys() and ( val == 'defer_to_model' or val is None): - model_dict[question_name][key] = model_defaults[question_name][key] + model_dict[question_name][key] = model_defaults[ + question_name][key] if question_name in platform_defaults.keys(): for key, val in platform_defaults[question_name].items(): @@ -279,7 +288,8 @@ def override_with_defaults(self, suite_task: QuestionType) -> None: if question['options'] == 'defer_to_code': question['options'] = self.possible_model_components - if question_name == 'experiment_id' and question['default_value'] == 'defer_to_code': + if question_name == 'experiment_id' and question[ + 'default_value'] == 'defer_to_code': question['default_value'] = f'swell-{self.suite}' # ---------------------------------------------------------------------------------------------- @@ -320,33 +330,40 @@ def override_with_external(self, suite_task: QuestionType) -> None: if question['question_type'] == suite_task: if model in override_dict['models']: if question_name in override_dict['models'][model]: - question['default_value'] = override_dict['models'][model][question_name] + question['default_value'] = override_dict[ + 'models'][model][question_name] # ---------------------------------------------------------------------------------------------- - def get_questions_of_type(self, suite_task: QuestionType, question_dictionary: Mapping) -> Mapping: + def get_questions_of_type(self, + suite_task: QuestionType, + question_dictionary: Mapping + ) -> Mapping: out_dict = {} - + if 'models' in question_dictionary.keys(): for model in self.possible_model_components: if model in question_dictionary.keys(): - out_dict[model] = self.get_questions_of_type(suite_task, question_dictionary[model]) + out_dict[model] = self.get_questions_of_type( + suite_task, question_dictionary[model]) else: for question_name, question in question_dictionary.items(): if question['question_type'] == suite_task: out_dict[question['question_name']] = question - + return out_dict # ---------------------------------------------------------------------------------------------- def ask_questions_and_configure(self, suite_task: QuestionType) -> Tuple[dict, dict]: - if self.config_client.__class__.__name__ == 'GetAnswerCli' and suite_task == QuestionType.SUITE: + if self.config_client.__class__.__name__ == 'GetAnswerCli' and ( + suite_task == QuestionType.SUITE): self.logger.info("Please answer the following questions to configure your experiment ") - for question_name, question in self.get_questions_of_type(suite_task, self.question_dictionary_model_ind).items(): + for question_name, question in self.get_questions_of_type( + suite_task, self.question_dictionary_model_ind).items(): self.ask_a_question(self.question_dictionary_model_ind, question_name) if self.suite_needs_model_components: @@ -356,11 +373,12 @@ def ask_questions_and_configure(self, suite_task: QuestionType) -> Tuple[dict, d for model in self.experiment_dict['model_components']: model_dict = self.question_dictionary_model_dep[model] - for question_name, question in self.get_questions_of_type(suite_task, model_dict).items(): + for question_name, question in self.get_questions_of_type( + suite_task, model_dict).items(): self.ask_a_question(model_dict, question_name, model) # ---------------------------------------------------------------------------------------------- - + def ask_a_question( self, full_question_dictionary: dict, diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 3ada33b0b..ada273f19 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -11,6 +11,7 @@ # -------------------------------------------------------------------------------------------------- + class Workflow_3dvar(CylcWorkflow): def define_description(self): description = self.comment_block(""" @@ -18,7 +19,9 @@ def define_description(self): """) return description - + + # -------------------------------------------------------------------------------------------------- + def define_graph_section(self): graph_str = '' @@ -34,7 +37,7 @@ def define_graph_section(self): # If not able to link to build create the build BuildJediByLinking:fail? => BuildJedi """ - + for model_component in self.experiment_dict['model_components']: r1 += """ @@ -57,7 +60,9 @@ def define_graph_section(self): GetObservations-{model_component} # GenerateBClimatology, for ocean it is cycle dependent - GenerateBClimatologyByLinking-{model_component}:fail? => GenerateBClimatology-{model_component} + GenerateBClimatologyByLinking-{model_component}:fail? => + GenerateBClimatology-{model_component} + GetBackground-{model_component} => GenerateBClimatology-{model_component} # Perform staging that is cycle dependent @@ -68,7 +73,11 @@ def define_graph_section(self): StageJedi-{model_component}[^] => RunJediVariationalExecutable-{model_component} StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} - GenerateBClimatologyByLinking-{model_component}? | GenerateBClimatology-{model_component} => RunJediVariationalExecutable-{model_component} + + GenerateBClimatologyByLinking-{model_component}? | + GenerateBClimatology-{model_component} => + RunJediVariationalExecutable-{model_component} + GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} # EvaObservations @@ -85,13 +94,13 @@ def define_graph_section(self): # Clean up large files EvaObservations-{model_component} & SaveObsDiags-{model_component} => - CleanCycle-{model_component} + CleanCycle-{model_component} """ cycle_str = cycle_str.format(model_component=model_component) - + graph_str += self.format_cycle(cycle_time, cycle_str) return self.create_new_section('graph', graph_str) - + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/all_suites.py b/src/swell/suites/all_suites.py index 10c47bd71..17919e5e2 100644 --- a/src/swell/suites/all_suites.py +++ b/src/swell/suites/all_suites.py @@ -37,11 +37,13 @@ def base_suite(cls, config: str) -> str: # -------------------------------------------------------------------------------------------------- + def format_suite_name(suite_name): return suite_name[1:] if suite_name[0] == '_' else suite_name # -------------------------------------------------------------------------------------------------- + def construct_suite_config_enum(): # Automatically construct enum of all suite configs @@ -86,10 +88,13 @@ def wrapper(suite_config_enum): # -------------------------------------------------------------------------------------------------- + @classmethod def get_workflow(cls, name): return getattr(cls, name).value +# -------------------------------------------------------------------------------------------------- + def construct_workflow_enum(): # Automatically construct enum of all suite configs @@ -127,6 +132,7 @@ def wrapper(workflow_enum): # -------------------------------------------------------------------------------------------------- + @construct_suite_config_enum() class SuiteConfigs(Enum): pass @@ -134,6 +140,7 @@ class SuiteConfigs(Enum): # -------------------------------------------------------------------------------------------------- + @construct_workflow_enum() class Workflows(Enum): pass diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 34268919b..c5ab16a9e 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -17,6 +17,7 @@ # -------------------------------------------------------------------------------------------------- + class TaskRuntimes(): @dataclass @@ -24,8 +25,7 @@ class root(Task): script: bool = False pre_script: str = "source $CYLC_SUITE_DEF_PATH/modules" environment: dict = mutable_field({'datetime': '$CYLC_TASK_CYCLE_POINT', - 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'}) - + 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'}) @dataclass class CloneJedi(Task): diff --git a/src/swell/utilities/cylc_formatting.py b/src/swell/utilities/cylc_formatting.py index 819579f79..0dff2ab0b 100644 --- a/src/swell/utilities/cylc_formatting.py +++ b/src/swell/utilities/cylc_formatting.py @@ -14,6 +14,7 @@ indent = ' ' + def format_dict(dictionary: Mapping): # Convert a dictionary into a string @@ -26,6 +27,7 @@ def format_dict(dictionary: Mapping): # -------------------------------------------------------------------------------------------------- + def indent_lines(string: str, level: int = 0, reset: bool = False): # Reset line indentation for string, and indent lines by level @@ -39,11 +41,12 @@ def indent_lines(string: str, level: int = 0, reset: bool = False): line = f'{indent*level}{line}' out_string += f'{line}\n' - + return out_string # -------------------------------------------------------------------------------------------------- + def format_section(section: Self, level: int = 0) -> str: # Format a string to match cylc's section syntax # format the header with the appropriate amount of enclosing brackets and indents @@ -66,17 +69,18 @@ def format_section(section: Self, level: int = 0) -> str: # -------------------------------------------------------------------------------------------------- + class CylcSection(): - ''' + ''' Holds the information contained in a section, including the name and contents, which can be a - string or dictionary. Also tracks child subsections, automatically handling indentation and syntax - at the time when the string is called. + string or dictionary. Also tracks child subsections, automatically handling indentation + and syntax at the time when the string is retrieved. ''' def __init__(self, name: Optional[str] = None, content: Union[str, dict] = '') -> None: self.name = name self.content = content - + self.subsections = [] def __format_section__(self, section: Self, level: int = 0) -> str: @@ -98,7 +102,7 @@ def __format_section__(self, section: Self, level: int = 0) -> str: section_str += f'# {"-"*98}\n\n' return section_str - + def add_subsection(self, subsection: Self) -> None: self.subsections.append(subsection) diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index d96345468..e6ab0ef49 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -17,10 +17,11 @@ indent = ' ' + @dataclass class Task: - ''' + ''' Contains the basic properties and information needed to format the cylc [runtime] section. ''' @@ -52,7 +53,7 @@ def __post_init__(self): self.scheduling_name += f'-{self.model}' elif self.is_model: - self.scheduling_name = self.scheduling_name.format(model = self.model) + self.scheduling_name = self.scheduling_name.format(model=self.model) if self.script is None: self.script = f'swell task {self.base_name} $config' @@ -69,17 +70,20 @@ def format_string_block(self, string: str) -> str: out_string += '"""' return out_string - + def match_platform(self, content: Union[str, dict], platform: str): if isinstance(content, Mapping): if platform in content.keys(): content = content[platform] elif 'all' in content.keys(): content = content['all'] - + return content - def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = '') -> CylcSection: + def create_new_section(self, + name: Optional[str] = None, + content: Union[str, dict] = '' + ) -> CylcSection: return CylcSection(name, content) def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): @@ -137,19 +141,20 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): **slurm_globals, **slurm_dict, **slurm_task - } + } slurm_section_dict = {} for key, value in slurm_dict.items(): slurm_section_dict[f'--{key}'] = value directive_section = self.create_new_section('directives', slurm_section_dict) - + runtime_section.add_subsection(directive_section) return runtime_section # -------------------------------------------------------------------------------------------------- + @dataclass class Model(Task): def __post_init__(self): @@ -158,6 +163,7 @@ def __post_init__(self): # -------------------------------------------------------------------------------------------------- + @dataclass class Cycling(Task): def __post_init__(self): @@ -166,6 +172,7 @@ def __post_init__(self): # -------------------------------------------------------------------------------------------------- + @dataclass class Slurm(Task): def __post_init__(self): diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 1c7c26a16..9469e2057 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -15,13 +15,14 @@ # -------------------------------------------------------------------------------------------------- + class CylcWorkflow(): - ''' + ''' Handles generating the flow.cylc file contents using the CylcSection syntax for each necessary section in the cylc file. Since Swell workflows share a lot of common language, this method has the convenience of automatically setting a lot of the contents. This means - that the graph section is the only part that will need to be adjusted in many cases, + that the graph section is the only part that will need to be adjusted in many cases, and tasks may need to be altered in src/swell/tasks/task_runtimes.py. ''' @@ -53,7 +54,7 @@ def format_cycle(self, name: str, cycle: str) -> str: return cycle_string # -------------------------------------------------------------------------------------------------- - + def reset_indentation(self, string: str) -> str: out_string = '' @@ -70,7 +71,7 @@ def reset_indentation(self, string: str) -> str: return out_string # -------------------------------------------------------------------------------------------------- - + def setup_workflow(self) -> None: self.header = self.define_header() self.description = self.define_description() @@ -87,17 +88,19 @@ def setup_workflow(self) -> None: # -------------------------------------------------------------------------------------------------- def define_header(self) -> str: - header = self.comment_block(string = """ - # (C) Copyright 2021- United States Government as represented by the Administrator of the - # National Aeronautics and Space Administration. All Rights Reserved. - # - # This software is licensed under the terms of the Apache Licence Version 2.0 - # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.""") + header = self.comment_block(string=""" + # (C) Copyright 2021- United States Government as represented by the Administrator of the + # National Aeronautics and Space Administration. All Rights Reserved. + # + # This software is licensed under the terms of the Apache Licence Version 2.0 + # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.""") return header - + def define_description(self) -> str: - description = self.comment_block("""# Cylc workflow auto-generated for suite {suite_to_run} by Swell.""".format(**self.experiment_dict)) + description = self.comment_block( + """# Cylc workflow auto-generated for suite {suite_to_run} by Swell.""" + .format(**self.experiment_dict)) return description # -------------------------------------------------------------------------------------------------- @@ -122,7 +125,6 @@ def comment_block(self, string, level: int = 0, section_break: bool = True): # -------------------------------------------------------------------------------------------------- - def define_scheduler(self) -> str: scheduler_dict = {'UTC mode': True, 'allow implicit tasks': False} scheduler = self.create_new_section('scheduler', scheduler_dict) @@ -130,7 +132,7 @@ def define_scheduler(self) -> str: return scheduler.get_section_str() # -------------------------------------------------------------------------------------------------- - + def define_scheduling(self) -> str: scheduling = self.define_scheduling_section() graph = self.define_graph_section() @@ -140,12 +142,12 @@ def define_scheduling(self) -> str: return scheduling.get_section_str() # -------------------------------------------------------------------------------------------------- - + def define_scheduling_section(self) -> CylcSection: scheduling_dict = {'initial cycle point': self.experiment_dict['start_cycle_point'], 'final cycle point': self.experiment_dict['final_cycle_point'], 'runahead limit': self.experiment_dict['runahead_limit']} - + scheduling_section = self.create_new_section('scheduling', scheduling_dict) return scheduling_section @@ -156,12 +158,12 @@ def define_graph_section(self) -> CylcSection: return self.create_new_section('graph') # -------------------------------------------------------------------------------------------------- - + def parse_graph_for_tasks(self) -> CylcSection: tasks = [] cylc_characters = [':', '[', ']', '?'] - + in_graph = False in_cycle = False @@ -194,7 +196,7 @@ def parse_graph_for_tasks(self) -> CylcSection: return tasks # -------------------------------------------------------------------------------------------------- - + def get_independent_and_model_tasks(self) -> Tuple[list, dict]: ind_tasks = [] model_tasks = {} @@ -221,17 +223,17 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: return ind_tasks, model_tasks # -------------------------------------------------------------------------------------------------- - + def define_runtime_task_overrides(self) -> dict: return {} # -------------------------------------------------------------------------------------------------- - + def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = ''): return CylcSection(name, content) # -------------------------------------------------------------------------------------------------- - + def define_runtime(self) -> str: runtime_section = self.create_new_section('runtime', '\n# Task defaults\n# -------------\n') @@ -253,9 +255,10 @@ def define_runtime(self) -> str: else: task_name = task model = None - + task_class = TaskRuntimes.get(task_name) - task_section = task_class(model=model).get_section(self.experiment_dict, self.slurm_external) + task_section = task_class(model=model).get_section( + self.experiment_dict, self.slurm_external) runtime_section.add_subsection(task_section) diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 9250efac0..9477f8870 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -16,12 +16,13 @@ from swell.utilities.logger import Logger + def prepare_slurm_defaults_and_overrides( logger: Logger, platform: str, slurm_file: Optional[str], ) -> dict: - + # Obtain platform-specific SLURM directives and set them as global defaults # Start by constructing the full platforms path # ------------------------------------------- @@ -35,7 +36,7 @@ def prepare_slurm_defaults_and_overrides( raise Exception(f"Platform '{platform}' has not been configured in SWELL") except Exception as err: raise err - + global_defaults = {} global_defaults['slurm_directives_global'] = {} @@ -92,6 +93,7 @@ def prepare_slurm_defaults_and_overrides( validate_directives(slurm_dict["slurm_directives_tasks"][task]) return slurm_dict + def validate_directives(directive_dict: dict) -> None: directive_pattern = r'(?<=--)[a-zA-Z-]+' # Parse sbatch docs and extract all directives (e.g., `--account`) diff --git a/src/swell/utilities/suite_utils.py b/src/swell/utilities/suite_utils.py index 5762d5ad9..2337e6bba 100644 --- a/src/swell/utilities/suite_utils.py +++ b/src/swell/utilities/suite_utils.py @@ -17,6 +17,7 @@ # -------------------------------------------------------------------------------------------------- + def get_model_components() -> list: # Path to model interfaces @@ -27,6 +28,7 @@ def get_model_components() -> list: # -------------------------------------------------------------------------------------------------- + def get_suites() -> list: # Path to platforms @@ -40,4 +42,3 @@ def get_suites() -> list: return suites # -------------------------------------------------------------------------------------------------- - diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index 13c4ea7ed..022065710 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -18,12 +18,14 @@ # -------------------------------------------------------------------------------------------------- + class QuestionType(StrEnum): SUITE = 'suite' TASK = 'task' # -------------------------------------------------------------------------------------------------- + class WidgetType(Enum): STRING = "string" STRING_CHECK_LIST = "string-check-list" From d0d6a1ccb9e033b59319a45858d97c02872b9f73 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 6 May 2025 09:46:39 -0400 Subject: [PATCH 011/299] Fix slurm formatting in runtime --- src/swell/utilities/cylc_runtime.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index e6ab0ef49..8fa50cb96 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -87,24 +87,31 @@ def create_new_section(self, return CylcSection(name, content) def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): + ''' Return the runtime section for the given task. ''' + platform = experiment_dict['platform'] runtime_dict = {} + # Set the pre_script only if it is specified if self.pre_script: runtime_dict['pre-script'] = self.format_string_block(self.pre_script) + # Set the script if self.script: runtime_dict['script'] = self.format_string_block(self.script) + # Specify the platform if this is a slurm task if self.slurm is not None: runtime_dict['platform'] = platform + # Set the time limit, default is 1 hour if self.time_limit is True: runtime_dict['execution time limit'] = 'PT1H' elif self.time_limit: time_limit = self.match_platform(self.time_limit, platform) runtime_dict['execution time limit'] = time_limit + # Set the retry if this task needs it if self.retry is True: runtime_dict['execution retry delays'] = '2*PT1M' if self.retry: @@ -113,16 +120,17 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) + # Set the environment dictionary if self.environment is not None: environment_section = self.create_new_section('environment', self.environment) runtime_section.add_subsection(environment_section) + # Specify the slurm dictionary with defaults from user and global settings if self.slurm is not None: slurm_dict = {} + for key, value in self.slurm.items(): slurm_dict[key] = self.match_platform(value, platform) - else: - slurm_dict = {} slurm_globals = slurm_external['slurm_directives_global'] slurm_task = {} @@ -137,15 +145,18 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): else: slurm_task = {} + # Set values from globals, task specifications, and overrides, in order of priority slurm_dict = {'job-name': self.scheduling_name, **slurm_globals, **slurm_dict, **slurm_task } + # Format the dictionary for cylc slurm_section_dict = {} for key, value in slurm_dict.items(): slurm_section_dict[f'--{key}'] = value + directive_section = self.create_new_section('directives', slurm_section_dict) runtime_section.add_subsection(directive_section) @@ -178,7 +189,6 @@ class Slurm(Task): def __post_init__(self): if not self.slurm: self.slurm = {} - super().__post_init__() # -------------------------------------------------------------------------------------------------- From 65d97a386be9352c0c2197f583c0b6d9f7450cdc Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 6 May 2025 12:34:13 -0400 Subject: [PATCH 012/299] Simplify task runtime definitions --- src/swell/tasks/task_runtimes.py | 68 +++++++++++++++++------------ src/swell/utilities/cylc_runtime.py | 28 ------------ 2 files changed, 41 insertions(+), 55 deletions(-) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index c5ab16a9e..bf99e19dd 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -13,7 +13,7 @@ from dataclasses import dataclass from swell.utilities.dataclass_utils import mutable_field -from swell.utilities.cylc_runtime import Task, Model, Cycling, Slurm +from swell.utilities.cylc_runtime import Task # -------------------------------------------------------------------------------------------------- @@ -36,57 +36,76 @@ class BuildJediByLinking(Task): pass @dataclass - class BuildJedi(Slurm): + class BuildJedi(Task): time_limit: bool = True + slurm: dict = mutable_field({}) @dataclass - class StageJedi(Model): - pass + class StageJedi(Task): + is_model: bool = True @dataclass - class StageJediCycle(Cycling, Model): + class StageJediCycle(Task): + is_cycling: bool = True + is_model: bool = True base_name: str = "StageJedi" scheduling_name: str = "StageJediCycle-{model}" @dataclass - class GetBackground(Cycling, Model): - pass + class GetBackground(Task): + is_cycling: bool = True + is_model: bool = True @dataclass - class GetObservations(Cycling, Model): - pass + class GetObservations(Task): + is_cycling: bool = True + is_model: bool = True @dataclass - class GenerateBClimatology(Cycling, Model, Slurm): + class GenerateBClimatology(Task): time_limit: bool = True + is_cycling: bool = True + is_model: bool = True + slurm: dict = mutable_field({}) @dataclass - class GenerateBClimatologyByLinking(Cycling, Model): - pass + class GenerateBClimatologyByLinking(Task): + is_cycling: bool = True + is_model: bool = True @dataclass - class RunJediVariationalExecutable(Cycling, Model, Slurm): + class RunJediVariationalExecutable(Task): time_limit: bool = True + is_cycling: bool = True + is_model: bool = True + slurm: dict = mutable_field({}) @dataclass - class EvaJediLog(Cycling, Model): - pass + class EvaJediLog(Task): + is_cycling: bool = True + is_model: bool = True @dataclass - class EvaIncrement(Cycling, Model): - pass + class EvaIncrement(Task): + is_cycling: bool = True + is_model: bool = True @dataclass - class EvaObservations(Cycling, Model, Slurm): + class EvaObservations(Task): time_limit: bool = True + is_cycling: bool = True + is_model: bool = True + slurm: dict = mutable_field({}) @dataclass - class SaveObsDiags(Cycling, Model): - pass + class SaveObsDiags(Task): + is_cycling: bool = True + is_model: bool = True @dataclass - class CleanCycle(Cycling, Model): - pass + class CleanCycle(Task): + is_cycling: bool = True + is_model: bool = True @classmethod def get(cls, name: str) -> Task: @@ -94,8 +113,3 @@ def get(cls, name: str) -> Task: # -------------------------------------------------------------------------------------------------- - -if __name__ == "__main__": - print(TaskRuntimes.get('RunJediVariationalExecutable')()) - r = TaskRuntimes.get('root')() - print(r.pre_script) diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 8fa50cb96..5cadb93a2 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -164,31 +164,3 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): return runtime_section # -------------------------------------------------------------------------------------------------- - - -@dataclass -class Model(Task): - def __post_init__(self): - self.is_model = True - super().__post_init__() - -# -------------------------------------------------------------------------------------------------- - - -@dataclass -class Cycling(Task): - def __post_init__(self): - self.is_cycling = True - super().__post_init__() - -# -------------------------------------------------------------------------------------------------- - - -@dataclass -class Slurm(Task): - def __post_init__(self): - if not self.slurm: - self.slurm = {} - super().__post_init__() - -# -------------------------------------------------------------------------------------------------- From c1bb1fc66a1545333da8fbb1499b4d287f84f045 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 6 May 2025 12:38:13 -0400 Subject: [PATCH 013/299] remove print statement --- src/swell/utilities/cylc_workflow.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 9469e2057..d6618360a 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -83,7 +83,6 @@ def setup_workflow(self) -> None: self.scheduling = self.define_scheduling() self.tasks = self.parse_graph_for_tasks() - print(self.tasks) # -------------------------------------------------------------------------------------------------- From 0ec63162898cb8da893d5db5c6433c4659436257 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 6 May 2025 13:50:48 -0400 Subject: [PATCH 014/299] Fix slurm setting --- src/swell/utilities/cylc_runtime.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 5cadb93a2..f921d8353 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -112,10 +112,12 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): runtime_dict['execution time limit'] = time_limit # Set the retry if this task needs it - if self.retry is True: - runtime_dict['execution retry delays'] = '2*PT1M' if self.retry: - retry = self.match_platform(self.retry, platform) + if self.retry is True: + retry = '2*PT1M' + else: + retry = self.match_platform(self.retry, platform) + runtime_dict['execution retry delays'] = retry runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) From d584c5d538957294aba8a5414ef14ba610befd9d Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 6 May 2025 13:54:38 -0400 Subject: [PATCH 015/299] Remove duplicate import --- src/swell/deployment/create_experiment.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index a94860937..432488c89 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -16,7 +16,6 @@ import yaml from typing import Union, Optional -from swell.suites.all_suites import SuiteConfigs from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import \ PrepareExperimentConfigAndSuite from swell.swell_path import get_swell_path From 21828f4b09e49cfaa7e9009f039292f9180d3d3b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 6 May 2025 16:40:43 -0400 Subject: [PATCH 016/299] Remove setter methods --- src/swell/deployment/create_experiment.py | 6 +++--- .../prepare_config_and_suite.py | 10 ---------- src/swell/utilities/cylc_workflow.py | 5 ----- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 432488c89..1804f1423 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -110,8 +110,8 @@ def prepare_config( # Set the tasks to be used in preparing the suite # ----------------------------------------------- - prepare_config_and_suite.set_model_independent_tasks(model_ind_tasks) - prepare_config_and_suite.set_model_dependent_tasks(model_dep_tasks) + prepare_config_and_suite.model_independent_tasks = model_ind_tasks + prepare_config_and_suite.model_dependent_tasks = model_dep_tasks # Ask the task questions # ---------------------- @@ -119,7 +119,7 @@ def prepare_config( # Update the workflow with the answered task questions # ---------------------------------------------------- - workflow.set_experiment_dict(experiment_dict) + workflow.experiment_dict = experiment_dict # Finalize the workflow by adding the runtime section, and get the contents # ------------------------------------------------------------------------- diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index dfd973a13..bd061a390 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -120,16 +120,6 @@ def configure_and_ask_task_questions(self) -> None: # ---------------------------------------------------------------------------------------------- - def set_model_independent_tasks(self, tasks: list) -> None: - self.model_independent_tasks = tasks - - # ---------------------------------------------------------------------------------------------- - - def set_model_dependent_tasks(self, model_task_dict: Mapping) -> None: - self.model_dependent_tasks = model_task_dict - - # ---------------------------------------------------------------------------------------------- - def get_experiment_dict(self) -> Mapping: return self.experiment_dict diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index d6618360a..d9d5be094 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -34,11 +34,6 @@ def __init__(self, experiment_dict, slurm_external) -> None: # -------------------------------------------------------------------------------------------------- - def set_experiment_dict(self, experiment_dict) -> None: - self.experiment_dict = experiment_dict - - # -------------------------------------------------------------------------------------------------- - def format_string_block(self, string) -> str: out_string = '"""\n' out_string += indent_lines(string, 1, True) From d00d382b09059a2e35bde513cc16dcfcd9b809a5 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 6 May 2025 17:44:15 -0400 Subject: [PATCH 017/299] Simplify formatting --- src/swell/utilities/cylc_formatting.py | 39 ++++---------------------- src/swell/utilities/slurm.py | 9 ++++-- 2 files changed, 12 insertions(+), 36 deletions(-) diff --git a/src/swell/utilities/cylc_formatting.py b/src/swell/utilities/cylc_formatting.py index 0dff2ab0b..63b5406a0 100644 --- a/src/swell/utilities/cylc_formatting.py +++ b/src/swell/utilities/cylc_formatting.py @@ -9,11 +9,10 @@ from typing import Union, Optional, Self from collections.abc import Mapping +import textwrap # -------------------------------------------------------------------------------------------------- -indent = ' ' - def format_dict(dictionary: Mapping): # Convert a dictionary into a string @@ -31,18 +30,12 @@ def format_dict(dictionary: Mapping): def indent_lines(string: str, level: int = 0, reset: bool = False): # Reset line indentation for string, and indent lines by level - out_string = '' - - for line in string.split('\n'): - if reset: - line = line.strip() - - if len(line) > 0: - line = f'{indent*level}{line}' + if reset: + string = textwrap.dedent(string) - out_string += f'{line}\n' + string = textwrap.indent(string, ' '*level) + '\n' - return out_string + return string # -------------------------------------------------------------------------------------------------- @@ -55,7 +48,7 @@ def format_section(section: Self, level: int = 0) -> str: name = section.name if name is not None: - section_str += f'{indent*level}{(level+1)*"["}{name}{"]"*(level+1)}\n' + section_str += textwrap.indent(f'{(level+1)*"["}{name}{"]"*(level+1)}\n', ' '*level) else: level -= 1 @@ -83,26 +76,6 @@ def __init__(self, name: Optional[str] = None, content: Union[str, dict] = '') - self.subsections = [] - def __format_section__(self, section: Self, level: int = 0) -> str: - section_str = '' - - name = section.name - if name is not None: - section_str += f'{indent*level}{(level+1)*"["}{name}{"]"*(level+1)}\n' - else: - level -= 1 - - content = section.content - if isinstance(content, Mapping): - content = format_dict(content) - - section_str += indent_lines(content, level+1, False) - - if level == 0: - section_str += f'# {"-"*98}\n\n' - - return section_str - def add_subsection(self, subsection: Self) -> None: self.subsections.append(subsection) diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 9477f8870..357291edd 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -58,9 +58,12 @@ def prepare_slurm_defaults_and_overrides( # ---------------------------------- if slurm_file is not None: logger.info(f"Reading SLURM directives from {slurm_file}.") - assert os.path.exists(slurm_file) - with open(slurm_file, "r") as slurmfile: - slurm_overrides = yaml.safe_load(slurmfile) + try: + with open(slurm_file, "r") as slurmfile: + slurm_overrides = yaml.safe_load(slurmfile) + except FileNotFoundError as err: + raise FileNotFoundError(f"Slurm config {slurm_file} not found.") + # Ensure that SLURM dict is _only_ used for SLURM directives. slurm_invalid_keys = set(slurm_overrides.keys()).difference({ "slurm_directives_global", From b699e877a325869b95eeaf28b95be889f3c6899d Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 7 May 2025 12:13:06 -0400 Subject: [PATCH 018/299] Simplify workflows and suite configs --- src/swell/deployment/create_experiment.py | 6 +- .../prepare_config_and_suite.py | 4 +- src/swell/suites/all_suites.py | 133 ++++++------------ src/swell/swell.py | 4 +- src/swell/utilities/config.py | 4 +- 5 files changed, 55 insertions(+), 96 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 1804f1423..1908cde60 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -23,7 +23,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.logger import Logger, get_logger from swell.utilities.slurm import prepare_slurm_defaults_and_overrides -from swell.suites.all_suites import SuiteConfigs, Workflows +from swell.suites.all_suites import suite_configs, workflows # -------------------------------------------------------------------------------------------------- @@ -102,7 +102,7 @@ def prepare_config( # Initialize the workflow # ----------------------- - workflow = Workflows.get_workflow(suite)(suite_dict, slurm_dict) + workflow = workflows.get_workflow(suite)(suite_dict, slurm_dict) # Get the list of tasks from the workflow's graph # ----------------------------------------------- @@ -158,7 +158,7 @@ def create_experiment_directory( # Get the base name of the suite # ------------------------------ - suite = SuiteConfigs.base_suite(suite_config) + suite = suite_configs.base_suite(suite_config) # Create a logger # --------------- diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index bd061a390..60ea7b7f5 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -25,7 +25,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.dictionary import update_dict, add_dict from swell.tasks.task_questions import TaskQuestions as task_questions -from swell.suites.all_suites import SuiteConfigs, Workflows +from swell.suites.all_suites import suite_configs from swell.utilities.swell_questions import QuestionType @@ -140,7 +140,7 @@ def prepare_suite_question_dictionary(self) -> None: question_dictionary_model_ind = {} question_dictionary_model_dep = {} - suite_config_obj = SuiteConfigs.get_config(self.suite_config) + suite_config_obj = suite_configs.get_config(self.suite_config) suite_question_list = suite_config_obj.expand_question_list() for model in self.possible_model_components: diff --git a/src/swell/suites/all_suites.py b/src/swell/suites/all_suites.py index 17919e5e2..483a37bfd 100644 --- a/src/swell/suites/all_suites.py +++ b/src/swell/suites/all_suites.py @@ -15,134 +15,93 @@ from swell.swell_path import get_swell_path from swell.utilities.suite_utils import get_suites from swell.suites.suite_questions import SuiteQuestions - -# -------------------------------------------------------------------------------------------------- - - -# Class methods for SuiteConfigs enum - -@classmethod -def get(cls, name): - return getattr(cls, name).value.value - - -@classmethod -def config_names(cls): - return cls._member_names_ - - -@classmethod -def base_suite(cls, config: str) -> str: - return cls.__config_suite_map__[config] +from swell.utilities.cylc_workflow import CylcWorkflow +from swell.utilities.swell_questions import QuestionList # -------------------------------------------------------------------------------------------------- def format_suite_name(suite_name): + # Format suite names starting with a digit return suite_name[1:] if suite_name[0] == '_' else suite_name # -------------------------------------------------------------------------------------------------- -def construct_suite_config_enum(): - # Automatically construct enum of all suite configs +class Workflows(): + # Maps suites to workflow objects - def wrapper(suite_config_enum): - # Dictionary used to create the enum - enum_dict = {} - # Map of config names to their parent suites - config_suite_map = {} + def __init__(self) -> None: + workflow_dict = {} - # Find all of the suite configs for suite in get_suites(): - config_path = os.path.join(get_swell_path(), 'suites', suite, 'suite_config.py') - if os.path.exists(config_path): - suite_container = getattr( - import_module(f'swell.suites.{suite}.suite_config'), 'SuiteConfig') - suite_configs = suite_container.get_all() - - for config in suite_configs: - enum_dict[format_suite_name(config)] = getattr(suite_container, config) - config_suite_map[format_suite_name(config)] = suite - else: - enum_dict[suite] = SuiteQuestions.all_suites.value - config_suite_map[suite] = suite + workflow_path = os.path.join(get_swell_path(), 'suites', suite, 'workflow.py') + if os.path.exists(workflow_path): + workflow = getattr( + import_module(f'swell.suites.{suite}.workflow'), f'Workflow_{suite}') - # Set the map dictionary to a hidden attribute - enum_dict['__config_suite_map__'] = config_suite_map + workflow_dict[suite] = workflow - # Override with manually specified keys in enum - for item in suite_config_enum: - enum_dict[item.name] = item.value + self.workflow_dict = workflow_dict - # Build the enum - enum_cls = Enum(suite_config_enum.__name__, enum_dict) + def get_workflow(self, suite: str) -> CylcWorkflow: + return self.workflow_dict[suite] - # Set classmethods for the enum - setattr(enum_cls, 'get_config', get) - setattr(enum_cls, 'config_names', config_names) - setattr(enum_cls, 'base_suite', base_suite) + def all_workflows(self) -> list: + return self.workflow_dict.keys() - return enum_cls - return wrapper # -------------------------------------------------------------------------------------------------- -@classmethod -def get_workflow(cls, name): - return getattr(cls, name).value +class SuiteConfigs(): + # Maps suite configuration objects -# -------------------------------------------------------------------------------------------------- + def __init__(self) -> None: - -def construct_workflow_enum(): - # Automatically construct enum of all suite configs - - def wrapper(workflow_enum): # Dictionary used to create the enum - enum_dict = {} + config_dict = {} # Map of config names to their parent suites - config_suite_map = {} + config_map = {} # Find all of the suite configs for suite in get_suites(): - config_path = os.path.join(get_swell_path(), 'suites', suite, 'workflow.py') + config_path = os.path.join(get_swell_path(), 'suites', suite, 'suite_config.py') if os.path.exists(config_path): - workflow = getattr( - import_module(f'swell.suites.{suite}.workflow'), f'Workflow_{suite}') - - enum_dict[suite] = workflow - - # Set the map dictionary to a hidden attribute - enum_dict['__config_suite_map__'] = config_suite_map + suite_container = getattr( + import_module(f'swell.suites.{suite}.suite_config'), 'SuiteConfig') + suite_configs = suite_container.get_all() - # Override with manually specified keys in enum - for item in workflow_enum: - enum_dict[item.name] = item.value + for config in suite_configs: + config_dict[format_suite_name(config)] = getattr(suite_container, config) + config_map[format_suite_name(config)] = suite + else: + config_dict[suite] = SuiteQuestions.all_suites.value + config_map[suite] = suite - # Build the enum - enum_cls = Enum(workflow_enum.__name__, enum_dict) + self.config_dict = config_dict + self.__config_map__ = config_map - # Set classmethods - setattr(enum_cls, 'get_workflow', get_workflow) + # -------------------------------------------------------------------------------------------------- - return enum_cls - return wrapper + def get_config(self, name: str) -> QuestionList: + return self.config_dict[name].value -# -------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------- + def all_configs(self) -> list: + return self.config_dict.keys() -@construct_suite_config_enum() -class SuiteConfigs(Enum): - pass + # -------------------------------------------------------------------------------------------------- + def base_suite(self, config: str) -> str: + return self.__config_map__[config] # -------------------------------------------------------------------------------------------------- -@construct_workflow_enum() -class Workflows(Enum): - pass +# Objects to reference in imports +suite_configs = SuiteConfigs() +workflows = Workflows() # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index 2d8da6e85..070e56867 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import task_wrapper, get_tasks from swell.test.test_driver import test_wrapper, valid_tests from swell.test.suite_tests.suite_tests import run_suite, TestSuite -from swell.suites.all_suites import SuiteConfigs +from swell.suites.all_suites import suite_configs from swell.utilities.welcome_message import write_welcome_message from swell.utilities.scripts.utility_driver import get_utilities, utility_wrapper @@ -87,7 +87,7 @@ def swell_driver() -> None: @swell_driver.command() -@click.argument('suite', type=click.Choice(SuiteConfigs.config_names())) +@click.argument('suite', type=click.Choice(suite_configs.all_configs())) @click.option('-m', '--input_method', 'input_method', default='defaults', type=click.Choice(['defaults', 'cli']), help=input_method_help) @click.option('-p', '--platform', 'platform', default='nccs_discover_sles15', diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 663709753..75c7172a2 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -12,7 +12,7 @@ from swell.tasks.task_questions import TaskQuestions as task_questions from swell.utilities.logger import Logger -from swell.suites.all_suites import SuiteConfigs +from swell.suites.all_suites import suite_configs # -------------------------------------------------------------------------------------------------- @@ -103,7 +103,7 @@ def __init__(self, input_file: str, logger: Logger, task_name: str, model: str) # ------------------------------------------------------------------------- # Check for suite questions - suite_questions = SuiteConfigs.get_config( + suite_questions = suite_configs.get_config( self.__suite_to_run__).get_all_question_names('suite') question_list = [] From 5be480c2a4137559bc11c406209a44279ebd555f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 9 Jun 2025 17:24:49 -0400 Subject: [PATCH 019/299] Clean up and introduce question order test --- src/swell/deployment/create_experiment.py | 1 - .../prepare_config_and_suite.py | 13 ---- src/swell/tasks/task_runtimes.py | 1 - src/swell/test/code_tests/code_tests.py | 4 + .../test/code_tests/question_order_test.py | 25 ++++++ src/swell/utilities/cylc_formatting.py | 6 +- src/swell/utilities/cylc_runtime.py | 12 ++- src/swell/utilities/cylc_workflow.py | 4 +- src/swell/utilities/question_defaults.py | 24 +++--- .../utilities/scripts/check_question_order.py | 77 +++++++++++++++++++ src/swell/utilities/suite_utils.py | 3 +- 11 files changed, 137 insertions(+), 33 deletions(-) create mode 100644 src/swell/test/code_tests/question_order_test.py create mode 100644 src/swell/utilities/scripts/check_question_order.py diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 1908cde60..592107401 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -9,7 +9,6 @@ import copy -import datetime import os import shutil import sys diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 60ea7b7f5..c3ecbf144 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -8,12 +8,10 @@ # -------------------------------------------------------------------------------------------------- -import copy import os import yaml from collections.abc import Mapping from typing import Union, Tuple, Optional -from enum import StrEnum import datetime from swell.swell_path import get_swell_path @@ -22,7 +20,6 @@ from swell.deployment.prepare_config_and_suite.question_and_answer_defaults import GetAnswerDefaults from swell.utilities.dictionary import dict_get from swell.utilities.logger import Logger -from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.dictionary import update_dict, add_dict from swell.tasks.task_questions import TaskQuestions as task_questions from swell.suites.all_suites import suite_configs @@ -125,16 +122,6 @@ def get_experiment_dict(self) -> Mapping: # ---------------------------------------------------------------------------------------------- - def get_workflow_file_str(self, slurm_dict): - self.workflow.set_experiment_dict(self.experiment_dict, slurm_dict) - self.workflow.set_runtime_str() - - self.workflow_str = self.workflow.get_workflow_str() - - return self.workflow_str - - # ---------------------------------------------------------------------------------------------- - def prepare_suite_question_dictionary(self) -> None: question_dictionary_model_ind = {} diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index bf99e19dd..593b4256b 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -111,5 +111,4 @@ class CleanCycle(Task): def get(cls, name: str) -> Task: return getattr(cls, name) - # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/test/code_tests/code_tests.py b/src/swell/test/code_tests/code_tests.py index ff3e4e4c2..c9ac63473 100644 --- a/src/swell/test/code_tests/code_tests.py +++ b/src/swell/test/code_tests/code_tests.py @@ -17,6 +17,7 @@ from swell.test.code_tests.unused_variables_test import UnusedVariablesTest from swell.test.code_tests.question_dictionary_comparison_test import QuestionDictionaryTest from swell.test.code_tests.test_generate_observing_system import GenerateObservingSystemTest +from swell.test.code_tests.question_order_test import QuestionOrderTest # -------------------------------------------------------------------------------------------------- @@ -43,6 +44,9 @@ def code_tests() -> None: # Load tests from UnusedVariablesTest test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(QuestionDictionaryTest)) + # Load question order test + test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(QuestionOrderTest)) + # Load SLURM tests test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(SLURMConfigTest)) diff --git a/src/swell/test/code_tests/question_order_test.py b/src/swell/test/code_tests/question_order_test.py new file mode 100644 index 000000000..66ffee8c3 --- /dev/null +++ b/src/swell/test/code_tests/question_order_test.py @@ -0,0 +1,25 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import unittest + +from swell.utilities.scripts.check_question_order import check_question_order + + +# -------------------------------------------------------------------------------------------------- + + +class QuestionOrderTest(unittest.TestCase): + + def test_question_order(self): + + check_question_order() + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_formatting.py b/src/swell/utilities/cylc_formatting.py index 63b5406a0..c86e3c1be 100644 --- a/src/swell/utilities/cylc_formatting.py +++ b/src/swell/utilities/cylc_formatting.py @@ -11,6 +11,8 @@ from collections.abc import Mapping import textwrap +INDENT = ' ' * 4 + # -------------------------------------------------------------------------------------------------- @@ -33,7 +35,7 @@ def indent_lines(string: str, level: int = 0, reset: bool = False): if reset: string = textwrap.dedent(string) - string = textwrap.indent(string, ' '*level) + '\n' + string = textwrap.indent(string, INDENT*level) + '\n' return string @@ -48,7 +50,7 @@ def format_section(section: Self, level: int = 0) -> str: name = section.name if name is not None: - section_str += textwrap.indent(f'{(level+1)*"["}{name}{"]"*(level+1)}\n', ' '*level) + section_str += textwrap.indent(f'{(level+1)*"["}{name}{"]"*(level+1)}\n', INDENT*level) else: level -= 1 diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index f921d8353..824512c51 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -15,8 +15,6 @@ # -------------------------------------------------------------------------------------------------- -indent = ' ' - @dataclass class Task: @@ -41,6 +39,8 @@ class Task: is_cycling: bool = False is_model: bool = False + # -------------------------------------------------------------------------------------------------- + def __post_init__(self): if self.base_name is None: @@ -64,6 +64,8 @@ def __post_init__(self): if self.is_model: self.script += f' -m {self.model}' + # -------------------------------------------------------------------------------------------------- + def format_string_block(self, string: str) -> str: out_string = '"""\n' out_string += indent_lines(string, 1) @@ -71,6 +73,8 @@ def format_string_block(self, string: str) -> str: return out_string + # -------------------------------------------------------------------------------------------------- + def match_platform(self, content: Union[str, dict], platform: str): if isinstance(content, Mapping): if platform in content.keys(): @@ -80,12 +84,16 @@ def match_platform(self, content: Union[str, dict], platform: str): return content + # -------------------------------------------------------------------------------------------------- + def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = '' ) -> CylcSection: return CylcSection(name, content) + # -------------------------------------------------------------------------------------------------- + def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): ''' Return the runtime section for the given task. ''' diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index d9d5be094..3b90458eb 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -91,6 +91,8 @@ def define_header(self) -> str: return header + # -------------------------------------------------------------------------------------------------- + def define_description(self) -> str: description = self.comment_block( """# Cylc workflow auto-generated for suite {suite_to_run} by Swell.""" @@ -168,7 +170,7 @@ def parse_graph_for_tasks(self) -> CylcSection: for sub_string in sub_strings: sub_string = sub_string.strip() - if '#' in sub_string: + if sub_string.startswith('#'): comment = True if not comment and in_graph and in_cycle: diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index af24ff3b9..a9e7c1595 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -922,6 +922,18 @@ class obs_provider(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class obs_thinning_rej_fraction(TaskQuestion): + default_value: float = 0.75 + question_name: str = "obs_thinning_rej_fraction" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the rejection fraction for obs thinning?" + widget_type: WType = WType.FLOAT + + # -------------------------------------------------------------------------------------------------- + @dataclass class observations(TaskQuestion): default_value: str = "defer_to_model" @@ -972,18 +984,6 @@ class observing_system_records_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- - @dataclass - class obs_thinning_rej_fraction(TaskQuestion): - default_value: float = 0.75 - question_name: str = "obs_thinning_rej_fraction" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the rejection fraction for obs thinning?" - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - @dataclass class path_to_ensemble(TaskQuestion): default_value: str = "defer_to_model" diff --git a/src/swell/utilities/scripts/check_question_order.py b/src/swell/utilities/scripts/check_question_order.py new file mode 100644 index 000000000..d36ae1e78 --- /dev/null +++ b/src/swell/utilities/scripts/check_question_order.py @@ -0,0 +1,77 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +from typing import Literal + +from swell.swell_path import get_swell_path + +# -------------------------------------------------------------------------------------------------- + + +def check_question_order(): + question_file = os.path.join(get_swell_path(), 'utilities', 'question_defaults.py') + + with open(question_file, 'r') as f: + lines = f.readlines() + + in_task_section = False + + task_questions = [] + suite_questions = [] + + for line in lines: + if 'class ' in line: + if '(SuiteQuestion)' in line: + if in_task_section: + raise Exception('Suite and Task questions are mixed up, please ensure ' + 'that suite questions come first, then task questions.') + + question = line.split('class ')[1].split('(SuiteQuestion)')[0].strip() + suite_questions.append(question) + elif '(TaskQuestion)' in line: + in_task_section = True + question = line.split('class ')[1].split('(TaskQuestion)')[0].strip() + + task_questions.append(question) + + check_order('task', task_questions) + check_order('suite', suite_questions) + +# -------------------------------------------------------------------------------------------------- + + +def check_order(qtype: Literal['task', 'suite'], check_list: list): + in_order = True + + sorted_list = sorted(check_list) + + max_chars = max([len(item) for item in sorted_list]) + 4 + + order_str = 'Order in file: ' + order_str += '{tabs}> Should be:\n'.format(tabs='-'*(max_chars-len(order_str))) + + for i, sorted_item in enumerate(sorted_list): + check_item = check_list[i] + + tab_char = ' ' + if check_item != sorted_item: + in_order = False + tab_char = '-' + + tabs = tab_char*((max_chars-len(check_item))) + + order_str += f'{check_item} {tabs}> {sorted_item}\n' + + if not in_order: + raise Exception(f'\nQuestions in the {qtype} section are not in order, ' + f'please rearrange according to the following:\n\n{order_str}') + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/suite_utils.py b/src/swell/utilities/suite_utils.py index 2337e6bba..c5daa9db8 100644 --- a/src/swell/utilities/suite_utils.py +++ b/src/swell/utilities/suite_utils.py @@ -37,7 +37,8 @@ def get_suites() -> list: # List of base suites suites = sorted([sdir for sdir in os.listdir(suites_directory) if (os.path.isdir(os.path.join(suites_directory, sdir)) - and os.path.exists(os.path.join(suites_directory, sdir, 'flow.cylc')))]) + and os.path.exists( + os.path.join(suites_directory, sdir, 'suite_config.py')))]) return suites From 7a888643b09302ade93bf7686348dc76aacabea4 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 10 Jun 2025 11:06:18 -0400 Subject: [PATCH 020/299] Initial commit --- src/swell/deployment/create_experiment.py | 1 + .../prepare_config_and_suite.py | 6 +- .../compare_variational/suite_config.py | 33 ++++++ .../suites/compare_variational/workflow.py | 109 ++++++++++++++++++ src/swell/swell.py | 46 +++++++- src/swell/tasks/base/task_base.py | 29 ++++- src/swell/tasks/f_grep_residual_norm.py | 57 +++++++++ src/swell/utilities/cylc_workflow.py | 13 +-- src/swell/utilities/question_defaults.py | 10 ++ src/swell/utilities/suite_utils.py | 3 +- 10 files changed, 290 insertions(+), 17 deletions(-) create mode 100644 src/swell/suites/compare_variational/suite_config.py create mode 100644 src/swell/suites/compare_variational/workflow.py create mode 100644 src/swell/tasks/f_grep_residual_norm.py diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 1908cde60..40812f668 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -119,6 +119,7 @@ def prepare_config( # Update the workflow with the answered task questions # ---------------------------------------------------- + workflow.experiment_dict = experiment_dict # Finalize the workflow by adding the runtime section, and get the contents diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 60ea7b7f5..da76713d1 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -170,6 +170,8 @@ def prepare_suite_question_dictionary(self) -> None: self.question_dictionary_model_ind = question_dictionary_model_ind self.question_dictionary_model_dep = question_dictionary_model_dep + # ---------------------------------------------------------------------------------------------- + def prepare_task_question_dictionary(self): for task in self.model_independent_tasks: if task in task_questions.get_all(): @@ -208,6 +210,8 @@ def prepare_task_question_dictionary(self): self.question_dictionary_model_dep = add_dict( self.question_dictionary_model_dep, {model: question_dict}) + # ---------------------------------------------------------------------------------------------- + def override_with_defaults(self, suite_task: QuestionType) -> None: # Perform a platform override on the model_ind dictionary @@ -293,7 +297,7 @@ def override_with_external(self, suite_task: QuestionType) -> None: override_dict = {} if isinstance(self.override, Mapping): - override_dict.update_dict(override_dict, self.override) + override_dict = update_dict(override_dict, self.override) elif isinstance(self.override, str): with open(self.override, 'r') as ymlfile: diff --git a/src/swell/suites/compare_variational/suite_config.py b/src/swell/suites/compare_variational/suite_config.py new file mode 100644 index 000000000..d6025b7f8 --- /dev/null +++ b/src/swell/suites/compare_variational/suite_config.py @@ -0,0 +1,33 @@ +# -------------------------------------------------------------------------------------------------- +# @package configuration +# +# Class containing the configuration. This is a dictionary that is converted from +# an input yaml configuration file. Various function are included for interacting with the +# dictionary. +# +# -------------------------------------------------------------------------------------------------- + + +from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.suites.suite_questions import SuiteQuestions as sq + +from enum import Enum + + +# -------------------------------------------------------------------------------------------------- + +class SuiteConfig(QuestionContainer, Enum): + + # -------------------------------------------------------------------------------------------------- + + compare_variational = QuestionList( + list_name="compare", + questions=[ + sq.all_suites, + qd.runahead_limit(), + qd.comparison_experiment_paths(), + ] + ) + + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py new file mode 100644 index 000000000..c39197d2a --- /dev/null +++ b/src/swell/suites/compare_variational/workflow.py @@ -0,0 +1,109 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import yaml +import os + +from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_compare_variational(CylcWorkflow): + + # -------------------------------------------------------------------------------------------------- + + def define_description(self): + description = self.comment_block(""" + # Cylc suite for running comparison tests on completed experiments + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_scheduling(self): + + paths = self.experiment_dict['comparison_experiment_paths'] + + if len(paths) > 0: + config_file = os.path.join(os.path.dirname(paths[0]), 'experiment.yaml') + with open(config_file, 'r') as f: + base_dict = yaml.safe_load(f) + + start_cycle_point = base_dict['start_cycle_point'] + final_cycle_point = base_dict['final_cycle_point'] + + cycle_model_times = {} + + for model in base_dict['models'].keys(): + cycle_times = base_dict['models'][model]['cycle_times'] + for cycle_time in cycle_times: + if cycle_time not in cycle_model_times: + cycle_model_times[cycle_time] = [] + cycle_model_times[cycle_time].append(model) + else: + raise Exception('No experiments have been specified') + + scheduling_section = self.create_new_section('scheduling', + {'initial cycle point': start_cycle_point, + 'final cycle point': final_cycle_point}) + + graph_str = '' + + for cycle_time, models in cycle_model_times.items(): + cycle_str = '' + + for model in models: + for i in range(len(paths)): + cycle_str += f"FGrepResidualNorm-{model}-{i}\n" + + graph_str += self.format_cycle(cycle_time, cycle_str) + + graph_section = self.create_new_section('graph', graph_str) + + scheduling_section.add_subsection(graph_section) + + return scheduling_section.get_section_str() + + # -------------------------------------------------------------------------------------------------- + + def define_runtime(self): + runtime_section = self.create_new_section('runtime', '\n# Task defaults\n# -------------\n') + + task_class = TaskRuntimes.get('root') + task_section = task_class().get_section( + self.experiment_dict, self.slurm_external) + + runtime_section.add_subsection(task_section) + + paths = self.experiment_dict['comparison_experiment_paths'] + + exp_root = os.path.expandvars(self.experiment_dict['experiment_root']) + exp_id = self.experiment_dict['experiment_id'] + + output_dir = os.path.join(exp_root, exp_id, 'comparison_tests') + + for i, path in enumerate(paths): + config_file = os.path.join(os.path.dirname(path), 'experiment.yaml') + with open(config_file, 'r') as f: + exp_dict = yaml.safe_load(f) + + for model in exp_dict['model_components']: + task_str = (f'script = """swell task FGrepResidualNorm {config_file} -d $datetime' + ' -m {model} -i {i} -o {output_dir}"""') + + task_section = self.create_new_section(f'FGrepResidualNorm-{model}-{i}', task_str) + runtime_section.add_subsection(task_section) + + return runtime_section.get_section_str() + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index 070e56867..9a705e69d 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -9,7 +9,8 @@ import click -from typing import Union, Optional, Literal +from collections.abc import Mapping +from typing import Union, Optional, Literal, Tuple from swell.deployment.platforms.platforms import get_platforms from swell.deployment.create_experiment import clone_config, create_experiment_directory @@ -18,6 +19,7 @@ from swell.test.test_driver import test_wrapper, valid_tests from swell.test.suite_tests.suite_tests import run_suite, TestSuite from swell.suites.all_suites import suite_configs +from swell.utilities.dictionary import update_dict from swell.utilities.welcome_message import write_welcome_message from swell.utilities.scripts.utility_driver import get_utilities, utility_wrapper @@ -82,6 +84,13 @@ def swell_driver() -> None: or for task-model combinations. """ +test_iteration_help = """ +(For diagnostic tasks only) - Set the number that is associated with the particular experiment +that is being compared. """ + +test_output_help = """ +(For diagnostic tasks only) - Define the output directory that diagnostics tests will send +their results. Used in comparison suites. """ # -------------------------------------------------------------------------------------------------- @@ -184,12 +193,16 @@ def launch( @click.option('-d', '--datetime', 'datetime', default=None, help=datetime_help) @click.option('-m', '--model', 'model', default=None, help=model_help) @click.option('-p', '--ensemblePacket', 'ensemblePacket', default=None, help=ensemble_help) +@click.option('-i', '--testIteration', 'testIteration', default=None, help=test_iteration_help) +@click.option('-o', '--testOutput', 'testOutput', default=None, help=test_output_help) def task( task: str, config: str, datetime: Optional[str], model: Optional[str], - ensemblePacket: Optional[str] + ensemblePacket: Optional[str], + testIteration: Optional[int], + testOutput: Optional[str] ) -> None: """ Run a workflow task @@ -201,7 +214,7 @@ def task( config (str): Path to the configuration file for the task.\n """ - task_wrapper(task, config, datetime, model, ensemblePacket) + task_wrapper(task, config, datetime, model, ensemblePacket, testIteration, testOutput) # -------------------------------------------------------------------------------------------------- @@ -224,6 +237,33 @@ def utility(utility: str) -> None: # -------------------------------------------------------------------------------------------------- +@swell_driver.command() +@click.argument('comparison_type', type=click.Choice(['variational'])) +@click.argument('experiments', type=click.Path(), nargs=-1) +@click.option('-p', '--platform', 'platform', default='nccs_discover_sles15', + type=click.Choice(get_platforms()), help=platform_help) +@click.option('-o', '--override', 'override', default=None, help=override_help) +@click.option('-a', '--advanced', 'advanced', default=False, help=advanced_help) +@click.option('-s', '--slurm', 'slurm', default=None, help=slurm_help) +def compare(comparison_type, experiments: Tuple[str], platform, override, advanced, slurm) -> None: + if override is not None: + if isinstance(override, str): + with open(override, 'r') as f: + override = yaml.safe_load(f) + else: + override = {} + + if 'comparison_experiment_paths' in override.keys(): + override['comparison_experiment_paths'] = ( + override['comparison_experiment_paths'].extend(list(experiments))) + else: + override['comparison_experiment_paths'] = list(experiments) + + create_experiment_directory(f'compare_{comparison_type}', 'defaults', + platform, override, advanced, slurm) + + +# -------------------------------------------------------------------------------------------------- @swell_driver.command() @click.argument('test', type=click.Choice(valid_tests)) diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 8479d9a78..de2ea3115 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -41,7 +41,9 @@ def __init__( datetime_input: Optional[str], model: str, ensemblePacket: Optional[str], - task_name: str + task_name: str, + test_iteration: Optional[int], + test_output: Optional[str], ) -> None: # Create message logger @@ -120,6 +122,17 @@ def __init__( self.da_window_params = DataAssimilationWindowParams(self.logger, self.__datetime__.string_iso()) + # Set the test iteration number + # ----------------------------- + self.test_iteration = test_iteration + + # Set the output directory for test results + # ----------------------------------------- + if test_output is not None: + self.test_output = test_output + else: + self.test_output = self.experiment_path() + # ---------------------------------------------------------------------------------------------- # Execute is the place where a task does its work. It's defined as abstract in the base class @@ -262,7 +275,9 @@ def create_task( config: str, datetime: Union[str, dt, None], model: str, - ensemblePacket: Optional[str] + ensemblePacket: Optional[str], + test_iteration: Optional[int], + test_output: Optional[str] ) -> taskBase: # Convert camel case string to snake case @@ -272,7 +287,8 @@ def create_task( task_class = getattr(importlib.import_module('swell.tasks.'+task_lower), task) # Return task object - return task_class(config, datetime, model, ensemblePacket, task) + return task_class(config, datetime, model, ensemblePacket, + task, test_iteration, test_output) # -------------------------------------------------------------------------------------------------- @@ -303,13 +319,16 @@ def task_wrapper( config: str, datetime: Union[str, dt, None], model: Optional[str], - ensemblePacket: Optional[str] + ensemblePacket: Optional[str], + test_iteration: Optional[int], + test_output: Optional[str] ) -> None: # Create the object constrc_start = time.perf_counter() creator = taskFactory() - task_object = creator.create_task(task, config, datetime, model, ensemblePacket) + task_object = creator.create_task(task, config, datetime, model, ensemblePacket, + test_iteration, test_output) constrc_final = time.perf_counter() constrc_time = f'Constructed in {constrc_final - constrc_start:0.4f} seconds' diff --git a/src/swell/tasks/f_grep_residual_norm.py b/src/swell/tasks/f_grep_residual_norm.py new file mode 100644 index 000000000..371a14364 --- /dev/null +++ b/src/swell/tasks/f_grep_residual_norm.py @@ -0,0 +1,57 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import re +import subprocess + +from swell.tasks.base.task_base import taskBase +from swell.utilities.suite_utils import get_model_components + +# -------------------------------------------------------------------------------------------------- + + +class FGrepResidualNorm(taskBase): + + def execute(self) -> None: + + cycle_dir = self.cycle_dir() + + cycle_time = self.__datetime__.string_directory() + model = self.__model__ + + # Build the command + command = ['fgrep', '"Residual norm"'] + [ + os.path.join(cycle_dir, 'jedi_variational_log.log')] + command = ' '.join(command) + + # Run the fgrep command + output = subprocess.run(command, capture_output=True, text=True, shell=True) + results = output.stdout + + # Create the output directory + os.makedirs(self.test_output, exist_ok=True) + + out_name = f'residual_norms_{model}_{cycle_time}' + if self.test_iteration is not None: + out_name += f'_{self.test_iteration}' + out_name += '.txt' + + # Write the output file + out_file = os.path.join(self.test_output, out_name) + with open(out_file, 'w') as f: + f.write(f'Residual norms output for experiment: {self.experiment_path()}:\n') + f.write(f'Model: {model}\n') + f.write(f'Cycle: {cycle_time}\n\n') + f.write(f'RESULTS:\n') + f.write(results) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index d9d5be094..60824575f 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -71,10 +71,6 @@ def setup_workflow(self) -> None: self.header = self.define_header() self.description = self.define_description() self.scheduler = self.define_scheduler() - - self.scheduling_section = self.define_scheduling_section() - self.graph_section = self.define_graph_section() - self.scheduling = self.define_scheduling() self.tasks = self.parse_graph_for_tasks() @@ -91,6 +87,8 @@ def define_header(self) -> str: return header + # -------------------------------------------------------------------------------------------------- + def define_description(self) -> str: description = self.comment_block( """# Cylc workflow auto-generated for suite {suite_to_run} by Swell.""" @@ -207,10 +205,11 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: for task in self.tasks: if '-' in task: task_name = task.split('-')[0] - model = task.split('-')[1] - if model in models: - model_tasks[model].append(task_name) + for entry in task.split('-'): + if entry in models: + model_tasks[model].append(task_name) + else: ind_tasks.append(task) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index af24ff3b9..5ce6b5fb7 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -38,6 +38,16 @@ class cycle_times(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class comparison_experiment_paths(SuiteQuestion): + default_value: list = mutable_field([]) + question_name: str = "comparison_experiment_paths" + ask_question: bool = True + prompt: str = "Provide paths to two or more experiments to run comparison tests on." + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_hofx_packets(SuiteQuestion): default_value: str = "defer_to_model" diff --git a/src/swell/utilities/suite_utils.py b/src/swell/utilities/suite_utils.py index 2337e6bba..82a305587 100644 --- a/src/swell/utilities/suite_utils.py +++ b/src/swell/utilities/suite_utils.py @@ -37,7 +37,8 @@ def get_suites() -> list: # List of base suites suites = sorted([sdir for sdir in os.listdir(suites_directory) if (os.path.isdir(os.path.join(suites_directory, sdir)) - and os.path.exists(os.path.join(suites_directory, sdir, 'flow.cylc')))]) + and os.path.exists(os.path.join(suites_directory, sdir, + 'suite_config.py')))]) return suites From 78303bdb5f4c5f105b506cc4974ceaaedc25b45e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 10 Jun 2025 16:06:02 -0400 Subject: [PATCH 021/299] remove non-task tasks from tasks --- src/swell/tasks/base/task_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 8479d9a78..9cf676139 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -289,7 +289,7 @@ def get_tasks() -> list: tasks = [] for task_file in task_files: base_name = os.path.basename(task_file) - if '__' not in base_name: + if '__' not in base_name and base_name not in ['task_questions.py', 'task_runtimes.py']: tasks.append(snake_case_to_camel_case(base_name[0:-3])) # Return list of valid task choices From 540e088bb89d0fdc5e2d07cfb42537a588cabbba Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 11 Jun 2025 17:35:15 -0400 Subject: [PATCH 022/299] add test for jedi --- src/swell/suites/compare_jedi/suite_config.py | 31 +++++++ src/swell/suites/compare_jedi/workflow.py | 82 +++++++++++++++++++ .../compare_variational/suite_config.py | 3 +- .../suites/compare_variational/workflow.py | 30 +++---- src/swell/suites/suite_questions.py | 10 +++ src/swell/tasks/compare_jedi_c_test_output.py | 70 ++++++++++++++++ src/swell/tasks/f_grep_residual_norm.py | 5 +- src/swell/tasks/jedi_c_test.py | 54 ++++++++++++ src/swell/tasks/task_questions.py | 11 ++- src/swell/tasks/task_runtimes.py | 4 + src/swell/utilities/cylc_workflow.py | 6 +- 11 files changed, 281 insertions(+), 25 deletions(-) create mode 100644 src/swell/suites/compare_jedi/suite_config.py create mode 100644 src/swell/suites/compare_jedi/workflow.py create mode 100644 src/swell/tasks/compare_jedi_c_test_output.py create mode 100644 src/swell/tasks/jedi_c_test.py diff --git a/src/swell/suites/compare_jedi/suite_config.py b/src/swell/suites/compare_jedi/suite_config.py new file mode 100644 index 000000000..da974e89a --- /dev/null +++ b/src/swell/suites/compare_jedi/suite_config.py @@ -0,0 +1,31 @@ +# -------------------------------------------------------------------------------------------------- +# @package configuration +# +# Class containing the configuration. This is a dictionary that is converted from +# an input yaml configuration file. Various function are included for interacting with the +# dictionary. +# +# -------------------------------------------------------------------------------------------------- + + +from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.suites.suite_questions import SuiteQuestions as sq + +from enum import Enum + + +# -------------------------------------------------------------------------------------------------- + +class SuiteConfig(QuestionContainer, Enum): + + # -------------------------------------------------------------------------------------------------- + + compare_jedi = QuestionList( + list_name="compare_jedi", + questions=[ + sq.compare, + ] + ) + + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_jedi/workflow.py b/src/swell/suites/compare_jedi/workflow.py new file mode 100644 index 000000000..4baed13f6 --- /dev/null +++ b/src/swell/suites/compare_jedi/workflow.py @@ -0,0 +1,82 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import yaml +import os + +from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes +from swell.utilities.cylc_runtime import Task + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_compare_jedi(CylcWorkflow): + + # -------------------------------------------------------------------------------------------------- + + def define_description(self): + description = self.comment_block(""" + # Cylc suite for running comparison tests on completed experiments + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_scheduling(self): + + paths = self.experiment_dict['comparison_experiment_paths'] + + if len(paths) == 2: + config_file = os.path.join(os.path.dirname(paths[0]), 'experiment.yaml') + with open(config_file, 'r') as f: + base_dict = yaml.safe_load(f) + else: + raise Exception('Please specify two experiments') + + scheduling_section = self.create_new_section('scheduling') + + r1_str = '' + dependency_str = '' + for i in range(len(paths)): + r1_str += f"JediCTest-{i}\n" + dependency_str += f"JediCTest-{i} & " + + dependency_str = dependency_str[:-3] + dependency_str += ' => CompareJediCTestOutput\n' + + r1_str += dependency_str + + graph_str = self.format_cycle('R1', r1_str) + + graph_section = self.create_new_section('graph', graph_str) + scheduling_section.add_subsection(graph_section) + + return scheduling_section.get_section_str() + + # -------------------------------------------------------------------------------------------------- + + def define_runtime_task_overrides(self): + overrides = {} + + exp_root = os.path.expandvars(self.experiment_dict['experiment_root']) + exp_id = os.path.expandvars(self.experiment_dict['experiment_id']) + + output_dir = os.path.join(exp_root, exp_id) + + for i, path in enumerate(self.experiment_dict['comparison_experiment_paths']): + config_file = os.path.join(os.path.dirname(path), 'experiment.yaml') + overrides[f'JediCTest-{i}'] = Task( + base_name=f'JediCTest', + scheduling_name=f'JediCTest-{i}', + script=f'swell task JediCTest {config_file} -i {i} -o {output_dir}', + slurm={}) + + return overrides diff --git a/src/swell/suites/compare_variational/suite_config.py b/src/swell/suites/compare_variational/suite_config.py index d6025b7f8..1fe9b002f 100644 --- a/src/swell/suites/compare_variational/suite_config.py +++ b/src/swell/suites/compare_variational/suite_config.py @@ -24,9 +24,8 @@ class SuiteConfig(QuestionContainer, Enum): compare_variational = QuestionList( list_name="compare", questions=[ - sq.all_suites, + sq.compare, qd.runahead_limit(), - qd.comparison_experiment_paths(), ] ) diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index d3a7122d9..f2666d2f6 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -12,6 +12,7 @@ from swell.utilities.cylc_workflow import CylcWorkflow from swell.tasks.task_runtimes import TaskRuntimes +from swell.utilities.cylc_runtime import Task # -------------------------------------------------------------------------------------------------- @@ -75,21 +76,15 @@ def define_scheduling(self): # -------------------------------------------------------------------------------------------------- - def define_runtime(self): - runtime_section = self.create_new_section('runtime', '\n# Task defaults\n# -------------\n') - - task_class = TaskRuntimes.get('root') - task_section = task_class().get_section( - self.experiment_dict, self.slurm_external) - - runtime_section.add_subsection(task_section) - - paths = self.experiment_dict['comparison_experiment_paths'] + def define_runtime_task_overrides(self): + overrides = {} exp_root = os.path.expandvars(self.experiment_dict['experiment_root']) exp_id = self.experiment_dict['experiment_id'] - output_dir = os.path.join(exp_root, exp_id, 'comparison_tests') + output_dir = os.path.join(exp_root, exp_id) + + paths = self.experiment_dict['comparison_experiment_paths'] for i, path in enumerate(paths): config_file = os.path.join(os.path.dirname(path), 'experiment.yaml') @@ -97,13 +92,12 @@ def define_runtime(self): exp_dict = yaml.safe_load(f) for model in exp_dict['model_components']: - task_str = (f'script = """swell task FGrepResidualNorm {config_file} -d $datetime' - f' -m {model} -i {i} -o {output_dir}"""') - - task_section = self.create_new_section(f'FGrepResidualNorm-{model}-{i}', task_str) - runtime_section.add_subsection(task_section) - - return runtime_section.get_section_str() + overrides[f'FGrepResidualNorm-{model}-{i}'] = Task( + base_name='FGrepResidualNorm', + scheduling_name=(f'FGrepResidualNorm-{model}-{i}'), + script=(f'swell task FGrepResidualNorm {config_file} -d $datetime' + f' -m {model} -i {i} -o {output_dir}')) + return overrides # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index 247bee869..ba31f9668 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -55,3 +55,13 @@ class SuiteQuestions(QuestionContainer, Enum): ) # -------------------------------------------------------------------------------------------------- + + compare = QuestionList( + list_name="compare", + questions=[ + all_suites, + qd.comparison_experiment_paths() + ] + ) + + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/compare_jedi_c_test_output.py b/src/swell/tasks/compare_jedi_c_test_output.py new file mode 100644 index 000000000..330283a2e --- /dev/null +++ b/src/swell/tasks/compare_jedi_c_test_output.py @@ -0,0 +1,70 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import re +import subprocess +import yaml +import glob +import re + +from swell.tasks.base.task_base import taskBase +from swell.utilities.suite_utils import get_model_components + +# -------------------------------------------------------------------------------------------------- + + +class CompareJediCTestOutput(taskBase): + + def execute(self) -> None: + + # Create dictionaries to track the ctest results + comparison_dict_0 = {} + comparison_dict_1 = {} + + # Find the results files + ctest_output_files = sorted(glob.glob(os.path.join( + self.experiment_path(), 'comparison_tests', 'jedi_ctest_results*txt'))) + + # Get the results from each file + for i, file in enumerate(ctest_output_files): + with open(file, 'r') as f: + lines = f.readlines() + + comparison_dict[str(i)] = {} + + for line in lines: + if re.match('[0-9]*/[0-9]* Test'): + test = int(line.split('/')[0].strip()) + if 'Passed' in line: + status = True + else: + status = False + + comparison_dict[str(i)][str(test)] = status + + # Make sure that the number of tests match + if len(comparison_dict_0) != len(comparison_dict_1): + self.logger.info('The two JEDI versions specified do not' + ' share the same number of ctests') + + else: + + # Match the tests + for test in comparison_dict_0.keys(): + if comparison_dict_0[test] and comparison_dict_1[test]: + test_0 = 'Passed' if comparison_dict_0[test] else 'Failed' + test_1 = 'Passed' if comparison_dict_1[test] else 'Failed' + + self.logger.info(f'Mismatch in test #{test}: Build 0' + f' {test_0}, Build 1 {test_1}') + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/f_grep_residual_norm.py b/src/swell/tasks/f_grep_residual_norm.py index 371a14364..ba76fe616 100644 --- a/src/swell/tasks/f_grep_residual_norm.py +++ b/src/swell/tasks/f_grep_residual_norm.py @@ -37,7 +37,8 @@ def execute(self) -> None: results = output.stdout # Create the output directory - os.makedirs(self.test_output, exist_ok=True) + out_dir = os.path.join(self.test_output, 'comparison_tests') + os.makedirs(out_dir, exist_ok=True) out_name = f'residual_norms_{model}_{cycle_time}' if self.test_iteration is not None: @@ -45,7 +46,7 @@ def execute(self) -> None: out_name += '.txt' # Write the output file - out_file = os.path.join(self.test_output, out_name) + out_file = os.path.join(out_dir, out_name) with open(out_file, 'w') as f: f.write(f'Residual norms output for experiment: {self.experiment_path()}:\n') f.write(f'Model: {model}\n') diff --git a/src/swell/tasks/jedi_c_test.py b/src/swell/tasks/jedi_c_test.py new file mode 100644 index 000000000..32c81cb2a --- /dev/null +++ b/src/swell/tasks/jedi_c_test.py @@ -0,0 +1,54 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import re +import subprocess + +from swell.tasks.base.task_base import taskBase +from swell.utilities.suite_utils import get_model_components + +# -------------------------------------------------------------------------------------------------- + + +class JediCTest(taskBase): + + def execute(self) -> None: + + # Locate the soca directory + build_dir = self.config.existing_jedi_build_directory() + soca_dir = os.path.join(build_dir, 'soca') + + # Run the ctests + cwd = os.getcwd() + os.chdir(soca_dir) + command = ['ctest', '-V', '-I'] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # Record the output + results, error = process.communicate() + os.chdir(cwd) + + # Get the output file name + out_name = f'jedi_ctest_results' + if self.test_iteration is not None: + out_name += f'-{self.test_iteration}' + out_name += '.txt' + + # Make the output directory + out_path = os.path.join(self.test_output, 'comparison_tests') + os.makedirs(out_path, exist_ok=True) + + # Write the results + with open(os.path.join(out_path, out_name), 'w') as f: + f.write(f'JEDI CTest results for build located at: {soca_dir}\n') + f.write(results.decode('utf-8')) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 9beed33ac..bce9fcfc1 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -1,4 +1,3 @@ -# -------------------------------------------------------------------------------------------------- # (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # @@ -491,6 +490,16 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + JediCTest = QuestionList( + list_name="JediCTest", + questions=[ + qd.existing_jedi_build_directory(), + qd.existing_jedi_source_directory() + ] + ) + + # -------------------------------------------------------------------------------------------------- + MoveForecastRestart = QuestionList( list_name="MoveForecastRestart", questions=[ diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 593b4256b..dd53a0dc2 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -31,6 +31,10 @@ class root(Task): class CloneJedi(Task): pass + @dataclass + class CompareJediCTestOutput(Task): + pass + @dataclass class BuildJediByLinking(Task): pass diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index b49213fbe..2a7980f02 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -234,7 +234,8 @@ def define_runtime(self) -> str: for task in ['root'] + self.tasks: if task in runtime_overrides.keys(): - task_section = self.define_runtime_task_overrides[task] + task_section = runtime_overrides[task].get_section( + self.experiment_dict, self.slurm_external) runtime_section.add_subsection(task_section) @@ -242,7 +243,8 @@ def define_runtime(self) -> str: if '-' in task: task_name = task.split('-')[0] model = task.split('-')[1] - if model not in self.experiment_dict['model_components']: + if 'model_components' not in self.experiment_dict or ( + model not in self.experiment_dict['model_components']): task_name = task model = None else: From f9b5cd5aaa24b1ce4f299c8a13fd7f69571ad3b8 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 12 Jun 2025 10:48:06 -0400 Subject: [PATCH 023/299] add compare command for jedi --- src/swell/swell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/swell.py b/src/swell/swell.py index 9a705e69d..b6148f496 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -238,7 +238,7 @@ def utility(utility: str) -> None: # -------------------------------------------------------------------------------------------------- @swell_driver.command() -@click.argument('comparison_type', type=click.Choice(['variational'])) +@click.argument('comparison_type', type=click.Choice(['variational', 'jedi'])) @click.argument('experiments', type=click.Path(), nargs=-1) @click.option('-p', '--platform', 'platform', default='nccs_discover_sles15', type=click.Choice(get_platforms()), help=platform_help) From 9c307f02465a6d03d29336e9dbb67fbd0912f97f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 12 Jun 2025 17:32:18 -0400 Subject: [PATCH 024/299] Add workflow pause and event status emails --- src/swell/deployment/create_experiment.py | 3 +- src/swell/deployment/launch_experiment.py | 18 ++++++++- .../prepare_config_and_suite.py | 26 ++++++++++++- src/swell/swell.py | 8 +++- src/swell/utilities/cylc_runtime.py | 39 ++++++++++++++++++- src/swell/utilities/cylc_workflow.py | 26 +++++++++++-- src/swell/utilities/question_defaults.py | 23 ++++++++++- 7 files changed, 133 insertions(+), 10 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 31691e8ee..d6c23e492 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -101,7 +101,8 @@ def prepare_config( # Initialize the workflow # ----------------------- - workflow = workflows.get_workflow(suite)(suite_dict, slurm_dict) + workflow_class = workflows.get_workflow(suite) + workflow = workflow_class(suite_dict, slurm_dict) # Get the list of tasks from the workflow's graph # ----------------------------------------------- diff --git a/src/swell/deployment/launch_experiment.py b/src/swell/deployment/launch_experiment.py index a3a3dd3de..e8f93fd50 100644 --- a/src/swell/deployment/launch_experiment.py +++ b/src/swell/deployment/launch_experiment.py @@ -103,7 +103,9 @@ def cylc_run_experiment(self) -> None: # NB: Could be a factory based on workfl def launch_experiment( suite_path: str, no_detach: bool, - log_path: str + log_path: str, + send_cylc_messages: bool = False, + allow_pause: bool = False ) -> None: # Get the path to where the suite files are located @@ -127,6 +129,20 @@ def launch_experiment( False) deploy_workflow.logger.info('Experiment name: ' + experiment_name) + # Set environment variable allowing for cylc email messaging + # ---------------------------------------------------------- + if send_cylc_messages: + os.environ['SWELL_SEND_MESSAGES'] = str(1) + else: + os.environ['SWELL_SEND_MESSAGES'] = str(0) + + # Set environment variable allowing for pausing on set tasks + # ---------------------------------------------------------- + if allow_pause: + os.environ['SWELL_PAUSE_WORKFLOW'] = str(1) + else: + os.environ['SWELL_PAUSE_WORKFLOW'] = str(0) + # Launch the workflow # ------------------- deploy_workflow.cylc_run_experiment() diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 2b23608a3..abae69b2d 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -13,6 +13,7 @@ from collections.abc import Mapping from typing import Union, Tuple, Optional import datetime +from dataclasses import asdict from swell.swell_path import get_swell_path from swell.utilities.suite_utils import get_model_components @@ -24,6 +25,7 @@ from swell.tasks.task_questions import TaskQuestions as task_questions from swell.suites.all_suites import suite_configs from swell.utilities.swell_questions import QuestionType +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- @@ -160,7 +162,13 @@ def prepare_suite_question_dictionary(self) -> None: # ---------------------------------------------------------------------------------------------- def prepare_task_question_dictionary(self): + + # Track all possible tasks + task_options = [] + + # Iterate through model independent tasks and update with defaults if not already set for task in self.model_independent_tasks: + task_options.append(task) if task in task_questions.get_all(): question_list = task_questions[task].value.expand_question_list() for question in question_list: @@ -183,8 +191,10 @@ def prepare_task_question_dictionary(self): self.question_dictionary_model_ind = add_dict( self.question_dictionary_model_ind, question_dict) + # Iterate through model dependent tasks and update if not already set for model, task_list in self.model_dependent_tasks.items(): for task in task_list: + task_options.append(task) if task in task_questions.get_all(): question_list = task_questions[task].value.expand_question_list() @@ -197,6 +207,20 @@ def prepare_task_question_dictionary(self): self.question_dictionary_model_dep = add_dict( self.question_dictionary_model_dep, {model: question_dict}) + # Set options for task email parameters + message_question_dict = {'task_email_parameters': + asdict(qd.task_email_parameters(options=task_options))} + + self.question_dictionary_model_ind = add_dict(self.question_dictionary_model_ind, + message_question_dict) + + # Set options for workflow pause + pause_question_dict = {'pause_on_tasks': + asdict(qd.pause_on_tasks(options=task_options))} + + self.question_dictionary_model_ind = add_dict(self.question_dictionary_model_ind, + pause_question_dict) + # ---------------------------------------------------------------------------------------------- def override_with_defaults(self, suite_task: QuestionType) -> None: @@ -305,7 +329,7 @@ def override_with_external(self, suite_task: QuestionType) -> None: # Iterate over the model_dep dictionary and override # -------------------------------------------------- - if self.suite_needs_model_components and override_dict['models'] is not None: + if self.suite_needs_model_components and 'models' in override_dict.keys(): for model, model_dict in self.question_dictionary_model_dep.items(): for question_name, question in model_dict.items(): if question['question_type'] == suite_task: diff --git a/src/swell/swell.py b/src/swell/swell.py index b6148f496..97cdd04b3 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -167,10 +167,14 @@ def clone( @click.argument('suite_path') @click.option('-b', '--no-detach', 'no_detach', is_flag=True, default=False, help=no_detach_help) @click.option('-l', '--log_path', 'log_path', default=None, help=log_path_help) +@click.option('-m', '--send-messages', 'send_messages', is_flag=True) +@click.option('-d', '--pause-workflow', 'pause_workflow', is_flag=True) def launch( suite_path: str, no_detach: bool, - log_path: str + log_path: str, + send_messages: bool, + pause_workflow: bool ) -> None: """ Launch an experiment with the cylc workflow manager @@ -181,7 +185,7 @@ def launch( suite_path (str): Path to where the flow.cylc and associated suite files are located. \n """ - launch_experiment(suite_path, no_detach, log_path) + launch_experiment(suite_path, no_detach, log_path, send_messages, pause_workflow) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 824512c51..e2de20714 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -7,6 +7,8 @@ # -------------------------------------------------------------------------------------------------- +import os +import yaml from typing import Union, Optional, Self from collections.abc import Mapping from dataclasses import dataclass @@ -106,7 +108,14 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): # Set the script if self.script: - runtime_dict['script'] = self.format_string_block(self.script) + script_str = self.script + + if 'pause_on_tasks' in experiment_dict.keys(): + if len(set([self.base_name, self.scheduling_name]) + & set(experiment_dict['pause_on_tasks'])) > 0: + script_str += '\ncylc pause $CYLC_WORKFLOW_ID' + + runtime_dict['script'] = self.format_string_block(script_str) # Specify the platform if this is a slurm task if self.slurm is not None: @@ -171,6 +180,34 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): runtime_section.add_subsection(directive_section) + # Check slurm messaging parameters + events = [] + if 'task_email_parameters' in experiment_dict.keys(): + if experiment_dict['task_email_parameters'] == 'auto': + # Set message status to fail or event fail + events = ['failed', 'submit-failed'] + elif self.scheduling_name in experiment_dict['task_email_parameters'].keys(): + events = experiment_dict['task_email_parameters'][self.scheduling_name] + elif self.base_name in experiment_dict['task_email_parameters'].keys(): + events = experiment_dict['task_email_parameters'][self.base_name] + + # Add messaging section + settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) + if os.path.exists(settings_file) and len(events) > 0: + with open(settings_file, 'r') as f: + settings_dict = yaml.safe_load(f) + if 'email_address' in settings_dict.keys(): + email_address = settings_dict['email_address'] + address_section = self.create_new_section('mail', f'mail events = {email_address}') + runtime_section.add_subsection(address_section) + + event_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" + event_str += "mail events = " + ', '.join(events) + event_str += "\n{% endif %}\n" + + event_section = self.create_new_section('events', event_str) + runtime_section.add_subsection(event_section) + return runtime_section # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 2a7980f02..2dda31821 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -9,6 +9,8 @@ from typing import Union, Optional, Self, Tuple from collections.abc import Mapping +import os +import yaml from swell.utilities.cylc_formatting import CylcSection, indent_lines from swell.tasks.task_runtimes import TaskRuntimes @@ -78,7 +80,8 @@ def setup_workflow(self) -> None: # -------------------------------------------------------------------------------------------------- def define_header(self) -> str: - header = self.comment_block(string=""" + header = '#!jinja2\n' + header += self.comment_block(string=""" # (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # @@ -118,8 +121,25 @@ def comment_block(self, string, level: int = 0, section_break: bool = True): # -------------------------------------------------------------------------------------------------- def define_scheduler(self) -> str: - scheduler_dict = {'UTC mode': True, 'allow implicit tasks': False} - scheduler = self.create_new_section('scheduler', scheduler_dict) + scheduler_str = 'UTC mode = True\nallow implicit tasks = False\n' + + settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) + if os.path.exists(settings_file): + with open(settings_file, 'r') as f: + settings_dict = yaml.safe_load(f) + if 'email_address' in settings_dict.keys(): + email_address = settings_dict['email_address'] + + message_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" + message_str += '[[events]]\n' + message_str += indent_lines('mail events = startup, shutdown\n', 1) + message_str += '[[mail]]\n' + message_str += indent_lines(f'to = {email_address}\n', 1) + message_str += '{% endif %}\n' + + scheduler_str += message_str + + scheduler = self.create_new_section('scheduler', scheduler_str) return scheduler.get_section_str() diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 271a99a9a..4f58fcffe 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -9,7 +9,7 @@ from dataclasses import dataclass -from typing import List, Dict +from typing import List, Dict, Union from swell.utilities.swell_questions import SuiteQuestion, TaskQuestion from swell.utilities.swell_questions import WidgetType as WType @@ -1047,6 +1047,17 @@ class path_to_gsi_nc_diags(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class pause_on_tasks(TaskQuestion): + default_value: list = mutable_field([]) + question_name: str = "pause_on_tasks" + ask_question: bool = False + prompt: str = ("Specify any tasks that the workflow should pause on " + "(for development purposes).") + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + @dataclass class perhost(TaskQuestion): default_value: str = None @@ -1152,6 +1163,16 @@ class swell_static_files_user(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class task_email_parameters(TaskQuestion): + default_value: Union[str, dict] = "auto" + question_name: str = "task_email_parameters" + prompt: str = ("Provide a dictionary mapping tasks to cylc event statuses, or 'auto' to " + "automatically configure these based on the graph.") + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + @dataclass class total_processors(TaskQuestion): default_value: str = "defer_to_model" From 375b40fa5b2800bedde999efe871473afac9d6f0 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 12 Jun 2025 17:49:50 -0400 Subject: [PATCH 025/299] Add message on pause and fix to messaging --- src/swell/deployment/launch_experiment.py | 6 ++++++ src/swell/utilities/cylc_runtime.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/swell/deployment/launch_experiment.py b/src/swell/deployment/launch_experiment.py index e8f93fd50..6384dfca9 100644 --- a/src/swell/deployment/launch_experiment.py +++ b/src/swell/deployment/launch_experiment.py @@ -87,6 +87,12 @@ def cylc_run_experiment(self) -> None: # NB: Could be a factory based on workfl self.logger.info(' \u001b[32mcylc stop --kill ' + self.experiment_name + '\033[0m') self.logger.info(' ', False) + send_messages = os.environ.get('SWELL_SEND_MESSAGES') + if send_messages == '1': + self.logger.info(' Workflow will pause on tasks configured to do so. To unpause:') + self.logger.info(' \u001b[32mcylc play ' + self.experiment_name + '\033[0m') + self.logger.info(' ', False) + # Launch the job monitor self.logger.critical('Launching the TUI, press \'q\' at any time to exit the TUI') input() diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index e2de20714..bab951f7e 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -198,7 +198,7 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): settings_dict = yaml.safe_load(f) if 'email_address' in settings_dict.keys(): email_address = settings_dict['email_address'] - address_section = self.create_new_section('mail', f'mail events = {email_address}') + address_section = self.create_new_section('mail', f'to = {email_address}') runtime_section.add_subsection(address_section) event_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" From b38e0acbea5994d67f0cd3b22571006c2ecdc628 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 13 Jun 2025 09:11:39 -0400 Subject: [PATCH 026/299] pycodestyle fix --- src/swell/utilities/cylc_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 2dda31821..35e93a26a 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -114,7 +114,7 @@ def comment_block(self, string, level: int = 0, section_break: bool = True): out_string += f'{line}\n' if section_break: - out_string += f'\n# {"-" *98}\n\n' + out_string += f'\n# {"-" * 98}\n\n' return out_string From eb83c5b0c599f7661b254ec290425a88f3b7d23e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 16 Jun 2025 16:12:13 -0400 Subject: [PATCH 027/299] Update slurm code test --- src/swell/swell.py | 2 - src/swell/tasks/compare_jedi_c_test_output.py | 3 - src/swell/tasks/task_runtimes.py | 9 ++- src/swell/test/code_tests/slurm_test.py | 58 +++++++++++----- src/swell/utilities/cylc_runtime.py | 69 ++++++++++++------- src/swell/utilities/question_defaults.py | 20 +++--- src/swell/utilities/slurm.py | 54 ++++++++++----- 7 files changed, 141 insertions(+), 74 deletions(-) diff --git a/src/swell/swell.py b/src/swell/swell.py index 97cdd04b3..e2a618e83 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -9,7 +9,6 @@ import click -from collections.abc import Mapping from typing import Union, Optional, Literal, Tuple from swell.deployment.platforms.platforms import get_platforms @@ -19,7 +18,6 @@ from swell.test.test_driver import test_wrapper, valid_tests from swell.test.suite_tests.suite_tests import run_suite, TestSuite from swell.suites.all_suites import suite_configs -from swell.utilities.dictionary import update_dict from swell.utilities.welcome_message import write_welcome_message from swell.utilities.scripts.utility_driver import get_utilities, utility_wrapper diff --git a/src/swell/tasks/compare_jedi_c_test_output.py b/src/swell/tasks/compare_jedi_c_test_output.py index 330283a2e..c7690d47b 100644 --- a/src/swell/tasks/compare_jedi_c_test_output.py +++ b/src/swell/tasks/compare_jedi_c_test_output.py @@ -10,13 +10,10 @@ import os import re -import subprocess -import yaml import glob import re from swell.tasks.base.task_base import taskBase -from swell.utilities.suite_utils import get_model_components # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index dd53a0dc2..65e6cae84 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -82,7 +82,7 @@ class RunJediVariationalExecutable(Task): time_limit: bool = True is_cycling: bool = True is_model: bool = True - slurm: dict = mutable_field({}) + slurm: dict = mutable_field({'nodes': 3}) @dataclass class EvaJediLog(Task): @@ -111,6 +111,13 @@ class CleanCycle(Task): is_cycling: bool = True is_model: bool = True + @dataclass + class RunJediUfoExecutable(Task): + is_cycling: bool = True + is_model: bool = True + slurm: dict = mutable_field({'ntasks-per-node': 1}) + time_limit: bool = True + @classmethod def get(cls, name: str) -> Task: return getattr(cls, name) diff --git a/src/swell/test/code_tests/slurm_test.py b/src/swell/test/code_tests/slurm_test.py index c8d75c1e7..1494b5d87 100644 --- a/src/swell/test/code_tests/slurm_test.py +++ b/src/swell/test/code_tests/slurm_test.py @@ -9,8 +9,9 @@ import unittest -from swell.utilities.slurm import prepare_scheduling_dict +from swell.utilities.slurm import prepare_slurm_defaults_and_overrides from swell.utilities.logger import get_logger +from swell.tasks.task_runtimes import TaskRuntimes from unittest.mock import patch, Mock # -------------------------------------------------------------------------------------------------- @@ -31,7 +32,6 @@ def test_slurm_config(self, platform_mocked: Mock, mock_global_defaults: Mock) - # Nested example experiment_dict = { - "model_components": ["geos_atmosphere", "geos_marine"], "slurm_directives_global": { "account": "x1234", }, @@ -50,30 +50,52 @@ def test_slurm_config(self, platform_mocked: Mock, mock_global_defaults: Mock) - } platform_mocked.return_value = "Linux-5.14.21" - sd_discover_sles15 = prepare_scheduling_dict(logger, experiment_dict, - platform="nccs_discover_sles15") - self.assertEqual(sd_discover_sles15["RunJediVariationalExecutable"]["directives"] - ["all"]["constraint"], "mil") - self.assertEqual(sd_discover_sles15["RunJediVariationalExecutable"]["directives"] - ["all"]["qos"], "dastest") + sd_discover_sles15 = prepare_slurm_defaults_and_overrides(logger, 'nccs_discover_sles15', + experiment_dict) + + run_jedi_var_class = TaskRuntimes.get('RunJediVariationalExecutable') + run_jedi_var_obj = run_jedi_var_class() + run_jedi_var_slurm = run_jedi_var_obj.generate_task_slurm_dict( + sd_discover_sles15, 'nccs_discover_sles15') + + self.assertEqual(run_jedi_var_slurm["constraint"], "mil") + self.assertEqual(run_jedi_var_slurm["qos"], "dastest") + + eva_obs_class = TaskRuntimes.get('EvaObservations') + build_jedi_class = TaskRuntimes.get('BuildJedi') + run_jedi_ufo_class = TaskRuntimes.get('RunJediUfoExecutable') # Platform generic tests for sd in [sd_discover_sles15]: for mc in ["all", "geos_atmosphere", "geos_marine"]: + run_jedi_var_obj = run_jedi_var_class(model=mc) + eva_obs_obj = eva_obs_class(model=mc) + build_jedi_obj = build_jedi_class(model=mc) + run_jedi_ufo_obj = run_jedi_ufo_class(model=mc) + + run_jedi_var_dict = run_jedi_var_obj.generate_task_slurm_dict( + sd, 'nccs_discover_sles15') + eva_obs_dict = eva_obs_obj.generate_task_slurm_dict( + sd, 'nccs_discover_sles15') + build_jedi_dict = build_jedi_obj.generate_task_slurm_dict( + sd, 'nccs_discover_sles15') + run_jedi_ufo_dict = run_jedi_ufo_obj.generate_task_slurm_dict( + sd, 'nccs_discover_sles15') + # Hard-coded task-specific defaults - self.assertEqual(sd["RunJediVariationalExecutable"]["directives"][mc]["nodes"], 3) - self.assertEqual(sd["RunJediUfoTestsExecutable"]["directives"][mc] - ["ntasks-per-node"], 1) + self.assertEqual(run_jedi_var_dict["nodes"], 3) + self.assertEqual(run_jedi_ufo_dict["ntasks-per-node"], 1) # Global defaults from experiment dict - self.assertEqual(sd["BuildJedi"]["directives"][mc]["account"], "x1234") - self.assertEqual(sd["RunJediUfoTestsExecutable"]["directives"][mc]["account"], - "x1234") + self.assertEqual(build_jedi_dict["account"], "x1234") + self.assertEqual(run_jedi_ufo_dict["account"], "x1234") # Task-specific, model-generic config - self.assertEqual(sd["EvaObservations"]["directives"][mc]["account"], "x5678") - self.assertEqual(sd["EvaObservations"]["directives"][mc]["ntasks-per-node"], 4) + self.assertEqual(eva_obs_dict["account"], "x5678") + self.assertEqual(eva_obs_dict["ntasks-per-node"], 4) # Task-specific, model-specific configs - self.assertEqual(sd["EvaObservations"]["directives"]["geos_marine"]["nodes"], 2) - self.assertEqual(sd["EvaObservations"]["directives"]["geos_atmosphere"]["nodes"], 4) + if mc == "geos_marine": + self.assertEqual(eva_obs_dict["nodes"], 2) + if mc == "geos_atmosphere": + self.assertEqual(eva_obs_dict["nodes"], 4) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index bab951f7e..912ddade3 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -14,6 +14,8 @@ from dataclasses import dataclass from swell.utilities.cylc_formatting import CylcSection, indent_lines +from swell.utilities.suite_utils import get_model_components +from swell.utilities.dictionary import update_dict # -------------------------------------------------------------------------------------------------- @@ -96,6 +98,48 @@ def create_new_section(self, # -------------------------------------------------------------------------------------------------- + def resolve_model(self, slurm_dict: Mapping) -> dict: + ''' Resolve "all" and "model" entries in slurm dictionary ''' + if 'all' in slurm_dict.keys() and isinstance(slurm_dict['all'], Mapping): + slurm_dict = update_dict(slurm_dict, slurm_dict['all']) + del slurm_dict['all'] + if self.model in slurm_dict.keys() and isinstance(slurm_dict[self.model], Mapping): + slurm_dict = update_dict(slurm_dict, slurm_dict[self.model]) + + for model in get_model_components(): + if model in slurm_dict.keys(): + del slurm_dict[model] + + return slurm_dict + + # -------------------------------------------------------------------------------------------------- + + def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Mapping: + slurm_dict = {} + if self.slurm is not None: + for key, value in self.slurm.items(): + slurm_dict[key] = self.match_platform(value, platform) + + slurm_globals = slurm_external['slurm_directives_global'] + slurm_task = {} + + if 'slurm_directives_tasks' in slurm_external.keys(): + task_directives = slurm_external['slurm_directives_tasks'] + + if self.base_name in task_directives: + slurm_task = task_directives[self.base_name] + if self.scheduling_name in task_directives: + slurm_task = task_directives[self.scheduling_name] + + slurm_dict = {'job-name': self.scheduling_name, + **self.resolve_model(slurm_globals), + **self.resolve_model(slurm_dict), + **self.resolve_model(slurm_task)} + + return slurm_dict + + # -------------------------------------------------------------------------------------------------- + def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): ''' Return the runtime section for the given task. ''' @@ -146,32 +190,9 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): # Specify the slurm dictionary with defaults from user and global settings if self.slurm is not None: - slurm_dict = {} - - for key, value in self.slurm.items(): - slurm_dict[key] = self.match_platform(value, platform) - - slurm_globals = slurm_external['slurm_directives_global'] - slurm_task = {} - - if 'slurm_directives_tasks' in slurm_external.keys(): - task_directives = slurm_external['slurm_directives_tasks'] - - if self.base_name in task_directives: - slurm_task = task_directives[self.base_name] - if self.scheduling_name in task_directives: - slurm_task = task_directives[self.scheduling_name] - else: - slurm_task = {} - # Set values from globals, task specifications, and overrides, in order of priority - slurm_dict = {'job-name': self.scheduling_name, - **slurm_globals, - **slurm_dict, - **slurm_task - } + slurm_dict = self.generate_task_slurm_dict(slurm_external, platform) - # Format the dictionary for cylc slurm_section_dict = {} for key, value in slurm_dict.items(): slurm_section_dict[f'--{key}'] = value diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 4f58fcffe..ff953f300 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -24,6 +24,16 @@ class QuestionDefaults(): # Suite question defaults go here # -------------------------------------------------------------------------------------------------- + @dataclass + class comparison_experiment_paths(SuiteQuestion): + default_value: list = mutable_field([]) + question_name: str = "comparison_experiment_paths" + ask_question: bool = True + prompt: str = "Provide paths to two or more experiments to run comparison tests on." + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + @dataclass class cycle_times(SuiteQuestion): default_value: str = "defer_to_model" @@ -38,16 +48,6 @@ class cycle_times(SuiteQuestion): # -------------------------------------------------------------------------------------------------- - @dataclass - class comparison_experiment_paths(SuiteQuestion): - default_value: list = mutable_field([]) - question_name: str = "comparison_experiment_paths" - ask_question: bool = True - prompt: str = "Provide paths to two or more experiments to run comparison tests on." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - @dataclass class ensemble_hofx_packets(SuiteQuestion): default_value: str = "defer_to_model" diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 357291edd..2739757a8 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -10,17 +10,20 @@ import os import re import yaml -from typing import Optional +from typing import Optional, Union +from collections.abc import Mapping from importlib import resources from swell.utilities.logger import Logger +# -------------------------------------------------------------------------------------------------- + def prepare_slurm_defaults_and_overrides( logger: Logger, platform: str, - slurm_file: Optional[str], + slurm_overrides: Union[Mapping, str, None], ) -> dict: # Obtain platform-specific SLURM directives and set them as global defaults @@ -56,13 +59,16 @@ def prepare_slurm_defaults_and_overrides( # everything in `experiment.yaml` and support it through the Questionary # infrastructure. # ---------------------------------- - if slurm_file is not None: - logger.info(f"Reading SLURM directives from {slurm_file}.") - try: - with open(slurm_file, "r") as slurmfile: - slurm_overrides = yaml.safe_load(slurmfile) - except FileNotFoundError as err: - raise FileNotFoundError(f"Slurm config {slurm_file} not found.") + if slurm_overrides is not None: + if isinstance(slurm_overrides, str): + logger.info(f"Reading SLURM directives from {slurm_overrides}.") + try: + with open(slurm_file, "r") as slurmfile: + slurm_overrides = yaml.safe_load(slurm_overrides) + except FileNotFoundError as err: + raise FileNotFoundError(f"Slurm config {slurm_overrides} not found.") + elif not isinstance(slurm_overrides, Mapping): + raise TypeError("Slurm overrides is not of type Mapping") # Ensure that SLURM dict is _only_ used for SLURM directives. slurm_invalid_keys = set(slurm_overrides.keys()).difference({ @@ -84,7 +90,7 @@ def prepare_slurm_defaults_and_overrides( slurm_dict['slurm_directives_global'] = { **global_defaults['slurm_directives_global'], - **user_globals['slurm_directives_global'], + **user_globals, **slurm_overrides['slurm_directives_global']} validate_directives(slurm_dict["slurm_directives_global"]) @@ -96,6 +102,8 @@ def prepare_slurm_defaults_and_overrides( validate_directives(slurm_dict["slurm_directives_tasks"][task]) return slurm_dict +# -------------------------------------------------------------------------------------------------- + def validate_directives(directive_dict: dict) -> None: directive_pattern = r'(?<=--)[a-zA-Z-]+' @@ -105,12 +113,19 @@ def validate_directives(directive_dict: dict) -> None: for s in man_sbatch.split("\n") if re.search(directive_pattern, s) } - # Make sure that everything in `directive_dict` is in `directive_list`; - # i.e., that all entries are valid slurm directives. - invalid_directives = set(directive_dict.keys()).difference(directive_list) - assert \ - len(invalid_directives) == 0, \ - f"The following are invalid SLURM directives: {invalid_directives}" + + for key, item in directive_dict.items(): + if isinstance(item, Mapping): + validate_directives(item) + else: + # Make sure that everything in `directive_dict` is in `directive_list`; + # i.e., that all entries are valid slurm directives. + invalid_directives = set(directive_dict.keys()).difference(directive_list) + assert \ + len(invalid_directives) == 0, \ + f"The following are invalid SLURM directives: {invalid_directives}" + +# -------------------------------------------------------------------------------------------------- def slurm_global_defaults( @@ -118,6 +133,7 @@ def slurm_global_defaults( yaml_path: str = "~/.swell/swell-slurm.yaml" ) -> dict: yaml_path = os.path.expanduser(yaml_path) + ''' user_globals = {} user_globals['slurm_directives_global'] = {} @@ -125,8 +141,14 @@ def slurm_global_defaults( logger.info(f"Loading SLURM user configuration from {yaml_path}") with open(yaml_path, "r") as yaml_file: user_globals['slurm_directives_global'] = yaml.safe_load(yaml_file) + ''' + with open(yaml_path, 'r') as yaml_file: + user_globals = yaml.safe_load(yaml_file) + return user_globals +# -------------------------------------------------------------------------------------------------- + man_sbatch = """ Parallel run options: From f92d4c6f6593989e3cc6a4c45f8089d7514936e9 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 17 Jun 2025 11:17:16 -0400 Subject: [PATCH 028/299] added some suites --- src/swell/suites/3dvar/workflow.py | 18 +++++++++++------ .../utilities/scripts/compare_questions.py | 20 ++++++++++++++++--- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index ada273f19..6a3ecdc8b 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -23,8 +23,10 @@ def define_description(self): # -------------------------------------------------------------------------------------------------- def define_graph_section(self): + # Define the string of the graph section graph_str = '' + # Define the string for the R1 (first non-cycling) section r1 = """ # Triggers for non cycle time dependent tasks # ------------------------------------------- @@ -39,18 +41,20 @@ def define_graph_section(self): """ for model_component in self.experiment_dict['model_components']: - r1 += """ + r1 += f""" # Stage JEDI static files CloneJedi => StageJedi-{model_component} - """.format(model_component=model_component) + """ + # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) + # Format the string for each cycle for model_component in self.experiment_dict['model_components']: if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = """ + cycle_str = f""" # Task triggers for: {model_component} # ------------------ # Get background @@ -97,10 +101,12 @@ def define_graph_section(self): CleanCycle-{model_component} """ - cycle_str = cycle_str.format(model_component=model_component) - + # Add the cycle string to the graph string graph_str += self.format_cycle(cycle_time, cycle_str) - return self.create_new_section('graph', graph_str) + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/scripts/compare_questions.py b/src/swell/utilities/scripts/compare_questions.py index 330de1122..f4383ebcf 100644 --- a/src/swell/utilities/scripts/compare_questions.py +++ b/src/swell/utilities/scripts/compare_questions.py @@ -18,6 +18,9 @@ from swell.tasks.task_questions import TaskQuestions as tq from swell.utilities.swell_questions import QuestionList from swell.utilities.case_switching import camel_case_to_snake_case +from swell.suites.all_suites import Workflows, SuiteConfigs +from swell.utilities.logger import get_logger +from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import PrepareExperimentConfigAndSuite # -------------------------------------------------------------------------------------------------- @@ -52,11 +55,22 @@ def read_cylc_lines(suite: str) -> list: def get_all_tasks(suite: str) -> list: """ Parse the suite's flow.cylc file and get all the tasks used by the suite. """ - lines = read_cylc_lines(suite) + logger = get_logger('CodeTests') + + prepare_config = PrepareExperimentConfigAndSuite(logger, + suite, + suite, + 'defaults', + 'nccs_discover_sles15', + False) + + suite_dict = prepare_config.get_experiment_dict() + + workflow_class = Workflows.get_workflow(suite) + workflow_obj = workflow_class(suite_dict) - tasks = [line.split('swell task ')[1].split(' ')[0] for line in lines if 'swell task' in line] + tasks = workflow_obj.parse_graph_for_tasks() - tasks = sorted(list(set(tasks))) return tasks From 86c580f94a120d2447ec8e74ad77b6bc64e077e1 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 17 Jun 2025 11:52:03 -0400 Subject: [PATCH 029/299] add workflows --- src/swell/suites/3dfgat_atmos/workflow.py | 117 ++++++++++++++++++ src/swell/suites/3dfgat_cycle/workflow.py | 143 ++++++++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 src/swell/suites/3dfgat_atmos/workflow.py create mode 100644 src/swell/suites/3dfgat_cycle/workflow.py diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py new file mode 100644 index 000000000..6c8c90fd9 --- /dev/null +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -0,0 +1,117 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.cylc_workflow import CylcWorkflow + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_3dvar(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for executing JEDI-based non-cycling variational data assimilation + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + """ + + for model_component in self.experiment_dict['model_components']: + r1 += f""" + + # Clone geos ana for generating observing system records + CloneGeosMksi-{model_component} + """ + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Format the string for each cycle + for model_component in self.experiment_dict['model_components']: + if 'cycle_times' in self.experiment_dict['models'][model_component]: + for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: + cycle_str = f""" + # Task triggers for: {model_component} + # ------------------ + # Generate satellite channel records + CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} + + # Get background, provide a way to get background directly from GEOS experiment + GetBackgroundGeosExperiment-{model_component} :fail? => GetBackground-{model_component} + + # Get observations + """ + if self.experiment_dict['cycling_varbc']: + cycle_str += f""" + # Cycling VarBC is active, biases from the previous cycle will be used + + RunJediVariationalExecutable-{model_component}[-PT6H] => GetObservations-{model_component} + """ + else: + cycle_str += """ + # Cycling VarBC is inactive, static bias files will be used + GetObservations-{model_component} + """ + + cycle_str += f""" + # Perform staging that is cycle dependent + StageJediCycle-{model_component} + + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} + CloneJedi[^] => StageJediCycle-{model_component} + StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} + GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} + GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} + GenerateObservingSystemRecords-{model_component} => RunJediVariationalExecutable-{model_component} + + # EvaObservations + RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} + + # EvaJediLog + RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} + + # EvaIncrement + RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} + + # Save observations + RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} + + # Clean up large files + EvaObservations-{model_component} & SaveObsDiags-{model_component} => + CleanCycle-{model_component} + """ + + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py new file mode 100644 index 000000000..a47e0caa7 --- /dev/null +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -0,0 +1,143 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.cylc_workflow import CylcWorkflow + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_3dvar(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for executing Geos forecast + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone Geos source code + CloneGeos + + # Clone JEDI source code + CloneJedi + + # Build Geos source code by linking + CloneGeos => BuildGeosByLinking? + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildGeosByLinking:fail? => BuildGeos + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + # Need first set of restarts to run model + GetGeosRestart => PrepGeosRunDir + + # Model cannot run without code + BuildGeosByLinking? | BuildGeos => RunGeosExecutable + """ + + for model_component in self.experiment_dict['model_components']: + r1 += f""" + # JEDI cannot run without code + BuildJediByLinking? | BuildJedi => RunJediFgatExecutable-{{model_component}} + + # Stage JEDI static files + CloneJedi => StageJedi-{{model_component}} => RunJediFgatExecutable-{{model_component}} + """ + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Format the string for each cycle + for model_component in self.experiment_dict['model_components']: + if 'cycle_times' in self.experiment_dict['models'][model_component]: + for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: + cycle_str = f""" + # Model preperation + # Run the forecast through two windows (need to output restarts at the end of the + # first window and backgrounds for the second window) + # MoveDaRestart-{model_component}[-P1D] => PrepGeosRunDir + MoveDaRestart-{model_component}[-PT6H] => PrepGeosRunDir + PrepGeosRunDir => RunGeosExecutable + + # Run the analysis + # RunGeosExecutable => StageJediCycle-{model_component} + RunGeosExecutable => LinkGeosOutput-{model_component} + LinkGeosOutput-{model_component} => GenerateBClimatology-{model_component} + + # Data assimilation preperation + GetObservations-{model_component} + GenerateBClimatologyByLinking-{model_component} :fail? => GenerateBClimatology-{model_component} + + LinkGeosOutput-{model_component} => RunJediFgatExecutable-{model_component} + StageJediCycle-{model_component} => RunJediFgatExecutable-{model_component} + GenerateBClimatologyByLinking-{model_component}? | GenerateBClimatology-{model_component} => RunJediFgatExecutable-{model_component} + GetObservations-{model_component} => RunJediFgatExecutable-{model_component} + + # Run analysis diagnostics + RunJediFgatExecutable-{model_component} => EvaObservations-{model_component} + RunJediFgatExecutable-{model_component} => EvaJediLog-{model_component} + EvaIncrement-{model_component} => PrepareAnalysis-{model_component} + + # Prepare analysis for next forecast + RunJediFgatExecutable-{model_component} => EvaIncrement-{model_component} + """ + if 'cice6' in self.experiment_dict['models']['geos_marine']['marine_models']: + cycle_str += f""" + PrepareAnalysis-{model_component} => RunJediConvertStateSoca2ciceExecutable-{model_component} + RunJediConvertStateSoca2ciceExecutable-{model_component} => SaveRestart-{model_component} + RunJediConvertStateSoca2ciceExecutable-{model_component} => CleanCycle-{model_component} + """ + else: + cycle_str += """ + PrepareAnalysis-{model_component} => SaveRestart-{model_component} + """ + + cycle_str += f""" + # Move restart to next cycle + SaveRestart-{model_component} => MoveDaRestart-{model_component} + + # Save analysis output + # RunJediFgatExecutable-{model_component} => SaveAnalysis-{model_component} + RunJediFgatExecutable-{model_component} => SaveObsDiags-{model_component} + + # Save model output + # MoveBackground-{model_component} => StoreBackground-{model_component} + + # Remove Run Directory + # MoveDaRestart-{model_component} & MoveBackground-{model_component} => RemoveForecastDir + MoveDaRestart-{model_component} => RemoveForecastDir + + # Clean up large files + EvaObservations-{model_component} & EvaJediLog-{model_component} & EvaIncrement-{model_component} & SaveObsDiags-{model_component} => + CleanCycle-{model_component} + """ + + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- From 776e2d8333c149aeaaca5228faab32f0682ed6a1 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 17 Jun 2025 15:06:25 -0400 Subject: [PATCH 030/299] add all suite workflows --- src/swell/suites/3dfgat_atmos/workflow.py | 2 +- src/swell/suites/3dfgat_cycle/workflow.py | 2 +- src/swell/suites/3dvar/workflow.py | 66 ++++----- src/swell/suites/3dvar_atmos/workflow.py | 118 +++++++++++++++ src/swell/suites/3dvar_cycle/workflow.py | 145 +++++++++++++++++++ src/swell/suites/build_geos/workflow.py | 80 ++++++++++ src/swell/suites/build_jedi/workflow.py | 80 ++++++++++ src/swell/suites/convert_ncdiags/workflow.py | 69 +++++++++ src/swell/suites/forecast_geos/workflow.py | 76 ++++++++++ src/swell/suites/geosadas/workflow.py | 89 ++++++++++++ src/swell/suites/hofx/workflow.py | 99 +++++++++++++ src/swell/suites/localensembleda/workflow.py | 116 +++++++++++++++ src/swell/suites/ufo_testing/workflow.py | 89 ++++++++++++ src/swell/utilities/cylc_workflow.py | 7 +- 14 files changed, 1001 insertions(+), 37 deletions(-) create mode 100644 src/swell/suites/3dvar_atmos/workflow.py create mode 100644 src/swell/suites/3dvar_cycle/workflow.py create mode 100644 src/swell/suites/build_geos/workflow.py create mode 100644 src/swell/suites/build_jedi/workflow.py create mode 100644 src/swell/suites/convert_ncdiags/workflow.py create mode 100644 src/swell/suites/forecast_geos/workflow.py create mode 100644 src/swell/suites/geosadas/workflow.py create mode 100644 src/swell/suites/hofx/workflow.py create mode 100644 src/swell/suites/localensembleda/workflow.py create mode 100644 src/swell/suites/ufo_testing/workflow.py diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 6c8c90fd9..1637a7360 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -12,7 +12,7 @@ # -------------------------------------------------------------------------------------------------- -class Workflow_3dvar(CylcWorkflow): +class Workflow_3dfgat_atmos(CylcWorkflow): def define_description(self): description = self.comment_block(""" # Cylc suite for executing JEDI-based non-cycling variational data assimilation diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index a47e0caa7..9f83d907f 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -12,7 +12,7 @@ # -------------------------------------------------------------------------------------------------- -class Workflow_3dvar(CylcWorkflow): +class Workflow_3dfgat_cycle(CylcWorkflow): def define_description(self): description = self.comment_block(""" # Cylc suite for executing Geos forecast diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 6a3ecdc8b..1f87091ff 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -55,51 +55,51 @@ def define_graph_section(self): if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: cycle_str = f""" - # Task triggers for: {model_component} - # ------------------ - # Get background - GetBackground-{model_component} + # Task triggers for: {model_component} + # ------------------ + # Get background + GetBackground-{model_component} - # Get observations - GetObservations-{model_component} + # Get observations + GetObservations-{model_component} - # GenerateBClimatology, for ocean it is cycle dependent - GenerateBClimatologyByLinking-{model_component}:fail? => - GenerateBClimatology-{model_component} + # GenerateBClimatology, for ocean it is cycle dependent + GenerateBClimatologyByLinking-{model_component}:fail? => + GenerateBClimatology-{model_component} - GetBackground-{model_component} => GenerateBClimatology-{model_component} + GetBackground-{model_component} => GenerateBClimatology-{model_component} - # Perform staging that is cycle dependent - StageJediCycle-{model_component} + # Perform staging that is cycle dependent + StageJediCycle-{model_component} - # Run Jedi variational executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} - StageJedi-{model_component}[^] => RunJediVariationalExecutable-{model_component} - StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} - GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} + StageJedi-{model_component}[^] => RunJediVariationalExecutable-{model_component} + StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} + GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} - GenerateBClimatologyByLinking-{model_component}? | - GenerateBClimatology-{model_component} => - RunJediVariationalExecutable-{model_component} + GenerateBClimatologyByLinking-{model_component}? | + GenerateBClimatology-{model_component} => + RunJediVariationalExecutable-{model_component} - GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} + GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} - # EvaObservations - RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} + # EvaObservations + RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} - # EvaJediLog - RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} + # EvaJediLog + RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} - # EvaIncrement - RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} + # EvaIncrement + RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} - # Save observations - RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} + # Save observations + RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} - # Clean up large files - EvaObservations-{model_component} & SaveObsDiags-{model_component} => - CleanCycle-{model_component} - """ + # Clean up large files + EvaObservations-{model_component} & SaveObsDiags-{model_component} => + CleanCycle-{model_component} + """ # Add the cycle string to the graph string graph_str += self.format_cycle(cycle_time, cycle_str) diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py new file mode 100644 index 000000000..7ecdb3360 --- /dev/null +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -0,0 +1,118 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.cylc_workflow import CylcWorkflow + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_3dvar_atmos(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for executing JEDI-based non-cycling variational data assimilation + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + """ + + for model_component in self.experiment_dict['model_components']: + r1 += f""" + + # Stage JEDI static files + CloneGeosMksi-{model_component} + """ + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Format the string for each cycle + for model_component in self.experiment_dict['model_components']: + if 'cycle_times' in self.experiment_dict['models'][model_component]: + for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: + cycle_str = f""" + # Task triggers for: {model_component} + # ------------------ + # Generate satellite channel records + CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} + + # Get background, provide a way to get background directly from GEOS experiment + GetBackgroundGeosExperiment-{model_component} :fail? => GetBackground-{model_component} + + # Get observations + """ + + if self.experiment_dict['cycling_varbc']: + cycle_str += f""" + # Cycling VarBC is active, biases from the previous cycle will be used + + RunJediVariationalExecutable-{model_component}[-PT6H] => GetObservations-{model_component} + """ + else: + cycle_str += f""" + # Cycling VarBC is inactive, static bias files will be used + GetObservations-{model_component} + """ + + cycle_str += f""" + # Perform staging that is cycle dependent + StageJediCycle-{model_component} + + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} + CloneJedi[^] => StageJediCycle-{model_component} + StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} + GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} + GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} + GenerateObservingSystemRecords-{model_component} => RunJediVariationalExecutable-{model_component} + + # EvaObservations + RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} + + # EvaJediLog + RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} + + # EvaIncrement + RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} + + # Save observations + RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} + + # Clean up large files + EvaObservations-{model_component} & SaveObsDiags-{model_component} => + CleanCycle-{model_component} + """ + + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py new file mode 100644 index 000000000..52af7d17c --- /dev/null +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -0,0 +1,145 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.cylc_workflow import CylcWorkflow + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_3dvar_cycle(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for executing JEDI-based non-cycling variational data assimilation + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone Geos source code + CloneGeos + + # Clone JEDI source code + CloneJedi + + # Build Geos source code by linking + CloneGeos => BuildGeosByLinking? + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildGeosByLinking:fail? => BuildGeos + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + # Need first set of restarts to run model + GetGeosRestart => PrepGeosRunDir + + # Model cannot run without code + BuildGeosByLinking? | BuildGeos => RunGeosExecutable + """ + + for model_component in self.experiment_dict['model_components']: + r1 += f""" + + # JEDI cannot run without code + BuildJediByLinking? | BuildJedi => RunJediVariationalExecutable-{model_component} + + # Stage JEDI static files + CloneJedi => StageJedi-{model_component} => RunJediVariationalExecutable-{model_component} + """ + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Format the string for each cycle + for model_component in self.experiment_dict['model_components']: + if 'cycle_times' in self.experiment_dict['models'][model_component]: + for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: + cycle_str = f""" + # Model things + # Run the forecast through two windows (need to output restarts at the end of the + # first window and backgrounds for the second window) + # MoveDaRestart-{model_component}[-P1D] => PrepGeosRunDir + MoveDaRestart-{model_component}[-PT6H] => PrepGeosRunDir + PrepGeosRunDir => RunGeosExecutable + + # Run the analysis + # RunGeosExecutable => StageJediCycle-{model_component} + RunGeosExecutable => LinkGeosOutput-{model_component} + LinkGeosOutput-{model_component} => GenerateBClimatology-{model_component} + + # Data assimilation things + GetObservations-{model_component} + GenerateBClimatologyByLinking-{model_component} :fail? => GenerateBClimatology-{model_component} + + LinkGeosOutput-{model_component} => RunJediVariationalExecutable-{model_component} + StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} + GenerateBClimatologyByLinking-{model_component}? | GenerateBClimatology-{model_component} => RunJediVariationalExecutable-{model_component} + GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} + + # Run analysis diagnostics + RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} + RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} + RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} + + # Prepare analysis for next forecast + EvaIncrement-{model_component} => PrepareAnalysis-{model_component} + """ + if 'cice6' in self.experiment_dict['models']['geos_marine']['marine_models']: + cycle_str += f""" + PrepareAnalysis-{model_component} => RunJediConvertStateSoca2ciceExecutable-{model_component} + RunJediConvertStateSoca2ciceExecutable-{model_component} => SaveRestart-{model_component} + RunJediConvertStateSoca2ciceExecutable-{model_component} => CleanCycle-{model_component} + """ + else: + cycle_str += f""" + PrepareAnalysis-{model_component} => SaveRestart-{model_component} + """ + + cycle_str += f""" + # Move restart to next cycle + SaveRestart-{model_component} => MoveDaRestart-{model_component} + + # Save analysis output + # RunJediVariationalExecutable-{model_component} => SaveAnalysis-{model_component} + RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} + + # Save model output + # MoveBackground-{model_component} => StoreBackground-{model_component} + + # Remove Run Directory + # MoveDaRestart-{model_component} & MoveBackground-{model_component} => RemoveForecastDir + MoveDaRestart-{model_component} => RemoveForecastDir + + # Clean up large files + # EvaObservations-{model_component} & EvaJediLog-{model_component} & SaveObsDiags-{model_component} & RemoveForecastDir => + EvaObservations-{model_component} & EvaJediLog-{model_component} & EvaIncrement-{model_component} & SaveObsDiags-{model_component} => + CleanCycle-{model_component} + """ + + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py new file mode 100644 index 000000000..1cf6cc962 --- /dev/null +++ b/src/swell/suites/build_geos/workflow.py @@ -0,0 +1,80 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +import yaml + +from swell.utilities.cylc_workflow import CylcWorkflow +from swell.utilities.cylc_formatting import indent_lines + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_build_geos(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for building the GEOS model + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_scheduler(self) -> str: + scheduler_str = 'allow implicit tasks = False\n' + + settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) + if os.path.exists(settings_file): + with open(settings_file, 'r') as f: + settings_dict = yaml.safe_load(f) + if 'email_address' in settings_dict.keys(): + email_address = settings_dict['email_address'] + + message_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" + message_str += '[[events]]\n' + message_str += indent_lines('mail events = startup, shutdown\n', 1) + message_str += '[[mail]]\n' + message_str += indent_lines(f'to = {email_address}\n', 1) + message_str += '{% endif %}\n' + + scheduler_str += message_str + + scheduler = self.create_new_section('scheduler', scheduler_str) + + return scheduler.get_section_str() + + # -------------------------------------------------------------------------------------------------- + + def define_scheduling_section(self): + scheduling_section = self.create_new_section('scheduling', '') + + return scheduling_section + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = """ + CloneGeos => BuildGeosByLinking? + + BuildGeosByLinking:fail? => BuildGeos + """ + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py new file mode 100644 index 000000000..dfa811a69 --- /dev/null +++ b/src/swell/suites/build_jedi/workflow.py @@ -0,0 +1,80 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +import yaml + +from swell.utilities.cylc_workflow import CylcWorkflow +from swell.utilities.cylc_formatting import indent_lines + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_build_jedi(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for building the JEDI code + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_scheduler(self) -> str: + scheduler_str = 'allow implicit tasks = False\n' + + settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) + if os.path.exists(settings_file): + with open(settings_file, 'r') as f: + settings_dict = yaml.safe_load(f) + if 'email_address' in settings_dict.keys(): + email_address = settings_dict['email_address'] + + message_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" + message_str += '[[events]]\n' + message_str += indent_lines('mail events = startup, shutdown\n', 1) + message_str += '[[mail]]\n' + message_str += indent_lines(f'to = {email_address}\n', 1) + message_str += '{% endif %}\n' + + scheduler_str += message_str + + scheduler = self.create_new_section('scheduler', scheduler_str) + + return scheduler.get_section_str() + + # -------------------------------------------------------------------------------------------------- + + def define_scheduling_section(self): + scheduling_section = self.create_new_section('scheduling', '') + + return scheduling_section + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = """ + CloneJedi => BuildJediByLinking? + + BuildJediByLinking:fail? => BuildJedi + """ + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py new file mode 100644 index 000000000..ff5b0806f --- /dev/null +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -0,0 +1,69 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.cylc_workflow import CylcWorkflow + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_convert_ncdiags(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for executing geos_atmosphere ObsFilters tests + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + """ + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Format the string for each cycle + for cycle_time in self.experiment_dict['cycle_times']: + cycle_str = f""" + # Convert bias correction to ioda + GetGsiBc + GetGsiBc => GsiBcToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda + + # Convert ncdiags to ioda + GetGsiNcdiag + GetGsiNcdiag => GsiNcdiagToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda + + # Clean up + GsiNcdiagToIoda => CleanCycle + """ + graph_str += self.format_cycle(cycle_time, cycle_str) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py new file mode 100644 index 000000000..9f1917302 --- /dev/null +++ b/src/swell/suites/forecast_geos/workflow.py @@ -0,0 +1,76 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.cylc_workflow import CylcWorkflow + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_forecast_geos(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for executing geos_atmosphere ObsFilters tests + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone Geos source code + CloneGeos + + # Build Geos source code by linking + CloneGeos => BuildGeosByLinking? + + # If not able to link to build create the build + BuildGeosByLinking:fail? => BuildGeos + + # Need first set of restarts to run model + GetGeosRestart => PrepGeosRunDir + + # Get first set of restarts + BuildGeosByLinking? | BuildGeos => RunGeosExecutable + """ + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Format the string for each cycle + for cycle_time in self.experiment_dict['cycle_times']: + cycle_str = f""" + + # Run Geos Executable + PrepGeosRunDir => RunGeosExecutable + MoveForecastRestart[-PT6H] => PrepGeosRunDir + + # Move restart to next cycle + RunGeosExecutable => MoveForecastRestart + + # Save restarts if requested + # MoveForecastRestart[-PT6H] => SaveRestart + + # Remove Run Directory + MoveForecastRestart => RemoveForecastDir + """ + graph_str += self.format_cycle(cycle_time, cycle_str) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py new file mode 100644 index 000000000..aab050cb0 --- /dev/null +++ b/src/swell/suites/geosadas/workflow.py @@ -0,0 +1,89 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.cylc_workflow import CylcWorkflow + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_geosadas(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for executing JEDI-based non-cycling variational data assimilation + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_scheduling_section(self): + scheduling_dict = {'initial cycle point': '2020-12-15T00:00:00Z', + 'final cycle point': '2020-12-15T00:00:00Z'} + + scheduling_section = self.create_new_section('scheduling', scheduling_dict) + return scheduling_section + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = """ + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + BuildJediByLinking? + + # Stage JEDI static files + CloneJedi => StageJedi + + # Clone geos ana for generating observing system records + CloneGeosMksi + """ + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Format the string for each cycle + for cycle_time in ['T00']: + cycle_str = f""" + # Generate satellite channel records + CloneGeosMksi[^] => GenerateObservingSystemRecords + + # Get and convert bias correction coefficients + GetGsiBc => GsiBcToIoda + + # Get and convert ncdiags + GetGsiNcdiag => GsiNcdiagToIoda + + # Get background + GetGeosAdasBackground + + # Run Jedi variational executable + GenerateObservingSystemRecords => RunJediVariationalExecutable + BuildJediByLinking[^] => RunJediVariationalExecutable + StageJedi[^] => RunJediVariationalExecutable + GsiBcToIoda => RunJediVariationalExecutable + GsiNcdiagToIoda => RunJediVariationalExecutable + GetGeosAdasBackground => RunJediVariationalExecutable + + # Clean cycle + RunJediVariationalExecutable => CleanCycle + """ + graph_str += self.format_cycle(cycle_time, cycle_str) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py new file mode 100644 index 000000000..d6736cbc2 --- /dev/null +++ b/src/swell/suites/hofx/workflow.py @@ -0,0 +1,99 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.cylc_workflow import CylcWorkflow + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_hofx(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for executing JEDI-based h(x) + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + """ + + for model_component in self.experiment_dict['model_components']: + r1 += f""" + + # Clone geos ana for generating observing system records + CloneGeosMksi-{model_component} + """ + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Format the string for each cycle + for model_component in self.experiment_dict['model_components']: + if 'cycle_times' in self.experiment_dict['models'][model_component]: + for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: + cycle_str = f""" + # Task triggers for: {model_component} + # ------------------ + # Generate satellite channel records + CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} + + # Get background, provide a way to get background directly from GEOS experiment + GetBackgroundGeosExperiment-{model_component} :fail? => GetBackground-{model_component} + + # Get observations + GetObservations-{model_component} + + # Perform staging that is cycle dependent + StageJediCycle-{model_component} + + # Run Jedi hofx executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediHofxExecutable-{model_component} + CloneJedi[^] => StageJediCycle-{model_component} + StageJediCycle-{model_component} => RunJediHofxExecutable-{model_component} + GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => RunJediHofxExecutable-{model_component} + GetObservations-{model_component} => RunJediHofxExecutable-{model_component} + GenerateObservingSystemRecords-{model_component} => RunJediHofxExecutable-{model_component} + + # EvaObservations + RunJediHofxExecutable-{model_component} => EvaObservations-{model_component} + + # Save observations + RunJediHofxExecutable-{model_component} => SaveObsDiags-{model_component} + + # Clean up large files + EvaObservations-{model_component} & SaveObsDiags-{model_component} => + CleanCycle-{model_component} + """ + + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py new file mode 100644 index 000000000..987d43e8a --- /dev/null +++ b/src/swell/suites/localensembleda/workflow.py @@ -0,0 +1,116 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.cylc_workflow import CylcWorkflow + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_localensembleda(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for executing JEDI-based h(x) + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + """ + + for model_component in self.experiment_dict['model_components']: + r1 += f""" + + # Clone geos ana for generating observing system records + CloneGeosMksi-{model_component} + """ + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Format the string for each cycle + for model_component in self.experiment_dict['model_components']: + if 'cycle_times' in self.experiment_dict['models'][model_component]: + for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: + cycle_str = f""" + # Task triggers for: {{model_component}} + # ------------------ + + # Perform staging that is cycle dependent + BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{{model_component}} => sync_point + + GetObservations-{{model_component}} => sync_point + + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} => sync_point + + GetEnsembleGeosExperiment-{{model_component}} => sync_point + + sync_point => ThinObs + """ + + if self.experiment_dict['skip_ensemble_hofx']: + cycle_str += """ + sync_point => ThinObs => RunJediLocalEnsembleDaExecutable-{{model_component}} + """ + else: + if self.experiment_dict['ensemble_hofx_strategy'] == 'serial': + cycle_str += """ + sync_point => RunJediEnsembleMeanVariance-{{model_component}} => RunJediHofxEnsembleExecutable-{{model_component}} + RunJediHofxEnsembleExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + """ + elif self.experiment_dict['ensemble_hofx_strategy'] == 'parallel': + for packet in range(self.experiment_dict['ensemble_hofx_packets']): + cycle_str += f""" + # When strategy is parallel, only proceed if all RunJediHofxEnsembleExecutable completes successfully for each packet + + # There is a need for a task to combine all hofx observations together, compute node preferred, put here as placeholder + # RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunEnsembleHofxCombiner-{{model_component}} + # RunEnsembleHofxCombiner-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + + sync_point => RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} + RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + """ + + cycle_str += """ + # EvaObservations + RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaObservations-{{model_component}} + + # Save observations + RunJediLocalEnsembleDaExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + + # Clean up large files + EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} + """ + + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py new file mode 100644 index 000000000..198d529f1 --- /dev/null +++ b/src/swell/suites/ufo_testing/workflow.py @@ -0,0 +1,89 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.cylc_workflow import CylcWorkflow + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_ufo_testing(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for executing geos_atmosphere ObsFilters tests + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + # Clone geos ana for generating observing system records + CloneGeosMksi + """ + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Format the string for each cycle + for cycle_time in self.experiment_dict['cycle_times']: + cycle_str = f""" + + # Generate satellite channel records + CloneGeosMksi[^] => GenerateObservingSystemRecords + + # Convert bias correction to ioda + GetGsiBc + GetGsiBc => GsiBcToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda + + # Convert ncdiags to ioda + GetGsiNcdiag + GetGsiNcdiag => GsiNcdiagToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda + + GetGeovals + + # Run Jedi hofx executable + GenerateObservingSystemRecords => RunJediUfoTestsExecutable + GsiNcdiagToIoda => RunJediUfoTestsExecutable + GsiBcToIoda => RunJediUfoTestsExecutable + GetGeovals => RunJediUfoTestsExecutable + + # EvaObservations + RunJediUfoTestsExecutable => EvaObservations + + # Clean up large files + EvaObservations => CleanCycle + """ + + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 35e93a26a..fc9f508c7 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -14,6 +14,7 @@ from swell.utilities.cylc_formatting import CylcSection, indent_lines from swell.tasks.task_runtimes import TaskRuntimes +from swell.utilities.dictionary import update_dict # -------------------------------------------------------------------------------------------------- @@ -157,8 +158,10 @@ def define_scheduling(self) -> str: def define_scheduling_section(self) -> CylcSection: scheduling_dict = {'initial cycle point': self.experiment_dict['start_cycle_point'], - 'final cycle point': self.experiment_dict['final_cycle_point'], - 'runahead limit': self.experiment_dict['runahead_limit']} + 'final cycle point': self.experiment_dict['final_cycle_point']} + + if 'runahead_limit' in self.experiment_dict: + scheduling_dict = update_dict(scheduling_dict, {'runahead limit': self.experiment_dict['runahead_limit']}) scheduling_section = self.create_new_section('scheduling', scheduling_dict) From 17430462ad3eebed18acf418301a4ac90f15c51e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 17 Jun 2025 15:28:07 -0400 Subject: [PATCH 031/299] fix cycle times --- src/swell/suites/convert_ncdiags/workflow.py | 34 +++++++++++--------- src/swell/suites/forecast_geos/workflow.py | 28 ++++++++-------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index ff5b0806f..0fd17c11d 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -44,22 +44,24 @@ def define_graph_section(self): graph_str += self.format_cycle('R1', r1) # Format the string for each cycle - for cycle_time in self.experiment_dict['cycle_times']: - cycle_str = f""" - # Convert bias correction to ioda - GetGsiBc - GetGsiBc => GsiBcToIoda - BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda - - # Convert ncdiags to ioda - GetGsiNcdiag - GetGsiNcdiag => GsiNcdiagToIoda - BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda - - # Clean up - GsiNcdiagToIoda => CleanCycle - """ - graph_str += self.format_cycle(cycle_time, cycle_str) + for model in self.experiment_dict['models'].keys(): + if 'cycle_times' in self.experiment_dict['models'][model]['cycle_times']: + for cycle_time in self.experiment_dict['models'][model]['cycle_times']: + cycle_str = f""" + # Convert bias correction to ioda + GetGsiBc + GetGsiBc => GsiBcToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda + + # Convert ncdiags to ioda + GetGsiNcdiag + GetGsiNcdiag => GsiNcdiagToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda + + # Clean up + GsiNcdiagToIoda => CleanCycle + """ + graph_str += self.format_cycle(cycle_time, cycle_str) # Create the graph section graph_section = self.create_new_section('graph', graph_str) diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index 9f1917302..8f01fb628 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -50,23 +50,25 @@ def define_graph_section(self): graph_str += self.format_cycle('R1', r1) # Format the string for each cycle - for cycle_time in self.experiment_dict['cycle_times']: - cycle_str = f""" + for model in self.experiment_dict['models'].keys(): + if 'cycle_times' in self.experiment_dict['models'][model]['cycle_times']: + for cycle_time in self.experiment_dict['models'][model]['cycle_times']: + cycle_str = f""" - # Run Geos Executable - PrepGeosRunDir => RunGeosExecutable - MoveForecastRestart[-PT6H] => PrepGeosRunDir + # Run Geos Executable + PrepGeosRunDir => RunGeosExecutable + MoveForecastRestart[-PT6H] => PrepGeosRunDir - # Move restart to next cycle - RunGeosExecutable => MoveForecastRestart + # Move restart to next cycle + RunGeosExecutable => MoveForecastRestart - # Save restarts if requested - # MoveForecastRestart[-PT6H] => SaveRestart + # Save restarts if requested + # MoveForecastRestart[-PT6H] => SaveRestart - # Remove Run Directory - MoveForecastRestart => RemoveForecastDir - """ - graph_str += self.format_cycle(cycle_time, cycle_str) + # Remove Run Directory + MoveForecastRestart => RemoveForecastDir + """ + graph_str += self.format_cycle(cycle_time, cycle_str) # Create the graph section graph_section = self.create_new_section('graph', graph_str) From 175210f4505467a15f0acac7e745331edc38f9c3 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 25 Jun 2025 16:39:46 -0400 Subject: [PATCH 032/299] add all suites --- .../geos_atmosphere/suite_questions.yaml | 6 + .../geos_atmosphere/task_questions.yaml | 6 - .../prepare_config_and_suite.py | 8 +- src/swell/suites/3dfgat_atmos/suite_config.py | 1 + src/swell/suites/3dfgat_atmos/workflow.py | 18 +- src/swell/suites/3dfgat_cycle/workflow.py | 14 +- src/swell/suites/3dvar_atmos/suite_config.py | 1 + src/swell/suites/3dvar_atmos/workflow.py | 2 +- src/swell/suites/localensembleda/workflow.py | 42 ++--- src/swell/suites/ufo_testing/workflow.py | 44 +++-- src/swell/tasks/task_questions.py | 1 - src/swell/tasks/task_runtimes.py | 178 ++++++++++++++++-- src/swell/utilities/cylc_runtime.py | 4 +- src/swell/utilities/question_defaults.py | 40 ++-- 14 files changed, 258 insertions(+), 107 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/suite_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/suite_questions.yaml index 3fcd0cf65..5f337d152 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/suite_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/suite_questions.yaml @@ -2,6 +2,12 @@ cycle_times: default_value: ['T00', 'T06', 'T12', 'T18'] options: ['T00', 'T06', 'T12', 'T18'] +cycling_varbc: + default_value: false + options: + - true + - false + ensemble_hofx_strategy: default_value: 'serial' diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index 3d86efe7d..a6db8c698 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -64,12 +64,6 @@ clean_patterns: - '*.txt' - logfile.*.out -cycling_varbc: - default_value: false - options: - - true - - false - ensemble_hofx_packets: default_value: 2 options: None diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index abae69b2d..77b9aa28e 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -156,6 +156,11 @@ def prepare_suite_question_dictionary(self) -> None: if 'model_components' not in question_dictionary_model_ind.keys(): self.suite_needs_model_components = False + for question in suite_question_list: + if question['question_name'] == 'cycle_times': + question['models'] = None + question_dictionary_model_ind['cycle_times'] = question + self.question_dictionary_model_ind = question_dictionary_model_ind self.question_dictionary_model_dep = question_dictionary_model_dep @@ -348,7 +353,7 @@ def get_questions_of_type(self, if 'models' in question_dictionary.keys(): for model in self.possible_model_components: - if model in question_dictionary.keys(): + if model in question_dictionary['models'].keys(): out_dict[model] = self.get_questions_of_type( suite_task, question_dictionary[model]) @@ -377,7 +382,6 @@ def ask_questions_and_configure(self, suite_task: QuestionType) -> Tuple[dict, d for model in self.experiment_dict['model_components']: model_dict = self.question_dictionary_model_dep[model] - for question_name, question in self.get_questions_of_type( suite_task, model_dict).items(): self.ask_a_question(model_dict, question_name, model) diff --git a/src/swell/suites/3dfgat_atmos/suite_config.py b/src/swell/suites/3dfgat_atmos/suite_config.py index f5302ba24..54be82cc1 100644 --- a/src/swell/suites/3dfgat_atmos/suite_config.py +++ b/src/swell/suites/3dfgat_atmos/suite_config.py @@ -30,6 +30,7 @@ class SuiteConfig(QuestionContainer, Enum): qd.jedi_build_method("use_existing"), qd.model_components(['geos_atmosphere']), qd.runahead_limit("P2"), + qd.cycling_varbc() ], geos_atmosphere=[ qd.cycle_times([ diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 1637a7360..c045cdad0 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -25,7 +25,7 @@ def define_description(self): def define_graph_section(self): # Define the string of the graph section graph_str = '' - + print('graph', self.experiment_dict) # Define the string for the R1 (first non-cycling) section r1 = """ # Triggers for non cycle time dependent tasks @@ -65,19 +65,19 @@ def define_graph_section(self): # Get observations """ - if self.experiment_dict['cycling_varbc']: - cycle_str += f""" + if self.experiment_dict['models'][model_component]['cycling_varbc']: + cycle_str += f""" # Cycling VarBC is active, biases from the previous cycle will be used RunJediVariationalExecutable-{model_component}[-PT6H] => GetObservations-{model_component} """ - else: - cycle_str += """ + else: + cycle_str += f""" # Cycling VarBC is inactive, static bias files will be used GetObservations-{model_component} """ - cycle_str += f""" + cycle_str += f""" # Perform staging that is cycle dependent StageJediCycle-{model_component} @@ -106,9 +106,9 @@ def define_graph_section(self): CleanCycle-{model_component} """ - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) - + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) + print(graph_str) # Create the graph section graph_section = self.create_new_section('graph', graph_str) diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index 9f83d907f..bae97230c 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -58,10 +58,10 @@ def define_graph_section(self): for model_component in self.experiment_dict['model_components']: r1 += f""" # JEDI cannot run without code - BuildJediByLinking? | BuildJedi => RunJediFgatExecutable-{{model_component}} + BuildJediByLinking? | BuildJedi => RunJediFgatExecutable-{model_component} # Stage JEDI static files - CloneJedi => StageJedi-{{model_component}} => RunJediFgatExecutable-{{model_component}} + CloneJedi => StageJedi-{model_component} => RunJediFgatExecutable-{model_component} """ # Format the R1 cycle and add it to the graph @@ -101,18 +101,18 @@ def define_graph_section(self): # Prepare analysis for next forecast RunJediFgatExecutable-{model_component} => EvaIncrement-{model_component} """ - if 'cice6' in self.experiment_dict['models']['geos_marine']['marine_models']: - cycle_str += f""" + if 'cice6' in self.experiment_dict['models']['geos_marine']['marine_models']: + cycle_str += f""" PrepareAnalysis-{model_component} => RunJediConvertStateSoca2ciceExecutable-{model_component} RunJediConvertStateSoca2ciceExecutable-{model_component} => SaveRestart-{model_component} RunJediConvertStateSoca2ciceExecutable-{model_component} => CleanCycle-{model_component} """ - else: - cycle_str += """ + else: + cycle_str += f""" PrepareAnalysis-{model_component} => SaveRestart-{model_component} """ - cycle_str += f""" + cycle_str += f""" # Move restart to next cycle SaveRestart-{model_component} => MoveDaRestart-{model_component} diff --git a/src/swell/suites/3dvar_atmos/suite_config.py b/src/swell/suites/3dvar_atmos/suite_config.py index 88f8777b5..ecb73703a 100644 --- a/src/swell/suites/3dvar_atmos/suite_config.py +++ b/src/swell/suites/3dvar_atmos/suite_config.py @@ -30,6 +30,7 @@ class SuiteConfig(QuestionContainer, Enum): qd.runahead_limit("P2"), qd.jedi_build_method("use_existing"), qd.model_components(['geos_atmosphere']), + qd.cycling_varbc(), ], geos_atmosphere=[ qd.cycle_times([ diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index 7ecdb3360..3eabe9726 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -66,7 +66,7 @@ def define_graph_section(self): # Get observations """ - if self.experiment_dict['cycling_varbc']: + if self.experiment_dict['models'][model_component]['cycling_varbc']: cycle_str += f""" # Cycling VarBC is active, biases from the previous cycle will be used diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 987d43e8a..cc556102d 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -27,7 +27,7 @@ def define_graph_section(self): graph_str = '' # Define the string for the R1 (first non-cycling) section - r1 = """ + r1 = f""" # Triggers for non cycle time dependent tasks # ------------------------------------------- # Clone JEDI source code @@ -55,30 +55,30 @@ def define_graph_section(self): if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: cycle_str = f""" - # Task triggers for: {{model_component}} + # Task triggers for: {model_component} # ------------------ # Perform staging that is cycle dependent - BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{{model_component}} => sync_point + BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{model_component} => sync_point - GetObservations-{{model_component}} => sync_point + GetObservations-{model_component} => sync_point - CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} => sync_point + CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} => sync_point - GetEnsembleGeosExperiment-{{model_component}} => sync_point + GetEnsembleGeosExperiment-{model_component} => sync_point sync_point => ThinObs """ - if self.experiment_dict['skip_ensemble_hofx']: - cycle_str += """ - sync_point => ThinObs => RunJediLocalEnsembleDaExecutable-{{model_component}} + if self.experiment_dict['models'][model_component]['skip_ensemble_hofx']: + cycle_str += f""" + sync_point => ThinObs => RunJediLocalEnsembleDaExecutable-{model_component} """ else: if self.experiment_dict['ensemble_hofx_strategy'] == 'serial': - cycle_str += """ - sync_point => RunJediEnsembleMeanVariance-{{model_component}} => RunJediHofxEnsembleExecutable-{{model_component}} - RunJediHofxEnsembleExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + cycle_str += f""" + sync_point => RunJediEnsembleMeanVariance-{model_component} => RunJediHofxEnsembleExecutable-{model_component} + RunJediHofxEnsembleExecutable-{model_component} => RunJediLocalEnsembleDaExecutable-{model_component} """ elif self.experiment_dict['ensemble_hofx_strategy'] == 'parallel': for packet in range(self.experiment_dict['ensemble_hofx_packets']): @@ -86,23 +86,23 @@ def define_graph_section(self): # When strategy is parallel, only proceed if all RunJediHofxEnsembleExecutable completes successfully for each packet # There is a need for a task to combine all hofx observations together, compute node preferred, put here as placeholder - # RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunEnsembleHofxCombiner-{{model_component}} - # RunEnsembleHofxCombiner-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + # RunJediHofxEnsembleExecutable-{model_component}_pack{packet} => RunEnsembleHofxCombiner-{model_component} + # RunEnsembleHofxCombiner-{model_component} => RunJediLocalEnsembleDaExecutable-{model_component} - sync_point => RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} - RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + sync_point => RunJediHofxEnsembleExecutable-{model_component}_pack{packet} + RunJediHofxEnsembleExecutable-{model_component}_pack{packet} => RunJediLocalEnsembleDaExecutable-{model_component} """ - cycle_str += """ + cycle_str += f""" # EvaObservations - RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaObservations-{{model_component}} + RunJediLocalEnsembleDaExecutable-{model_component} => EvaObservations-{model_component} # Save observations - RunJediLocalEnsembleDaExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + RunJediLocalEnsembleDaExecutable-{model_component} => SaveObsDiags-{model_component} # Clean up large files - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} + EvaObservations-{model_component} & SaveObsDiags-{model_component} => + CleanCycle-{model_component} """ # Add the cycle string to the graph string diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index 198d529f1..d8481c409 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -38,48 +38,54 @@ def define_graph_section(self): # If not able to link to build create the build BuildJediByLinking:fail? => BuildJedi + """ + + for model_component in self.experiment_dict['models']: + r1 += f""" # Clone geos ana for generating observing system records - CloneGeosMksi + CloneGeosMksi-{model_component} """ # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) # Format the string for each cycle - for cycle_time in self.experiment_dict['cycle_times']: - cycle_str = f""" + for model_component in self.experiment_dict['models']: + if 'cycle_times' in self.experiment_dict['models'][model_component].keys(): + for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: + cycle_str = f""" # Generate satellite channel records - CloneGeosMksi[^] => GenerateObservingSystemRecords + CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} # Convert bias correction to ioda - GetGsiBc - GetGsiBc => GsiBcToIoda - BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda + GetGsiBc-{model_component} + GetGsiBc-{model_component} => GsiBcToIoda-{model_component} + BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda-{model_component} # Convert ncdiags to ioda - GetGsiNcdiag - GetGsiNcdiag => GsiNcdiagToIoda - BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda + GetGsiNcdiag-{model_component} + GetGsiNcdiag-{model_component} => GsiNcdiagToIoda-{model_component} + BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda-{model_component} - GetGeovals + GetGeovals-{model_component} # Run Jedi hofx executable - GenerateObservingSystemRecords => RunJediUfoTestsExecutable - GsiNcdiagToIoda => RunJediUfoTestsExecutable - GsiBcToIoda => RunJediUfoTestsExecutable - GetGeovals => RunJediUfoTestsExecutable + GenerateObservingSystemRecords-{model_component} => RunJediUfoTestsExecutable-{model_component} + GsiNcdiagToIoda-{model_component} => RunJediUfoTestsExecutable-{model_component} + GsiBcToIoda-{model_component} => RunJediUfoTestsExecutable-{model_component} + GetGeovals-{model_component} => RunJediUfoTestsExecutable-{model_component} # EvaObservations - RunJediUfoTestsExecutable => EvaObservations + RunJediUfoTestsExecutable-{model_component} => EvaObservations-{model_component} # Clean up large files - EvaObservations => CleanCycle + EvaObservations-{model_component} => CleanCycle-{model_component} """ - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) # Create the graph section graph_section = self.create_new_section('graph', graph_str) diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index bce9fcfc1..aff113f73 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -325,7 +325,6 @@ class TaskQuestions(QuestionContainer, Enum): list_name="GetObservations", questions=[ background_crtm_obs, - qd.cycling_varbc(), qd.obs_experiment(), qd.obs_provider(), qd.observing_system_records_path(), diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 65e6cae84..e167bc380 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -28,11 +28,11 @@ class root(Task): 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'}) @dataclass - class CloneJedi(Task): + class BuildGeos(Task): pass @dataclass - class CompareJediCTestOutput(Task): + class BuildGeosByLinking(Task): pass @dataclass @@ -45,21 +45,92 @@ class BuildJedi(Task): slurm: dict = mutable_field({}) @dataclass - class StageJedi(Task): + class CleanCycle(Task): + is_cycling: bool = True is_model: bool = True @dataclass - class StageJediCycle(Task): + class CloneGeos(Task): + pass + + @dataclass + class CloneJedi(Task): + pass + + @dataclass + class CloneGeosMksi(Task): + is_model: bool = True + + @dataclass + class CompareJediCTestOutput(Task): + pass + + @dataclass + class EvaJediLog(Task): + is_cycling: bool = True + is_model: bool = True + + @dataclass + class EvaIncrement(Task): is_cycling: bool = True is_model: bool = True - base_name: str = "StageJedi" - scheduling_name: str = "StageJediCycle-{model}" + + @dataclass + class EvaObservations(Task): + time_limit: bool = True + is_cycling: bool = True + is_model: bool = True + slurm: dict = mutable_field({}) @dataclass class GetBackground(Task): is_cycling: bool = True is_model: bool = True + @dataclass + class GetBackgroundGeosExperiment(Task): + is_cycling: bool = True + is_model: bool = True + + @dataclass + class GetEnsembleGeosExperiment(Task): + is_cycling: bool = True + is_model: bool = True + + @dataclass + class GetGeosRestart(Task): + is_cycling: bool = True + + @dataclass + class GetGeovals(Task): + is_cycling: bool = True + is_model: bool = True + + @dataclass + class GetGsiBc(Task): + is_cycling: bool = True + is_model: bool = True + + @dataclass + class GsiBcToIoda(Task): + is_cycling: bool = True + is_model: bool = True + + @dataclass + class GetGsiNcdiag(Task): + is_cycling: bool = True + is_model: bool = True + + @dataclass + class GsiNcdiagToIoda(Task): + is_cycling: bool = True + is_model: bool = True + + @dataclass + class GetGeosAdasBackground(Task): + is_cycling: bool = True + is_model: bool = True + @dataclass class GetObservations(Task): is_cycling: bool = True @@ -77,6 +148,55 @@ class GenerateBClimatologyByLinking(Task): is_cycling: bool = True is_model: bool = True + @dataclass + class GenerateObservingSystemRecords(Task): + is_cycling: bool = True + is_model: bool = True + + @dataclass + class LinkGeosOutput(Task): + is_cycling: bool = True + is_model: bool = True + + @dataclass + class MoveDaRestart(Task): + is_cycling: bool = True + is_model: bool = True + + @dataclass + class MoveForecastRestart(Task): + is_cycling: bool = True + + @dataclass + class PrepGeosRunDir(Task): + is_cycling: bool = True + + @dataclass + class PrepareAnalysis(Task): + is_cycling: bool = True + is_model: bool = True + + @dataclass + class RunJediFgatExecutable(Task): + is_cycling: bool = True + is_model: bool = True + time_limit: bool = True + slurm: dict = mutable_field({}) + + @dataclass + class RunJediHofxExecutable(Task): + is_cycling: bool = True + is_model: bool = True + time_limit: bool = True + slurm: dict = mutable_field({}) + + @dataclass + class RunJediLocalEnsembleDaExecutable(Task): + is_cycling: bool = True + is_model: bool = True + time_limit: bool = True + slurm: dict = mutable_field({}) + @dataclass class RunJediVariationalExecutable(Task): time_limit: bool = True @@ -85,20 +205,39 @@ class RunJediVariationalExecutable(Task): slurm: dict = mutable_field({'nodes': 3}) @dataclass - class EvaJediLog(Task): + class RemoveForecastDir(Task): + is_cycling: bool = True + + @dataclass + class RunGeosExecutable(Task): + is_cycling: bool = True + + @dataclass + class RunJediUfoExecutable(Task): is_cycling: bool = True is_model: bool = True + slurm: dict = mutable_field({}) + time_limit: bool = True @dataclass - class EvaIncrement(Task): + class RunJediUfoTestsExecutable(Task): + time_limit: bool = True is_cycling: bool = True is_model: bool = True + slurm: dict = mutable_field({'ntasks-per-node': 1}) @dataclass - class EvaObservations(Task): + class RunJediConvertStateSoca2ciceExecutable(Task): + is_cycling: bool = True + is_model: bool = True time_limit: bool = True + slurm: dict = mutable_field({'nodes': 1}) + + @dataclass + class RunJediFgatExecutable(Task): is_cycling: bool = True is_model: bool = True + time_limit: bool = True slurm: dict = mutable_field({}) @dataclass @@ -107,16 +246,31 @@ class SaveObsDiags(Task): is_model: bool = True @dataclass - class CleanCycle(Task): + class SaveRestart(Task): is_cycling: bool = True is_model: bool = True @dataclass - class RunJediUfoExecutable(Task): + class StageJedi(Task): + is_model: bool = True + + @dataclass + class StageJediCycle(Task): + is_cycling: bool = True + is_model: bool = True + base_name: str = "StageJedi" + scheduling_name: str = "StageJediCycle-{model}" + + @dataclass + class sync_point(Task): + script = "true" + + @dataclass + class ThinObs(Task): is_cycling: bool = True is_model: bool = True - slurm: dict = mutable_field({'ntasks-per-node': 1}) time_limit: bool = True + slurm: dict = mutable_field({}) @classmethod def get(cls, name: str) -> Task: diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 912ddade3..118f9fac5 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -53,7 +53,7 @@ def __post_init__(self): if self.scheduling_name is None: self.scheduling_name = self.base_name - if self.is_model: + if self.is_model and self.model is not None: self.scheduling_name += f'-{self.model}' elif self.is_model: @@ -65,7 +65,7 @@ def __post_init__(self): if self.is_cycling: self.script += ' -d $datetime' - if self.is_model: + if self.is_model and self.model is not None: self.script += f' -m {self.model}' # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index ff953f300..0b5deb30c 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -48,6 +48,19 @@ class cycle_times(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class cycling_varbc(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "cycling_varbc" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Do you want to use cycling VarBC option?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_hofx_packets(SuiteQuestion): default_value: str = "defer_to_model" @@ -311,19 +324,6 @@ class crtm_coeff_dir(TaskQuestion): # -------------------------------------------------------------------------------------------------- - @dataclass - class cycling_varbc(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "cycling_varbc" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Do you want to use cycling VarBC option?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - @dataclass class ensemble_hofx_packets(TaskQuestion): default_value: str = "defer_to_model" @@ -1131,20 +1131,6 @@ class single_observations(TaskQuestion): # -------------------------------------------------------------------------------------------------- - @dataclass - class skip_ensemble_hofx(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "skip_ensemble_hofx" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which local ensemble solver type should be implemented?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - @dataclass class swell_static_files(TaskQuestion): default_value: str = "defer_to_platform" From b8ea4caa26bd441adcb8c1b6fd55f85a40b668b6 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 7 Jul 2025 10:49:59 -0400 Subject: [PATCH 033/299] fixes --- .../jedi/interfaces/geos_marine/suite_questions.yaml | 10 ++++++++++ .../jedi/interfaces/geos_marine/task_questions.yaml | 10 ---------- src/swell/suites/3dfgat_atmos/workflow.py | 12 ++++++------ src/swell/suites/3dfgat_cycle/workflow.py | 12 ++++++------ src/swell/tasks/task_questions.py | 9 +++++++++ 5 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_marine/suite_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_marine/suite_questions.yaml index e2474eb4d..cd68810e9 100644 --- a/src/swell/configuration/jedi/interfaces/geos_marine/suite_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_marine/suite_questions.yaml @@ -10,3 +10,13 @@ ensemble_hofx_packets: skip_ensemble_hofx: default_value: true + +marine_models: + default_value: + - mom6 + - cice6 + options: + - mom6 + - cice6 + - bgc + - ww3 diff --git a/src/swell/configuration/jedi/interfaces/geos_marine/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_marine/task_questions.yaml index ec333b376..e70e41f56 100644 --- a/src/swell/configuration/jedi/interfaces/geos_marine/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_marine/task_questions.yaml @@ -57,16 +57,6 @@ jedi_forecast_model: options: - NA -marine_models: - default_value: - - mom6 - - cice6 - options: - - mom6 - - cice6 - - bgc - - ww3 - minimizer: default_value: RPCG options: diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index c045cdad0..066a2ad00 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -25,7 +25,7 @@ def define_description(self): def define_graph_section(self): # Define the string of the graph section graph_str = '' - print('graph', self.experiment_dict) + # Define the string for the R1 (first non-cycling) section r1 = """ # Triggers for non cycle time dependent tasks @@ -67,15 +67,15 @@ def define_graph_section(self): """ if self.experiment_dict['models'][model_component]['cycling_varbc']: cycle_str += f""" - # Cycling VarBC is active, biases from the previous cycle will be used + # Cycling VarBC is active, biases from the previous cycle will be used - RunJediVariationalExecutable-{model_component}[-PT6H] => GetObservations-{model_component} + RunJediVariationalExecutable-{model_component}[-PT6H] => GetObservations-{model_component} """ else: cycle_str += f""" - # Cycling VarBC is inactive, static bias files will be used - GetObservations-{model_component} - """ + # Cycling VarBC is inactive, static bias files will be used + GetObservations-{model_component} + """ cycle_str += f""" # Perform staging that is cycle dependent diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index bae97230c..c0fe120ea 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -103,14 +103,14 @@ def define_graph_section(self): """ if 'cice6' in self.experiment_dict['models']['geos_marine']['marine_models']: cycle_str += f""" - PrepareAnalysis-{model_component} => RunJediConvertStateSoca2ciceExecutable-{model_component} - RunJediConvertStateSoca2ciceExecutable-{model_component} => SaveRestart-{model_component} - RunJediConvertStateSoca2ciceExecutable-{model_component} => CleanCycle-{model_component} - """ + PrepareAnalysis-{model_component} => RunJediConvertStateSoca2ciceExecutable-{model_component} + RunJediConvertStateSoca2ciceExecutable-{model_component} => SaveRestart-{model_component} + RunJediConvertStateSoca2ciceExecutable-{model_component} => CleanCycle-{model_component} + """ else: cycle_str += f""" - PrepareAnalysis-{model_component} => SaveRestart-{model_component} - """ + PrepareAnalysis-{model_component} => SaveRestart-{model_component} + """ cycle_str += f""" # Move restart to next cycle diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index aff113f73..24ec679ce 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -539,6 +539,15 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + StageJediCycle = QuestionList( + list_name="StageJediCycle", + questions=[ + StageJedi, + ] + ) + + # -------------------------------------------------------------------------------------------------- + EvaIncrement = QuestionList( list_name="EvaIncrement", questions=[ From 336a2c99bf002c02bbbff7eb15c87ec25653a732 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 8 Jul 2025 09:58:51 -0400 Subject: [PATCH 034/299] add cycling_varbc --- src/swell/suites/3dfgat_cycle/suite_config.py | 1 + src/swell/suites/3dvar/suite_config.py | 1 + src/swell/suites/3dvar_cycle/suite_config.py | 1 + src/swell/suites/hofx/suite_config.py | 1 + src/swell/suites/localensembleda/suite_config.py | 1 + 5 files changed, 5 insertions(+) diff --git a/src/swell/suites/3dfgat_cycle/suite_config.py b/src/swell/suites/3dfgat_cycle/suite_config.py index da5878636..cf62c7081 100644 --- a/src/swell/suites/3dfgat_cycle/suite_config.py +++ b/src/swell/suites/3dfgat_cycle/suite_config.py @@ -25,6 +25,7 @@ class SuiteConfig(QuestionContainer, Enum): list_name="3dfgat_cycle", questions=[ sq.marine, + qd.cycling_varbc(), qd.start_cycle_point("2021-07-02T06:00:00Z"), qd.final_cycle_point("2021-07-02T12:00:00Z"), qd.runahead_limit("P2"), diff --git a/src/swell/suites/3dvar/suite_config.py b/src/swell/suites/3dvar/suite_config.py index 53c7b5f62..3eaed8d70 100644 --- a/src/swell/suites/3dvar/suite_config.py +++ b/src/swell/suites/3dvar/suite_config.py @@ -25,6 +25,7 @@ class SuiteConfig(QuestionContainer, Enum): list_name="3dvar", questions=[ sq.marine, + qd.cycling_varbc(), qd.start_cycle_point("2021-07-01T12:00:00Z"), qd.final_cycle_point("2021-07-01T12:00:00Z"), qd.jedi_build_method("use_existing"), diff --git a/src/swell/suites/3dvar_cycle/suite_config.py b/src/swell/suites/3dvar_cycle/suite_config.py index 4309affa7..f6795972d 100644 --- a/src/swell/suites/3dvar_cycle/suite_config.py +++ b/src/swell/suites/3dvar_cycle/suite_config.py @@ -25,6 +25,7 @@ class SuiteConfig(QuestionContainer, Enum): list_name="3dvar_cycle", questions=[ sq.marine, + qd.cycling_varbc(), qd.start_cycle_point("2021-07-02T06:00:00Z"), qd.final_cycle_point("2021-07-02T12:00:00Z"), qd.runahead_limit("P2"), diff --git a/src/swell/suites/hofx/suite_config.py b/src/swell/suites/hofx/suite_config.py index 90723ef84..2448ab528 100644 --- a/src/swell/suites/hofx/suite_config.py +++ b/src/swell/suites/hofx/suite_config.py @@ -25,6 +25,7 @@ class SuiteConfig(QuestionContainer, Enum): list_name="hofx", questions=[ sq.marine, + qd.cycling_varbc(), qd.window_type(), qd.jedi_build_method("use_existing"), qd.save_geovals(True), diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index d4a53db43..776a1eaea 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -25,6 +25,7 @@ class SuiteConfig(QuestionContainer, Enum): list_name="localensembleda", questions=[ sq.marine, + qd.cycling_varbc(), qd.ensemble_hofx_packets(), qd.ensemble_hofx_strategy(), qd.skip_ensemble_hofx(), From bb5984d215eef94e5242072e3ca567bd44be8081 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 8 Jul 2025 10:00:36 -0400 Subject: [PATCH 035/299] remove print statement --- src/swell/suites/3dfgat_atmos/workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 066a2ad00..e8819dcbd 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -108,7 +108,7 @@ def define_graph_section(self): # Add the cycle string to the graph string graph_str += self.format_cycle(cycle_time, cycle_str) - print(graph_str) + # Create the graph section graph_section = self.create_new_section('graph', graph_str) From 904ffffeaa1a95ac821d43eeb7b880fe6a73f16e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 8 Jul 2025 10:28:34 -0400 Subject: [PATCH 036/299] revise templating --- src/swell/suites/3dfgat_atmos/workflow.py | 145 ++++++++++++---------- 1 file changed, 80 insertions(+), 65 deletions(-) diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index e8819dcbd..8ee7b42e1 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -11,6 +11,79 @@ # -------------------------------------------------------------------------------------------------- +r1_template = """ +# Triggers for non cycle time dependent tasks +# ------------------------------------------- +# Clone JEDI source code +CloneJedi + +# Build JEDI source code by linking +CloneJedi => BuildJediByLinking? + +# If not able to link to build create the build +BuildJediByLinking:fail? => BuildJedi +""" + +r1_model = """ +# Clone geos ana for generating observing system records +CloneGeosMksi-{model_component} +""" + +cycle_template_1 = """ +# Task triggers for: {model_component} +# ------------------ +# Generate satellite channel records +CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} + +# Get background, provide a way to get background directly from GEOS experiment +GetBackgroundGeosExperiment-{model_component} :fail? => GetBackground-{model_component} + +# Get observations +""" + +cycle_template_2 = """ +# Cycling VarBC is active, biases from the previous cycle will be used + +RunJediVariationalExecutable-{model_component}[-PT6H] => GetObservations-{model_component} +""" + +cycle_template_3 = """ +# Cycling VarBC is inactive, static bias files will be used +GetObservations-{model_component} +""" + +cycling_template_4 = """ +# Perform staging that is cycle dependent +StageJediCycle-{model_component} + +# Run Jedi variational executable +BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} +CloneJedi[^] => StageJediCycle-{model_component} +StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} +GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => +RunJediVariationalExecutable-{model_component} + +GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} +GenerateObservingSystemRecords-{model_component} => RunJediVariationalExecutable-{model_component} + +# EvaObservations +RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} + +# EvaJediLog +RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} + +# EvaIncrement +RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} + +# Save observations +RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} + +# Clean up large files +EvaObservations-{model_component} & SaveObsDiags-{model_component} => +CleanCycle-{model_component} +""" + +# -------------------------------------------------------------------------------------------------- class Workflow_3dfgat_atmos(CylcWorkflow): def define_description(self): @@ -27,25 +100,10 @@ def define_graph_section(self): graph_str = '' # Define the string for the R1 (first non-cycling) section - r1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - """ + r1 = r1_template for model_component in self.experiment_dict['model_components']: - r1 += f""" - - # Clone geos ana for generating observing system records - CloneGeosMksi-{model_component} - """ + r1 += r1_model.format(model_component=model_component) # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) @@ -54,57 +112,14 @@ def define_graph_section(self): for model_component in self.experiment_dict['model_components']: if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = f""" - # Task triggers for: {model_component} - # ------------------ - # Generate satellite channel records - CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} + cycle_str = cycle_template_1.format(model_component=model_component) - # Get background, provide a way to get background directly from GEOS experiment - GetBackgroundGeosExperiment-{model_component} :fail? => GetBackground-{model_component} - - # Get observations - """ if self.experiment_dict['models'][model_component]['cycling_varbc']: - cycle_str += f""" - # Cycling VarBC is active, biases from the previous cycle will be used - - RunJediVariationalExecutable-{model_component}[-PT6H] => GetObservations-{model_component} - """ + cycle_str += cycle_template_2.format(model_component=model_component) else: - cycle_str += f""" - # Cycling VarBC is inactive, static bias files will be used - GetObservations-{model_component} - """ - - cycle_str += f""" - # Perform staging that is cycle dependent - StageJediCycle-{model_component} - - # Run Jedi variational executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} - CloneJedi[^] => StageJediCycle-{model_component} - StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} - GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} - GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} - GenerateObservingSystemRecords-{model_component} => RunJediVariationalExecutable-{model_component} - - # EvaObservations - RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} - - # EvaJediLog - RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} - - # EvaIncrement - RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} - - # Save observations - RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} - - # Clean up large files - EvaObservations-{model_component} & SaveObsDiags-{model_component} => - CleanCycle-{model_component} - """ + cycle_str += cycle_template_3.format(model_component=model_component) + + cycle_str += cycling_template_4.format(model_component=model_component) # Add the cycle string to the graph string graph_str += self.format_cycle(cycle_time, cycle_str) From 07111f25ee8db4e849b644e6e50fc45c1a3c1d91 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 8 Jul 2025 11:22:33 -0400 Subject: [PATCH 037/299] revise templating for all suites --- src/swell/suites/3dvar/workflow.py | 132 ++++++------ src/swell/suites/3dvar_atmos/workflow.py | 145 +++++++------ src/swell/suites/3dvar_cycle/workflow.py | 202 ++++++++++--------- src/swell/suites/convert_ncdiags/workflow.py | 57 +++--- src/swell/suites/forecast_geos/workflow.py | 70 ++++--- src/swell/suites/geosadas/workflow.py | 79 ++++---- src/swell/suites/hofx/workflow.py | 107 +++++----- src/swell/suites/localensembleda/workflow.py | 135 +++++++------ src/swell/suites/ufo_testing/workflow.py | 98 ++++----- 9 files changed, 550 insertions(+), 475 deletions(-) diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 1f87091ff..380fba0a2 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -11,6 +11,72 @@ # -------------------------------------------------------------------------------------------------- +r1_template = """ +# Triggers for non cycle time dependent tasks +# ------------------------------------------- +# Clone JEDI source code +CloneJedi + +# Build JEDI source code by linking +CloneJedi => BuildJediByLinking? + +# If not able to link to build create the build +BuildJediByLinking:fail? => BuildJedi +""" + +r1_model = """ +# Stage JEDI static files +CloneJedi => StageJedi-{model_component} +""" + +cycle_template = """ +# Task triggers for: {model_component} +# ------------------ +# Get background +GetBackground-{model_component} + +# Get observations +GetObservations-{model_component} + +# GenerateBClimatology, for ocean it is cycle dependent +GenerateBClimatologyByLinking-{model_component}:fail? => +GenerateBClimatology-{model_component} + +GetBackground-{model_component} => GenerateBClimatology-{model_component} + +# Perform staging that is cycle dependent +StageJediCycle-{model_component} + +# Run Jedi variational executable +BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} +StageJedi-{model_component}[^] => RunJediVariationalExecutable-{model_component} +StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} +GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} + +GenerateBClimatologyByLinking-{model_component}? | +GenerateBClimatology-{model_component} => +RunJediVariationalExecutable-{model_component} + +GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} + +# EvaObservations +RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} + +# EvaJediLog +RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} + +# EvaIncrement +RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} + +# Save observations +RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} + +# Clean up large files +EvaObservations-{model_component} & SaveObsDiags-{model_component} => +CleanCycle-{model_component} +""" + +# -------------------------------------------------------------------------------------------------- class Workflow_3dvar(CylcWorkflow): def define_description(self): @@ -27,25 +93,10 @@ def define_graph_section(self): graph_str = '' # Define the string for the R1 (first non-cycling) section - r1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - """ + r1 = r1_template for model_component in self.experiment_dict['model_components']: - r1 += f""" - - # Stage JEDI static files - CloneJedi => StageJedi-{model_component} - """ + r1 += r1_model.format(model_component=model_component) # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) @@ -54,52 +105,7 @@ def define_graph_section(self): for model_component in self.experiment_dict['model_components']: if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = f""" - # Task triggers for: {model_component} - # ------------------ - # Get background - GetBackground-{model_component} - - # Get observations - GetObservations-{model_component} - - # GenerateBClimatology, for ocean it is cycle dependent - GenerateBClimatologyByLinking-{model_component}:fail? => - GenerateBClimatology-{model_component} - - GetBackground-{model_component} => GenerateBClimatology-{model_component} - - # Perform staging that is cycle dependent - StageJediCycle-{model_component} - - # Run Jedi variational executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} - StageJedi-{model_component}[^] => RunJediVariationalExecutable-{model_component} - StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} - GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} - - GenerateBClimatologyByLinking-{model_component}? | - GenerateBClimatology-{model_component} => - RunJediVariationalExecutable-{model_component} - - GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} - - # EvaObservations - RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} - - # EvaJediLog - RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} - - # EvaIncrement - RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} - - # Save observations - RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} - - # Clean up large files - EvaObservations-{model_component} & SaveObsDiags-{model_component} => - CleanCycle-{model_component} - """ + cycle_str = cycle_template.format(model_component=model_component) # Add the cycle string to the graph string graph_str += self.format_cycle(cycle_time, cycle_str) diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index 3eabe9726..a7a28f634 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -11,6 +11,78 @@ # -------------------------------------------------------------------------------------------------- +r1_template = """ +# Triggers for non cycle time dependent tasks +# ------------------------------------------- +# Clone JEDI source code +CloneJedi + +# Build JEDI source code by linking +CloneJedi => BuildJediByLinking? + +# If not able to link to build create the build +BuildJediByLinking:fail? => BuildJedi +""" + +r1_model = """ + +# Stage JEDI static files +CloneGeosMksi-{model_component} +""" + +cycle_template_1 = """ +# Task triggers for: {model_component} +# ------------------ +# Generate satellite channel records +CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} + +# Get background, provide a way to get background directly from GEOS experiment +GetBackgroundGeosExperiment-{model_component} :fail? => GetBackground-{model_component} + +# Get observations +""" + +cycle_template_2 = """ +# Cycling VarBC is active, biases from the previous cycle will be used + +RunJediVariationalExecutable-{model_component}[-PT6H] => GetObservations-{model_component} +""" + +cycle_template_3 = """ +# Cycling VarBC is inactive, static bias files will be used +GetObservations-{model_component} +""" + +cycle_template_4 = """ +# Perform staging that is cycle dependent +StageJediCycle-{model_component} + +# Run Jedi variational executable +BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} +CloneJedi[^] => StageJediCycle-{model_component} +StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} +GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} +GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} +GenerateObservingSystemRecords-{model_component} => RunJediVariationalExecutable-{model_component} + +# EvaObservations +RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} + +# EvaJediLog +RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} + +# EvaIncrement +RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} + +# Save observations +RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} + +# Clean up large files +EvaObservations-{model_component} & SaveObsDiags-{model_component} => +CleanCycle-{model_component} +""" + +# -------------------------------------------------------------------------------------------------- class Workflow_3dvar_atmos(CylcWorkflow): def define_description(self): @@ -27,25 +99,10 @@ def define_graph_section(self): graph_str = '' # Define the string for the R1 (first non-cycling) section - r1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - """ + r1 = r1_template for model_component in self.experiment_dict['model_components']: - r1 += f""" - - # Stage JEDI static files - CloneGeosMksi-{model_component} - """ + r1 += r1_model.format(model_component=model_component) # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) @@ -54,58 +111,14 @@ def define_graph_section(self): for model_component in self.experiment_dict['model_components']: if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = f""" - # Task triggers for: {model_component} - # ------------------ - # Generate satellite channel records - CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} - - # Get background, provide a way to get background directly from GEOS experiment - GetBackgroundGeosExperiment-{model_component} :fail? => GetBackground-{model_component} - - # Get observations - """ + cycle_str = cycle_template_1.format(model_component=model_component) if self.experiment_dict['models'][model_component]['cycling_varbc']: - cycle_str += f""" - # Cycling VarBC is active, biases from the previous cycle will be used - - RunJediVariationalExecutable-{model_component}[-PT6H] => GetObservations-{model_component} - """ + cycle_str += cycle_template_2.format(model_component=model_component) else: - cycle_str += f""" - # Cycling VarBC is inactive, static bias files will be used - GetObservations-{model_component} - """ - - cycle_str += f""" - # Perform staging that is cycle dependent - StageJediCycle-{model_component} - - # Run Jedi variational executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} - CloneJedi[^] => StageJediCycle-{model_component} - StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} - GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} - GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} - GenerateObservingSystemRecords-{model_component} => RunJediVariationalExecutable-{model_component} - - # EvaObservations - RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} - - # EvaJediLog - RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} - - # EvaIncrement - RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} - - # Save observations - RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} - - # Clean up large files - EvaObservations-{model_component} & SaveObsDiags-{model_component} => - CleanCycle-{model_component} - """ + cycle_str += cycle_template_3.format(model_component=model_component) + + cycle_str += cycle_template_4.format(model_component=model_component) # Add the cycle string to the graph string graph_str += self.format_cycle(cycle_time, cycle_str) diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index 52af7d17c..8d6505015 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -11,6 +11,106 @@ # -------------------------------------------------------------------------------------------------- +r1_template = """ +# Triggers for non cycle time dependent tasks +# ------------------------------------------- +# Clone Geos source code +CloneGeos + +# Clone JEDI source code +CloneJedi + +# Build Geos source code by linking +CloneGeos => BuildGeosByLinking? + +# Build JEDI source code by linking +CloneJedi => BuildJediByLinking? + +# If not able to link to build create the build +BuildGeosByLinking:fail? => BuildGeos + +# If not able to link to build create the build +BuildJediByLinking:fail? => BuildJedi + +# Need first set of restarts to run model +GetGeosRestart => PrepGeosRunDir + +# Model cannot run without code +BuildGeosByLinking? | BuildGeos => RunGeosExecutable +""" + +r1_model = """ + +# JEDI cannot run without code +BuildJediByLinking? | BuildJedi => RunJediVariationalExecutable-{model_component} + +# Stage JEDI static files +CloneJedi => StageJedi-{model_component} => RunJediVariationalExecutable-{model_component} +""" + +cycle_template_1 = """ +# Model things +# Run the forecast through two windows (need to output restarts at the end of the +# first window and backgrounds for the second window) +# MoveDaRestart-{model_component}[-P1D] => PrepGeosRunDir +MoveDaRestart-{model_component}[-PT6H] => PrepGeosRunDir +PrepGeosRunDir => RunGeosExecutable + +# Run the analysis +# RunGeosExecutable => StageJediCycle-{model_component} +RunGeosExecutable => LinkGeosOutput-{model_component} +LinkGeosOutput-{model_component} => GenerateBClimatology-{model_component} + +# Data assimilation things +GetObservations-{model_component} +GenerateBClimatologyByLinking-{model_component} :fail? => GenerateBClimatology-{model_component} + +LinkGeosOutput-{model_component} => RunJediVariationalExecutable-{model_component} +StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} +GenerateBClimatologyByLinking-{model_component}? | GenerateBClimatology-{model_component} => RunJediVariationalExecutable-{model_component} +GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} + +# Run analysis diagnostics +RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} +RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} +RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} + +# Prepare analysis for next forecast +EvaIncrement-{model_component} => PrepareAnalysis-{model_component} +""" + +cycle_template_2 = """ +PrepareAnalysis-{model_component} => RunJediConvertStateSoca2ciceExecutable-{model_component} +RunJediConvertStateSoca2ciceExecutable-{model_component} => SaveRestart-{model_component} +RunJediConvertStateSoca2ciceExecutable-{model_component} => CleanCycle-{model_component} +""" + +cycle_template_3 = """ +PrepareAnalysis-{model_component} => SaveRestart-{model_component} +""" + +cycle_template_4 = """ +# Move restart to next cycle +SaveRestart-{model_component} => MoveDaRestart-{model_component} + +# Save analysis output +# RunJediVariationalExecutable-{model_component} => SaveAnalysis-{model_component} +RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} + +# Save model output +# MoveBackground-{model_component} => StoreBackground-{model_component} + +# Remove Run Directory +# MoveDaRestart-{model_component} & MoveBackground-{model_component} => RemoveForecastDir +MoveDaRestart-{model_component} => RemoveForecastDir + +# Clean up large files +# EvaObservations-{model_component} & EvaJediLog-{model_component} & SaveObsDiags-{model_component} & RemoveForecastDir => +EvaObservations-{model_component} & EvaJediLog-{model_component} & EvaIncrement-{model_component} & SaveObsDiags-{model_component} => +CleanCycle-{model_component} +""" + +# -------------------------------------------------------------------------------------------------- class Workflow_3dvar_cycle(CylcWorkflow): def define_description(self): @@ -27,43 +127,10 @@ def define_graph_section(self): graph_str = '' # Define the string for the R1 (first non-cycling) section - r1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone Geos source code - CloneGeos - - # Clone JEDI source code - CloneJedi - - # Build Geos source code by linking - CloneGeos => BuildGeosByLinking? - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildGeosByLinking:fail? => BuildGeos - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - # Need first set of restarts to run model - GetGeosRestart => PrepGeosRunDir - - # Model cannot run without code - BuildGeosByLinking? | BuildGeos => RunGeosExecutable - """ + r1 = r1_template for model_component in self.experiment_dict['model_components']: - r1 += f""" - - # JEDI cannot run without code - BuildJediByLinking? | BuildJedi => RunJediVariationalExecutable-{model_component} - - # Stage JEDI static files - CloneJedi => StageJedi-{model_component} => RunJediVariationalExecutable-{model_component} - """ + r1 += r1_model.format(model_component=model_component) # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) @@ -72,67 +139,14 @@ def define_graph_section(self): for model_component in self.experiment_dict['model_components']: if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = f""" - # Model things - # Run the forecast through two windows (need to output restarts at the end of the - # first window and backgrounds for the second window) - # MoveDaRestart-{model_component}[-P1D] => PrepGeosRunDir - MoveDaRestart-{model_component}[-PT6H] => PrepGeosRunDir - PrepGeosRunDir => RunGeosExecutable - - # Run the analysis - # RunGeosExecutable => StageJediCycle-{model_component} - RunGeosExecutable => LinkGeosOutput-{model_component} - LinkGeosOutput-{model_component} => GenerateBClimatology-{model_component} - - # Data assimilation things - GetObservations-{model_component} - GenerateBClimatologyByLinking-{model_component} :fail? => GenerateBClimatology-{model_component} - - LinkGeosOutput-{model_component} => RunJediVariationalExecutable-{model_component} - StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} - GenerateBClimatologyByLinking-{model_component}? | GenerateBClimatology-{model_component} => RunJediVariationalExecutable-{model_component} - GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} - - # Run analysis diagnostics - RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} - RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} - RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} - - # Prepare analysis for next forecast - EvaIncrement-{model_component} => PrepareAnalysis-{model_component} - """ + cycle_str = cycle_template_1.format(model_component=model_component) + if 'cice6' in self.experiment_dict['models']['geos_marine']['marine_models']: - cycle_str += f""" - PrepareAnalysis-{model_component} => RunJediConvertStateSoca2ciceExecutable-{model_component} - RunJediConvertStateSoca2ciceExecutable-{model_component} => SaveRestart-{model_component} - RunJediConvertStateSoca2ciceExecutable-{model_component} => CleanCycle-{model_component} - """ + cycle_str += cycle_template_2.format(model_component=model_component) else: - cycle_str += f""" - PrepareAnalysis-{model_component} => SaveRestart-{model_component} - """ - - cycle_str += f""" - # Move restart to next cycle - SaveRestart-{model_component} => MoveDaRestart-{model_component} - - # Save analysis output - # RunJediVariationalExecutable-{model_component} => SaveAnalysis-{model_component} - RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} - - # Save model output - # MoveBackground-{model_component} => StoreBackground-{model_component} - - # Remove Run Directory - # MoveDaRestart-{model_component} & MoveBackground-{model_component} => RemoveForecastDir - MoveDaRestart-{model_component} => RemoveForecastDir - - # Clean up large files - # EvaObservations-{model_component} & EvaJediLog-{model_component} & SaveObsDiags-{model_component} & RemoveForecastDir => - EvaObservations-{model_component} & EvaJediLog-{model_component} & EvaIncrement-{model_component} & SaveObsDiags-{model_component} => - CleanCycle-{model_component} - """ + cycle_str += cycle_template_3.format(model_component=model_component) + + cycle_str += cycle_template_4.format(model_component=model_component) # Add the cycle string to the graph string graph_str += self.format_cycle(cycle_time, cycle_str) diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index 0fd17c11d..c60fd5121 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -11,6 +11,35 @@ # -------------------------------------------------------------------------------------------------- +r1_template = """ +# Triggers for non cycle time dependent tasks +# ------------------------------------------- +# Clone JEDI source code +CloneJedi + +# Build JEDI source code by linking +CloneJedi => BuildJediByLinking? + +# If not able to link to build create the build +BuildJediByLinking:fail? => BuildJedi +""" + +cycle_template = """ +# Convert bias correction to ioda +GetGsiBc +GetGsiBc => GsiBcToIoda +BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda + +# Convert ncdiags to ioda +GetGsiNcdiag +GetGsiNcdiag => GsiNcdiagToIoda +BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda + +# Clean up +GsiNcdiagToIoda => CleanCycle +""" + +# -------------------------------------------------------------------------------------------------- class Workflow_convert_ncdiags(CylcWorkflow): def define_description(self): @@ -27,18 +56,7 @@ def define_graph_section(self): graph_str = '' # Define the string for the R1 (first non-cycling) section - r1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - """ + r1 = r1_template # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) @@ -47,20 +65,7 @@ def define_graph_section(self): for model in self.experiment_dict['models'].keys(): if 'cycle_times' in self.experiment_dict['models'][model]['cycle_times']: for cycle_time in self.experiment_dict['models'][model]['cycle_times']: - cycle_str = f""" - # Convert bias correction to ioda - GetGsiBc - GetGsiBc => GsiBcToIoda - BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda - - # Convert ncdiags to ioda - GetGsiNcdiag - GetGsiNcdiag => GsiNcdiagToIoda - BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda - - # Clean up - GsiNcdiagToIoda => CleanCycle - """ + cycle_str = cycle_template graph_str += self.format_cycle(cycle_time, cycle_str) # Create the graph section diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index 8f01fb628..cdca1c623 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -11,6 +11,41 @@ # -------------------------------------------------------------------------------------------------- +r1_template = """ +# Triggers for non cycle time dependent tasks +# ------------------------------------------- +# Clone Geos source code +CloneGeos + +# Build Geos source code by linking +CloneGeos => BuildGeosByLinking? + +# If not able to link to build create the build +BuildGeosByLinking:fail? => BuildGeos + +# Need first set of restarts to run model +GetGeosRestart => PrepGeosRunDir + +# Get first set of restarts +BuildGeosByLinking? | BuildGeos => RunGeosExecutable +""" + +cycle_template = """ +# Run Geos Executable +PrepGeosRunDir => RunGeosExecutable +MoveForecastRestart[-PT6H] => PrepGeosRunDir + +# Move restart to next cycle +RunGeosExecutable => MoveForecastRestart + +# Save restarts if requested +# MoveForecastRestart[-PT6H] => SaveRestart + +# Remove Run Directory +MoveForecastRestart => RemoveForecastDir +""" + +# -------------------------------------------------------------------------------------------------- class Workflow_forecast_geos(CylcWorkflow): def define_description(self): @@ -27,24 +62,7 @@ def define_graph_section(self): graph_str = '' # Define the string for the R1 (first non-cycling) section - r1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone Geos source code - CloneGeos - - # Build Geos source code by linking - CloneGeos => BuildGeosByLinking? - - # If not able to link to build create the build - BuildGeosByLinking:fail? => BuildGeos - - # Need first set of restarts to run model - GetGeosRestart => PrepGeosRunDir - - # Get first set of restarts - BuildGeosByLinking? | BuildGeos => RunGeosExecutable - """ + r1 = r1_template # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) @@ -53,21 +71,7 @@ def define_graph_section(self): for model in self.experiment_dict['models'].keys(): if 'cycle_times' in self.experiment_dict['models'][model]['cycle_times']: for cycle_time in self.experiment_dict['models'][model]['cycle_times']: - cycle_str = f""" - - # Run Geos Executable - PrepGeosRunDir => RunGeosExecutable - MoveForecastRestart[-PT6H] => PrepGeosRunDir - - # Move restart to next cycle - RunGeosExecutable => MoveForecastRestart - - # Save restarts if requested - # MoveForecastRestart[-PT6H] => SaveRestart - - # Remove Run Directory - MoveForecastRestart => RemoveForecastDir - """ + cycle_str = cycle_template graph_str += self.format_cycle(cycle_time, cycle_str) # Create the graph section diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index aab050cb0..ce419e8f0 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -11,6 +11,46 @@ # -------------------------------------------------------------------------------------------------- +r1_template = """ +# Clone JEDI source code +CloneJedi + +# Build JEDI source code by linking +BuildJediByLinking? + +# Stage JEDI static files +CloneJedi => StageJedi + +# Clone geos ana for generating observing system records +CloneGeosMksi +""" + +cycle_template = """ +# Generate satellite channel records +CloneGeosMksi[^] => GenerateObservingSystemRecords + +# Get and convert bias correction coefficients +GetGsiBc => GsiBcToIoda + +# Get and convert ncdiags +GetGsiNcdiag => GsiNcdiagToIoda + +# Get background +GetGeosAdasBackground + +# Run Jedi variational executable +GenerateObservingSystemRecords => RunJediVariationalExecutable +BuildJediByLinking[^] => RunJediVariationalExecutable +StageJedi[^] => RunJediVariationalExecutable +GsiBcToIoda => RunJediVariationalExecutable +GsiNcdiagToIoda => RunJediVariationalExecutable +GetGeosAdasBackground => RunJediVariationalExecutable + +# Clean cycle +RunJediVariationalExecutable => CleanCycle +""" + +# -------------------------------------------------------------------------------------------------- class Workflow_geosadas(CylcWorkflow): def define_description(self): @@ -36,49 +76,14 @@ def define_graph_section(self): graph_str = '' # Define the string for the R1 (first non-cycling) section - r1 = """ - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - BuildJediByLinking? - - # Stage JEDI static files - CloneJedi => StageJedi - - # Clone geos ana for generating observing system records - CloneGeosMksi - """ + r1 = r1_template # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) # Format the string for each cycle for cycle_time in ['T00']: - cycle_str = f""" - # Generate satellite channel records - CloneGeosMksi[^] => GenerateObservingSystemRecords - - # Get and convert bias correction coefficients - GetGsiBc => GsiBcToIoda - - # Get and convert ncdiags - GetGsiNcdiag => GsiNcdiagToIoda - - # Get background - GetGeosAdasBackground - - # Run Jedi variational executable - GenerateObservingSystemRecords => RunJediVariationalExecutable - BuildJediByLinking[^] => RunJediVariationalExecutable - StageJedi[^] => RunJediVariationalExecutable - GsiBcToIoda => RunJediVariationalExecutable - GsiNcdiagToIoda => RunJediVariationalExecutable - GetGeosAdasBackground => RunJediVariationalExecutable - - # Clean cycle - RunJediVariationalExecutable => CleanCycle - """ + cycle_str = cycle_template graph_str += self.format_cycle(cycle_time, cycle_str) # Create the graph section diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index d6736cbc2..85bf1cc36 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -11,6 +11,60 @@ # -------------------------------------------------------------------------------------------------- +r1_template = """ +# Triggers for non cycle time dependent tasks +# ------------------------------------------- +# Clone JEDI source code +CloneJedi + +# Build JEDI source code by linking +CloneJedi => BuildJediByLinking? + +# If not able to link to build create the build +BuildJediByLinking:fail? => BuildJedi +""" + +r1_model = """ + +# Clone geos ana for generating observing system records +CloneGeosMksi-{model_component} +""" + +cycle_template_1 = """ +# Task triggers for: {model_component} +# ------------------ +# Generate satellite channel records +CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} + +# Get background, provide a way to get background directly from GEOS experiment +GetBackgroundGeosExperiment-{model_component} :fail? => GetBackground-{model_component} + +# Get observations +GetObservations-{model_component} + +# Perform staging that is cycle dependent +StageJediCycle-{model_component} + +# Run Jedi hofx executable +BuildJediByLinking[^]? | BuildJedi[^] => RunJediHofxExecutable-{model_component} +CloneJedi[^] => StageJediCycle-{model_component} +StageJediCycle-{model_component} => RunJediHofxExecutable-{model_component} +GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => RunJediHofxExecutable-{model_component} +GetObservations-{model_component} => RunJediHofxExecutable-{model_component} +GenerateObservingSystemRecords-{model_component} => RunJediHofxExecutable-{model_component} + +# EvaObservations +RunJediHofxExecutable-{model_component} => EvaObservations-{model_component} + +# Save observations +RunJediHofxExecutable-{model_component} => SaveObsDiags-{model_component} + +# Clean up large files +EvaObservations-{model_component} & SaveObsDiags-{model_component} => +CleanCycle-{model_component} +""" + +# -------------------------------------------------------------------------------------------------- class Workflow_hofx(CylcWorkflow): def define_description(self): @@ -27,25 +81,10 @@ def define_graph_section(self): graph_str = '' # Define the string for the R1 (first non-cycling) section - r1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - """ + r1 = r1_template for model_component in self.experiment_dict['model_components']: - r1 += f""" - - # Clone geos ana for generating observing system records - CloneGeosMksi-{model_component} - """ + r1 += r1_model.format(model_component=model_component) # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) @@ -54,39 +93,7 @@ def define_graph_section(self): for model_component in self.experiment_dict['model_components']: if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = f""" - # Task triggers for: {model_component} - # ------------------ - # Generate satellite channel records - CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} - - # Get background, provide a way to get background directly from GEOS experiment - GetBackgroundGeosExperiment-{model_component} :fail? => GetBackground-{model_component} - - # Get observations - GetObservations-{model_component} - - # Perform staging that is cycle dependent - StageJediCycle-{model_component} - - # Run Jedi hofx executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediHofxExecutable-{model_component} - CloneJedi[^] => StageJediCycle-{model_component} - StageJediCycle-{model_component} => RunJediHofxExecutable-{model_component} - GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => RunJediHofxExecutable-{model_component} - GetObservations-{model_component} => RunJediHofxExecutable-{model_component} - GenerateObservingSystemRecords-{model_component} => RunJediHofxExecutable-{model_component} - - # EvaObservations - RunJediHofxExecutable-{model_component} => EvaObservations-{model_component} - - # Save observations - RunJediHofxExecutable-{model_component} => SaveObsDiags-{model_component} - - # Clean up large files - EvaObservations-{model_component} & SaveObsDiags-{model_component} => - CleanCycle-{model_component} - """ + cycle_str = cycle_template_1.format(model_component=model_component) # Add the cycle string to the graph string graph_str += self.format_cycle(cycle_time, cycle_str) diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index cc556102d..233c96d42 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -11,6 +11,74 @@ # -------------------------------------------------------------------------------------------------- +r1_template = """ +# Triggers for non cycle time dependent tasks +# ------------------------------------------- +# Clone JEDI source code +CloneJedi + +# Build JEDI source code by linking +CloneJedi => BuildJediByLinking? + +# If not able to link to build create the build +BuildJediByLinking:fail? => BuildJedi +""" + +r1_model = """ + +# Clone geos ana for generating observing system records +CloneGeosMksi-{model_component} +""" + +cycle_template_1 = """ +# Task triggers for: {model_component} +# ------------------ + +# Perform staging that is cycle dependent +BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{model_component} => sync_point + +GetObservations-{model_component} => sync_point + +CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} => sync_point + +GetEnsembleGeosExperiment-{model_component} => sync_point + +sync_point => ThinObs +""" + +cycle_template_2 = """ +sync_point => ThinObs => RunJediLocalEnsembleDaExecutable-{model_component} +""" + +cycle_template_3 = """ +sync_point => RunJediEnsembleMeanVariance-{model_component} => RunJediHofxEnsembleExecutable-{model_component} +RunJediHofxEnsembleExecutable-{model_component} => RunJediLocalEnsembleDaExecutable-{model_component} +""" + +cycle_template_4 = """ +# When strategy is parallel, only proceed if all RunJediHofxEnsembleExecutable completes successfully for each packet + +# There is a need for a task to combine all hofx observations together, compute node preferred, put here as placeholder +# RunJediHofxEnsembleExecutable-{model_component}_pack{packet} => RunEnsembleHofxCombiner-{model_component} +# RunEnsembleHofxCombiner-{model_component} => RunJediLocalEnsembleDaExecutable-{model_component} + +sync_point => RunJediHofxEnsembleExecutable-{model_component}_pack{packet} +RunJediHofxEnsembleExecutable-{model_component}_pack{packet} => RunJediLocalEnsembleDaExecutable-{model_component} +""" + +cycle_template_5 = """ +# EvaObservations +RunJediLocalEnsembleDaExecutable-{model_component} => EvaObservations-{model_component} + +# Save observations +RunJediLocalEnsembleDaExecutable-{model_component} => SaveObsDiags-{model_component} + +# Clean up large files +EvaObservations-{model_component} & SaveObsDiags-{model_component} => +CleanCycle-{model_component} +""" + +# -------------------------------------------------------------------------------------------------- class Workflow_localensembleda(CylcWorkflow): def define_description(self): @@ -27,25 +95,10 @@ def define_graph_section(self): graph_str = '' # Define the string for the R1 (first non-cycling) section - r1 = f""" - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - """ + r1 = r1_template for model_component in self.experiment_dict['model_components']: - r1 += f""" - - # Clone geos ana for generating observing system records - CloneGeosMksi-{model_component} - """ + r1 += r1_model.format(model_component=model_component) # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) @@ -54,56 +107,18 @@ def define_graph_section(self): for model_component in self.experiment_dict['model_components']: if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = f""" - # Task triggers for: {model_component} - # ------------------ - - # Perform staging that is cycle dependent - BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{model_component} => sync_point - - GetObservations-{model_component} => sync_point - - CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} => sync_point - - GetEnsembleGeosExperiment-{model_component} => sync_point - - sync_point => ThinObs - """ + cycle_str = cycle_template_1.format(model_component=model_component) if self.experiment_dict['models'][model_component]['skip_ensemble_hofx']: - cycle_str += f""" - sync_point => ThinObs => RunJediLocalEnsembleDaExecutable-{model_component} - """ + cycle_str += cycle_template_2.format(model_component=model_component) else: if self.experiment_dict['ensemble_hofx_strategy'] == 'serial': - cycle_str += f""" - sync_point => RunJediEnsembleMeanVariance-{model_component} => RunJediHofxEnsembleExecutable-{model_component} - RunJediHofxEnsembleExecutable-{model_component} => RunJediLocalEnsembleDaExecutable-{model_component} - """ + cycle_str += cycle_template_3.format(model_component=model_component) elif self.experiment_dict['ensemble_hofx_strategy'] == 'parallel': for packet in range(self.experiment_dict['ensemble_hofx_packets']): - cycle_str += f""" - # When strategy is parallel, only proceed if all RunJediHofxEnsembleExecutable completes successfully for each packet - - # There is a need for a task to combine all hofx observations together, compute node preferred, put here as placeholder - # RunJediHofxEnsembleExecutable-{model_component}_pack{packet} => RunEnsembleHofxCombiner-{model_component} - # RunEnsembleHofxCombiner-{model_component} => RunJediLocalEnsembleDaExecutable-{model_component} - - sync_point => RunJediHofxEnsembleExecutable-{model_component}_pack{packet} - RunJediHofxEnsembleExecutable-{model_component}_pack{packet} => RunJediLocalEnsembleDaExecutable-{model_component} - """ - - cycle_str += f""" - # EvaObservations - RunJediLocalEnsembleDaExecutable-{model_component} => EvaObservations-{model_component} - - # Save observations - RunJediLocalEnsembleDaExecutable-{model_component} => SaveObsDiags-{model_component} + cycle_str += cycle_template_4.format(model_component=model_component, packet=packet) - # Clean up large files - EvaObservations-{model_component} & SaveObsDiags-{model_component} => - CleanCycle-{model_component} - """ + cycle_str += cycle_template_5.format(model_component=model_component) # Add the cycle string to the graph string graph_str += self.format_cycle(cycle_time, cycle_str) diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index d8481c409..8bd4a320e 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -11,6 +11,55 @@ # -------------------------------------------------------------------------------------------------- +r1_template = """ +# Triggers for non cycle time dependent tasks +# ------------------------------------------- +# Clone JEDI source code +CloneJedi + +# Build JEDI source code by linking +CloneJedi => BuildJediByLinking? + +# If not able to link to build create the build +BuildJediByLinking:fail? => BuildJedi +""" + +r1_model = """ + +# Clone geos ana for generating observing system records +CloneGeosMksi-{model_component} +""" + +cycle_template_1 = """ +# Generate satellite channel records +CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} + +# Convert bias correction to ioda +GetGsiBc-{model_component} +GetGsiBc-{model_component} => GsiBcToIoda-{model_component} +BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda-{model_component} + +# Convert ncdiags to ioda +GetGsiNcdiag-{model_component} +GetGsiNcdiag-{model_component} => GsiNcdiagToIoda-{model_component} +BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda-{model_component} + +GetGeovals-{model_component} + +# Run Jedi hofx executable +GenerateObservingSystemRecords-{model_component} => RunJediUfoTestsExecutable-{model_component} +GsiNcdiagToIoda-{model_component} => RunJediUfoTestsExecutable-{model_component} +GsiBcToIoda-{model_component} => RunJediUfoTestsExecutable-{model_component} +GetGeovals-{model_component} => RunJediUfoTestsExecutable-{model_component} + +# EvaObservations +RunJediUfoTestsExecutable-{model_component} => EvaObservations-{model_component} + +# Clean up large files +EvaObservations-{model_component} => CleanCycle-{model_component} +""" + +# -------------------------------------------------------------------------------------------------- class Workflow_ufo_testing(CylcWorkflow): def define_description(self): @@ -27,25 +76,10 @@ def define_graph_section(self): graph_str = '' # Define the string for the R1 (first non-cycling) section - r1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - """ + r1 = r1_template for model_component in self.experiment_dict['models']: - r1 += f""" - - # Clone geos ana for generating observing system records - CloneGeosMksi-{model_component} - """ + r1 += r1_model.format(model_component=model_component) # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) @@ -54,35 +88,7 @@ def define_graph_section(self): for model_component in self.experiment_dict['models']: if 'cycle_times' in self.experiment_dict['models'][model_component].keys(): for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = f""" - - # Generate satellite channel records - CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} - - # Convert bias correction to ioda - GetGsiBc-{model_component} - GetGsiBc-{model_component} => GsiBcToIoda-{model_component} - BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda-{model_component} - - # Convert ncdiags to ioda - GetGsiNcdiag-{model_component} - GetGsiNcdiag-{model_component} => GsiNcdiagToIoda-{model_component} - BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda-{model_component} - - GetGeovals-{model_component} - - # Run Jedi hofx executable - GenerateObservingSystemRecords-{model_component} => RunJediUfoTestsExecutable-{model_component} - GsiNcdiagToIoda-{model_component} => RunJediUfoTestsExecutable-{model_component} - GsiBcToIoda-{model_component} => RunJediUfoTestsExecutable-{model_component} - GetGeovals-{model_component} => RunJediUfoTestsExecutable-{model_component} - - # EvaObservations - RunJediUfoTestsExecutable-{model_component} => EvaObservations-{model_component} - - # Clean up large files - EvaObservations-{model_component} => CleanCycle-{model_component} - """ + cycle_str = cycle_template_1.format(model_component=model_component) # Add the cycle string to the graph string graph_str += self.format_cycle(cycle_time, cycle_str) From 70e5e4bf4e06ebf91315237b5d27821b42f708ab Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 8 Jul 2025 14:46:31 -0400 Subject: [PATCH 038/299] fixes for code tests --- src/swell/suites/compare_jedi/workflow.py | 8 +- .../suites/compare_variational/workflow.py | 6 +- src/swell/suites/forecast_geos/workflow.py | 8 +- src/swell/test/code_tests/slurm_test.py | 2 +- src/swell/utilities/cylc_workflow.py | 3 + .../utilities/scripts/compare_questions.py | 91 +++++++++++-------- 6 files changed, 67 insertions(+), 51 deletions(-) diff --git a/src/swell/suites/compare_jedi/workflow.py b/src/swell/suites/compare_jedi/workflow.py index 4baed13f6..54a8dbd3b 100644 --- a/src/swell/suites/compare_jedi/workflow.py +++ b/src/swell/suites/compare_jedi/workflow.py @@ -34,12 +34,8 @@ def define_scheduling(self): paths = self.experiment_dict['comparison_experiment_paths'] - if len(paths) == 2: - config_file = os.path.join(os.path.dirname(paths[0]), 'experiment.yaml') - with open(config_file, 'r') as f: - base_dict = yaml.safe_load(f) - else: - raise Exception('Please specify two experiments') + if len(paths) < 2: + self.logger.info('Please specify two experiments') scheduling_section = self.create_new_section('scheduling') diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index f2666d2f6..39aa4b9b2 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -51,7 +51,11 @@ def define_scheduling(self): cycle_model_times[cycle_time] = [] cycle_model_times[cycle_time].append(model) else: - raise Exception('No experiments have been specified') + start_cycle_point = 'None' + final_cycle_point = 'None' + cycle_model_times = {} + + self.logger.info('No experiments have been specified') scheduling_section = self.create_new_section('scheduling', {'initial cycle point': start_cycle_point, diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index cdca1c623..f79b0a2a4 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -68,11 +68,9 @@ def define_graph_section(self): graph_str += self.format_cycle('R1', r1) # Format the string for each cycle - for model in self.experiment_dict['models'].keys(): - if 'cycle_times' in self.experiment_dict['models'][model]['cycle_times']: - for cycle_time in self.experiment_dict['models'][model]['cycle_times']: - cycle_str = cycle_template - graph_str += self.format_cycle(cycle_time, cycle_str) + for cycle_time in self.experiment_dict['cycle_times']: + cycle_str = cycle_template + graph_str += self.format_cycle(cycle_time, cycle_str) # Create the graph section graph_section = self.create_new_section('graph', graph_str) diff --git a/src/swell/test/code_tests/slurm_test.py b/src/swell/test/code_tests/slurm_test.py index 1494b5d87..af5033424 100644 --- a/src/swell/test/code_tests/slurm_test.py +++ b/src/swell/test/code_tests/slurm_test.py @@ -63,7 +63,7 @@ def test_slurm_config(self, platform_mocked: Mock, mock_global_defaults: Mock) - eva_obs_class = TaskRuntimes.get('EvaObservations') build_jedi_class = TaskRuntimes.get('BuildJedi') - run_jedi_ufo_class = TaskRuntimes.get('RunJediUfoExecutable') + run_jedi_ufo_class = TaskRuntimes.get('RunJediUfoTestsExecutable') # Platform generic tests for sd in [sd_discover_sles15]: diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index fc9f508c7..f2bedde99 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -15,6 +15,7 @@ from swell.utilities.cylc_formatting import CylcSection, indent_lines from swell.tasks.task_runtimes import TaskRuntimes from swell.utilities.dictionary import update_dict +from swell.utilities.logger import get_logger # -------------------------------------------------------------------------------------------------- @@ -33,6 +34,8 @@ def __init__(self, experiment_dict, slurm_external) -> None: self.experiment_dict = experiment_dict self.slurm_external = slurm_external + self.logger = get_logger(self.__class__.__name__) + self.setup_workflow() # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/scripts/compare_questions.py b/src/swell/utilities/scripts/compare_questions.py index 35052d285..08f240c60 100644 --- a/src/swell/utilities/scripts/compare_questions.py +++ b/src/swell/utilities/scripts/compare_questions.py @@ -18,7 +18,7 @@ from swell.tasks.task_questions import TaskQuestions as tq from swell.utilities.swell_questions import QuestionList from swell.utilities.case_switching import camel_case_to_snake_case -from swell.suites.all_suites import Workflows, SuiteConfigs +from swell.suites.all_suites import workflows, suite_configs from swell.utilities.logger import get_logger from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import PrepareExperimentConfigAndSuite @@ -38,41 +38,52 @@ def filter_list(cls, lst: list) -> list: # -------------------------------------------------------------------------------------------------- +def get_workflow(suite: str): + """ Parse the suite's flow.cylc file and get all the tasks used by the suite. """ + + logger = get_logger('CodeTests') -def read_cylc_lines(suite: str) -> list: - """ Get lines from the suite's flow.cylc file, in list seperated by newline. """ + prepare_config = PrepareExperimentConfigAndSuite(logger, + suite, + suite, + 'nccs_discover_sles15', + 'defaults', + None) - suite_file = os.path.join(get_swell_path(), 'suites', suite, 'flow.cylc') + suite_dict = prepare_config.get_experiment_dict() - with open(suite_file, 'r') as f: - lines = f.readlines() + workflow_class = workflows.get_workflow(suite) + workflow_obj = workflow_class(suite_dict, {}) - return lines + return workflow_obj # -------------------------------------------------------------------------------------------------- +def get_scheduling(suite: str): + + workflow_obj = get_workflow(suite) -def get_all_tasks(suite: str) -> list: - """ Parse the suite's flow.cylc file and get all the tasks used by the suite. """ + scheduling_section = workflow_obj.define_scheduling() - logger = get_logger('CodeTests') + return scheduling_section - prepare_config = PrepareExperimentConfigAndSuite(logger, - suite, - suite, - 'defaults', - 'nccs_discover_sles15', - False) +# -------------------------------------------------------------------------------------------------- - suite_dict = prepare_config.get_experiment_dict() +def get_all_tasks(suite: str) -> list: + """ Parse the suite's flow.cylc file and get all the tasks used by the suite. """ - workflow_class = Workflows.get_workflow(suite) - workflow_obj = workflow_class(suite_dict) + workflow_obj = get_workflow(suite) tasks = workflow_obj.parse_graph_for_tasks() + base_tasks = [] - return tasks + for task in tasks: + base_task = task.split('-')[0] + if base_task not in ['StageJediCycle', 'sync_point']: + base_tasks.append(base_task) + + return base_tasks # -------------------------------------------------------------------------------------------------- @@ -89,7 +100,9 @@ def questions_in_cylc(suite: str) -> list: cylc_questions = [] - lines = read_cylc_lines(suite) + scheduling = get_scheduling(suite) + + lines = scheduling.split('\n') for line in lines: line = line.strip() @@ -183,24 +196,26 @@ def compare_used_and_set_questions() -> Tuple[dict, dict]: task_file = os.path.join(get_swell_path(), 'tasks', camel_case_to_snake_case(task) + '.py') - with open(task_file, 'r') as f: - config_lines = [line for line in f.readlines() if 'self.config.' in line] - for line in config_lines: - if 'get_key_for_model' in line: - field = line.split( - 'self.config.get_key_for_model(')[1].split(')')[0].strip() + ')' - if len(field.split(',')) == 1: - field = field.split(',')[0] + '()' + if os.path.exists(task_file): + with open(task_file, 'r') as f: + config_lines = [line for line in f.readlines() if 'self.config.' in line] + for line in config_lines: + if 'get_key_for_model' in line: + field = line.split( + 'self.config.get_key_for_model(')[1].split(')')[0].strip() + ')' + if len(field.split(',')) == 1: + field = field.split(',')[0] + '()' + else: + field = field.split(',')[ + 0].strip() + '(' + field.split(',')[-1].strip() + ')' + + field = field.replace('"', '') + field = field.replace("'", '') else: - field = field.split(',')[ - 0].strip() + '(' + field.split(',')[-1].strip() + ')' - - field = field.replace('"', '') - field = field.replace("'", '') - else: - field = line.split('self.config.')[1].split(')')[0].strip() + ')' - # Include the parentheses, so we can later assess whether the key is optional - used_task.append(field) + field = line.split('self.config.')[1].split(')')[0].strip() + ')' + + # Include the parentheses, so we can later assess whether the key is optional + used_task.append(field) set_task = sorted(list(set(set_task))) used_task = sorted(list(set(used_task))) From 53498efc2c0607801b25c5afc1a897e78e1d210c Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 8 Jul 2025 15:00:43 -0400 Subject: [PATCH 039/299] try to fix unused variables --- src/swell/tasks/task_runtimes.py | 48 ++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index e167bc380..30f6ea692 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -21,6 +21,7 @@ class TaskRuntimes(): @dataclass + @member class root(Task): script: bool = False pre_script: str = "source $CYLC_SUITE_DEF_PATH/modules" @@ -28,54 +29,66 @@ class root(Task): 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'}) @dataclass + @member class BuildGeos(Task): pass @dataclass + @member class BuildGeosByLinking(Task): pass @dataclass + @member class BuildJediByLinking(Task): pass @dataclass + @member class BuildJedi(Task): time_limit: bool = True slurm: dict = mutable_field({}) @dataclass + @member class CleanCycle(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class CloneGeos(Task): pass @dataclass + @member class CloneJedi(Task): pass @dataclass + @member class CloneGeosMksi(Task): is_model: bool = True @dataclass + @member class CompareJediCTestOutput(Task): pass @dataclass + @member class EvaJediLog(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class EvaIncrement(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class EvaObservations(Task): time_limit: bool = True is_cycling: bool = True @@ -83,60 +96,72 @@ class EvaObservations(Task): slurm: dict = mutable_field({}) @dataclass + @member class GetBackground(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class GetBackgroundGeosExperiment(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class GetEnsembleGeosExperiment(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class GetGeosRestart(Task): is_cycling: bool = True @dataclass + @member class GetGeovals(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class GetGsiBc(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class GsiBcToIoda(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class GetGsiNcdiag(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class GsiNcdiagToIoda(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class GetGeosAdasBackground(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class GetObservations(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class GenerateBClimatology(Task): time_limit: bool = True is_cycling: bool = True @@ -144,39 +169,47 @@ class GenerateBClimatology(Task): slurm: dict = mutable_field({}) @dataclass + @member class GenerateBClimatologyByLinking(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class GenerateObservingSystemRecords(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class LinkGeosOutput(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class MoveDaRestart(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class MoveForecastRestart(Task): is_cycling: bool = True @dataclass + @member class PrepGeosRunDir(Task): is_cycling: bool = True @dataclass + @member class PrepareAnalysis(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class RunJediFgatExecutable(Task): is_cycling: bool = True is_model: bool = True @@ -184,6 +217,7 @@ class RunJediFgatExecutable(Task): slurm: dict = mutable_field({}) @dataclass + @member class RunJediHofxExecutable(Task): is_cycling: bool = True is_model: bool = True @@ -191,6 +225,7 @@ class RunJediHofxExecutable(Task): slurm: dict = mutable_field({}) @dataclass + @member class RunJediLocalEnsembleDaExecutable(Task): is_cycling: bool = True is_model: bool = True @@ -198,6 +233,7 @@ class RunJediLocalEnsembleDaExecutable(Task): slurm: dict = mutable_field({}) @dataclass + @member class RunJediVariationalExecutable(Task): time_limit: bool = True is_cycling: bool = True @@ -205,14 +241,17 @@ class RunJediVariationalExecutable(Task): slurm: dict = mutable_field({'nodes': 3}) @dataclass + @member class RemoveForecastDir(Task): is_cycling: bool = True @dataclass + @member class RunGeosExecutable(Task): is_cycling: bool = True @dataclass + @member class RunJediUfoExecutable(Task): is_cycling: bool = True is_model: bool = True @@ -220,6 +259,7 @@ class RunJediUfoExecutable(Task): time_limit: bool = True @dataclass + @member class RunJediUfoTestsExecutable(Task): time_limit: bool = True is_cycling: bool = True @@ -227,6 +267,7 @@ class RunJediUfoTestsExecutable(Task): slurm: dict = mutable_field({'ntasks-per-node': 1}) @dataclass + @member class RunJediConvertStateSoca2ciceExecutable(Task): is_cycling: bool = True is_model: bool = True @@ -234,6 +275,7 @@ class RunJediConvertStateSoca2ciceExecutable(Task): slurm: dict = mutable_field({'nodes': 1}) @dataclass + @member class RunJediFgatExecutable(Task): is_cycling: bool = True is_model: bool = True @@ -241,20 +283,24 @@ class RunJediFgatExecutable(Task): slurm: dict = mutable_field({}) @dataclass + @member class SaveObsDiags(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class SaveRestart(Task): is_cycling: bool = True is_model: bool = True @dataclass + @member class StageJedi(Task): is_model: bool = True @dataclass + @member class StageJediCycle(Task): is_cycling: bool = True is_model: bool = True @@ -262,10 +308,12 @@ class StageJediCycle(Task): scheduling_name: str = "StageJediCycle-{model}" @dataclass + @member class sync_point(Task): script = "true" @dataclass + @member class ThinObs(Task): is_cycling: bool = True is_model: bool = True From 56862f3173c3962115e32d67159a619b29383d63 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 8 Jul 2025 15:47:28 -0400 Subject: [PATCH 040/299] functional code tests --- src/swell/suites/all_suites.py | 1 - src/swell/suites/compare_jedi/suite_config.py | 1 - src/swell/suites/compare_jedi/workflow.py | 2 -- src/swell/suites/compare_variational/workflow.py | 1 - src/swell/tasks/f_grep_residual_norm.py | 2 -- src/swell/tasks/jedi_c_test.py | 2 -- src/swell/test/code_tests/unused_variables_test.py | 4 +++- src/swell/utilities/cylc_runtime.py | 2 +- src/swell/utilities/cylc_workflow.py | 3 +-- src/swell/utilities/scripts/compare_questions.py | 2 +- src/swell/utilities/slurm.py | 8 ++++---- src/swell/utilities/suite_utils.py | 2 -- 12 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/swell/suites/all_suites.py b/src/swell/suites/all_suites.py index 483a37bfd..3ac94ae3e 100644 --- a/src/swell/suites/all_suites.py +++ b/src/swell/suites/all_suites.py @@ -9,7 +9,6 @@ # -------------------------------------------------------------------------------------------------- import os -from enum import Enum from importlib import import_module from swell.swell_path import get_swell_path diff --git a/src/swell/suites/compare_jedi/suite_config.py b/src/swell/suites/compare_jedi/suite_config.py index da974e89a..23950be55 100644 --- a/src/swell/suites/compare_jedi/suite_config.py +++ b/src/swell/suites/compare_jedi/suite_config.py @@ -9,7 +9,6 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/compare_jedi/workflow.py b/src/swell/suites/compare_jedi/workflow.py index 54a8dbd3b..ab3b91de1 100644 --- a/src/swell/suites/compare_jedi/workflow.py +++ b/src/swell/suites/compare_jedi/workflow.py @@ -7,11 +7,9 @@ # -------------------------------------------------------------------------------------------------- -import yaml import os from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes from swell.utilities.cylc_runtime import Task # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 39aa4b9b2..344343152 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -11,7 +11,6 @@ import os from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes from swell.utilities.cylc_runtime import Task # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/f_grep_residual_norm.py b/src/swell/tasks/f_grep_residual_norm.py index ba76fe616..b8b066691 100644 --- a/src/swell/tasks/f_grep_residual_norm.py +++ b/src/swell/tasks/f_grep_residual_norm.py @@ -9,11 +9,9 @@ import os -import re import subprocess from swell.tasks.base.task_base import taskBase -from swell.utilities.suite_utils import get_model_components # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/jedi_c_test.py b/src/swell/tasks/jedi_c_test.py index 32c81cb2a..8c115c519 100644 --- a/src/swell/tasks/jedi_c_test.py +++ b/src/swell/tasks/jedi_c_test.py @@ -9,11 +9,9 @@ import os -import re import subprocess from swell.tasks.base.task_base import taskBase -from swell.utilities.suite_utils import get_model_components # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/test/code_tests/unused_variables_test.py b/src/swell/test/code_tests/unused_variables_test.py index 4d348b08e..7ad27ae8c 100644 --- a/src/swell/test/code_tests/unused_variables_test.py +++ b/src/swell/test/code_tests/unused_variables_test.py @@ -34,7 +34,9 @@ def test_unused_variables(self) -> None: for root, _, files in os.walk(get_swell_path()): for filename in files: - if filename.endswith('.py'): # Only process Python files + # Only process Python files + # Ignore results from task_runtimes.py + if filename.endswith('.py') and filename not in ['task_runtimes.py']: file_path = os.path.join(root, filename) flake8_output = run_flake8(file_path) diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 118f9fac5..be99c69a4 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -9,7 +9,7 @@ import os import yaml -from typing import Union, Optional, Self +from typing import Union, Optional from collections.abc import Mapping from dataclasses import dataclass diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index f2bedde99..25dfc9f11 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -7,8 +7,7 @@ # -------------------------------------------------------------------------------------------------- -from typing import Union, Optional, Self, Tuple -from collections.abc import Mapping +from typing import Union, Optional, Tuple import os import yaml diff --git a/src/swell/utilities/scripts/compare_questions.py b/src/swell/utilities/scripts/compare_questions.py index 08f240c60..d56f4464a 100644 --- a/src/swell/utilities/scripts/compare_questions.py +++ b/src/swell/utilities/scripts/compare_questions.py @@ -18,7 +18,7 @@ from swell.tasks.task_questions import TaskQuestions as tq from swell.utilities.swell_questions import QuestionList from swell.utilities.case_switching import camel_case_to_snake_case -from swell.suites.all_suites import workflows, suite_configs +from swell.suites.all_suites import workflows from swell.utilities.logger import get_logger from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import PrepareExperimentConfigAndSuite diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 2739757a8..7c21adfed 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -10,7 +10,7 @@ import os import re import yaml -from typing import Optional, Union +from typing import Union from collections.abc import Mapping from importlib import resources @@ -63,9 +63,9 @@ def prepare_slurm_defaults_and_overrides( if isinstance(slurm_overrides, str): logger.info(f"Reading SLURM directives from {slurm_overrides}.") try: - with open(slurm_file, "r") as slurmfile: - slurm_overrides = yaml.safe_load(slurm_overrides) - except FileNotFoundError as err: + with open(slurm_overrides, "r") as slurmfile: + slurm_overrides = yaml.safe_load(slurmfile) + except FileNotFoundError: raise FileNotFoundError(f"Slurm config {slurm_overrides} not found.") elif not isinstance(slurm_overrides, Mapping): raise TypeError("Slurm overrides is not of type Mapping") diff --git a/src/swell/utilities/suite_utils.py b/src/swell/utilities/suite_utils.py index 82a305587..0d63ce745 100644 --- a/src/swell/utilities/suite_utils.py +++ b/src/swell/utilities/suite_utils.py @@ -8,9 +8,7 @@ # -------------------------------------------------------------------------------------------------- -import glob import os -import importlib from swell.swell_path import get_swell_path From 54ecaef28f263ac720a47ae81d07d700c3d1c2d7 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 8 Jul 2025 16:10:54 -0400 Subject: [PATCH 041/299] pycodestyle fixes --- src/swell/suites/3dfgat_atmos/workflow.py | 1 + src/swell/suites/3dfgat_cycle/workflow.py | 25 ++++++++++++----- src/swell/suites/3dvar/workflow.py | 1 + src/swell/suites/3dvar_atmos/workflow.py | 7 +++-- src/swell/suites/3dvar_cycle/workflow.py | 13 ++++++--- src/swell/suites/build_geos/workflow.py | 2 +- src/swell/suites/build_jedi/workflow.py | 2 +- src/swell/suites/convert_ncdiags/workflow.py | 1 + src/swell/suites/forecast_geos/workflow.py | 1 + src/swell/suites/geosadas/workflow.py | 5 ++-- src/swell/suites/hofx/workflow.py | 5 +++- src/swell/suites/localensembleda/workflow.py | 28 +++++++++++++------ src/swell/suites/ufo_testing/workflow.py | 1 + .../test/code_tests/unused_variables_test.py | 2 +- src/swell/utilities/cylc_workflow.py | 6 ++-- .../utilities/scripts/compare_questions.py | 11 ++++++-- 16 files changed, 79 insertions(+), 32 deletions(-) diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 8ee7b42e1..efe0c408c 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -85,6 +85,7 @@ # -------------------------------------------------------------------------------------------------- + class Workflow_3dfgat_atmos(CylcWorkflow): def define_description(self): description = self.comment_block(""" diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index c0fe120ea..6c3004723 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -86,11 +86,14 @@ def define_graph_section(self): # Data assimilation preperation GetObservations-{model_component} - GenerateBClimatologyByLinking-{model_component} :fail? => GenerateBClimatology-{model_component} + GenerateBClimatologyByLinking-{model_component} :fail? => + GenerateBClimatology-{model_component} LinkGeosOutput-{model_component} => RunJediFgatExecutable-{model_component} StageJediCycle-{model_component} => RunJediFgatExecutable-{model_component} - GenerateBClimatologyByLinking-{model_component}? | GenerateBClimatology-{model_component} => RunJediFgatExecutable-{model_component} + GenerateBClimatologyByLinking-{model_component}? | + GenerateBClimatology-{model_component} => RunJediFgatExecutable-{model_component} + GetObservations-{model_component} => RunJediFgatExecutable-{model_component} # Run analysis diagnostics @@ -103,9 +106,14 @@ def define_graph_section(self): """ if 'cice6' in self.experiment_dict['models']['geos_marine']['marine_models']: cycle_str += f""" - PrepareAnalysis-{model_component} => RunJediConvertStateSoca2ciceExecutable-{model_component} - RunJediConvertStateSoca2ciceExecutable-{model_component} => SaveRestart-{model_component} - RunJediConvertStateSoca2ciceExecutable-{model_component} => CleanCycle-{model_component} + PrepareAnalysis-{model_component} => + RunJediConvertStateSoca2ciceExecutable-{model_component} + + RunJediConvertStateSoca2ciceExecutable-{model_component} => + SaveRestart-{model_component} + + RunJediConvertStateSoca2ciceExecutable-{model_component} => + CleanCycle-{model_component} """ else: cycle_str += f""" @@ -124,11 +132,14 @@ def define_graph_section(self): # MoveBackground-{model_component} => StoreBackground-{model_component} # Remove Run Directory - # MoveDaRestart-{model_component} & MoveBackground-{model_component} => RemoveForecastDir + # MoveDaRestart-{model_component} & MoveBackground-{model_component} => + RemoveForecastDir + MoveDaRestart-{model_component} => RemoveForecastDir # Clean up large files - EvaObservations-{model_component} & EvaJediLog-{model_component} & EvaIncrement-{model_component} & SaveObsDiags-{model_component} => + EvaObservations-{model_component} & EvaJediLog-{model_component} & + EvaIncrement-{model_component} & SaveObsDiags-{model_component} => CleanCycle-{model_component} """ diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 380fba0a2..c18036a49 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -78,6 +78,7 @@ # -------------------------------------------------------------------------------------------------- + class Workflow_3dvar(CylcWorkflow): def define_description(self): description = self.comment_block(""" diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index a7a28f634..d555d4314 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -61,7 +61,9 @@ BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} CloneJedi[^] => StageJediCycle-{model_component} StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} -GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} +GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => +RunJediVariationalExecutable-{model_component} + GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} GenerateObservingSystemRecords-{model_component} => RunJediVariationalExecutable-{model_component} @@ -84,6 +86,7 @@ # -------------------------------------------------------------------------------------------------- + class Workflow_3dvar_atmos(CylcWorkflow): def define_description(self): description = self.comment_block(""" @@ -112,7 +115,7 @@ def define_graph_section(self): if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: cycle_str = cycle_template_1.format(model_component=model_component) - + if self.experiment_dict['models'][model_component]['cycling_varbc']: cycle_str += cycle_template_2.format(model_component=model_component) else: diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index 8d6505015..a3a523b89 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -67,7 +67,9 @@ LinkGeosOutput-{model_component} => RunJediVariationalExecutable-{model_component} StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} -GenerateBClimatologyByLinking-{model_component}? | GenerateBClimatology-{model_component} => RunJediVariationalExecutable-{model_component} +GenerateBClimatologyByLinking-{model_component}? | GenerateBClimatology-{model_component} => +RunJediVariationalExecutable-{model_component} + GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} # Run analysis diagnostics @@ -105,13 +107,16 @@ MoveDaRestart-{model_component} => RemoveForecastDir # Clean up large files -# EvaObservations-{model_component} & EvaJediLog-{model_component} & SaveObsDiags-{model_component} & RemoveForecastDir => -EvaObservations-{model_component} & EvaJediLog-{model_component} & EvaIncrement-{model_component} & SaveObsDiags-{model_component} => +# EvaObservations-{model_component} & EvaJediLog-{model_component} & +SaveObsDiags-{model_component} & RemoveForecastDir => +EvaObservations-{model_component} & EvaJediLog-{model_component} & +EvaIncrement-{model_component} & SaveObsDiags-{model_component} => CleanCycle-{model_component} """ # -------------------------------------------------------------------------------------------------- + class Workflow_3dvar_cycle(CylcWorkflow): def define_description(self): description = self.comment_block(""" @@ -140,7 +145,7 @@ def define_graph_section(self): if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: cycle_str = cycle_template_1.format(model_component=model_component) - + if 'cice6' in self.experiment_dict['models']['geos_marine']['marine_models']: cycle_str += cycle_template_2.format(model_component=model_component) else: diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py index 1cf6cc962..15386f253 100644 --- a/src/swell/suites/build_geos/workflow.py +++ b/src/swell/suites/build_geos/workflow.py @@ -48,7 +48,7 @@ def define_scheduler(self) -> str: scheduler = self.create_new_section('scheduler', scheduler_str) return scheduler.get_section_str() - + # -------------------------------------------------------------------------------------------------- def define_scheduling_section(self): diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index dfa811a69..1f066cacf 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -48,7 +48,7 @@ def define_scheduler(self) -> str: scheduler = self.create_new_section('scheduler', scheduler_str) return scheduler.get_section_str() - + # -------------------------------------------------------------------------------------------------- def define_scheduling_section(self): diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index c60fd5121..613bb876d 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -41,6 +41,7 @@ # -------------------------------------------------------------------------------------------------- + class Workflow_convert_ncdiags(CylcWorkflow): def define_description(self): description = self.comment_block(""" diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index f79b0a2a4..ddf024c81 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -47,6 +47,7 @@ # -------------------------------------------------------------------------------------------------- + class Workflow_forecast_geos(CylcWorkflow): def define_description(self): description = self.comment_block(""" diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index ce419e8f0..2b5154212 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -52,6 +52,7 @@ # -------------------------------------------------------------------------------------------------- + class Workflow_geosadas(CylcWorkflow): def define_description(self): description = self.comment_block(""" @@ -61,11 +62,11 @@ def define_description(self): return description # -------------------------------------------------------------------------------------------------- - + def define_scheduling_section(self): scheduling_dict = {'initial cycle point': '2020-12-15T00:00:00Z', 'final cycle point': '2020-12-15T00:00:00Z'} - + scheduling_section = self.create_new_section('scheduling', scheduling_dict) return scheduling_section diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index 85bf1cc36..a9b2df315 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -49,7 +49,9 @@ BuildJediByLinking[^]? | BuildJedi[^] => RunJediHofxExecutable-{model_component} CloneJedi[^] => StageJediCycle-{model_component} StageJediCycle-{model_component} => RunJediHofxExecutable-{model_component} -GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => RunJediHofxExecutable-{model_component} +GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => +RunJediHofxExecutable-{model_component} + GetObservations-{model_component} => RunJediHofxExecutable-{model_component} GenerateObservingSystemRecords-{model_component} => RunJediHofxExecutable-{model_component} @@ -66,6 +68,7 @@ # -------------------------------------------------------------------------------------------------- + class Workflow_hofx(CylcWorkflow): def define_description(self): description = self.comment_block(""" diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 233c96d42..0b56dafca 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -51,19 +51,29 @@ """ cycle_template_3 = """ -sync_point => RunJediEnsembleMeanVariance-{model_component} => RunJediHofxEnsembleExecutable-{model_component} -RunJediHofxEnsembleExecutable-{model_component} => RunJediLocalEnsembleDaExecutable-{model_component} +sync_point => RunJediEnsembleMeanVariance-{model_component} => +RunJediHofxEnsembleExecutable-{model_component} + +RunJediHofxEnsembleExecutable-{model_component} => +RunJediLocalEnsembleDaExecutable-{model_component} """ cycle_template_4 = """ -# When strategy is parallel, only proceed if all RunJediHofxEnsembleExecutable completes successfully for each packet +# When strategy is parallel, only proceed if all RunJediHofxEnsembleExecutable completes +# successfully for each packet + +# There is a need for a task to combine all hofx observations together, compute node preferred, +# put here as placeholder -# There is a need for a task to combine all hofx observations together, compute node preferred, put here as placeholder -# RunJediHofxEnsembleExecutable-{model_component}_pack{packet} => RunEnsembleHofxCombiner-{model_component} -# RunEnsembleHofxCombiner-{model_component} => RunJediLocalEnsembleDaExecutable-{model_component} +# RunJediHofxEnsembleExecutable-{model_component}_pack{packet} => +# RunEnsembleHofxCombiner-{model_component} + +# RunEnsembleHofxCombiner-{model_component} => +# RunJediLocalEnsembleDaExecutable-{model_component} sync_point => RunJediHofxEnsembleExecutable-{model_component}_pack{packet} -RunJediHofxEnsembleExecutable-{model_component}_pack{packet} => RunJediLocalEnsembleDaExecutable-{model_component} +RunJediHofxEnsembleExecutable-{model_component}_pack{packet} => +RunJediLocalEnsembleDaExecutable-{model_component} """ cycle_template_5 = """ @@ -80,6 +90,7 @@ # -------------------------------------------------------------------------------------------------- + class Workflow_localensembleda(CylcWorkflow): def define_description(self): description = self.comment_block(""" @@ -116,7 +127,8 @@ def define_graph_section(self): cycle_str += cycle_template_3.format(model_component=model_component) elif self.experiment_dict['ensemble_hofx_strategy'] == 'parallel': for packet in range(self.experiment_dict['ensemble_hofx_packets']): - cycle_str += cycle_template_4.format(model_component=model_component, packet=packet) + cycle_str += cycle_template_4.format( + model_component=model_component, packet=packet) cycle_str += cycle_template_5.format(model_component=model_component) diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index 8bd4a320e..1de3395e2 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -61,6 +61,7 @@ # -------------------------------------------------------------------------------------------------- + class Workflow_ufo_testing(CylcWorkflow): def define_description(self): description = self.comment_block(""" diff --git a/src/swell/test/code_tests/unused_variables_test.py b/src/swell/test/code_tests/unused_variables_test.py index 7ad27ae8c..ebf5daf63 100644 --- a/src/swell/test/code_tests/unused_variables_test.py +++ b/src/swell/test/code_tests/unused_variables_test.py @@ -36,7 +36,7 @@ def test_unused_variables(self) -> None: for filename in files: # Only process Python files # Ignore results from task_runtimes.py - if filename.endswith('.py') and filename not in ['task_runtimes.py']: + if filename.endswith('.py') and filename not in ['task_runtimes.py']: file_path = os.path.join(root, filename) flake8_output = run_flake8(file_path) diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 25dfc9f11..7a93e5cfa 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -161,9 +161,11 @@ def define_scheduling(self) -> str: def define_scheduling_section(self) -> CylcSection: scheduling_dict = {'initial cycle point': self.experiment_dict['start_cycle_point'], 'final cycle point': self.experiment_dict['final_cycle_point']} - + if 'runahead_limit' in self.experiment_dict: - scheduling_dict = update_dict(scheduling_dict, {'runahead limit': self.experiment_dict['runahead_limit']}) + scheduling_dict = update_dict( + scheduling_dict, + {'runahead limit': self.experiment_dict['runahead_limit']}) scheduling_section = self.create_new_section('scheduling', scheduling_dict) diff --git a/src/swell/utilities/scripts/compare_questions.py b/src/swell/utilities/scripts/compare_questions.py index d56f4464a..a046a54ec 100644 --- a/src/swell/utilities/scripts/compare_questions.py +++ b/src/swell/utilities/scripts/compare_questions.py @@ -20,7 +20,8 @@ from swell.utilities.case_switching import camel_case_to_snake_case from swell.suites.all_suites import workflows from swell.utilities.logger import get_logger -from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import PrepareExperimentConfigAndSuite +from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import ( + PrepareExperimentConfigAndSuite) # -------------------------------------------------------------------------------------------------- @@ -38,6 +39,7 @@ def filter_list(cls, lst: list) -> list: # -------------------------------------------------------------------------------------------------- + def get_workflow(suite: str): """ Parse the suite's flow.cylc file and get all the tasks used by the suite. """ @@ -59,8 +61,9 @@ def get_workflow(suite: str): # -------------------------------------------------------------------------------------------------- + def get_scheduling(suite: str): - + workflow_obj = get_workflow(suite) scheduling_section = workflow_obj.define_scheduling() @@ -69,6 +72,7 @@ def get_scheduling(suite: str): # -------------------------------------------------------------------------------------------------- + def get_all_tasks(suite: str) -> list: """ Parse the suite's flow.cylc file and get all the tasks used by the suite. """ @@ -214,7 +218,8 @@ def compare_used_and_set_questions() -> Tuple[dict, dict]: else: field = line.split('self.config.')[1].split(')')[0].strip() + ')' - # Include the parentheses, so we can later assess whether the key is optional + # Include the parentheses, so we can later + # assess whether the key is optional used_task.append(field) set_task = sorted(list(set(set_task))) From 24396f361fa11d61ac5cbad77da09e85eaf0dab4 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 8 Jul 2025 16:37:37 -0400 Subject: [PATCH 042/299] revise templating for 3dfgat_cycle --- src/swell/suites/3dfgat_cycle/workflow.py | 217 ++++++++++++---------- 1 file changed, 115 insertions(+), 102 deletions(-) diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index 6c3004723..f1b469641 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -11,6 +11,115 @@ # -------------------------------------------------------------------------------------------------- +r1_template = """ +# Triggers for non cycle time dependent tasks +# ------------------------------------------- +# Clone Geos source code +CloneGeos + +# Clone JEDI source code +CloneJedi + +# Build Geos source code by linking +CloneGeos => BuildGeosByLinking? + +# Build JEDI source code by linking +CloneJedi => BuildJediByLinking? + +# If not able to link to build create the build +BuildGeosByLinking:fail? => BuildGeos + +# If not able to link to build create the build +BuildJediByLinking:fail? => BuildJedi + +# Need first set of restarts to run model +GetGeosRestart => PrepGeosRunDir + +# Model cannot run without code +BuildGeosByLinking? | BuildGeos => RunGeosExecutable +""" + +r1_model = """ +# JEDI cannot run without code +BuildJediByLinking? | BuildJedi => RunJediFgatExecutable-{model_component} + +# Stage JEDI static files +CloneJedi => StageJedi-{model_component} => RunJediFgatExecutable-{model_component} +""" + +cycle_template_1 = """ +# Model preperation +# Run the forecast through two windows (need to output restarts at the end of the +# first window and backgrounds for the second window) +# MoveDaRestart-{model_component}[-P1D] => PrepGeosRunDir +MoveDaRestart-{model_component}[-PT6H] => PrepGeosRunDir +PrepGeosRunDir => RunGeosExecutable + +# Run the analysis +# RunGeosExecutable => StageJediCycle-{model_component} +RunGeosExecutable => LinkGeosOutput-{model_component} +LinkGeosOutput-{model_component} => GenerateBClimatology-{model_component} + +# Data assimilation preperation +GetObservations-{model_component} +GenerateBClimatologyByLinking-{model_component} :fail? => +GenerateBClimatology-{model_component} + +LinkGeosOutput-{model_component} => RunJediFgatExecutable-{model_component} +StageJediCycle-{model_component} => RunJediFgatExecutable-{model_component} +GenerateBClimatologyByLinking-{model_component}? | +GenerateBClimatology-{model_component} => RunJediFgatExecutable-{model_component} + +GetObservations-{model_component} => RunJediFgatExecutable-{model_component} + +# Run analysis diagnostics +RunJediFgatExecutable-{model_component} => EvaObservations-{model_component} +RunJediFgatExecutable-{model_component} => EvaJediLog-{model_component} +EvaIncrement-{model_component} => PrepareAnalysis-{model_component} + +# Prepare analysis for next forecast +RunJediFgatExecutable-{model_component} => EvaIncrement-{model_component} +""" + +cycle_template_2 = """ +PrepareAnalysis-{model_component} => +RunJediConvertStateSoca2ciceExecutable-{model_component} + +RunJediConvertStateSoca2ciceExecutable-{model_component} => +SaveRestart-{model_component} + +RunJediConvertStateSoca2ciceExecutable-{model_component} => +CleanCycle-{model_component} +""" + +cycle_template_3 = """ +PrepareAnalysis-{model_component} => SaveRestart-{model_component} +""" + +cycle_template_4 = """ +# Move restart to next cycle +SaveRestart-{model_component} => MoveDaRestart-{model_component} + +# Save analysis output +# RunJediFgatExecutable-{model_component} => SaveAnalysis-{model_component} +RunJediFgatExecutable-{model_component} => SaveObsDiags-{model_component} + +# Save model output +# MoveBackground-{model_component} => StoreBackground-{model_component} + +# Remove Run Directory +# MoveDaRestart-{model_component} & MoveBackground-{model_component} => +RemoveForecastDir + +MoveDaRestart-{model_component} => RemoveForecastDir + +# Clean up large files +EvaObservations-{model_component} & EvaJediLog-{model_component} & +EvaIncrement-{model_component} & SaveObsDiags-{model_component} => +CleanCycle-{model_component} +""" + +# -------------------------------------------------------------------------------------------------- class Workflow_3dfgat_cycle(CylcWorkflow): def define_description(self): @@ -27,42 +136,10 @@ def define_graph_section(self): graph_str = '' # Define the string for the R1 (first non-cycling) section - r1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone Geos source code - CloneGeos - - # Clone JEDI source code - CloneJedi - - # Build Geos source code by linking - CloneGeos => BuildGeosByLinking? - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildGeosByLinking:fail? => BuildGeos - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - # Need first set of restarts to run model - GetGeosRestart => PrepGeosRunDir - - # Model cannot run without code - BuildGeosByLinking? | BuildGeos => RunGeosExecutable - """ + r1 = r1_template for model_component in self.experiment_dict['model_components']: - r1 += f""" - # JEDI cannot run without code - BuildJediByLinking? | BuildJedi => RunJediFgatExecutable-{model_component} - - # Stage JEDI static files - CloneJedi => StageJedi-{model_component} => RunJediFgatExecutable-{model_component} - """ + r1 += r1_template.format(model_component=model_component) # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) @@ -71,77 +148,13 @@ def define_graph_section(self): for model_component in self.experiment_dict['model_components']: if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = f""" - # Model preperation - # Run the forecast through two windows (need to output restarts at the end of the - # first window and backgrounds for the second window) - # MoveDaRestart-{model_component}[-P1D] => PrepGeosRunDir - MoveDaRestart-{model_component}[-PT6H] => PrepGeosRunDir - PrepGeosRunDir => RunGeosExecutable - - # Run the analysis - # RunGeosExecutable => StageJediCycle-{model_component} - RunGeosExecutable => LinkGeosOutput-{model_component} - LinkGeosOutput-{model_component} => GenerateBClimatology-{model_component} - - # Data assimilation preperation - GetObservations-{model_component} - GenerateBClimatologyByLinking-{model_component} :fail? => - GenerateBClimatology-{model_component} - - LinkGeosOutput-{model_component} => RunJediFgatExecutable-{model_component} - StageJediCycle-{model_component} => RunJediFgatExecutable-{model_component} - GenerateBClimatologyByLinking-{model_component}? | - GenerateBClimatology-{model_component} => RunJediFgatExecutable-{model_component} - - GetObservations-{model_component} => RunJediFgatExecutable-{model_component} - - # Run analysis diagnostics - RunJediFgatExecutable-{model_component} => EvaObservations-{model_component} - RunJediFgatExecutable-{model_component} => EvaJediLog-{model_component} - EvaIncrement-{model_component} => PrepareAnalysis-{model_component} - - # Prepare analysis for next forecast - RunJediFgatExecutable-{model_component} => EvaIncrement-{model_component} - """ + cycle_str = cycle_template_1.format(model_component=model_component) if 'cice6' in self.experiment_dict['models']['geos_marine']['marine_models']: - cycle_str += f""" - PrepareAnalysis-{model_component} => - RunJediConvertStateSoca2ciceExecutable-{model_component} - - RunJediConvertStateSoca2ciceExecutable-{model_component} => - SaveRestart-{model_component} - - RunJediConvertStateSoca2ciceExecutable-{model_component} => - CleanCycle-{model_component} - """ + cycle_str += cycle_template_2.format(model_component=model_component) else: - cycle_str += f""" - PrepareAnalysis-{model_component} => SaveRestart-{model_component} - """ - - cycle_str += f""" - # Move restart to next cycle - SaveRestart-{model_component} => MoveDaRestart-{model_component} - - # Save analysis output - # RunJediFgatExecutable-{model_component} => SaveAnalysis-{model_component} - RunJediFgatExecutable-{model_component} => SaveObsDiags-{model_component} - - # Save model output - # MoveBackground-{model_component} => StoreBackground-{model_component} - - # Remove Run Directory - # MoveDaRestart-{model_component} & MoveBackground-{model_component} => - RemoveForecastDir - - MoveDaRestart-{model_component} => RemoveForecastDir + cycle_str += cycle_template_3.format(model_component=model_component) - # Clean up large files - EvaObservations-{model_component} & EvaJediLog-{model_component} & - EvaIncrement-{model_component} & SaveObsDiags-{model_component} => - CleanCycle-{model_component} - """ + cycle_str += cycle_template_4.format(model_component=model_component) # Add the cycle string to the graph string graph_str += self.format_cycle(cycle_time, cycle_str) From 14ca73311488ff4c7c650bfb63e4f8bb78178c77 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 9 Jul 2025 08:41:40 -0400 Subject: [PATCH 043/299] fix task_runtimes --- src/swell/tasks/task_runtimes.py | 49 -------------------------------- 1 file changed, 49 deletions(-) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 30f6ea692..2f4133388 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -9,7 +9,6 @@ from typing import Union, Optional, Self from collections.abc import Mapping -from enum import Enum, member from dataclasses import dataclass from swell.utilities.dataclass_utils import mutable_field @@ -21,7 +20,6 @@ class TaskRuntimes(): @dataclass - @member class root(Task): script: bool = False pre_script: str = "source $CYLC_SUITE_DEF_PATH/modules" @@ -29,66 +27,54 @@ class root(Task): 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'}) @dataclass - @member class BuildGeos(Task): pass @dataclass - @member class BuildGeosByLinking(Task): pass @dataclass - @member class BuildJediByLinking(Task): pass @dataclass - @member class BuildJedi(Task): time_limit: bool = True slurm: dict = mutable_field({}) @dataclass - @member class CleanCycle(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class CloneGeos(Task): pass @dataclass - @member class CloneJedi(Task): pass @dataclass - @member class CloneGeosMksi(Task): is_model: bool = True @dataclass - @member class CompareJediCTestOutput(Task): pass @dataclass - @member class EvaJediLog(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class EvaIncrement(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class EvaObservations(Task): time_limit: bool = True is_cycling: bool = True @@ -96,72 +82,60 @@ class EvaObservations(Task): slurm: dict = mutable_field({}) @dataclass - @member class GetBackground(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class GetBackgroundGeosExperiment(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class GetEnsembleGeosExperiment(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class GetGeosRestart(Task): is_cycling: bool = True @dataclass - @member class GetGeovals(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class GetGsiBc(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class GsiBcToIoda(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class GetGsiNcdiag(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class GsiNcdiagToIoda(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class GetGeosAdasBackground(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class GetObservations(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class GenerateBClimatology(Task): time_limit: bool = True is_cycling: bool = True @@ -169,47 +143,39 @@ class GenerateBClimatology(Task): slurm: dict = mutable_field({}) @dataclass - @member class GenerateBClimatologyByLinking(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class GenerateObservingSystemRecords(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class LinkGeosOutput(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class MoveDaRestart(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class MoveForecastRestart(Task): is_cycling: bool = True @dataclass - @member class PrepGeosRunDir(Task): is_cycling: bool = True @dataclass - @member class PrepareAnalysis(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class RunJediFgatExecutable(Task): is_cycling: bool = True is_model: bool = True @@ -217,7 +183,6 @@ class RunJediFgatExecutable(Task): slurm: dict = mutable_field({}) @dataclass - @member class RunJediHofxExecutable(Task): is_cycling: bool = True is_model: bool = True @@ -225,7 +190,6 @@ class RunJediHofxExecutable(Task): slurm: dict = mutable_field({}) @dataclass - @member class RunJediLocalEnsembleDaExecutable(Task): is_cycling: bool = True is_model: bool = True @@ -233,7 +197,6 @@ class RunJediLocalEnsembleDaExecutable(Task): slurm: dict = mutable_field({}) @dataclass - @member class RunJediVariationalExecutable(Task): time_limit: bool = True is_cycling: bool = True @@ -241,17 +204,14 @@ class RunJediVariationalExecutable(Task): slurm: dict = mutable_field({'nodes': 3}) @dataclass - @member class RemoveForecastDir(Task): is_cycling: bool = True @dataclass - @member class RunGeosExecutable(Task): is_cycling: bool = True @dataclass - @member class RunJediUfoExecutable(Task): is_cycling: bool = True is_model: bool = True @@ -259,7 +219,6 @@ class RunJediUfoExecutable(Task): time_limit: bool = True @dataclass - @member class RunJediUfoTestsExecutable(Task): time_limit: bool = True is_cycling: bool = True @@ -267,7 +226,6 @@ class RunJediUfoTestsExecutable(Task): slurm: dict = mutable_field({'ntasks-per-node': 1}) @dataclass - @member class RunJediConvertStateSoca2ciceExecutable(Task): is_cycling: bool = True is_model: bool = True @@ -275,7 +233,6 @@ class RunJediConvertStateSoca2ciceExecutable(Task): slurm: dict = mutable_field({'nodes': 1}) @dataclass - @member class RunJediFgatExecutable(Task): is_cycling: bool = True is_model: bool = True @@ -283,24 +240,20 @@ class RunJediFgatExecutable(Task): slurm: dict = mutable_field({}) @dataclass - @member class SaveObsDiags(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class SaveRestart(Task): is_cycling: bool = True is_model: bool = True @dataclass - @member class StageJedi(Task): is_model: bool = True @dataclass - @member class StageJediCycle(Task): is_cycling: bool = True is_model: bool = True @@ -308,12 +261,10 @@ class StageJediCycle(Task): scheduling_name: str = "StageJediCycle-{model}" @dataclass - @member class sync_point(Task): script = "true" @dataclass - @member class ThinObs(Task): is_cycling: bool = True is_model: bool = True From f1dd07bd07794c5fb4f02e347b8e22c94c6dbebe Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 9 Jul 2025 09:04:35 -0400 Subject: [PATCH 044/299] fix cycling --- src/swell/suites/3dfgat_cycle/workflow.py | 5 +++-- src/swell/suites/3dvar/workflow.py | 4 ++-- src/swell/suites/hofx/workflow.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index f1b469641..8f4f6d83b 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -148,6 +148,7 @@ def define_graph_section(self): for model_component in self.experiment_dict['model_components']: if 'cycle_times' in self.experiment_dict['models'][model_component]: for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: + cycle_str = cycle_template_1.format(model_component=model_component) if 'cice6' in self.experiment_dict['models']['geos_marine']['marine_models']: cycle_str += cycle_template_2.format(model_component=model_component) @@ -156,8 +157,8 @@ def define_graph_section(self): cycle_str += cycle_template_4.format(model_component=model_component) - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) # Create the graph section graph_section = self.create_new_section('graph', graph_str) diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index c18036a49..489e3e7b8 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -108,8 +108,8 @@ def define_graph_section(self): for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: cycle_str = cycle_template.format(model_component=model_component) - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) # Create the graph section graph_section = self.create_new_section('graph', graph_str) diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index a9b2df315..9dd458faf 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -98,8 +98,8 @@ def define_graph_section(self): for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: cycle_str = cycle_template_1.format(model_component=model_component) - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) # Create the graph section graph_section = self.create_new_section('graph', graph_str) From ab1fa20716b707b40fa2f1232f90e8ad396a07e2 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 9 Jul 2025 10:15:13 -0400 Subject: [PATCH 045/299] fix for forecast_geos --- .../prepare_config_and_suite/prepare_config_and_suite.py | 9 +++++++-- src/swell/suites/3dfgat_cycle/workflow.py | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 77b9aa28e..dab2a6888 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -171,6 +171,11 @@ def prepare_task_question_dictionary(self): # Track all possible tasks task_options = [] + # Model components used by the experiment + model_components = [] + if 'model_components' in self.experiment_dict.keys(): + model_components = self.experiment_dict['model_components'] + # Iterate through model independent tasks and update with defaults if not already set for task in self.model_independent_tasks: task_options.append(task) @@ -184,9 +189,9 @@ def prepare_task_question_dictionary(self): for question_model in question['models']: if question_model == 'all_models': - for model in self.experiment_dict['model_components']: + for model in model_components: model_dict[model] = question_dict - elif question_model in self.experiment_dict['model_components']: + elif question_model in model_components: model_dict[question_model] = question_dict self.question_dictionary_model_dep = add_dict( diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index 8f4f6d83b..c18bdb9c7 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -121,6 +121,7 @@ # -------------------------------------------------------------------------------------------------- + class Workflow_3dfgat_cycle(CylcWorkflow): def define_description(self): description = self.comment_block(""" From ab2079151772daa0ba268bce7c4088a6f166ff21 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 9 Jul 2025 11:32:33 -0400 Subject: [PATCH 046/299] fixes to suites --- src/swell/suites/3dfgat_cycle/workflow.py | 2 +- src/swell/suites/convert_ncdiags/workflow.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index c18bdb9c7..c14ad1447 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -140,7 +140,7 @@ def define_graph_section(self): r1 = r1_template for model_component in self.experiment_dict['model_components']: - r1 += r1_template.format(model_component=model_component) + r1 += r1_model.format(model_component=model_component) # Format the R1 cycle and add it to the graph graph_str += self.format_cycle('R1', r1) diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index 613bb876d..2cdd11ee2 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -64,7 +64,7 @@ def define_graph_section(self): # Format the string for each cycle for model in self.experiment_dict['models'].keys(): - if 'cycle_times' in self.experiment_dict['models'][model]['cycle_times']: + if 'cycle_times' in self.experiment_dict['models'][model].keys(): for cycle_time in self.experiment_dict['models'][model]['cycle_times']: cycle_str = cycle_template graph_str += self.format_cycle(cycle_time, cycle_str) From ddb8f5665fbaf3eaef62dd79eda25c2c0ba03b27 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 9 Jul 2025 14:03:47 -0400 Subject: [PATCH 047/299] add comments --- .../prepare_config_and_suite.py | 104 ++---------------- src/swell/utilities/cylc_runtime.py | 5 + src/swell/utilities/cylc_workflow.py | 28 +++++ 3 files changed, 43 insertions(+), 94 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index dab2a6888..2c9397568 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -100,8 +100,11 @@ def __init__( self.comment_dict['suite_to_run'] = 'Record of the suite being executed' # Get list of all possible models + # ------------------------------- self.possible_model_components = get_model_components() + # Start initializing the suite questions first + # -------------------------------------------- self.prepare_suite_question_dictionary() self.override_with_defaults(QuestionType.SUITE) self.override_with_external(QuestionType.SUITE) @@ -110,6 +113,8 @@ def __init__( # ---------------------------------------------------------------------------------------------- def configure_and_ask_task_questions(self) -> None: + # Finalize the experiment config with task questions + self.prepare_task_question_dictionary() self.override_with_defaults(QuestionType.TASK) self.override_with_external(QuestionType.TASK) @@ -125,6 +130,7 @@ def get_experiment_dict(self) -> Mapping: # ---------------------------------------------------------------------------------------------- def prepare_suite_question_dictionary(self) -> None: + # Get questions from the suite config question_dictionary_model_ind = {} question_dictionary_model_dep = {} @@ -167,6 +173,7 @@ def prepare_suite_question_dictionary(self) -> None: # ---------------------------------------------------------------------------------------------- def prepare_task_question_dictionary(self): + # Fill in the question dictionaries with questions from the tasks # Track all possible tasks task_options = [] @@ -354,6 +361,8 @@ def get_questions_of_type(self, suite_task: QuestionType, question_dictionary: Mapping ) -> Mapping: + + # Get all questions of a certain type out_dict = {} if 'models' in question_dictionary.keys(): @@ -372,6 +381,7 @@ def get_questions_of_type(self, # ---------------------------------------------------------------------------------------------- def ask_questions_and_configure(self, suite_task: QuestionType) -> Tuple[dict, dict]: + # Handle asking questions for either suites or tasks if self.config_client.__class__.__name__ == 'GetAnswerCli' and ( suite_task == QuestionType.SUITE): @@ -460,98 +470,4 @@ def question_not_been_asked(self, question_key: str, model: str) -> bool: return True - # ---------------------------------------------------------------------------------------------- - - def get_suite_task_list_model_ind(self, suite_str: str) -> list: - - # Search the suite string for lines containing 'swell task' and not '-m' - swell_task_lines = [line for line in suite_str.split('\n') if 'swell task' in line and - '-m' not in line] - - # Now get the task part - tasks = [] - for line in swell_task_lines: - # Split by 'swell task' - # Remove any leading spaces - # Split by space - tasks.append(line.split('swell task')[1].strip().split(' ')[0]) - - # Ensure there are no duplicate tasks - tasks = list(set(tasks)) - - # Return tasks - return tasks - - # ---------------------------------------------------------------------------------------------- - - def get_all_model_dep_tasks(self, suite_str: str) -> list: - - # Search the suite string for lines containing 'swell task' and '-m' - swell_task_lines = [line for line in suite_str.split('\n') if 'swell task' in line and - '-m' in line] - - # Strip " and spaces from all lines - swell_task_lines = [line.replace('"', '') for line in swell_task_lines] - swell_task_lines = [line.strip() for line in swell_task_lines] - - # All tasks - all_tasks = [] - - for line in swell_task_lines: - all_tasks.append(line.split('swell task ')[1].split(' ')[0]) - - # Ensure all_tasks are unique - all_tasks = list(set(all_tasks)) - - return all_tasks - - # ---------------------------------------------------------------------------------------------- - - def get_suite_task_list_model_dep(self, suite_str: str) -> dict: - - # Search the suite string for lines containing 'swell task' and '-m' - swell_task_lines = [line for line in suite_str.split('\n') if 'swell task' in line and - '-m' in line] - - # Strip " and spaces from all lines - swell_task_lines = [line.replace('"', '') for line in swell_task_lines] - swell_task_lines = [line.strip() for line in swell_task_lines] - - # Now get the model part - models = [] - for line in swell_task_lines: - models.append(line.split('-m')[1].split('0')[0].strip()) - - # Unique models - models = list(set(models)) - - # Assemble dictionary where key is model and val is the tasks that model is associated with - model_tasks = {} - for model in models: - - # Get all elements of swell_task_lines that contains "-m {model}" - model_tasks_this_model = [line for line in swell_task_lines if f'-m {model}' in line] - - # Get task name - tasks = [] - for line in model_tasks_this_model: - tasks.append(line.split('swell task ')[1].split(' ')[0]) - - # Unique model tasks - model_tasks[model] = list(set(tasks)) - - # Return the dictionary - return model_tasks - - # ---------------------------------------------------------------------------------------------- - - def get_dynamic_tasks(self, question_list: list) -> list: - tasks = [] - - for question in question_list: - if question['question_name'] == 'dynamic_task_list': - tasks.extend(question['default_value']) - - return tasks - # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index be99c69a4..619b45ed3 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -80,6 +80,8 @@ def format_string_block(self, string: str) -> str: # -------------------------------------------------------------------------------------------------- def match_platform(self, content: Union[str, dict], platform: str): + # Resolve platform-specific entries in the task object + if isinstance(content, Mapping): if platform in content.keys(): content = content[platform] @@ -115,6 +117,9 @@ def resolve_model(self, slurm_dict: Mapping) -> dict: # -------------------------------------------------------------------------------------------------- def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Mapping: + # Take the external slurm dictionary and merge it with the task's parameters + # to get the dict that will be output in the flow.cylc + slurm_dict = {} if self.slurm is not None: for key, value in self.slurm.items(): diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 7a93e5cfa..10bd11d9d 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -40,6 +40,8 @@ def __init__(self, experiment_dict, slurm_external) -> None: # -------------------------------------------------------------------------------------------------- def format_string_block(self, string) -> str: + # Format a string block with proper indentation + out_string = '"""\n' out_string += indent_lines(string, 1, True) out_string += '"""\n' @@ -49,6 +51,8 @@ def format_string_block(self, string) -> str: # -------------------------------------------------------------------------------------------------- def format_cycle(self, name: str, cycle: str) -> str: + # Format a cycle in the graph section + cycle_string = f'{name} = ' cycle_string += self.format_string_block(cycle) return cycle_string @@ -56,6 +60,7 @@ def format_cycle(self, name: str, cycle: str) -> str: # -------------------------------------------------------------------------------------------------- def reset_indentation(self, string: str) -> str: + out_string = '' start = False @@ -73,6 +78,8 @@ def reset_indentation(self, string: str) -> str: # -------------------------------------------------------------------------------------------------- def setup_workflow(self) -> None: + # Initial setup for the workflow, includes everything but the runtime section + self.header = self.define_header() self.description = self.define_description() self.scheduler = self.define_scheduler() @@ -83,6 +90,8 @@ def setup_workflow(self) -> None: # -------------------------------------------------------------------------------------------------- def define_header(self) -> str: + # Define a 'header', usually this includes any copyright information + header = '#!jinja2\n' header += self.comment_block(string=""" # (C) Copyright 2021- United States Government as represented by the Administrator of the @@ -104,6 +113,8 @@ def define_description(self) -> str: # -------------------------------------------------------------------------------------------------- def comment_block(self, string, level: int = 0, section_break: bool = True): + # Format a comment block with proper indentation + out_string = '' string = indent_lines(string, level, reset=True) @@ -124,6 +135,8 @@ def comment_block(self, string, level: int = 0, section_break: bool = True): # -------------------------------------------------------------------------------------------------- def define_scheduler(self) -> str: + # Define a scheduler section that includes email infrastructure + scheduler_str = 'UTC mode = True\nallow implicit tasks = False\n' settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) @@ -149,6 +162,8 @@ def define_scheduler(self) -> str: # -------------------------------------------------------------------------------------------------- def define_scheduling(self) -> str: + # Get the string for the entire scheduling section + scheduling = self.define_scheduling_section() graph = self.define_graph_section() @@ -159,6 +174,8 @@ def define_scheduling(self) -> str: # -------------------------------------------------------------------------------------------------- def define_scheduling_section(self) -> CylcSection: + # Define a CylcSection object that comprises the initial contents of the scheduling section + # This will be appended to the graph section to create the overall section scheduling_dict = {'initial cycle point': self.experiment_dict['start_cycle_point'], 'final cycle point': self.experiment_dict['final_cycle_point']} @@ -174,11 +191,14 @@ def define_scheduling_section(self) -> CylcSection: # -------------------------------------------------------------------------------------------------- def define_graph_section(self) -> CylcSection: + # Define a CylcSection object that will be used to build the scheduling section return self.create_new_section('graph') # -------------------------------------------------------------------------------------------------- def parse_graph_for_tasks(self) -> CylcSection: + # Iterate through the graph section and determine all the tasks used by the suite + tasks = [] cylc_characters = [':', '[', ']', '?'] @@ -217,6 +237,8 @@ def parse_graph_for_tasks(self) -> CylcSection: # -------------------------------------------------------------------------------------------------- def get_independent_and_model_tasks(self) -> Tuple[list, dict]: + # Separate the tasks into model independent and dependent + ind_tasks = [] model_tasks = {} @@ -245,18 +267,23 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: # -------------------------------------------------------------------------------------------------- def define_runtime_task_overrides(self) -> dict: + # Override in suite file to set any custom runtimes as needed by the suite return {} # -------------------------------------------------------------------------------------------------- def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = ''): + # Create a new section with indentation and content return CylcSection(name, content) # -------------------------------------------------------------------------------------------------- def define_runtime(self) -> str: + # Handle adding runtime sections for all tasks + runtime_section = self.create_new_section('runtime', '\n# Task defaults\n# -------------\n') + # Grab any overrides for certain tasks runtime_overrides = self.define_runtime_task_overrides() for task in ['root'] + self.tasks: @@ -291,6 +318,7 @@ def define_runtime(self) -> str: # -------------------------------------------------------------------------------------------------- def get_workflow_str(self) -> str: + # Get the whole string to go into the flow.cylc file workflow_str = '' From e09e98e767b8352799f8edcda7c548985365f770 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 15 Jul 2025 15:54:38 -0400 Subject: [PATCH 048/299] test plots --- .../eva/compare_increment-geos_marine.yaml | 275 ++++++++++++++++++ src/swell/tasks/eva_comparison_increment.py | 40 +++ 2 files changed, 315 insertions(+) create mode 100644 src/swell/suites/compare_variational/eva/compare_increment-geos_marine.yaml create mode 100644 src/swell/tasks/eva_comparison_increment.py diff --git a/src/swell/suites/compare_variational/eva/compare_increment-geos_marine.yaml b/src/swell/suites/compare_variational/eva/compare_increment-geos_marine.yaml new file mode 100644 index 000000000..ff008999a --- /dev/null +++ b/src/swell/suites/compare_variational/eva/compare_increment-geos_marine.yaml @@ -0,0 +1,275 @@ +datasets: + + - name: experiment_increment_1 + type: SocaRestart + soca_filenames: {{increment_file_path_1}} + geometry_file: {{cycle_dir_1}}/INPUT/soca_gridspec.nc + + variables: [Temp, Salt, ave_ssh] + coordinate variables: [lon, lat] + + - name: experiment_increment_2 + type: SocaRestart + soca_filenames: {{increment_file_path_2}} + geometry_file: {{cycle_dir_2}}/INPUT/soca_gridspec.nc + + variables: [Temp, Salt, ave_ssh] + coordinate variables: [lon, lat] + +graphics: + + plotting_backend: Emcpy + figure_list: + + - batch figure: + variables: [ave_ssh] + figure: + figure size: [60,10] + layout: [3,1] + title: 'SOCA Increment' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}.png' + + # ave_ssh plot 1 + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: SSH Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::SOCAgrid::lon + latitude: + variable: experiment_increment::SOCAgrid::lat + data: + variable: experiment_increment::SOCAVars::ave_ssh + label: ave_ssh increment 1 + colorbar: true + cmap: 'bwr' + vmin: &vmin_ave_ssh -0.25 + vmax: &vmax_ave_ssh 0.25 + + # ave_ssh plot 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: SSH Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::SOCAgrid::lon + latitude: + variable: experiment_increment_2::SOCAgrid::lat + data: + variable: experiment_increment_2::SOCAVars::ave_ssh + label: ave_ssh increment 2 + colorbar: true + cmap: 'bwr' + vmin: *vmin_ave_ssh + vmax: *vmax_ave_ssh + + # ave_ssh diff plot + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: SSH Increment (1-2) Diff + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::SOCAgrid::lon + latitude: + variable: experiment_increment_1::SOCAgrid::lat + data: + variable: experiment_increment_1::SOCAVars::ave_ssh_diff + label: ave_ssh increment + colorbar: true + cmap: 'bwr' + vmin: *vmin_ave_ssh + vmax: *vmax_ave_ssh + + - batch figure: + variables: [Temp] + figure: + figure size: [60,10] + layout: [3,1] + title: 'Soca Increment' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}.png' + plots: + # Temp plot 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: SST Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::SOCAgrid::lon + latitude: + variable: experiment_increment_1::SOCAgrid::lat + data: + variable: experiment_increment_1::SOCAVars::Temp + slices: '[0,...]' + label: SST increment 1 + colorbar: true + cmap: 'bwr' + vmin: &vmin_temp -3 + vmax: &vmax_temp 3 + + # Temp plot 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: SST Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::SOCAgrid::lon + latitude: + variable: experiment_increment_2::SOCAgrid::lat + data: + variable: experiment_increment_2::SOCAVars::Temp + slices: '[0,...]' + label: SST increment + colorbar: true + cmap: 'bwr' + vmin: *vmin_temp + vmax: *vmax_temp + + # Temp diff plot + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: SST Increment (1-2) Diff + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::SOCAgrid::lon + latitude: + variable: experiment_increment_1::SOCAgrid::lat + data: + variable: experiment_increment_1::SOCAVars::Temp_diff + slices: '[0,...]' + label: SST increment + colorbar: true + cmap: 'bwr' + vmin: *vmin_temp + vmax: *vmax_temp + + - batch figure: + variables: [Salt] + figure: + figure size: [60,10] + layout: [3,1] + title: 'Soca Increment' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}.png' + plots: + # Salt plot 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: SSS Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::SOCAgrid::lon + latitude: + variable: experiment_increment_1::SOCAgrid::lat + data: + variable: experiment_increment_1::SOCAVars::Salt + slices: '[0,...]' + label: SSS increment 1 + colorbar: true + cmap: 'bwr' + vmin: &vmin_salt -1 + vmax: &vmax_salt 1 + + # Salt plot 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: SSS Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::SOCAgrid::lon + latitude: + variable: experiment_increment_2::SOCAgrid::lat + data: + variable: experiment_increment_2::SOCAVars::Salt + slices: '[0,...]' + label: SSS increment 2 + colorbar: true + cmap: 'bwr' + vmin: *vmin_salt + vmax: *vmin_salt + + # Salt Diff plot + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: SSS Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::SOCAgrid::lon + latitude: + variable: experiment_increment_1::SOCAgrid::lat + data: + variable: experiment_increment_1::SOCAVars::Salt_diff + slices: '[0,...]' + label: SSS increment (1-2) Diff + colorbar: true + cmap: 'bwr' + vmin: *vmin_salt + vmax: *vmin_salt + +# ssh diff +transforms: +- equals: experiment_increment_1::SOCAVars::ave_ssh-experiment_increment_2::SOCAVars::ave_ssh + for: + variable: ave_ssh + new name: experiment_increment_1::SOCAVars::ave_ssh_diff + transform: arithmetic + +# Temp diff +transforms: +- equals: experiment_increment_1::SOCAVars::Temp-experiment_increment_2::SOCAVars::Temp + for: + variable: Temp + new name: experiment_increment_1::SOCAVars::Temp_diff + transform: arithmetic + +# Salt diff +transforms: +- equals: experiment_increment_1::SOCAVars::Salt-experiment_increment_2::SOCAVars::Salt + for: + variable: Salt + new name: experiment_increment_1::SOCAVars::Salt_diff + transform: arithmetic \ No newline at end of file diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py new file mode 100644 index 000000000..0d3809ca5 --- /dev/null +++ b/src/swell/tasks/eva_comparison_increment.py @@ -0,0 +1,40 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. + +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import yaml + +from eva.eva_driver import eva + +from swell.tasks.base.task_base import taskBase +from swell.utilities.jinja2 import template_string_jinja2 + +# -------------------------------------------------------------------------------------------------- + + +class EvaComparisonIncrement(taskBase): + + def execute(self) -> None: + + experiment_paths = self.config.comparison_experiment_paths() + + experiment_path_1 = experiment_paths[0] + + experiment_file_path_1 = os.path.join(os.path.basename(experiment_path_1), 'experiment.yaml') + + task_1 = taskBase(config_input=experiment_file_path_1, + datetime_input=self.cycle_time(), + model=self.get_model(), + ensemblePacket=self.get_ensemble_packet(), + task_name='taskBase', + test_iteration=None, + test_output=None) + + print(task_1.cycle_dir()) \ No newline at end of file From 337110c34fd901f435ea25979d43ba61e7cefb73 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 15 Jul 2025 17:16:51 -0400 Subject: [PATCH 049/299] add imcrement comparison --- src/swell/tasks/eva_comparison_increment.py | 90 ++++++++++++++++++--- 1 file changed, 79 insertions(+), 11 deletions(-) diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index 0d3809ca5..fe69b97c4 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -10,6 +10,8 @@ import os import yaml +import glob +import datetime from eva.eva_driver import eva @@ -21,20 +23,86 @@ class EvaComparisonIncrement(taskBase): + def cycles_in_experiment(self, path): + cycle_glob = os.path.join(os.path.dirname(path), '..', 'run', '*Z', self.get_model()) + cycle_paths = glob.glob(cycle_glob) + + cycle_times = [] + + for cycle_path in cycle_paths: + cycle_time = cycle_path.split(f'Z/{self.get_model()}')[0].split('run/')[1] + cycle_times.append(cycle_time) + + return cycle_times + def execute(self) -> None: + model = self.get_model() + + eva_path = os.path.join(self.experiment_path(), self.experiment_id()+'-suite', 'eva') + eva_config_file = os.path.join(eva_path, f'comparison_increment-{model}.yaml') + with open(eva_config_file, 'r') as eva_config_file_open: + eva_str_template = eva_config_file_open.read() + experiment_paths = self.config.comparison_experiment_paths() experiment_path_1 = experiment_paths[0] + experiment_path_2 = experiment_paths[1] + + experiment_id_1 = os.path.basename(os.path.dirname(experiment_path_1)) + experiment_id_2 = os.path.basename(os.path.dirname(experiment_path_2)) + + cycle_times_1 = self.cycles_in_experiment(experiment_path_1) + cycle_times_2 = self.cycles_in_experiment(experiment_path_2) + + cycle_times = list(set(cycle_times_1) & set(cycle_times_2)) + + for cycle_time in cycle_times: + cycle_dir = os.path.join(self.experiment_root(), 'run', cycle_time, self.get_model(), 'eva') + os.makedirs(cycle_dir, exist_ok=True) + + cycle_time_dto = datetime.datetime.strptime(cycle_time, '%Y%m%dT%H%M%SZ') + + # Info to task log + info_string = 'Running Eva to plot from the increment file' + self.logger.info('') + self.logger.info(info_string) + self.logger.info('-'*len(info_string)) + + # Create time strings for eva_override directory + cycle_time_reformat = cycle_time_dto.strftime('%Y%m%d_%H%M%Sz') + + # Create dictionary used to override the eva config + eva_override = {} + + # Soca case + if model == 'geos_marine': + ocn_cycle_time = cycle_time_dto.strftime('%Y-%m-%dT%H:%M:%SZ') + incr_file_1 = f'ocn.{experiment_id_1}.incr.{ocn_cycle_time}.nc' + incr_file_2 = f'ocn.{experiment_id_2}.incr.{ocn_cycle_time}.nc' + + cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', cycle_time, self.get_model()) + cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', cycle_time, self.get_model()) + + increment_file_path_1 = os.path.join(cycle_dir_1, incr_file_1) + increment_file_path_2 = os.path.join(cycle_dir_2, incr_file_2) + + eva_override['cycle_dir'] = self.cycle_dir() + eva_override['cycle_time'] = cycle_time_reformat + eva_override['increment_file_path_1'] = increment_file_path_1 + eva_override['increment_file_path_2'] = increment_file_path_2 + + # Override the eva dictionary + eva_str = template_string_jinja2(self.logger, eva_str_template, eva_override) + eva_dict = yaml.safe_load(eva_str) + + # Write eva dictionary to file + # ---------------------------- + conf_output = os.path.join(self.cycle_dir(), 'eva', 'increment', 'comparison_increment_eva.yaml') + os.makedirs(os.path.dirname(conf_output), exist_ok=True) + with open(conf_output, 'w') as outfile: + yaml.dump(eva_dict, outfile, default_flow_style=False) - experiment_file_path_1 = os.path.join(os.path.basename(experiment_path_1), 'experiment.yaml') - - task_1 = taskBase(config_input=experiment_file_path_1, - datetime_input=self.cycle_time(), - model=self.get_model(), - ensemblePacket=self.get_ensemble_packet(), - task_name='taskBase', - test_iteration=None, - test_output=None) - - print(task_1.cycle_dir()) \ No newline at end of file + # Call eva + # -------- + eva(eva_dict) From 7e5d22330e2aadae63920670f47687cdd056852f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 16 Jul 2025 11:29:49 -0400 Subject: [PATCH 050/299] Functional comparison plots --- ... => comparison_increment-geos_marine.yaml} | 34 +++++++------------ .../compare_variational/suite_config.py | 16 +++++++++ src/swell/tasks/eva_comparison_increment.py | 16 +++++---- src/swell/utilities/question_defaults.py | 2 +- 4 files changed, 40 insertions(+), 28 deletions(-) rename src/swell/suites/compare_variational/eva/{compare_increment-geos_marine.yaml => comparison_increment-geos_marine.yaml} (91%) diff --git a/src/swell/suites/compare_variational/eva/compare_increment-geos_marine.yaml b/src/swell/suites/compare_variational/eva/comparison_increment-geos_marine.yaml similarity index 91% rename from src/swell/suites/compare_variational/eva/compare_increment-geos_marine.yaml rename to src/swell/suites/compare_variational/eva/comparison_increment-geos_marine.yaml index ff008999a..7168a27e6 100644 --- a/src/swell/suites/compare_variational/eva/compare_increment-geos_marine.yaml +++ b/src/swell/suites/compare_variational/eva/comparison_increment-geos_marine.yaml @@ -36,16 +36,16 @@ graphics: domain: global add_map_features: ['coastline'] add_colorbar: - label: SSH Increment + label: SSH Increment 1 add_grid: layers: - type: MapGridded longitude: - variable: experiment_increment::SOCAgrid::lon + variable: experiment_increment_1::SOCAgrid::lon latitude: - variable: experiment_increment::SOCAgrid::lat + variable: experiment_increment_1::SOCAgrid::lat data: - variable: experiment_increment::SOCAVars::ave_ssh + variable: experiment_increment_1::SOCAVars::ave_ssh label: ave_ssh increment 1 colorbar: true cmap: 'bwr' @@ -58,7 +58,7 @@ graphics: domain: global add_map_features: ['coastline'] add_colorbar: - label: SSH Increment + label: SSH Increment 2 add_grid: layers: - type: MapGridded @@ -93,8 +93,6 @@ graphics: label: ave_ssh increment colorbar: true cmap: 'bwr' - vmin: *vmin_ave_ssh - vmax: *vmax_ave_ssh - batch figure: variables: [Temp] @@ -110,7 +108,7 @@ graphics: domain: global add_map_features: ['coastline'] add_colorbar: - label: SST Increment + label: SST Increment 1 add_grid: layers: - type: MapGridded @@ -144,7 +142,7 @@ graphics: data: variable: experiment_increment_2::SOCAVars::Temp slices: '[0,...]' - label: SST increment + label: SST increment 2 colorbar: true cmap: 'bwr' vmin: *vmin_temp @@ -167,11 +165,9 @@ graphics: data: variable: experiment_increment_1::SOCAVars::Temp_diff slices: '[0,...]' - label: SST increment + label: SST increment (1-2) Diff colorbar: true cmap: 'bwr' - vmin: *vmin_temp - vmax: *vmax_temp - batch figure: variables: [Salt] @@ -187,7 +183,7 @@ graphics: domain: global add_map_features: ['coastline'] add_colorbar: - label: SSS Increment + label: SSS Increment 1 add_grid: layers: - type: MapGridded @@ -210,7 +206,7 @@ graphics: domain: global add_map_features: ['coastline'] add_colorbar: - label: SSS Increment + label: SSS Increment 2 add_grid: layers: - type: MapGridded @@ -225,7 +221,7 @@ graphics: colorbar: true cmap: 'bwr' vmin: *vmin_salt - vmax: *vmin_salt + vmax: *vmax_salt # Salt Diff plot - mapping: @@ -233,7 +229,7 @@ graphics: domain: global add_map_features: ['coastline'] add_colorbar: - label: SSS Increment + label: SSS Increment (1-2) Diff add_grid: layers: - type: MapGridded @@ -247,8 +243,6 @@ graphics: label: SSS increment (1-2) Diff colorbar: true cmap: 'bwr' - vmin: *vmin_salt - vmax: *vmin_salt # ssh diff transforms: @@ -259,7 +253,6 @@ transforms: transform: arithmetic # Temp diff -transforms: - equals: experiment_increment_1::SOCAVars::Temp-experiment_increment_2::SOCAVars::Temp for: variable: Temp @@ -267,9 +260,8 @@ transforms: transform: arithmetic # Salt diff -transforms: - equals: experiment_increment_1::SOCAVars::Salt-experiment_increment_2::SOCAVars::Salt for: variable: Salt new name: experiment_increment_1::SOCAVars::Salt_diff - transform: arithmetic \ No newline at end of file + transform: arithmetic diff --git a/src/swell/suites/compare_variational/suite_config.py b/src/swell/suites/compare_variational/suite_config.py index 1fe9b002f..e728a3101 100644 --- a/src/swell/suites/compare_variational/suite_config.py +++ b/src/swell/suites/compare_variational/suite_config.py @@ -7,6 +7,7 @@ # # -------------------------------------------------------------------------------------------------- +import importlib from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd @@ -14,6 +15,9 @@ from enum import Enum +_3dvar_container = importlib.import_module(f'swell.suites.3dvar.suite_config') +_3dvar_config = getattr(_3dvar_container, 'SuiteConfig') +_3dvar = getattr(_3dvar_config, '_3dvar') # -------------------------------------------------------------------------------------------------- @@ -30,3 +34,15 @@ class SuiteConfig(QuestionContainer, Enum): ) # -------------------------------------------------------------------------------------------------- + + compare_3dvar = QuestionList( + list_name="compare_3dvar", + questions=[ + compare_variational, + qd.model_components(['geos_marine']), + _3dvar, + + ] + ) + + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index fe69b97c4..c32bfc807 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -30,7 +30,7 @@ def cycles_in_experiment(self, path): cycle_times = [] for cycle_path in cycle_paths: - cycle_time = cycle_path.split(f'Z/{self.get_model()}')[0].split('run/')[1] + cycle_time = cycle_path.split(f'/{self.get_model()}')[0].split('run/')[1] cycle_times.append(cycle_time) return cycle_times @@ -49,8 +49,8 @@ def execute(self) -> None: experiment_path_1 = experiment_paths[0] experiment_path_2 = experiment_paths[1] - experiment_id_1 = os.path.basename(os.path.dirname(experiment_path_1)) - experiment_id_2 = os.path.basename(os.path.dirname(experiment_path_2)) + experiment_id_1 = os.path.basename(os.path.dirname(experiment_path_1)).replace('-suite', '') + experiment_id_2 = os.path.basename(os.path.dirname(experiment_path_2)).replace('-suite', '') cycle_times_1 = self.cycles_in_experiment(experiment_path_1) cycle_times_2 = self.cycles_in_experiment(experiment_path_2) @@ -58,8 +58,9 @@ def execute(self) -> None: cycle_times = list(set(cycle_times_1) & set(cycle_times_2)) for cycle_time in cycle_times: - cycle_dir = os.path.join(self.experiment_root(), 'run', cycle_time, self.get_model(), 'eva') + cycle_dir = os.path.join(self.experiment_path(), 'run', cycle_time, self.get_model()) os.makedirs(cycle_dir, exist_ok=True) + print(cycle_dir) cycle_time_dto = datetime.datetime.strptime(cycle_time, '%Y%m%dT%H%M%SZ') @@ -87,7 +88,10 @@ def execute(self) -> None: increment_file_path_1 = os.path.join(cycle_dir_1, incr_file_1) increment_file_path_2 = os.path.join(cycle_dir_2, incr_file_2) - eva_override['cycle_dir'] = self.cycle_dir() + eva_override['cycle_dir'] = cycle_dir + eva_override['cycle_dir_1'] = cycle_dir_1 + eva_override['cycle_dir_2'] = cycle_dir_2 + eva_override['cycle_time'] = cycle_time_reformat eva_override['increment_file_path_1'] = increment_file_path_1 eva_override['increment_file_path_2'] = increment_file_path_2 @@ -98,7 +102,7 @@ def execute(self) -> None: # Write eva dictionary to file # ---------------------------- - conf_output = os.path.join(self.cycle_dir(), 'eva', 'increment', 'comparison_increment_eva.yaml') + conf_output = os.path.join(cycle_dir, 'eva', 'increment', 'comparison_increment_eva.yaml') os.makedirs(os.path.dirname(conf_output), exist_ok=True) with open(conf_output, 'w') as outfile: yaml.dump(eva_dict, outfile, default_flow_style=False) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 91b8edb86..322c4e3f2 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -29,7 +29,7 @@ class comparison_experiment_paths(SuiteQuestion): default_value: list = mutable_field([]) question_name: str = "comparison_experiment_paths" ask_question: bool = True - prompt: str = "Provide paths to two or more experiments to run comparison tests on." + prompt: str = "Provide paths to two experiments to run comparison tests on." widget_type: WType = WType.STRING_CHECK_LIST # -------------------------------------------------------------------------------------------------- From 43e35f196d3114fe2047749ea3884bb899b0d647 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 16 Jul 2025 14:31:09 -0400 Subject: [PATCH 051/299] Add comments for comparison plots --- src/swell/suites/3dvar_cycle/workflow.py | 2 +- .../compare_variational/suite_config.py | 2 + .../suites/compare_variational/workflow.py | 3 ++ src/swell/tasks/eva_comparison_increment.py | 51 +++++++++++++++++-- src/swell/tasks/task_questions.py | 9 ++++ src/swell/tasks/task_runtimes.py | 4 ++ 6 files changed, 65 insertions(+), 6 deletions(-) diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index a3a523b89..def9de928 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -108,7 +108,7 @@ # Clean up large files # EvaObservations-{model_component} & EvaJediLog-{model_component} & -SaveObsDiags-{model_component} & RemoveForecastDir => +# SaveObsDiags-{model_component} & RemoveForecastDir => EvaObservations-{model_component} & EvaJediLog-{model_component} & EvaIncrement-{model_component} & SaveObsDiags-{model_component} => CleanCycle-{model_component} diff --git a/src/swell/suites/compare_variational/suite_config.py b/src/swell/suites/compare_variational/suite_config.py index e728a3101..4f5ee5987 100644 --- a/src/swell/suites/compare_variational/suite_config.py +++ b/src/swell/suites/compare_variational/suite_config.py @@ -21,6 +21,7 @@ # -------------------------------------------------------------------------------------------------- + class SuiteConfig(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- @@ -29,6 +30,7 @@ class SuiteConfig(QuestionContainer, Enum): list_name="compare", questions=[ sq.compare, + qd.model_components(), qd.runahead_limit(), ] ) diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 344343152..e0d8ccd51 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -62,6 +62,9 @@ def define_scheduling(self): graph_str = '' + for model in self.experiment_dict['model_components']: + graph_str += self.format_cycle('R1', f"EvaComparisonIncrement-{model}\n") + for cycle_time, models in cycle_model_times.items(): cycle_str = '' diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index c32bfc807..c565927b6 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -17,6 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.utilities.jinja2 import template_string_jinja2 +from swell.utilities.data_assimilation_window_params import DataAssimilationWindowParams # -------------------------------------------------------------------------------------------------- @@ -39,31 +40,47 @@ def execute(self) -> None: model = self.get_model() + # Open the eva configuration file eva_path = os.path.join(self.experiment_path(), self.experiment_id()+'-suite', 'eva') eva_config_file = os.path.join(eva_path, f'comparison_increment-{model}.yaml') with open(eva_config_file, 'r') as eva_config_file_open: eva_str_template = eva_config_file_open.read() + # Get the paths for the two experiments experiment_paths = self.config.comparison_experiment_paths() experiment_path_1 = experiment_paths[0] experiment_path_2 = experiment_paths[1] + # Experiment IDs for the two experiments experiment_id_1 = os.path.basename(os.path.dirname(experiment_path_1)).replace('-suite', '') experiment_id_2 = os.path.basename(os.path.dirname(experiment_path_2)).replace('-suite', '') + # Window information + window_offset = self.config.window_offset() + window_type = self.config.window_type() + + # Cycle times in the run directory for each experiment cycle_times_1 = self.cycles_in_experiment(experiment_path_1) cycle_times_2 = self.cycles_in_experiment(experiment_path_2) + # Shared cycle times for each experiment cycle_times = list(set(cycle_times_1) & set(cycle_times_2)) for cycle_time in cycle_times: + + # Create the cycle dir for this experiment cycle_dir = os.path.join(self.experiment_path(), 'run', cycle_time, self.get_model()) os.makedirs(cycle_dir, exist_ok=True) - print(cycle_dir) cycle_time_dto = datetime.datetime.strptime(cycle_time, '%Y%m%dT%H%M%SZ') + da_window_params = DataAssimilationWindowParams(self.logger, cycle_time_dto.strftime( + '%Y-%m-%dT%H:%M:%SZ')) + + window_begin_dto = da_window_params.window_begin(window_offset, dto=True) + window_begin = window_begin_dto.strftime('%Y%m%d_%H%M%Sz') + # Info to task log info_string = 'Running Eva to plot from the increment file' self.logger.info('') @@ -73,6 +90,24 @@ def execute(self) -> None: # Create time strings for eva_override directory cycle_time_reformat = cycle_time_dto.strftime('%Y%m%d_%H%M%Sz') + # Define the increment filename and path + # For 3D-Var and 3D-FGAT, the increment file is in the middle of the DA window + # For 3D-FGAT atmos, the increment is at the beginning of the DA window. This + # is just a limited implementation of 4DVAR in SWELL by turning the linear operator off, + # where the cost-type is 4D-VAR in OOPS folder. + # TODO: This really complicates the increment file naming and path for now, this + # is a temporary solution (I'm refering to window type and atmos if statement) + # TODO: Increment iteration number may change according to outer iteration loops + # which is currenly manually set in varincrement1.yaml + # For now we are only plotting the first one + iter_no = 1 + incr_file_1 = f'{experiment_id_1}.increment-iter{iter_no}.{cycle_time_reformat}.nc4' + incr_file_2 = f'{experiment_id_2}.increment-iter{iter_no}.{cycle_time_reformat}.nc4' + + if window_type == '4D' and 'atmos' in self.suite_name(): + incr_file_1 = f'{experiment_id_1}.increment-iter{iter_no}.{window_begin}.nc4' + incr_file_2 = f'{experiment_id_2}.increment-iter{iter_no}.{window_begin}.nc4' + # Create dictionary used to override the eva config eva_override = {} @@ -82,13 +117,18 @@ def execute(self) -> None: incr_file_1 = f'ocn.{experiment_id_1}.incr.{ocn_cycle_time}.nc' incr_file_2 = f'ocn.{experiment_id_2}.incr.{ocn_cycle_time}.nc' - cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', cycle_time, self.get_model()) - cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', cycle_time, self.get_model()) - + cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', + cycle_time, self.get_model()) + cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', + cycle_time, self.get_model()) + + # Files to fill the template in the config file increment_file_path_1 = os.path.join(cycle_dir_1, incr_file_1) increment_file_path_2 = os.path.join(cycle_dir_2, incr_file_2) eva_override['cycle_dir'] = cycle_dir + eva_override['window_begin'] = window_begin + eva_override['cycle_dir_1'] = cycle_dir_1 eva_override['cycle_dir_2'] = cycle_dir_2 @@ -102,7 +142,8 @@ def execute(self) -> None: # Write eva dictionary to file # ---------------------------- - conf_output = os.path.join(cycle_dir, 'eva', 'increment', 'comparison_increment_eva.yaml') + conf_output = os.path.join(cycle_dir, 'eva', 'increment', + 'comparison_increment_eva.yaml') os.makedirs(os.path.dirname(conf_output), exist_ok=True) with open(conf_output, 'w') as outfile: yaml.dump(eva_dict, outfile, default_flow_style=False) diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 110cd4ad0..00b065f91 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -565,6 +565,15 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + EvaComparisonIncrement = QuestionList( + list_name="EvaIncrement", + questions=[ + EvaIncrement + ] + ) + + # -------------------------------------------------------------------------------------------------- + GenerateObservingSystemRecords = QuestionList( list_name="GenerateObservingSystemRecords", questions=[ diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 2f4133388..29ddec8ff 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -69,6 +69,10 @@ class EvaJediLog(Task): is_cycling: bool = True is_model: bool = True + @dataclass + class EvaComparisonIncrement(Task): + is_model: bool = True + @dataclass class EvaIncrement(Task): is_cycling: bool = True From 8164a426845dd6f6073bd434e8158e098f66d005 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 16 Jul 2025 14:34:29 -0400 Subject: [PATCH 052/299] pycodestyle fixes --- src/swell/tasks/eva_comparison_increment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index c565927b6..9ead0bd54 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -121,7 +121,7 @@ def execute(self) -> None: cycle_time, self.get_model()) cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', cycle_time, self.get_model()) - + # Files to fill the template in the config file increment_file_path_1 = os.path.join(cycle_dir_1, incr_file_1) increment_file_path_2 = os.path.join(cycle_dir_2, incr_file_2) From b8029e2bf236ffbc4751e103663d59d79852f69e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 18 Jul 2025 09:58:44 -0400 Subject: [PATCH 053/299] fix workflow --- src/swell/suites/compare_variational/workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index e0d8ccd51..485d6d1b9 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -50,8 +50,8 @@ def define_scheduling(self): cycle_model_times[cycle_time] = [] cycle_model_times[cycle_time].append(model) else: - start_cycle_point = 'None' - final_cycle_point = 'None' + start_cycle_point = '2021-07-01T12:00:00Z' + final_cycle_point = '2021-07-01T12:00:00Z' cycle_model_times = {} self.logger.info('No experiments have been specified') From 7060a77b5dbea909e31b4e563941895995208d1b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 18 Jul 2025 16:24:14 -0400 Subject: [PATCH 054/299] check for fgrep status --- src/swell/tasks/f_grep_residual_norm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/tasks/f_grep_residual_norm.py b/src/swell/tasks/f_grep_residual_norm.py index b8b066691..19a4ac598 100644 --- a/src/swell/tasks/f_grep_residual_norm.py +++ b/src/swell/tasks/f_grep_residual_norm.py @@ -31,7 +31,7 @@ def execute(self) -> None: command = ' '.join(command) # Run the fgrep command - output = subprocess.run(command, capture_output=True, text=True, shell=True) + output = subprocess.run(command, capture_output=True, text=True, shell=True, check=True) results = output.stdout # Create the output directory From cb29859417dea46f345d15e0956cdfd46d79ad97 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 30 Jul 2025 09:26:19 -0400 Subject: [PATCH 055/299] use cylc cycling --- .../compare_variational/suite_config.py | 9 +- .../suites/compare_variational/workflow.py | 57 ++++----- src/swell/tasks/eva_comparison_increment.py | 20 ++- src/swell/utilities/check_da_params.py | 120 ++++++++++++++++++ 4 files changed, 159 insertions(+), 47 deletions(-) create mode 100644 src/swell/utilities/check_da_params.py diff --git a/src/swell/suites/compare_variational/suite_config.py b/src/swell/suites/compare_variational/suite_config.py index 4f5ee5987..ee0568d16 100644 --- a/src/swell/suites/compare_variational/suite_config.py +++ b/src/swell/suites/compare_variational/suite_config.py @@ -9,7 +9,7 @@ import importlib -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionContainer, QuestionList, WidgetType from swell.utilities.question_defaults import QuestionDefaults as qd from swell.suites.suite_questions import SuiteQuestions as sq @@ -30,6 +30,8 @@ class SuiteConfig(QuestionContainer, Enum): list_name="compare", questions=[ sq.compare, + qd.start_cycle_point(default_value=None, widget_type=WidgetType.STRING), + qd.final_cycle_point(default_value=None, widget_type=WidgetType.STRING), qd.model_components(), qd.runahead_limit(), ] @@ -40,10 +42,11 @@ class SuiteConfig(QuestionContainer, Enum): compare_3dvar = QuestionList( list_name="compare_3dvar", questions=[ + _3dvar, compare_variational, qd.model_components(['geos_marine']), - _3dvar, - + qd.start_cycle_point(default_value=None, widget_type=WidgetType.STRING), + qd.final_cycle_point(default_value=None, widget_type=WidgetType.STRING), ] ) diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 485d6d1b9..96224f99d 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -12,6 +12,7 @@ from swell.utilities.cylc_workflow import CylcWorkflow from swell.utilities.cylc_runtime import Task +from swell.utilities.check_da_params import check_da_params # -------------------------------------------------------------------------------------------------- @@ -31,48 +32,40 @@ def define_description(self): def define_scheduling(self): - paths = self.experiment_dict['comparison_experiment_paths'] + config_list = self.experiment_dict['comparison_experiment_paths'] - if len(paths) > 0: - config_file = os.path.join(os.path.dirname(paths[0]), 'experiment.yaml') - with open(config_file, 'r') as f: - base_dict = yaml.safe_load(f) + start_cycle_point = self.experiment_dict['start_cycle_point'] + final_cycle_point = self.experiment_dict['final_cycle_point'] - start_cycle_point = base_dict['start_cycle_point'] - final_cycle_point = base_dict['final_cycle_point'] + graph_str = '' - cycle_model_times = {} + if 'models' in self.experiment_dict: + for model in self.experiment_dict['models'].keys(): + cycle_times = self.experiment_dict['models'][model]['cycle_times'] - for model in base_dict['models'].keys(): - cycle_times = base_dict['models'][model]['cycle_times'] - for cycle_time in cycle_times: - if cycle_time not in cycle_model_times: - cycle_model_times[cycle_time] = [] - cycle_model_times[cycle_time].append(model) - else: - start_cycle_point = '2021-07-01T12:00:00Z' - final_cycle_point = '2021-07-01T12:00:00Z' - cycle_model_times = {} + start_cycle_point, final_cycle_point, cycle_times = check_da_params( + config_list, + model, + start_cycle_point, + final_cycle_point, + cycle_times) - self.logger.info('No experiments have been specified') + scheduling_section = self.create_new_section( + 'scheduling', {'initial cycle point': start_cycle_point, + 'final cycle point': final_cycle_point}) - scheduling_section = self.create_new_section('scheduling', - {'initial cycle point': start_cycle_point, - 'final cycle point': final_cycle_point}) - - graph_str = '' + for cycle_time in cycle_times: + cycle_str = '' - for model in self.experiment_dict['model_components']: - graph_str += self.format_cycle('R1', f"EvaComparisonIncrement-{model}\n") + cycle_str += f"EvaComparisonIncrement-{model}\n" - for cycle_time, models in cycle_model_times.items(): - cycle_str = '' + for i in range(len(config_list)): + cycle_str += f"FGrepResidualNorm-{model}-{i}\n" - for model in models: - for i in range(len(paths)): - cycle_str += f"FGrepResidualNorm-{model}-{i}\n" + graph_str += self.format_cycle(cycle_time, cycle_str) - graph_str += self.format_cycle(cycle_time, cycle_str) + else: + scheduling_section = self.create_new_section('scheduling', {}) graph_section = self.create_new_section('graph', graph_str) diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index 9ead0bd54..ab3802643 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -52,10 +52,6 @@ def execute(self) -> None: experiment_path_1 = experiment_paths[0] experiment_path_2 = experiment_paths[1] - # Experiment IDs for the two experiments - experiment_id_1 = os.path.basename(os.path.dirname(experiment_path_1)).replace('-suite', '') - experiment_id_2 = os.path.basename(os.path.dirname(experiment_path_2)).replace('-suite', '') - # Window information window_offset = self.config.window_offset() window_type = self.config.window_type() @@ -101,12 +97,12 @@ def execute(self) -> None: # which is currenly manually set in varincrement1.yaml # For now we are only plotting the first one iter_no = 1 - incr_file_1 = f'{experiment_id_1}.increment-iter{iter_no}.{cycle_time_reformat}.nc4' - incr_file_2 = f'{experiment_id_2}.increment-iter{iter_no}.{cycle_time_reformat}.nc4' + incr_file_1 = f'*.increment-iter{iter_no}.{cycle_time_reformat}.nc4' + incr_file_2 = f'*.increment-iter{iter_no}.{cycle_time_reformat}.nc4' if window_type == '4D' and 'atmos' in self.suite_name(): - incr_file_1 = f'{experiment_id_1}.increment-iter{iter_no}.{window_begin}.nc4' - incr_file_2 = f'{experiment_id_2}.increment-iter{iter_no}.{window_begin}.nc4' + incr_file_1 = f'*.increment-iter{iter_no}.{window_begin}.nc4' + incr_file_2 = f'*.increment-iter{iter_no}.{window_begin}.nc4' # Create dictionary used to override the eva config eva_override = {} @@ -114,8 +110,8 @@ def execute(self) -> None: # Soca case if model == 'geos_marine': ocn_cycle_time = cycle_time_dto.strftime('%Y-%m-%dT%H:%M:%SZ') - incr_file_1 = f'ocn.{experiment_id_1}.incr.{ocn_cycle_time}.nc' - incr_file_2 = f'ocn.{experiment_id_2}.incr.{ocn_cycle_time}.nc' + incr_file_1 = f'ocn.*.incr.{ocn_cycle_time}.nc' + incr_file_2 = f'ocn.*.incr.{ocn_cycle_time}.nc' cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', cycle_time, self.get_model()) @@ -123,8 +119,8 @@ def execute(self) -> None: cycle_time, self.get_model()) # Files to fill the template in the config file - increment_file_path_1 = os.path.join(cycle_dir_1, incr_file_1) - increment_file_path_2 = os.path.join(cycle_dir_2, incr_file_2) + increment_file_path_1 = glob.glob(os.path.join(cycle_dir_1, incr_file_1))[0] + increment_file_path_2 = glob.glob(os.path.join(cycle_dir_2, incr_file_2))[0] eva_override['cycle_dir'] = cycle_dir eva_override['window_begin'] = window_begin diff --git a/src/swell/utilities/check_da_params.py b/src/swell/utilities/check_da_params.py new file mode 100644 index 000000000..d79e92abf --- /dev/null +++ b/src/swell/utilities/check_da_params.py @@ -0,0 +1,120 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +import yaml +import os +from typing import Optional +from datetime import datetime + +from swell.utilities.logger import get_logger +from swell.utilities.datetime_util import datetime_formats + +# -------------------------------------------------------------------------------------------------- + + +def check_da_params(config_list: list, + model_component: str, + start_cycle_point_in: Optional[str], + final_cycle_point_in: Optional[str], + cycle_times_in: Optional[str]) -> None: + + # From two or more experiments, check that the window parameters are the same, and gather the + # common cycle times between the two. Returns times between the start and final cycle points, + # or pick automatically if set to None. + + # Dictionary containing window and cycles for all experiments + cycle_dicts = {} + + logger = get_logger('CheckDAParams') + + for i, config_file in enumerate(config_list): + + # Name the config after its index + cycle_dict = cycle_dicts[str(i)] = {} + + # Abort if file not found + if not os.path.isfile(config_file): + logger.abort(f'Config file {config_file} does not exist.') + + # Open the file for location and window information + with open(config_file, 'r') as f: + config_dict = yaml.safe_load(f) + + # Window params + start_cycle_point = config_dict['start_cycle_point'] + final_cycle_point = config_dict['final_cycle_point'] + + window_length = config_dict['models'][model_component]['window_length'] + window_offset = config_dict['models'][model_component]['window_offset'] + cycle_times = config_dict['models'][model_component]['cycle_times'] + + # Save the information + cycle_dict['start_cycle_dto'] = datetime.strptime(start_cycle_point, + datetime_formats['iso_format']) + cycle_dict['final_cycle_dto'] = datetime.strptime(final_cycle_point, + datetime_formats['iso_format']) + + cycle_dict['window_length'] = window_length + cycle_dict['window_offset'] = window_offset + cycle_dict['cycle_times'] = cycle_times + + start_cycle_dtos = [] + final_cycle_dtos = [] + + window_lengths = [] + window_offsets = [] + + # Gather all the window params into a list + for value in cycle_dicts.values(): + start_cycle_dtos.append(value['start_cycle_dto']) + final_cycle_dtos.append(value['final_cycle_dto']) + + window_lengths.append(value['window_length']) + window_offsets.append(value['window_offset']) + + # Check to make sure the window params match + if len(set(window_lengths)) > 1 or len(set(window_offsets)) > 1: + logger.abort('The experiments specified have mismatched window parameters.') + + # Iterate through all the cycle times and find the common ones. + common_cycle_times = None + for value in cycle_dicts.values(): + cycle_times = value['cycle_times'] + + if common_cycle_times is not None: + common_cycle_times = list(set(cycle_times) & set(common_cycle_times)) + else: + common_cycle_times = cycle_times + + cycle_times_out = [] + + if cycle_times_in is None or cycle_times_in == 'None': + if common_cycle_times is None: + cycle_times_out = [] + else: + cycle_times_out = common_cycle_times + else: + cycle_times_out = cycle_times_in + + # Get the start cycle dto object, or set automatically + if start_cycle_point_in is None or start_cycle_point_in == 'None': + start_cycle_dto = max(start_cycle_dtos) + start_cycle_point_out = start_cycle_dto.strftime(datetime_formats['iso_format']) + else: + start_cycle_point_out = start_cycle_point_in + + # Get the final cycle dto object, or set automatically + if final_cycle_point_in is None or final_cycle_point_in == 'None': + final_cycle_dto = min(final_cycle_dtos) + final_cycle_point_out = final_cycle_dto.strftime(datetime_formats['iso_format']) + else: + final_cycle_point_out = final_cycle_point_in + + return start_cycle_point_out, final_cycle_point_out, cycle_times_out + +# -------------------------------------------------------------------------------------------------- From 34568721b7245cfc38ab8b0d00a20c990de58490 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 30 Jul 2025 11:26:59 -0400 Subject: [PATCH 056/299] rework fgrep task --- src/swell/suites/3dvar/suite_config.py | 1 + src/swell/suites/compare_variational/workflow.py | 10 +++++----- ..._grep_residual_norm.py => jedi_oops_log_parser.py} | 10 ++++++++-- src/swell/tasks/task_questions.py | 9 +++++++++ src/swell/tasks/task_runtimes.py | 5 +++++ src/swell/utilities/question_defaults.py | 11 +++++++++++ 6 files changed, 39 insertions(+), 7 deletions(-) rename src/swell/tasks/{f_grep_residual_norm.py => jedi_oops_log_parser.py} (88%) diff --git a/src/swell/suites/3dvar/suite_config.py b/src/swell/suites/3dvar/suite_config.py index 3eaed8d70..b3a2dad78 100644 --- a/src/swell/suites/3dvar/suite_config.py +++ b/src/swell/suites/3dvar/suite_config.py @@ -30,6 +30,7 @@ class SuiteConfig(QuestionContainer, Enum): qd.final_cycle_point("2021-07-01T12:00:00Z"), qd.jedi_build_method("use_existing"), qd.model_components(['geos_marine']), + qd.parser_options(), ], geos_marine=[ qd.cycle_times(['T12']), diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 96224f99d..3525f0f00 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -60,7 +60,7 @@ def define_scheduling(self): cycle_str += f"EvaComparisonIncrement-{model}\n" for i in range(len(config_list)): - cycle_str += f"FGrepResidualNorm-{model}-{i}\n" + cycle_str += f"JediOopsLogParser-{model}-{i}\n" graph_str += self.format_cycle(cycle_time, cycle_str) @@ -91,10 +91,10 @@ def define_runtime_task_overrides(self): exp_dict = yaml.safe_load(f) for model in exp_dict['model_components']: - overrides[f'FGrepResidualNorm-{model}-{i}'] = Task( - base_name='FGrepResidualNorm', - scheduling_name=(f'FGrepResidualNorm-{model}-{i}'), - script=(f'swell task FGrepResidualNorm {config_file} -d $datetime' + overrides[f'JediOopsLogParser-{model}-{i}'] = Task( + base_name='JediOopsLogParser', + scheduling_name=(f'JediOopsLogParser-{model}-{i}'), + script=(f'swell task JediOopsLogParser {config_file} -d $datetime' f' -m {model} -i {i} -o {output_dir}')) return overrides diff --git a/src/swell/tasks/f_grep_residual_norm.py b/src/swell/tasks/jedi_oops_log_parser.py similarity index 88% rename from src/swell/tasks/f_grep_residual_norm.py rename to src/swell/tasks/jedi_oops_log_parser.py index 19a4ac598..4174fe171 100644 --- a/src/swell/tasks/f_grep_residual_norm.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -16,9 +16,9 @@ # -------------------------------------------------------------------------------------------------- -class FGrepResidualNorm(taskBase): +class JediOopsLogParser(taskBase): - def execute(self) -> None: + def fgrep_residual_norm(self): cycle_dir = self.cycle_dir() @@ -52,5 +52,11 @@ def execute(self) -> None: f.write(f'RESULTS:\n') f.write(results) + def execute(self) -> None: + + for parser_option in self.config.parser_options(['fgrep_residual_norm']): + if parser_option == 'fgrep_residual_norm': + self.fgrep_residual_norm() + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 00b065f91..19cd9b541 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -102,6 +102,15 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + JediOopsLogParser = QuestionList( + list_name="JediOopsLogParser", + questions=[ + qd.parser_options(), + ] + ) + + # -------------------------------------------------------------------------------------------------- + MoveDaRestart = QuestionList( list_name="MoveDaRestart", questions=[ diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 29ddec8ff..92a7bc4dd 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -85,6 +85,11 @@ class EvaObservations(Task): is_model: bool = True slurm: dict = mutable_field({}) + @dataclass + class JediOopsLogParser(Task): + is_cycling: bool = True + is_model: bool = True + @dataclass class GetBackground(Task): is_cycling: bool = True diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 322c4e3f2..9f2d31d0f 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -141,6 +141,17 @@ class model_components(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class parser_options(SuiteQuestion): + default_value: list = mutable_field(['fgrep_residual_norm']) + question_name: str = "parser_options" + ask_question: bool = True + options: list = mutable_field(['fgrep_residual_norm']) + prompt: str = "List the test types to run on the JEDI oops log." + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + @dataclass class runahead_limit(SuiteQuestion): default_value: str = "P4" From 7eec497fc4d0b508f35c25c41526eea709ed9f3e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 31 Jul 2025 14:41:49 -0400 Subject: [PATCH 057/299] Fix for missing cycle times --- src/swell/utilities/check_da_params.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/swell/utilities/check_da_params.py b/src/swell/utilities/check_da_params.py index d79e92abf..f87d947b1 100644 --- a/src/swell/utilities/check_da_params.py +++ b/src/swell/utilities/check_da_params.py @@ -103,15 +103,21 @@ def check_da_params(config_list: list, # Get the start cycle dto object, or set automatically if start_cycle_point_in is None or start_cycle_point_in == 'None': - start_cycle_dto = max(start_cycle_dtos) - start_cycle_point_out = start_cycle_dto.strftime(datetime_formats['iso_format']) + if len(start_cycle_dtos) > 0: + start_cycle_dto = max(start_cycle_dtos) + start_cycle_point_out = start_cycle_dto.strftime(datetime_formats['iso_format']) + else: + start_cycle_point_out = None else: start_cycle_point_out = start_cycle_point_in # Get the final cycle dto object, or set automatically if final_cycle_point_in is None or final_cycle_point_in == 'None': - final_cycle_dto = min(final_cycle_dtos) - final_cycle_point_out = final_cycle_dto.strftime(datetime_formats['iso_format']) + if len(final_cycle_dtos) > 0: + final_cycle_dto = min(final_cycle_dtos) + final_cycle_point_out = final_cycle_dto.strftime(datetime_formats['iso_format']) + else: + final_cycle_point_out = None else: final_cycle_point_out = final_cycle_point_in From e2b3bcad6c25ded64db44c7232543b5eca59cb50 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 19 Aug 2025 16:25:38 -0400 Subject: [PATCH 058/299] Fix for localensembleda --- src/swell/tasks/task_runtimes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 92a7bc4dd..971b8ef1b 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -271,10 +271,11 @@ class StageJediCycle(Task): @dataclass class sync_point(Task): - script = "true" + script: str = "true" @dataclass class ThinObs(Task): + script: str = "swell task RunJediObsfiltersExecutable $config -d $datetime -m {model_component}" is_cycling: bool = True is_model: bool = True time_limit: bool = True From a2e4d40d2223c626bdf7d3b5a95f99879ada738b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 14:01:17 -0400 Subject: [PATCH 059/299] Use datetime interval --- src/swell/tasks/eva_comparison_increment.py | 194 +++++++++----------- 1 file changed, 91 insertions(+), 103 deletions(-) diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index ab3802643..a9b9a8583 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -24,17 +24,17 @@ class EvaComparisonIncrement(taskBase): - def cycles_in_experiment(self, path): - cycle_glob = os.path.join(os.path.dirname(path), '..', 'run', '*Z', self.get_model()) - cycle_paths = glob.glob(cycle_glob) + def window_info_from_config(self, path: str): - cycle_times = [] + config_file = os.path.join(os.path.dirname(path), 'experiment.yaml') - for cycle_path in cycle_paths: - cycle_time = cycle_path.split(f'/{self.get_model()}')[0].split('run/')[1] - cycle_times.append(cycle_time) + with open(config_file, 'r') as f: + config_dict = yaml.safe_load(f) - return cycle_times + window_type = config_dict['models'][self.get_model()]['window_type'] + window_offset = config_dict['models'][self.get_model()]['window_offset'] + + return window_type, window_offset def execute(self) -> None: @@ -52,98 +52,86 @@ def execute(self) -> None: experiment_path_1 = experiment_paths[0] experiment_path_2 = experiment_paths[1] - # Window information - window_offset = self.config.window_offset() - window_type = self.config.window_type() - - # Cycle times in the run directory for each experiment - cycle_times_1 = self.cycles_in_experiment(experiment_path_1) - cycle_times_2 = self.cycles_in_experiment(experiment_path_2) - - # Shared cycle times for each experiment - cycle_times = list(set(cycle_times_1) & set(cycle_times_2)) - - for cycle_time in cycle_times: - - # Create the cycle dir for this experiment - cycle_dir = os.path.join(self.experiment_path(), 'run', cycle_time, self.get_model()) - os.makedirs(cycle_dir, exist_ok=True) - - cycle_time_dto = datetime.datetime.strptime(cycle_time, '%Y%m%dT%H%M%SZ') - - da_window_params = DataAssimilationWindowParams(self.logger, cycle_time_dto.strftime( - '%Y-%m-%dT%H:%M:%SZ')) - - window_begin_dto = da_window_params.window_begin(window_offset, dto=True) - window_begin = window_begin_dto.strftime('%Y%m%d_%H%M%Sz') - - # Info to task log - info_string = 'Running Eva to plot from the increment file' - self.logger.info('') - self.logger.info(info_string) - self.logger.info('-'*len(info_string)) - - # Create time strings for eva_override directory - cycle_time_reformat = cycle_time_dto.strftime('%Y%m%d_%H%M%Sz') - - # Define the increment filename and path - # For 3D-Var and 3D-FGAT, the increment file is in the middle of the DA window - # For 3D-FGAT atmos, the increment is at the beginning of the DA window. This - # is just a limited implementation of 4DVAR in SWELL by turning the linear operator off, - # where the cost-type is 4D-VAR in OOPS folder. - # TODO: This really complicates the increment file naming and path for now, this - # is a temporary solution (I'm refering to window type and atmos if statement) - # TODO: Increment iteration number may change according to outer iteration loops - # which is currenly manually set in varincrement1.yaml - # For now we are only plotting the first one - iter_no = 1 - incr_file_1 = f'*.increment-iter{iter_no}.{cycle_time_reformat}.nc4' - incr_file_2 = f'*.increment-iter{iter_no}.{cycle_time_reformat}.nc4' - - if window_type == '4D' and 'atmos' in self.suite_name(): - incr_file_1 = f'*.increment-iter{iter_no}.{window_begin}.nc4' - incr_file_2 = f'*.increment-iter{iter_no}.{window_begin}.nc4' - - # Create dictionary used to override the eva config - eva_override = {} - - # Soca case - if model == 'geos_marine': - ocn_cycle_time = cycle_time_dto.strftime('%Y-%m-%dT%H:%M:%SZ') - incr_file_1 = f'ocn.*.incr.{ocn_cycle_time}.nc' - incr_file_2 = f'ocn.*.incr.{ocn_cycle_time}.nc' - - cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', - cycle_time, self.get_model()) - cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', - cycle_time, self.get_model()) - - # Files to fill the template in the config file - increment_file_path_1 = glob.glob(os.path.join(cycle_dir_1, incr_file_1))[0] - increment_file_path_2 = glob.glob(os.path.join(cycle_dir_2, incr_file_2))[0] - - eva_override['cycle_dir'] = cycle_dir - eva_override['window_begin'] = window_begin - - eva_override['cycle_dir_1'] = cycle_dir_1 - eva_override['cycle_dir_2'] = cycle_dir_2 - - eva_override['cycle_time'] = cycle_time_reformat - eva_override['increment_file_path_1'] = increment_file_path_1 - eva_override['increment_file_path_2'] = increment_file_path_2 - - # Override the eva dictionary - eva_str = template_string_jinja2(self.logger, eva_str_template, eva_override) - eva_dict = yaml.safe_load(eva_str) - - # Write eva dictionary to file - # ---------------------------- - conf_output = os.path.join(cycle_dir, 'eva', 'increment', - 'comparison_increment_eva.yaml') - os.makedirs(os.path.dirname(conf_output), exist_ok=True) - with open(conf_output, 'w') as outfile: - yaml.dump(eva_dict, outfile, default_flow_style=False) - - # Call eva - # -------- - eva(eva_dict) + window_type, window_offset = self.window_info_from_config(experiment_path_1) + + # Create the cycle dir for this experiment + cycle_dir = self.cycle_dir() + + cycle_time_dto = self.cycle_time_dto() + + da_window_params = DataAssimilationWindowParams(self.logger, cycle_time_dto.strftime( + '%Y-%m-%dT%H:%M:%SZ')) + + window_begin_dto = da_window_params.window_begin(window_offset, dto=True) + window_begin = window_begin_dto.strftime('%Y%m%d_%H%M%Sz') + + # Info to task log + info_string = 'Running Eva to plot from the increment file' + self.logger.info('') + self.logger.info(info_string) + self.logger.info('-'*len(info_string)) + + # Create time strings for eva_override directory + cycle_time_reformat = cycle_time_dto.strftime('%Y%m%d_%H%M%Sz') + + # Define the increment filename and path + # For 3D-Var and 3D-FGAT, the increment file is in the middle of the DA window + # For 3D-FGAT atmos, the increment is at the beginning of the DA window. This + # is just a limited implementation of 4DVAR in SWELL by turning the linear operator off, + # where the cost-type is 4D-VAR in OOPS folder. + # TODO: This really complicates the increment file naming and path for now, this + # is a temporary solution (I'm refering to window type and atmos if statement) + # TODO: Increment iteration number may change according to outer iteration loops + # which is currenly manually set in varincrement1.yaml + # For now we are only plotting the first one + iter_no = 1 + incr_file_1 = f'*.increment-iter{iter_no}.{cycle_time_reformat}.nc4' + incr_file_2 = f'*.increment-iter{iter_no}.{cycle_time_reformat}.nc4' + + if window_type == '4D' and 'atmos' in self.suite_name(): + incr_file_1 = f'*.increment-iter{iter_no}.{window_begin}.nc4' + incr_file_2 = f'*.increment-iter{iter_no}.{window_begin}.nc4' + + # Create dictionary used to override the eva config + eva_override = {} + + # Soca case + if model == 'geos_marine': + ocn_cycle_time = cycle_time_dto.strftime('%Y-%m-%dT%H:%M:%SZ') + incr_file_1 = f'ocn.*.incr.{ocn_cycle_time}.nc' + incr_file_2 = f'ocn.*.incr.{ocn_cycle_time}.nc' + + cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', + self.cycle_time(), self.get_model()) + cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', + self.cycle_time(), self.get_model()) + + # Files to fill the template in the config file + increment_file_path_1 = glob.glob(os.path.join(cycle_dir_1, incr_file_1))[0] + increment_file_path_2 = glob.glob(os.path.join(cycle_dir_2, incr_file_2))[0] + + eva_override['cycle_dir'] = cycle_dir + eva_override['window_begin'] = window_begin + + eva_override['cycle_dir_1'] = cycle_dir_1 + eva_override['cycle_dir_2'] = cycle_dir_2 + + eva_override['cycle_time'] = cycle_time_reformat + eva_override['increment_file_path_1'] = increment_file_path_1 + eva_override['increment_file_path_2'] = increment_file_path_2 + + # Override the eva dictionary + eva_str = template_string_jinja2(self.logger, eva_str_template, eva_override) + eva_dict = yaml.safe_load(eva_str) + + # Write eva dictionary to file + # ---------------------------- + conf_output = os.path.join(cycle_dir, 'eva', 'increment', + 'comparison_increment_eva.yaml') + os.makedirs(os.path.dirname(conf_output), exist_ok=True) + with open(conf_output, 'w') as outfile: + yaml.dump(eva_dict, outfile, default_flow_style=False) + + # Call eva + # -------- + eva(eva_dict) From 921719097630a539096c23857657a35fb5cdec76 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 14:02:18 -0400 Subject: [PATCH 060/299] fixes for localensembleda --- src/swell/tasks/task_runtimes.py | 2 +- src/swell/utilities/cylc_runtime.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 971b8ef1b..0438929bd 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -275,7 +275,7 @@ class sync_point(Task): @dataclass class ThinObs(Task): - script: str = "swell task RunJediObsfiltersExecutable $config -d $datetime -m {model_component}" + script: str = "swell task RunJediObsfiltersExecutable $config -d $datetime -m geos_atmosphere" is_cycling: bool = True is_model: bool = True time_limit: bool = True diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 619b45ed3..19859cc0a 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -56,9 +56,6 @@ def __post_init__(self): if self.is_model and self.model is not None: self.scheduling_name += f'-{self.model}' - elif self.is_model: - self.scheduling_name = self.scheduling_name.format(model=self.model) - if self.script is None: self.script = f'swell task {self.base_name} $config' @@ -66,7 +63,11 @@ def __post_init__(self): self.script += ' -d $datetime' if self.is_model and self.model is not None: - self.script += f' -m {self.model}' + self.script += ' -m {model}' + + if self.is_model and self.model is not None: + self.script = self.script.format(model=self.model) + self.scheduling_name = self.scheduling_name.format(model=self.model) # -------------------------------------------------------------------------------------------------- From 5f39fd118a05da423e6c941f6f9d22aec4bbd814 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 15:01:15 -0400 Subject: [PATCH 061/299] Clean up eva comparison plots --- src/swell/suites/compare_variational/workflow.py | 4 ++++ src/swell/tasks/eva_comparison_increment.py | 8 ++++---- src/swell/tasks/task_runtimes.py | 4 +++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 3525f0f00..76a619fbd 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -50,6 +50,10 @@ def define_scheduling(self): final_cycle_point, cycle_times) + # Set the values in the config + self.experiment_dict['start_cycle_point'] = start_cycle_point + self.experiment_dict['final_cycle_point'] = final_cycle_point + scheduling_section = self.create_new_section( 'scheduling', {'initial cycle point': start_cycle_point, 'final cycle point': final_cycle_point}) diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index a9b9a8583..c20beb4f4 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -11,7 +11,6 @@ import os import yaml import glob -import datetime from eva.eva_driver import eva @@ -102,9 +101,9 @@ def execute(self) -> None: incr_file_2 = f'ocn.*.incr.{ocn_cycle_time}.nc' cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', - self.cycle_time(), self.get_model()) + self.__datetime__.string_directory(), self.get_model()) cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', - self.cycle_time(), self.get_model()) + self.__datetime__.string_directory(), self.get_model()) # Files to fill the template in the config file increment_file_path_1 = glob.glob(os.path.join(cycle_dir_1, incr_file_1))[0] @@ -127,7 +126,8 @@ def execute(self) -> None: # Write eva dictionary to file # ---------------------------- conf_output = os.path.join(cycle_dir, 'eva', 'increment', - 'comparison_increment_eva.yaml') + 'comparison_increment_eva.yaml') + os.makedirs(os.path.dirname(conf_output), exist_ok=True) with open(conf_output, 'w') as outfile: yaml.dump(eva_dict, outfile, default_flow_style=False) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 0438929bd..c664ae4b1 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -71,6 +71,7 @@ class EvaJediLog(Task): @dataclass class EvaComparisonIncrement(Task): + is_cycling: bool = True is_model: bool = True @dataclass @@ -275,7 +276,8 @@ class sync_point(Task): @dataclass class ThinObs(Task): - script: str = "swell task RunJediObsfiltersExecutable $config -d $datetime -m geos_atmosphere" + script: str = ("swell task RunJediObsfiltersExecutable $config" + " -d $datetime -m geos_atmosphere") is_cycling: bool = True is_model: bool = True time_limit: bool = True From c1f8ddbc944c6a452f4b8e81283038a7d3b06e48 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 15:27:54 -0400 Subject: [PATCH 062/299] Add Jedi-log comparison plots --- .../eva/comparison_jedi_log-geos_marine.yaml | 93 +++++++++++++++++++ .../compare_variational/suite_config.py | 21 +++-- .../suites/compare_variational/workflow.py | 1 + src/swell/tasks/eva_comparison_jedi_log.py | 74 +++++++++++++++ src/swell/tasks/task_runtimes.py | 5 + 5 files changed, 185 insertions(+), 9 deletions(-) create mode 100644 src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml create mode 100644 src/swell/tasks/eva_comparison_jedi_log.py diff --git a/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml b/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml new file mode 100644 index 000000000..d84cf2af2 --- /dev/null +++ b/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml @@ -0,0 +1,93 @@ +datasets: + +- type: JediLog + collection_name: jedi_log_test_1 + jedi_log_to_parse: '{{cycle_dir_1}}/jedi_variational_log.log' + data_to_parse: + convergence: true + +- type: JediLog + collection_name: jedi_log_test_2 + jedi_log_to_parse: '{{cycle_dir_2}}/jedi_variational_log.log' + data_to_parse: + convergence: true + +graphics: + + plotting_backend: Emcpy + figure_list: + + - figure: + layout: [3,1] + figure size: [12,10] + title: 'Residual Norm and Norm Reduction Plots' + output name: '{{cycle_dir}}/eva/jedi_log/convergence/residual_norm_reduction.png' + plots: + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Residual norm' + layers: + - type: LinePlot + x: + variable: jedi_log_test_1::convergence::total_iteration + y: + variable: jedi_log_test_1::convergence::residual_norm + color: 'red' + label: 'Experiment 1' + - type: LinePlot + x: + variable: jedi_log_test_2::convergence::total_iteration + y: + variable: jedi_log_test_2::convergence::residual_norm + color: 'blue' + label: 'Experiment 2' + + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Norm reduction' + layers: + - type: LinePlot + x: + variable: jedi_log_test_1::convergence::total_iteration + y: + variable: jedi_log_test_1::convergence::norm_reduction + color: 'red' + label: 'Experiment 1' + - type: LinePlot + x: + variable: jedi_log_test_2::convergence::total_iteration + y: + variable: jedi_log_test_2::convergence::norm_reduction + color: 'blue' + label: 'Experiment 2' + + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Normalized Value' + add_legend: + layers: + - type: LinePlot + x: + variable: jedi_log_test_1::convergence::total_iteration + y: + variable: jedi_log_test_1::convergence::residual_norm_normalized + color: 'red' + label: 'Normalized residual norm 1' + - type: LinePlot + x: + variable: jedi_log_test_2::convergence::total_iteration + y: + variable: jedi_log_test_2::convergence::residual_norm_normalized + color: 'blue' + label: 'Normalized residual norm 2' + - type: LinePlot + x: + variable: jedi_log_test_1::convergence::total_iteration + y: + variable: jedi_log_test_1::convergence::norm_reduction_normalized + color: 'yellow' + label: 'Normalized norm reduction 1' + - type: LinePlot + x: + variable: jedi_log_test_2::convergence::total_iteration + y: + variable: jedi_log_test_2::convergence::norm_reduction_normalized + color: 'green' + label: 'Normalized norm reduction 2' diff --git a/src/swell/suites/compare_variational/suite_config.py b/src/swell/suites/compare_variational/suite_config.py index ee0568d16..e956d6322 100644 --- a/src/swell/suites/compare_variational/suite_config.py +++ b/src/swell/suites/compare_variational/suite_config.py @@ -15,10 +15,6 @@ from enum import Enum -_3dvar_container = importlib.import_module(f'swell.suites.3dvar.suite_config') -_3dvar_config = getattr(_3dvar_container, 'SuiteConfig') -_3dvar = getattr(_3dvar_config, '_3dvar') - # -------------------------------------------------------------------------------------------------- @@ -39,14 +35,21 @@ class SuiteConfig(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- - compare_3dvar = QuestionList( - list_name="compare_3dvar", + compare_variational_marine = QuestionList( + list_name="compare_variational_marine", questions=[ - _3dvar, compare_variational, qd.model_components(['geos_marine']), - qd.start_cycle_point(default_value=None, widget_type=WidgetType.STRING), - qd.final_cycle_point(default_value=None, widget_type=WidgetType.STRING), + ] + ) + + # -------------------------------------------------------------------------------------------------- + + compare_variational_marine = QuestionList( + list_name="compare_variational_atmosphere", + questions=[ + compare_variational, + qd.model_components(['geos_atmosphere']), ] ) diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 76a619fbd..8a438ffc1 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -62,6 +62,7 @@ def define_scheduling(self): cycle_str = '' cycle_str += f"EvaComparisonIncrement-{model}\n" + cycle_str += f"EvaComparisonJediLog-{model}\n" for i in range(len(config_list)): cycle_str += f"JediOopsLogParser-{model}-{i}\n" diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py new file mode 100644 index 000000000..046fea0b1 --- /dev/null +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -0,0 +1,74 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. + +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import yaml + +from eva.eva_driver import eva + +from swell.tasks.base.task_base import taskBase +from swell.utilities.jinja2 import template_string_jinja2 + + +# -------------------------------------------------------------------------------------------------- + + +class EvaJediLog(taskBase): + + def execute(self) -> None: + + # Get the model + # ------------- + model = self.get_model() + + # Read Eva template file into dictionary + # -------------------------------------- + eva_path = os.path.join(self.experiment_path(), self.experiment_id()+'-suite', 'eva') + eva_config_file = os.path.join(eva_path, f'jedi_log-{model}.yaml') + with open(eva_config_file, 'r') as eva_config_file_open: + eva_str_template = eva_config_file_open.read() + + # Get the paths for the two experiments + experiment_paths = self.config.comparison_experiment_paths() + + experiment_path_1 = experiment_paths[0] + experiment_path_2 = experiment_paths[1] + + cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', + self.__datetime__.string_directory(), self.get_model()) + cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', + self.__datetime__.string_directory(), self.get_model()) + + # Info to task log + info_string = 'Running Eva to plot from the jedi_log file' + self.logger.info('') + self.logger.info(info_string) + self.logger.info('-'*len(info_string)) + + # Create dictionary used to override the eva config + eva_override = {} + eva_override['cycle_dir'] = self.cycle_dir() + eva_override['cycle_dir_1'] = cycle_dir_1 + eva_override['cycle_dir_2'] = cycle_dir_2 + + # Override the eva dictionary + eva_str = template_string_jinja2(self.logger, eva_str_template, eva_override) + eva_dict = yaml.safe_load(eva_str) + + # Write eva dictionary to file + # ---------------------------- + conf_output = os.path.join(self.cycle_dir(), 'eva', 'jedi_log', 'jedi_log_eva.yaml') + os.makedirs(os.path.dirname(conf_output), exist_ok=True) + with open(conf_output, 'w') as outfile: + yaml.dump(eva_dict, outfile, default_flow_style=False) + + # Call eva + # -------- + eva(eva_dict) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index c664ae4b1..07f24d0bb 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -73,6 +73,11 @@ class EvaJediLog(Task): class EvaComparisonIncrement(Task): is_cycling: bool = True is_model: bool = True + + @dataclass + class EvaComparisonJediLog(Task): + is_cycling: bool = True + is_model: bool = True @dataclass class EvaIncrement(Task): From 19f607852c8cadd36fe9d5e8e13d55b4a2d002c5 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 15:53:49 -0400 Subject: [PATCH 063/299] fixes for jedi_log --- src/swell/suites/compare_variational/suite_config.py | 3 ++- src/swell/suites/compare_variational/workflow.py | 1 + src/swell/tasks/eva_comparison_jedi_log.py | 6 +++--- src/swell/utilities/check_da_params.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/swell/suites/compare_variational/suite_config.py b/src/swell/suites/compare_variational/suite_config.py index e956d6322..3821ddd26 100644 --- a/src/swell/suites/compare_variational/suite_config.py +++ b/src/swell/suites/compare_variational/suite_config.py @@ -28,6 +28,7 @@ class SuiteConfig(QuestionContainer, Enum): sq.compare, qd.start_cycle_point(default_value=None, widget_type=WidgetType.STRING), qd.final_cycle_point(default_value=None, widget_type=WidgetType.STRING), + qd.cycle_times(default_value=[None], widget_type=WidgetType.STRING_CHECK_LIST), qd.model_components(), qd.runahead_limit(), ] @@ -45,7 +46,7 @@ class SuiteConfig(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- - compare_variational_marine = QuestionList( + compare_variational_atmosphere = QuestionList( list_name="compare_variational_atmosphere", questions=[ compare_variational, diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 8a438ffc1..7097ba6f8 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -41,6 +41,7 @@ def define_scheduling(self): if 'models' in self.experiment_dict: for model in self.experiment_dict['models'].keys(): + print(model) cycle_times = self.experiment_dict['models'][model]['cycle_times'] start_cycle_point, final_cycle_point, cycle_times = check_da_params( diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 046fea0b1..5cd7c726c 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -20,7 +20,7 @@ # -------------------------------------------------------------------------------------------------- -class EvaJediLog(taskBase): +class EvaComparisonJediLog(taskBase): def execute(self) -> None: @@ -31,7 +31,7 @@ def execute(self) -> None: # Read Eva template file into dictionary # -------------------------------------- eva_path = os.path.join(self.experiment_path(), self.experiment_id()+'-suite', 'eva') - eva_config_file = os.path.join(eva_path, f'jedi_log-{model}.yaml') + eva_config_file = os.path.join(eva_path, f'comparison_jedi_log-{model}.yaml') with open(eva_config_file, 'r') as eva_config_file_open: eva_str_template = eva_config_file_open.read() @@ -64,7 +64,7 @@ def execute(self) -> None: # Write eva dictionary to file # ---------------------------- - conf_output = os.path.join(self.cycle_dir(), 'eva', 'jedi_log', 'jedi_log_eva.yaml') + conf_output = os.path.join(self.cycle_dir(), 'eva', 'jedi_log', 'comparison_jedi_log_eva.yaml') os.makedirs(os.path.dirname(conf_output), exist_ok=True) with open(conf_output, 'w') as outfile: yaml.dump(eva_dict, outfile, default_flow_style=False) diff --git a/src/swell/utilities/check_da_params.py b/src/swell/utilities/check_da_params.py index f87d947b1..eeb18564d 100644 --- a/src/swell/utilities/check_da_params.py +++ b/src/swell/utilities/check_da_params.py @@ -93,7 +93,7 @@ def check_da_params(config_list: list, cycle_times_out = [] - if cycle_times_in is None or cycle_times_in == 'None': + if cycle_times_in == [None] or cycle_times_in == 'None': if common_cycle_times is None: cycle_times_out = [] else: From 8e74f79c4a9bccec514414ce7be4c2be164fff5d Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 15:57:50 -0400 Subject: [PATCH 064/299] add legend --- .../eva/comparison_jedi_log-geos_marine.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml b/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml index d84cf2af2..ed5bf1dbf 100644 --- a/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml +++ b/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml @@ -25,6 +25,7 @@ graphics: plots: - add_xlabel: 'Total inner iteration number' add_ylabel: 'Residual norm' + add_legend: layers: - type: LinePlot x: @@ -43,6 +44,7 @@ graphics: - add_xlabel: 'Total inner iteration number' add_ylabel: 'Norm reduction' + add_legend: layers: - type: LinePlot x: From 80714518eeb633edc773caf69067139ca6938464 Mon Sep 17 00:00:00 2001 From: Yonggang Yu Date: Tue, 29 Jul 2025 07:26:48 -0600 Subject: [PATCH 065/299] Link akbk*.nc4 to the fv3files/ directory (#602) --- .../jedi/interfaces/geos_atmosphere/model/stage.yaml | 1 + .../jedi/interfaces/geos_atmosphere/model/stage_cycle.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml index a3cc3d220..5d770b8da 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml @@ -2,6 +2,7 @@ directories: - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/GEOS_CRTM_Surface/geos.crtmsrf.{{horizontal_resolution}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/'] - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi/test/Data/fv3files/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/'] + - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi-data/testinput_tier_1/inputs/fv3files/*', '{{cycle_dir}}/fv3-jedi/fv3files/'] - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/1.0.1/gsi-coeffs-gmao-global-l{{vertical_resolution}}x{{gsibec_nlons}}y{{gsibec_nlats}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/'] - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/1.0.1/{{gsibec_configuration}}_l{{vertical_resolution}}x{{gsibec_nlons}}y{{gsibec_nlats}}.nml', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/'] - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/rcov/1.0.0/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/rcov/'] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.yaml index df3136234..b41beaeba 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.yaml @@ -2,6 +2,7 @@ directories: - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/GEOS_CRTM_Surface/geos.crtmsrf.{{horizontal_resolution}}.nc4', '{{cycle_dir}}/fv3-jedi/bkg/'] - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi/test/Data/fv3files/*', '{{cycle_dir}}/fv3-jedi/fv3files/'] + - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi-data/testinput_tier_1/inputs/fv3files/*', '{{cycle_dir}}/fv3-jedi/fv3files/'] - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/1.0.1/gsi-coeffs-gmao-global-l{{vertical_resolution}}x{{gsibec_nlons}}y{{gsibec_nlats}}.nc4', '{{cycle_dir}}/fv3-jedi/gsibec/'] - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec//1.0.1/{{gsibec_configuration}}_l{{vertical_resolution}}x{{gsibec_nlons}}y{{gsibec_nlats}}.nml', '{{cycle_dir}}/fv3-jedi/gsibec/'] - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/rcov/1.0.0/*', '{{cycle_dir}}/fv3-jedi/rcov/'] From 62878d920d2abac24de6efd13f9fd77e85e7c125 Mon Sep 17 00:00:00 2001 From: rtodling Date: Wed, 30 Jul 2025 06:54:25 -0600 Subject: [PATCH 066/299] SWELL now controls on peripheral files (#603) --- .../jedi/interfaces/geos_atmosphere/model/stage.yaml | 3 +-- .../jedi/interfaces/geos_atmosphere/model/stage_cycle.yaml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml index 5d770b8da..bc28cd121 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml @@ -1,8 +1,7 @@ - link_files: directories: - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/GEOS_CRTM_Surface/geos.crtmsrf.{{horizontal_resolution}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/'] - - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi/test/Data/fv3files/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/'] - - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi-data/testinput_tier_1/inputs/fv3files/*', '{{cycle_dir}}/fv3-jedi/fv3files/'] + - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/fv3files/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/'] - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/1.0.1/gsi-coeffs-gmao-global-l{{vertical_resolution}}x{{gsibec_nlons}}y{{gsibec_nlats}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/'] - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/1.0.1/{{gsibec_configuration}}_l{{vertical_resolution}}x{{gsibec_nlons}}y{{gsibec_nlats}}.nml', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/'] - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/rcov/1.0.0/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/rcov/'] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.yaml index b41beaeba..a064701db 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.yaml @@ -1,8 +1,7 @@ - link_files: directories: - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/GEOS_CRTM_Surface/geos.crtmsrf.{{horizontal_resolution}}.nc4', '{{cycle_dir}}/fv3-jedi/bkg/'] - - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi/test/Data/fv3files/*', '{{cycle_dir}}/fv3-jedi/fv3files/'] - - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi-data/testinput_tier_1/inputs/fv3files/*', '{{cycle_dir}}/fv3-jedi/fv3files/'] + - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/fv3files/*', '{{cycle_dir}}/fv3-jedi/fv3files/'] - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/1.0.1/gsi-coeffs-gmao-global-l{{vertical_resolution}}x{{gsibec_nlons}}y{{gsibec_nlats}}.nc4', '{{cycle_dir}}/fv3-jedi/gsibec/'] - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec//1.0.1/{{gsibec_configuration}}_l{{vertical_resolution}}x{{gsibec_nlons}}y{{gsibec_nlats}}.nml', '{{cycle_dir}}/fv3-jedi/gsibec/'] - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/rcov/1.0.0/*', '{{cycle_dir}}/fv3-jedi/rcov/'] From f478178cc2bc30bf4476e8377151ec98da215493 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 30 Jul 2025 14:40:20 -0400 Subject: [PATCH 067/299] Add task to get observations from GEOS location (#577) * Initial commit * intermediate * rename task and add toggle * restore get_observations * fix toggle * Updated to link * Update naming and add task to 3dvar_atmos * add task * Add task to other suites * add runtime for getobsnotinr2d2 --- .../nccs_discover_cascade/task_questions.yaml | 2 + .../nccs_discover_sles15/task_questions.yaml | 2 + src/swell/suites/3dfgat_atmos/flow.cylc | 6 +- src/swell/suites/3dvar_atmos/flow.cylc | 7 +- src/swell/suites/hofx/flow.cylc | 7 +- src/swell/suites/localensembleda/flow.cylc | 7 +- src/swell/tasks/get_obs_not_in_r2d2.py | 84 +++++++++++++++++++ src/swell/tasks/task_questions.py | 11 ++- src/swell/utilities/question_defaults.py | 13 +++ 9 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 src/swell/tasks/get_obs_not_in_r2d2.py diff --git a/src/swell/deployment/platforms/nccs_discover_cascade/task_questions.yaml b/src/swell/deployment/platforms/nccs_discover_cascade/task_questions.yaml index ca040e27d..cd3edcf88 100644 --- a/src/swell/deployment/platforms/nccs_discover_cascade/task_questions.yaml +++ b/src/swell/deployment/platforms/nccs_discover_cascade/task_questions.yaml @@ -31,3 +31,5 @@ r2d2_local_path: swell_static_files: default_value: /discover/nobackup/projects/gmao/advda/SwellStaticFiles +ioda_locations_not_in_r2d2: + default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/542/prePP/ioda diff --git a/src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml b/src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml index c2df4ee3a..ea40d746c 100644 --- a/src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml +++ b/src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml @@ -31,3 +31,5 @@ r2d2_local_path: swell_static_files: default_value: /discover/nobackup/projects/gmao/advda/SwellStaticFiles +ioda_locations_not_in_r2d2: + default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/542/prePP/ioda diff --git a/src/swell/suites/3dfgat_atmos/flow.cylc b/src/swell/suites/3dfgat_atmos/flow.cylc index b1b3971ea..984445c17 100644 --- a/src/swell/suites/3dfgat_atmos/flow.cylc +++ b/src/swell/suites/3dfgat_atmos/flow.cylc @@ -73,7 +73,8 @@ CloneJedi[^] => StageJediCycle-{{model_component}} StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} GenerateObservingSystemRecords-{{model_component}} => RunJediVariationalExecutable-{{model_component}} # EvaObservations @@ -132,6 +133,9 @@ [[CloneGeosMksi-{{model_component}}]] script = "swell task CloneGeosMksi $config -m {{model_component}}" + [[GetObsNotInR2d2-{{model_component}}]] + script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" + [[GenerateObservingSystemRecords-{{model_component}}]] script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc index b1b3971ea..19ffe8892 100644 --- a/src/swell/suites/3dvar_atmos/flow.cylc +++ b/src/swell/suites/3dvar_atmos/flow.cylc @@ -62,7 +62,7 @@ {% else %} # Cycling VarBC is inactive, static bias files will be used - GetObservations-{{model_component}} + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} {% endif %} # Perform staging that is cycle dependent @@ -73,7 +73,7 @@ CloneJedi[^] => StageJediCycle-{{model_component}} StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} GenerateObservingSystemRecords-{{model_component}} => RunJediVariationalExecutable-{{model_component}} # EvaObservations @@ -135,6 +135,9 @@ [[GenerateObservingSystemRecords-{{model_component}}]] script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" + [[GetObsNotInR2d2-{{model_component}}]] + script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" + [[StageJediCycle-{{model_component}}]] script = "swell task StageJedi $config -d $datetime -m {{model_component}}" diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc index 9922fef76..af199788d 100644 --- a/src/swell/suites/hofx/flow.cylc +++ b/src/swell/suites/hofx/flow.cylc @@ -55,7 +55,7 @@ GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} # Get observations - GetObservations-{{model_component}} + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} # Perform staging that is cycle dependent StageJediCycle-{{model_component}} @@ -65,7 +65,7 @@ CloneJedi[^] => StageJediCycle-{{model_component}} StageJediCycle-{{model_component}} => RunJediHofxExecutable-{{model_component}} GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediHofxExecutable-{{model_component}} - GetObservations-{{model_component}} => RunJediHofxExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediHofxExecutable-{{model_component}} GenerateObservingSystemRecords-{{model_component}} => RunJediHofxExecutable-{{model_component}} # EvaObservations @@ -133,6 +133,9 @@ [[GetObservations-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + [[GetObsNotInR2d2-{{model_component}}]] + script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" + [[RunJediHofxExecutable-{{model_component}}]] script = "swell task RunJediHofxExecutable $config -d $datetime -m {{model_component}}" platform = {{platform}} diff --git a/src/swell/suites/localensembleda/flow.cylc b/src/swell/suites/localensembleda/flow.cylc index 08d28d5e8..e130348bf 100644 --- a/src/swell/suites/localensembleda/flow.cylc +++ b/src/swell/suites/localensembleda/flow.cylc @@ -51,7 +51,9 @@ # Perform staging that is cycle dependent BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{{model_component}} => sync_point - GetObservations-{{model_component}} => sync_point + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => sync_point CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} => sync_point @@ -165,6 +167,9 @@ [[GetObservations-{{model_component}}]] script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + [[GetObsNotInR2d2-{{model_component}}]] + script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" + {% if not skip_ensemble_hofx %} {% if ensemble_hofx_strategy == 'serial' %} [[RunJediHofxEnsembleExecutable-{{model_component}}]] diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py new file mode 100644 index 000000000..450204cc0 --- /dev/null +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -0,0 +1,84 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import glob +import os +import subprocess + +from swell.tasks.base.task_base import taskBase + + +# -------------------------------------------------------------------------------------------------- + + +class GetObsNotInR2d2(taskBase): + + def execute(self) -> None: + + # Get the cycle string + # -------------------- + cycle_date = self.__datetime__.string_directory() + + # Get the path and pattern for the background files + # ------------------------------------------------- + existing_path = self.config.ioda_locations_not_in_r2d2() + + # Point to the model directory + # ---------------------------- + existing_path = os.path.join(existing_path, cycle_date, self.__model__) + + # Create the list containing the files to process + # ----------------------------------------------- + existing_path_files = [] + + # Set the file patterns to search for + # ----------------------------------- + file_patterns = ['*nc4', '*txt'] + + for file_pattern in file_patterns: + + # Get the full paths of all files + # ------------------------------- + existing_path_files.extend(glob.glob(os.path.join(existing_path, file_pattern))) + + # Assert that some files were found + # --------------------------------- + self.logger.assert_abort(len(existing_path_files) > 0, f'No background ' + 'files matching cycle in background directory.') + + # Loop over all the files + # ----------------------- + for existing_path_file in existing_path_files: + + # Get filename from full path + # --------------------------- + existing_file = os.path.basename(existing_path_file) + + # Set the target file + # ------------------- + existing_path_file_target = os.path.join(self.cycle_dir(), existing_file) + + # Remove the file if it exists + # ---------------------------- + if os.path.exists(existing_path_file_target): + os.remove(existing_path_file_target) + + # Build the copy command + # ---------------------- + command = ['ln', '-s', existing_path_file, existing_path_file_target] + + self.logger.info(f'Linking {existing_path_file} ' + f'to {existing_path_file_target}', wrap=False) + + # Copy the file + # ------------- + subprocess.run(command) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 19cd9b541..e85b04588 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -319,6 +319,15 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + GetObsNotInR2d2 = QuestionList( + list_name="GetExistingObservations", + questions=[ + qd.ioda_locations_not_in_r2d2(), + ] + ) + + # -------------------------------------------------------------------------------------------------- + GetGeovals = QuestionList( list_name="GetGeovals", questions=[ @@ -342,7 +351,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.observing_system_records_path(), qd.r2d2_local_path(), qd.window_length(), - qd.window_offset() + qd.window_offset(), ] ) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 9f2d31d0f..b79c72264 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -708,6 +708,19 @@ class horizontal_resolution(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class ioda_locations_not_in_r2d2(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "ioda_locations_not_in_r2d2" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("Provide a path that contains observation files not in r2d2.") + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + @dataclass class jedi_build_method(TaskQuestion): default_value: str = "create" From 293ef9f706cb26055a21c82188efd64b20168f7a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 1 Aug 2025 19:08:14 -0400 Subject: [PATCH 068/299] Automatically set window length for cycling (#604) * Automatically set window length * Update 3dfgat_cycle, remove comment --- src/swell/suites/3dfgat_cycle/flow.cylc | 5 ++--- src/swell/suites/3dvar_cycle/flow.cylc | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/swell/suites/3dfgat_cycle/flow.cylc b/src/swell/suites/3dfgat_cycle/flow.cylc index 175d06120..9dbdf1b89 100644 --- a/src/swell/suites/3dfgat_cycle/flow.cylc +++ b/src/swell/suites/3dfgat_cycle/flow.cylc @@ -68,8 +68,7 @@ # Model preperation # Run the forecast through two windows (need to output restarts at the end of the # first window and backgrounds for the second window) - # MoveDaRestart-{{model_component}}[-P1D] => PrepGeosRunDir - MoveDaRestart-{{model_component}}[-PT6H] => PrepGeosRunDir + MoveDaRestart-{{model_component}}[-{{models[model_component]["window_length"]}}] => PrepGeosRunDir PrepGeosRunDir => RunGeosExecutable # Run the analysis @@ -93,7 +92,7 @@ # Prepare analysis for next forecast RunJediFgatExecutable-{{model_component}} => EvaIncrement-{{model_component}} - {% if 'cice6' in models["geos_marine"]["marine_models"] %} + {% if 'cice6' in models[model_component]["marine_models"] %} PrepareAnalysis-{{model_component}} => RunJediConvertStateSoca2ciceExecutable-{{model_component}} RunJediConvertStateSoca2ciceExecutable-{{model_component}} => SaveRestart-{{model_component}} RunJediConvertStateSoca2ciceExecutable-{{model_component}} => CleanCycle-{{model_component}} diff --git a/src/swell/suites/3dvar_cycle/flow.cylc b/src/swell/suites/3dvar_cycle/flow.cylc index 08371c68f..8630756b0 100644 --- a/src/swell/suites/3dvar_cycle/flow.cylc +++ b/src/swell/suites/3dvar_cycle/flow.cylc @@ -67,8 +67,7 @@ # Model things # Run the forecast through two windows (need to output restarts at the end of the # first window and backgrounds for the second window) - # MoveDaRestart-{{model_component}}[-P1D] => PrepGeosRunDir - MoveDaRestart-{{model_component}}[-PT6H] => PrepGeosRunDir + MoveDaRestart-{{model_component}}[-{{models[model_component]["window_length"]}}] => PrepGeosRunDir PrepGeosRunDir => RunGeosExecutable # Run the analysis @@ -92,7 +91,7 @@ # Prepare analysis for next forecast EvaIncrement-{{model_component}} => PrepareAnalysis-{{model_component}} - {% if 'cice6' in models["geos_marine"]["marine_models"] %} + {% if 'cice6' in models[model_component]["marine_models"] %} PrepareAnalysis-{{model_component}} => RunJediConvertStateSoca2ciceExecutable-{{model_component}} RunJediConvertStateSoca2ciceExecutable-{{model_component}} => SaveRestart-{{model_component}} RunJediConvertStateSoca2ciceExecutable-{{model_component}} => CleanCycle-{{model_component}} From 176fda9325c286d2b006cafb7ff3052c213dcdd7 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 19 Aug 2025 16:25:38 -0400 Subject: [PATCH 069/299] Fix for localensembleda --- src/swell/tasks/task_runtimes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 92a7bc4dd..971b8ef1b 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -271,10 +271,11 @@ class StageJediCycle(Task): @dataclass class sync_point(Task): - script = "true" + script: str = "true" @dataclass class ThinObs(Task): + script: str = "swell task RunJediObsfiltersExecutable $config -d $datetime -m {model_component}" is_cycling: bool = True is_model: bool = True time_limit: bool = True From 2c8bd5cca658ec1a0038075a471ddabf686a3883 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 14:01:17 -0400 Subject: [PATCH 070/299] Use datetime interval --- src/swell/tasks/eva_comparison_increment.py | 194 +++++++++----------- 1 file changed, 91 insertions(+), 103 deletions(-) diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index ab3802643..a9b9a8583 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -24,17 +24,17 @@ class EvaComparisonIncrement(taskBase): - def cycles_in_experiment(self, path): - cycle_glob = os.path.join(os.path.dirname(path), '..', 'run', '*Z', self.get_model()) - cycle_paths = glob.glob(cycle_glob) + def window_info_from_config(self, path: str): - cycle_times = [] + config_file = os.path.join(os.path.dirname(path), 'experiment.yaml') - for cycle_path in cycle_paths: - cycle_time = cycle_path.split(f'/{self.get_model()}')[0].split('run/')[1] - cycle_times.append(cycle_time) + with open(config_file, 'r') as f: + config_dict = yaml.safe_load(f) - return cycle_times + window_type = config_dict['models'][self.get_model()]['window_type'] + window_offset = config_dict['models'][self.get_model()]['window_offset'] + + return window_type, window_offset def execute(self) -> None: @@ -52,98 +52,86 @@ def execute(self) -> None: experiment_path_1 = experiment_paths[0] experiment_path_2 = experiment_paths[1] - # Window information - window_offset = self.config.window_offset() - window_type = self.config.window_type() - - # Cycle times in the run directory for each experiment - cycle_times_1 = self.cycles_in_experiment(experiment_path_1) - cycle_times_2 = self.cycles_in_experiment(experiment_path_2) - - # Shared cycle times for each experiment - cycle_times = list(set(cycle_times_1) & set(cycle_times_2)) - - for cycle_time in cycle_times: - - # Create the cycle dir for this experiment - cycle_dir = os.path.join(self.experiment_path(), 'run', cycle_time, self.get_model()) - os.makedirs(cycle_dir, exist_ok=True) - - cycle_time_dto = datetime.datetime.strptime(cycle_time, '%Y%m%dT%H%M%SZ') - - da_window_params = DataAssimilationWindowParams(self.logger, cycle_time_dto.strftime( - '%Y-%m-%dT%H:%M:%SZ')) - - window_begin_dto = da_window_params.window_begin(window_offset, dto=True) - window_begin = window_begin_dto.strftime('%Y%m%d_%H%M%Sz') - - # Info to task log - info_string = 'Running Eva to plot from the increment file' - self.logger.info('') - self.logger.info(info_string) - self.logger.info('-'*len(info_string)) - - # Create time strings for eva_override directory - cycle_time_reformat = cycle_time_dto.strftime('%Y%m%d_%H%M%Sz') - - # Define the increment filename and path - # For 3D-Var and 3D-FGAT, the increment file is in the middle of the DA window - # For 3D-FGAT atmos, the increment is at the beginning of the DA window. This - # is just a limited implementation of 4DVAR in SWELL by turning the linear operator off, - # where the cost-type is 4D-VAR in OOPS folder. - # TODO: This really complicates the increment file naming and path for now, this - # is a temporary solution (I'm refering to window type and atmos if statement) - # TODO: Increment iteration number may change according to outer iteration loops - # which is currenly manually set in varincrement1.yaml - # For now we are only plotting the first one - iter_no = 1 - incr_file_1 = f'*.increment-iter{iter_no}.{cycle_time_reformat}.nc4' - incr_file_2 = f'*.increment-iter{iter_no}.{cycle_time_reformat}.nc4' - - if window_type == '4D' and 'atmos' in self.suite_name(): - incr_file_1 = f'*.increment-iter{iter_no}.{window_begin}.nc4' - incr_file_2 = f'*.increment-iter{iter_no}.{window_begin}.nc4' - - # Create dictionary used to override the eva config - eva_override = {} - - # Soca case - if model == 'geos_marine': - ocn_cycle_time = cycle_time_dto.strftime('%Y-%m-%dT%H:%M:%SZ') - incr_file_1 = f'ocn.*.incr.{ocn_cycle_time}.nc' - incr_file_2 = f'ocn.*.incr.{ocn_cycle_time}.nc' - - cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', - cycle_time, self.get_model()) - cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', - cycle_time, self.get_model()) - - # Files to fill the template in the config file - increment_file_path_1 = glob.glob(os.path.join(cycle_dir_1, incr_file_1))[0] - increment_file_path_2 = glob.glob(os.path.join(cycle_dir_2, incr_file_2))[0] - - eva_override['cycle_dir'] = cycle_dir - eva_override['window_begin'] = window_begin - - eva_override['cycle_dir_1'] = cycle_dir_1 - eva_override['cycle_dir_2'] = cycle_dir_2 - - eva_override['cycle_time'] = cycle_time_reformat - eva_override['increment_file_path_1'] = increment_file_path_1 - eva_override['increment_file_path_2'] = increment_file_path_2 - - # Override the eva dictionary - eva_str = template_string_jinja2(self.logger, eva_str_template, eva_override) - eva_dict = yaml.safe_load(eva_str) - - # Write eva dictionary to file - # ---------------------------- - conf_output = os.path.join(cycle_dir, 'eva', 'increment', - 'comparison_increment_eva.yaml') - os.makedirs(os.path.dirname(conf_output), exist_ok=True) - with open(conf_output, 'w') as outfile: - yaml.dump(eva_dict, outfile, default_flow_style=False) - - # Call eva - # -------- - eva(eva_dict) + window_type, window_offset = self.window_info_from_config(experiment_path_1) + + # Create the cycle dir for this experiment + cycle_dir = self.cycle_dir() + + cycle_time_dto = self.cycle_time_dto() + + da_window_params = DataAssimilationWindowParams(self.logger, cycle_time_dto.strftime( + '%Y-%m-%dT%H:%M:%SZ')) + + window_begin_dto = da_window_params.window_begin(window_offset, dto=True) + window_begin = window_begin_dto.strftime('%Y%m%d_%H%M%Sz') + + # Info to task log + info_string = 'Running Eva to plot from the increment file' + self.logger.info('') + self.logger.info(info_string) + self.logger.info('-'*len(info_string)) + + # Create time strings for eva_override directory + cycle_time_reformat = cycle_time_dto.strftime('%Y%m%d_%H%M%Sz') + + # Define the increment filename and path + # For 3D-Var and 3D-FGAT, the increment file is in the middle of the DA window + # For 3D-FGAT atmos, the increment is at the beginning of the DA window. This + # is just a limited implementation of 4DVAR in SWELL by turning the linear operator off, + # where the cost-type is 4D-VAR in OOPS folder. + # TODO: This really complicates the increment file naming and path for now, this + # is a temporary solution (I'm refering to window type and atmos if statement) + # TODO: Increment iteration number may change according to outer iteration loops + # which is currenly manually set in varincrement1.yaml + # For now we are only plotting the first one + iter_no = 1 + incr_file_1 = f'*.increment-iter{iter_no}.{cycle_time_reformat}.nc4' + incr_file_2 = f'*.increment-iter{iter_no}.{cycle_time_reformat}.nc4' + + if window_type == '4D' and 'atmos' in self.suite_name(): + incr_file_1 = f'*.increment-iter{iter_no}.{window_begin}.nc4' + incr_file_2 = f'*.increment-iter{iter_no}.{window_begin}.nc4' + + # Create dictionary used to override the eva config + eva_override = {} + + # Soca case + if model == 'geos_marine': + ocn_cycle_time = cycle_time_dto.strftime('%Y-%m-%dT%H:%M:%SZ') + incr_file_1 = f'ocn.*.incr.{ocn_cycle_time}.nc' + incr_file_2 = f'ocn.*.incr.{ocn_cycle_time}.nc' + + cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', + self.cycle_time(), self.get_model()) + cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', + self.cycle_time(), self.get_model()) + + # Files to fill the template in the config file + increment_file_path_1 = glob.glob(os.path.join(cycle_dir_1, incr_file_1))[0] + increment_file_path_2 = glob.glob(os.path.join(cycle_dir_2, incr_file_2))[0] + + eva_override['cycle_dir'] = cycle_dir + eva_override['window_begin'] = window_begin + + eva_override['cycle_dir_1'] = cycle_dir_1 + eva_override['cycle_dir_2'] = cycle_dir_2 + + eva_override['cycle_time'] = cycle_time_reformat + eva_override['increment_file_path_1'] = increment_file_path_1 + eva_override['increment_file_path_2'] = increment_file_path_2 + + # Override the eva dictionary + eva_str = template_string_jinja2(self.logger, eva_str_template, eva_override) + eva_dict = yaml.safe_load(eva_str) + + # Write eva dictionary to file + # ---------------------------- + conf_output = os.path.join(cycle_dir, 'eva', 'increment', + 'comparison_increment_eva.yaml') + os.makedirs(os.path.dirname(conf_output), exist_ok=True) + with open(conf_output, 'w') as outfile: + yaml.dump(eva_dict, outfile, default_flow_style=False) + + # Call eva + # -------- + eva(eva_dict) From 9a65fb4895f64e37cf4b86ac5a0dfd33e9143c6f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 14:02:18 -0400 Subject: [PATCH 071/299] fixes for localensembleda --- src/swell/tasks/task_runtimes.py | 2 +- src/swell/utilities/cylc_runtime.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 971b8ef1b..0438929bd 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -275,7 +275,7 @@ class sync_point(Task): @dataclass class ThinObs(Task): - script: str = "swell task RunJediObsfiltersExecutable $config -d $datetime -m {model_component}" + script: str = "swell task RunJediObsfiltersExecutable $config -d $datetime -m geos_atmosphere" is_cycling: bool = True is_model: bool = True time_limit: bool = True diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 619b45ed3..19859cc0a 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -56,9 +56,6 @@ def __post_init__(self): if self.is_model and self.model is not None: self.scheduling_name += f'-{self.model}' - elif self.is_model: - self.scheduling_name = self.scheduling_name.format(model=self.model) - if self.script is None: self.script = f'swell task {self.base_name} $config' @@ -66,7 +63,11 @@ def __post_init__(self): self.script += ' -d $datetime' if self.is_model and self.model is not None: - self.script += f' -m {self.model}' + self.script += ' -m {model}' + + if self.is_model and self.model is not None: + self.script = self.script.format(model=self.model) + self.scheduling_name = self.scheduling_name.format(model=self.model) # -------------------------------------------------------------------------------------------------- From 2272a65674e22b13acbbd42d2acbb600df47a05f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 15:01:15 -0400 Subject: [PATCH 072/299] Clean up eva comparison plots --- src/swell/suites/compare_variational/workflow.py | 4 ++++ src/swell/tasks/eva_comparison_increment.py | 8 ++++---- src/swell/tasks/task_runtimes.py | 4 +++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 3525f0f00..76a619fbd 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -50,6 +50,10 @@ def define_scheduling(self): final_cycle_point, cycle_times) + # Set the values in the config + self.experiment_dict['start_cycle_point'] = start_cycle_point + self.experiment_dict['final_cycle_point'] = final_cycle_point + scheduling_section = self.create_new_section( 'scheduling', {'initial cycle point': start_cycle_point, 'final cycle point': final_cycle_point}) diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index a9b9a8583..c20beb4f4 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -11,7 +11,6 @@ import os import yaml import glob -import datetime from eva.eva_driver import eva @@ -102,9 +101,9 @@ def execute(self) -> None: incr_file_2 = f'ocn.*.incr.{ocn_cycle_time}.nc' cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', - self.cycle_time(), self.get_model()) + self.__datetime__.string_directory(), self.get_model()) cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', - self.cycle_time(), self.get_model()) + self.__datetime__.string_directory(), self.get_model()) # Files to fill the template in the config file increment_file_path_1 = glob.glob(os.path.join(cycle_dir_1, incr_file_1))[0] @@ -127,7 +126,8 @@ def execute(self) -> None: # Write eva dictionary to file # ---------------------------- conf_output = os.path.join(cycle_dir, 'eva', 'increment', - 'comparison_increment_eva.yaml') + 'comparison_increment_eva.yaml') + os.makedirs(os.path.dirname(conf_output), exist_ok=True) with open(conf_output, 'w') as outfile: yaml.dump(eva_dict, outfile, default_flow_style=False) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 0438929bd..c664ae4b1 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -71,6 +71,7 @@ class EvaJediLog(Task): @dataclass class EvaComparisonIncrement(Task): + is_cycling: bool = True is_model: bool = True @dataclass @@ -275,7 +276,8 @@ class sync_point(Task): @dataclass class ThinObs(Task): - script: str = "swell task RunJediObsfiltersExecutable $config -d $datetime -m geos_atmosphere" + script: str = ("swell task RunJediObsfiltersExecutable $config" + " -d $datetime -m geos_atmosphere") is_cycling: bool = True is_model: bool = True time_limit: bool = True From 69584352072d05318ff7ee828ba5c0c490c68d21 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 15:27:54 -0400 Subject: [PATCH 073/299] Add Jedi-log comparison plots --- .../eva/comparison_jedi_log-geos_marine.yaml | 93 +++++++++++++++++++ .../compare_variational/suite_config.py | 21 +++-- .../suites/compare_variational/workflow.py | 1 + src/swell/tasks/eva_comparison_jedi_log.py | 74 +++++++++++++++ src/swell/tasks/task_runtimes.py | 5 + 5 files changed, 185 insertions(+), 9 deletions(-) create mode 100644 src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml create mode 100644 src/swell/tasks/eva_comparison_jedi_log.py diff --git a/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml b/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml new file mode 100644 index 000000000..d84cf2af2 --- /dev/null +++ b/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml @@ -0,0 +1,93 @@ +datasets: + +- type: JediLog + collection_name: jedi_log_test_1 + jedi_log_to_parse: '{{cycle_dir_1}}/jedi_variational_log.log' + data_to_parse: + convergence: true + +- type: JediLog + collection_name: jedi_log_test_2 + jedi_log_to_parse: '{{cycle_dir_2}}/jedi_variational_log.log' + data_to_parse: + convergence: true + +graphics: + + plotting_backend: Emcpy + figure_list: + + - figure: + layout: [3,1] + figure size: [12,10] + title: 'Residual Norm and Norm Reduction Plots' + output name: '{{cycle_dir}}/eva/jedi_log/convergence/residual_norm_reduction.png' + plots: + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Residual norm' + layers: + - type: LinePlot + x: + variable: jedi_log_test_1::convergence::total_iteration + y: + variable: jedi_log_test_1::convergence::residual_norm + color: 'red' + label: 'Experiment 1' + - type: LinePlot + x: + variable: jedi_log_test_2::convergence::total_iteration + y: + variable: jedi_log_test_2::convergence::residual_norm + color: 'blue' + label: 'Experiment 2' + + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Norm reduction' + layers: + - type: LinePlot + x: + variable: jedi_log_test_1::convergence::total_iteration + y: + variable: jedi_log_test_1::convergence::norm_reduction + color: 'red' + label: 'Experiment 1' + - type: LinePlot + x: + variable: jedi_log_test_2::convergence::total_iteration + y: + variable: jedi_log_test_2::convergence::norm_reduction + color: 'blue' + label: 'Experiment 2' + + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Normalized Value' + add_legend: + layers: + - type: LinePlot + x: + variable: jedi_log_test_1::convergence::total_iteration + y: + variable: jedi_log_test_1::convergence::residual_norm_normalized + color: 'red' + label: 'Normalized residual norm 1' + - type: LinePlot + x: + variable: jedi_log_test_2::convergence::total_iteration + y: + variable: jedi_log_test_2::convergence::residual_norm_normalized + color: 'blue' + label: 'Normalized residual norm 2' + - type: LinePlot + x: + variable: jedi_log_test_1::convergence::total_iteration + y: + variable: jedi_log_test_1::convergence::norm_reduction_normalized + color: 'yellow' + label: 'Normalized norm reduction 1' + - type: LinePlot + x: + variable: jedi_log_test_2::convergence::total_iteration + y: + variable: jedi_log_test_2::convergence::norm_reduction_normalized + color: 'green' + label: 'Normalized norm reduction 2' diff --git a/src/swell/suites/compare_variational/suite_config.py b/src/swell/suites/compare_variational/suite_config.py index ee0568d16..e956d6322 100644 --- a/src/swell/suites/compare_variational/suite_config.py +++ b/src/swell/suites/compare_variational/suite_config.py @@ -15,10 +15,6 @@ from enum import Enum -_3dvar_container = importlib.import_module(f'swell.suites.3dvar.suite_config') -_3dvar_config = getattr(_3dvar_container, 'SuiteConfig') -_3dvar = getattr(_3dvar_config, '_3dvar') - # -------------------------------------------------------------------------------------------------- @@ -39,14 +35,21 @@ class SuiteConfig(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- - compare_3dvar = QuestionList( - list_name="compare_3dvar", + compare_variational_marine = QuestionList( + list_name="compare_variational_marine", questions=[ - _3dvar, compare_variational, qd.model_components(['geos_marine']), - qd.start_cycle_point(default_value=None, widget_type=WidgetType.STRING), - qd.final_cycle_point(default_value=None, widget_type=WidgetType.STRING), + ] + ) + + # -------------------------------------------------------------------------------------------------- + + compare_variational_marine = QuestionList( + list_name="compare_variational_atmosphere", + questions=[ + compare_variational, + qd.model_components(['geos_atmosphere']), ] ) diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 76a619fbd..8a438ffc1 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -62,6 +62,7 @@ def define_scheduling(self): cycle_str = '' cycle_str += f"EvaComparisonIncrement-{model}\n" + cycle_str += f"EvaComparisonJediLog-{model}\n" for i in range(len(config_list)): cycle_str += f"JediOopsLogParser-{model}-{i}\n" diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py new file mode 100644 index 000000000..046fea0b1 --- /dev/null +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -0,0 +1,74 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. + +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import yaml + +from eva.eva_driver import eva + +from swell.tasks.base.task_base import taskBase +from swell.utilities.jinja2 import template_string_jinja2 + + +# -------------------------------------------------------------------------------------------------- + + +class EvaJediLog(taskBase): + + def execute(self) -> None: + + # Get the model + # ------------- + model = self.get_model() + + # Read Eva template file into dictionary + # -------------------------------------- + eva_path = os.path.join(self.experiment_path(), self.experiment_id()+'-suite', 'eva') + eva_config_file = os.path.join(eva_path, f'jedi_log-{model}.yaml') + with open(eva_config_file, 'r') as eva_config_file_open: + eva_str_template = eva_config_file_open.read() + + # Get the paths for the two experiments + experiment_paths = self.config.comparison_experiment_paths() + + experiment_path_1 = experiment_paths[0] + experiment_path_2 = experiment_paths[1] + + cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', + self.__datetime__.string_directory(), self.get_model()) + cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', + self.__datetime__.string_directory(), self.get_model()) + + # Info to task log + info_string = 'Running Eva to plot from the jedi_log file' + self.logger.info('') + self.logger.info(info_string) + self.logger.info('-'*len(info_string)) + + # Create dictionary used to override the eva config + eva_override = {} + eva_override['cycle_dir'] = self.cycle_dir() + eva_override['cycle_dir_1'] = cycle_dir_1 + eva_override['cycle_dir_2'] = cycle_dir_2 + + # Override the eva dictionary + eva_str = template_string_jinja2(self.logger, eva_str_template, eva_override) + eva_dict = yaml.safe_load(eva_str) + + # Write eva dictionary to file + # ---------------------------- + conf_output = os.path.join(self.cycle_dir(), 'eva', 'jedi_log', 'jedi_log_eva.yaml') + os.makedirs(os.path.dirname(conf_output), exist_ok=True) + with open(conf_output, 'w') as outfile: + yaml.dump(eva_dict, outfile, default_flow_style=False) + + # Call eva + # -------- + eva(eva_dict) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index c664ae4b1..07f24d0bb 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -73,6 +73,11 @@ class EvaJediLog(Task): class EvaComparisonIncrement(Task): is_cycling: bool = True is_model: bool = True + + @dataclass + class EvaComparisonJediLog(Task): + is_cycling: bool = True + is_model: bool = True @dataclass class EvaIncrement(Task): From 2eb77e71158e0d4d64612d95996fd7bbbd60a9ff Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 15:53:49 -0400 Subject: [PATCH 074/299] fixes for jedi_log --- src/swell/suites/compare_variational/suite_config.py | 3 ++- src/swell/suites/compare_variational/workflow.py | 1 + src/swell/tasks/eva_comparison_jedi_log.py | 6 +++--- src/swell/utilities/check_da_params.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/swell/suites/compare_variational/suite_config.py b/src/swell/suites/compare_variational/suite_config.py index e956d6322..3821ddd26 100644 --- a/src/swell/suites/compare_variational/suite_config.py +++ b/src/swell/suites/compare_variational/suite_config.py @@ -28,6 +28,7 @@ class SuiteConfig(QuestionContainer, Enum): sq.compare, qd.start_cycle_point(default_value=None, widget_type=WidgetType.STRING), qd.final_cycle_point(default_value=None, widget_type=WidgetType.STRING), + qd.cycle_times(default_value=[None], widget_type=WidgetType.STRING_CHECK_LIST), qd.model_components(), qd.runahead_limit(), ] @@ -45,7 +46,7 @@ class SuiteConfig(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- - compare_variational_marine = QuestionList( + compare_variational_atmosphere = QuestionList( list_name="compare_variational_atmosphere", questions=[ compare_variational, diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 8a438ffc1..7097ba6f8 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -41,6 +41,7 @@ def define_scheduling(self): if 'models' in self.experiment_dict: for model in self.experiment_dict['models'].keys(): + print(model) cycle_times = self.experiment_dict['models'][model]['cycle_times'] start_cycle_point, final_cycle_point, cycle_times = check_da_params( diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 046fea0b1..5cd7c726c 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -20,7 +20,7 @@ # -------------------------------------------------------------------------------------------------- -class EvaJediLog(taskBase): +class EvaComparisonJediLog(taskBase): def execute(self) -> None: @@ -31,7 +31,7 @@ def execute(self) -> None: # Read Eva template file into dictionary # -------------------------------------- eva_path = os.path.join(self.experiment_path(), self.experiment_id()+'-suite', 'eva') - eva_config_file = os.path.join(eva_path, f'jedi_log-{model}.yaml') + eva_config_file = os.path.join(eva_path, f'comparison_jedi_log-{model}.yaml') with open(eva_config_file, 'r') as eva_config_file_open: eva_str_template = eva_config_file_open.read() @@ -64,7 +64,7 @@ def execute(self) -> None: # Write eva dictionary to file # ---------------------------- - conf_output = os.path.join(self.cycle_dir(), 'eva', 'jedi_log', 'jedi_log_eva.yaml') + conf_output = os.path.join(self.cycle_dir(), 'eva', 'jedi_log', 'comparison_jedi_log_eva.yaml') os.makedirs(os.path.dirname(conf_output), exist_ok=True) with open(conf_output, 'w') as outfile: yaml.dump(eva_dict, outfile, default_flow_style=False) diff --git a/src/swell/utilities/check_da_params.py b/src/swell/utilities/check_da_params.py index f87d947b1..eeb18564d 100644 --- a/src/swell/utilities/check_da_params.py +++ b/src/swell/utilities/check_da_params.py @@ -93,7 +93,7 @@ def check_da_params(config_list: list, cycle_times_out = [] - if cycle_times_in is None or cycle_times_in == 'None': + if cycle_times_in == [None] or cycle_times_in == 'None': if common_cycle_times is None: cycle_times_out = [] else: From 0ff8e73e4bab4f7e2129029f0c23e48d1649b5a3 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 15:57:50 -0400 Subject: [PATCH 075/299] add legend --- .../eva/comparison_jedi_log-geos_marine.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml b/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml index d84cf2af2..ed5bf1dbf 100644 --- a/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml +++ b/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_marine.yaml @@ -25,6 +25,7 @@ graphics: plots: - add_xlabel: 'Total inner iteration number' add_ylabel: 'Residual norm' + add_legend: layers: - type: LinePlot x: @@ -43,6 +44,7 @@ graphics: - add_xlabel: 'Total inner iteration number' add_ylabel: 'Norm reduction' + add_legend: layers: - type: LinePlot x: From 59c96e3d09e51af24b9e95f0de42224aa855369b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 16:15:16 -0400 Subject: [PATCH 076/299] Add jedi log parser for atmos --- .../comparison_jedi_log-geos_atmosphere.yaml | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_atmosphere.yaml diff --git a/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_atmosphere.yaml b/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_atmosphere.yaml new file mode 100644 index 000000000..5ae7bc073 --- /dev/null +++ b/src/swell/suites/compare_variational/eva/comparison_jedi_log-geos_atmosphere.yaml @@ -0,0 +1,108 @@ +datasets: + +- type: JediLog + collection_name: JediLogTest_1 + jedi_log_to_parse: '{{cycle_dir_1}}/jedi_variational_log.log' + data_to_parse: + convergence: true + +- type: JediLog + collection_name: JediLogTest_2 + jedi_log_to_parse: '{{cycle_dir_2}}/jedi_variational_log.log' + data_to_parse: + convergence: true + +transforms: +- transform: arithmetic + new name: JediLogTest_1::convergence::${variable}_log + equals: log(JediLogTest_1::convergence::${variable}) + for: + variable: [residual_norm, norm_reduction] + +- transform: arithmetic + new name: JediLogTest_2::convergence::${variable}_log + equals: log(JediLogTest_2::convergence::${variable}) + for: + variable: [residual_norm, norm_reduction] + +graphics: + + plotting_backend: Emcpy + figure_list: + + - figure: + layout: [3,1] + figure size: [12,10] + title: 'Residual Norm and Norm Reduction Plots' + output name: '{{cycle_dir}}/eva/jedi_log/convergence/residual_norm_reduction.png' + plots: + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Residual norm' + add_legend: + layers: + - type: LinePlot + x: + variable: JediLogTest_1::convergence::total_iteration + y: + variable: JediLogTest_1::convergence::residual_norm + color: 'red' + label: 'Experiment 1' + - type: LinePlot + x: + variable: JediLogTest_2::convergence::total_iteration + y: + variable: JediLogTest_2::convergence::residual_norm + color: 'blue' + label: 'Experiment 2' + + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Log(norm reduction)' + add_legend: + layers: + - type: LinePlot + x: + variable: JediLogTest_1::convergence::total_iteration + y: + variable: JediLogTest_1::convergence::norm_reduction + color: 'red' + label: 'Experiment 1' + - type: LinePlot + x: + variable: JediLogTest_2::convergence::total_iteration + y: + variable: JediLogTest_2::convergence::norm_reduction + color: 'blue' + label: 'Experiment 2' + + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Log(reduction)' + add_legend: + layers: + - type: LinePlot + x: + variable: JediLogTest_1::convergence::total_iteration + y: + variable: JediLogTest_1::convergence::residual_norm_log + color: 'red' + label: 'Log(residual norm) 1' + - type: LinePlot + x: + variable: JediLogTest_2::convergence::total_iteration + y: + variable: JediLogTest_2::convergence::residual_norm_log + color: 'blue' + label: 'Log(residual norm) 2' + - type: LinePlot + x: + variable: JediLogTest_1::convergence::total_iteration + y: + variable: JediLogTest_1::convergence::norm_reduction_log + color: 'yellow' + label: 'Log norm reduction 1' + - type: LinePlot + x: + variable: JediLogTest_2::convergence::total_iteration + y: + variable: JediLogTest_2::convergence::norm_reduction_log + color: 'green' + label: 'Log norm reduction 2' From edab5f6f3a43590a932ac687033cc28934b0154a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 17:23:46 -0400 Subject: [PATCH 077/299] add increment for atmos --- .../comparison_increment-geos_atmosphere.yaml | 1911 +++++++++++++++++ 1 file changed, 1911 insertions(+) create mode 100644 src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml diff --git a/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml b/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml new file mode 100644 index 000000000..62fa90a45 --- /dev/null +++ b/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml @@ -0,0 +1,1911 @@ +datasets: +- group: increment + type: LatLon + filename: {{increment_file_path_1}} + name: experiment_increment + variables: [ps, ua, va, t, q, lat, lon] +- group: increment + type: LatLon + filename: {{increment_file_path_2}} + name: experiment_increment + variables: [ps, ua, va, t, q, lat, lon] + +graphics: + + plotting_backend: Emcpy + figure_list: + + #map plot for surface pressure increment + - batch figure: + variables: [ps] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Surface Pressure Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ps + label: PS increment 1 + colorbar: true + cmap: 'bwr' + vmin: &vmin_ps -100 + vmax: &vmax_ps 100 + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Surface Pressure Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::ps + label: PS increment 2 + colorbar: true + cmap: 'bwr' + vmin: -100 + vmax: 100 + + # Experiment Diff (1-2) + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Surface Pressure Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ps_diff + label: PS increment Diff + colorbar: true + cmap: 'bwr' + vmin: -100 + vmax: 100 + + #map plot for temperature increment (lowest level) + - batch figure: + variables: [t] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::t + slices: '[71,...]' + label: T increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: &vmin_t -1 + vmax: &vmax_t 1 + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::t + slices: '[71,...]' + label: T increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + # Experiment Diff (1-2) + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::t_diff + slices: '[71,...]' + label: T increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::t + slices: '[62,...]' + label: T increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::t + slices: '[62,...]' + label: T increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + # Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::t_diff + slices: '[62,...]' + label: T increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::t + slices: '[49,...]' + label: T increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::t + slices: '[49,...]' + label: T increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::t_diff + slices: '[49,...]' + label: T increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::t + slices: '[42,...]' + label: T increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::t + slices: '[42,...]' + label: T increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::t_diff + slices: '[42,...]' + label: T increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::t + slices: '[24,...]' + label: T increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::t + slices: '[24,...]' + label: T increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::t_diff + slices: '[24,...]' + label: T increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::t + slices: '[14,...]' + label: T increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::t + slices: '[14,...]' + label: T increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::t_diff + slices: '[14,...]' + label: T increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + #map plot for Zonal Wind increment (lowest level) + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ua + slices: '[71,...]' + label: U increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: &vmin_ua -1 + vmax: &vmax_ua 1 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::ua + slices: '[71,...]' + label: U increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment Diff (1-2) + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ua_diff + slices: '[71,...]' + label: U increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ua + slices: '[62,...]' + label: U increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::ua + slices: '[62,...]' + label: U increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ua_diff + slices: '[62,...]' + label: U increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ua + slices: '[49,...]' + label: U increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::ua + slices: '[49,...]' + label: U increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ua_diff + slices: '[49,...]' + label: U increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ua + slices: '[42,...]' + label: U increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::ua + slices: '[42,...]' + label: U increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ua_diff + slices: '[42,...]' + label: U increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ua + slices: '[24,...]' + label: U increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::ua + slices: '[24,...]' + label: U increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment Diff + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ua_diff + slices: '[24,...]' + label: U increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ua + slices: '[14,...]' + label: U increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::ua + slices: '[14,...]' + label: U increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + # Experiment Diff (1-2) + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::ua_diff + slices: '[14,...]' + label: U increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_ua + vmax: 1 + + #map plot for Meridional Wind increment (lowest level) + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::va + slices: '[71,...]' + label: V increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: &vmin_va -1 + vmax: &vmax_va 1 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::va + slices: '[71,...]' + label: V increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_va + vmax: *vmax_va + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::va_diff + slices: '[71,...]' + label: V increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_va + vmax: *vmax_va + + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::va + slices: '[62,...]' + label: V increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_va + vmax: *vmax_va + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::va + slices: '[62,...]' + label: V increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_va + vmax: *vmax_va + + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::va_diff + slices: '[62,...]' + label: V increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_va + vmax: *vmax_va + + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::va + slices: '[49,...]' + label: V increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_va + vmax: *vmax_va + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::va + slices: '[49,...]' + label: V increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_va + vmax: *vmax_va + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::va_diff + slices: '[49,...]' + label: V increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_va + vmax: *vmax_va + + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::va + slices: '[42,...]' + label: V increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_va + vmax: *vmax_va + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::va + slices: '[42,...]' + label: V increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_va + vmax: *vmax_va + + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::va_diff + slices: '[42,...]' + label: V increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: *vmin_va + vmax: *vmax_va + + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::va + slices: '[24,...]' + label: V increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::va + slices: '[24,...]' + label: V increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::va_diff + slices: '[24,...]' + label: V increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::va + slices: '[14,...]' + label: V increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::va + slices: '[14,...]' + label: V increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment Diff + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::va_diff + slices: '[14,...]' + label: V increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + + + #map plot for Specific Humidity increment (lowest level) + - batch figure: + variables: [q] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::q + slices: '[71,...]' + label: Q increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::q + slices: '[71,...]' + label: Q increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::q_diff + slices: '[71,...]' + label: Q increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + + #map plot for Specific Humidity increment + - batch figure: + variables: [q] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::q + slices: '[62,...]' + label: Q increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::q + slices: '[62,...]' + label: Q increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::q_diff + slices: '[62,...]' + label: Q increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + + #map plot for Specific Humidity increment + - batch figure: + variables: [q] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::q + slices: '[49,...]' + label: Q increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::q + slices: '[49,...]' + label: Q increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + + # Experiment Diff (1-2) + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::q_diff + slices: '[49,...]' + label: Q increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + + #map plot for Specific Humidity increment + - batch figure: + variables: [q] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::q + slices: '[42,...]' + label: Q increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::q + slices: '[42,...]' + label: Q increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::q_diff + slices: '[42,...]' + label: Q increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + + #map plot for Specific Humidity increment + - batch figure: + variables: [q] + figure: + figure size: [60,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' + plots: + # Experiment 1 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment 1 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::q + slices: '[24,...]' + label: Q increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + + # Experiment 2 + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment 2 + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::q + slices: '[24,...]' + label: Q increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + + # Experiment Diff + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment Diff (1-2) + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::q_diff + slices: '[24,...]' + label: Q increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + +transforms: +- equals: experiment_increment_1::increment::ps-experiment_increment_2::increment::ps + for: + variable: ps + new name: experiment_increment_1::increment::ps_diff + transform: arithmetic + +transforms: +- equals: experiment_increment_1::increment::t-experiment_increment_2::increment::t + for: + variable: t + new name: experiment_increment_1::increment::t_diff + transform: arithmetic + +transforms: +- equals: experiment_increment_1::increment::ua-experiment_increment_2::increment::ua + for: + variable: ua + new name: experiment_increment_1::increment::ua_diff + transform: arithmetic + +transforms: +- equals: experiment_increment_1::increment::va-experiment_increment_2::increment::va + for: + variable: va + new name: experiment_increment_1::increment::va_diff + transform: arithmetic + +transforms: +- equals: experiment_increment_1::increment::q-experiment_increment_2::increment::q + for: + variable: q + new name: experiment_increment_1::increment::q_diff + transform: arithmetic \ No newline at end of file From e89cee42405e738e656502326b6d39803ed699d0 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 17:33:18 -0400 Subject: [PATCH 078/299] fix --- .../eva/comparison_increment-geos_atmosphere.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml b/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml index 62fa90a45..388021f51 100644 --- a/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml +++ b/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml @@ -2,12 +2,12 @@ datasets: - group: increment type: LatLon filename: {{increment_file_path_1}} - name: experiment_increment + name: experiment_increment_1 variables: [ps, ua, va, t, q, lat, lon] - group: increment type: LatLon filename: {{increment_file_path_2}} - name: experiment_increment + name: experiment_increment_2 variables: [ps, ua, va, t, q, lat, lon] graphics: From ab12f72b5ca881e90d74495bb1bd5c453c35fc45 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 17:46:55 -0400 Subject: [PATCH 079/299] fix transform --- .../eva/comparison_increment-geos_atmosphere.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml b/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml index 388021f51..70a719e1c 100644 --- a/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml +++ b/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml @@ -1882,30 +1882,26 @@ transforms: new name: experiment_increment_1::increment::ps_diff transform: arithmetic -transforms: - equals: experiment_increment_1::increment::t-experiment_increment_2::increment::t for: variable: t new name: experiment_increment_1::increment::t_diff transform: arithmetic -transforms: - equals: experiment_increment_1::increment::ua-experiment_increment_2::increment::ua for: variable: ua new name: experiment_increment_1::increment::ua_diff transform: arithmetic -transforms: - equals: experiment_increment_1::increment::va-experiment_increment_2::increment::va for: variable: va new name: experiment_increment_1::increment::va_diff transform: arithmetic -transforms: - equals: experiment_increment_1::increment::q-experiment_increment_2::increment::q for: variable: q new name: experiment_increment_1::increment::q_diff - transform: arithmetic \ No newline at end of file + transform: arithmetic From 41f5277355301291b7c713a4593de9fd5528829e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 22 Aug 2025 17:48:44 -0400 Subject: [PATCH 080/299] Fix layout --- .../comparison_increment-geos_atmosphere.yaml | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml b/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml index 70a719e1c..5b0dd828f 100644 --- a/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml +++ b/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml @@ -20,7 +20,7 @@ graphics: variables: [ps] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}.png' plots: @@ -94,7 +94,7 @@ graphics: variables: [t] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' plots: @@ -169,7 +169,7 @@ graphics: variables: [t] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' plots: @@ -244,7 +244,7 @@ graphics: variables: [t] figure: figure size: [20,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' plots: @@ -321,7 +321,7 @@ graphics: variables: [t] figure: figure size: [20,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' plots: @@ -399,7 +399,7 @@ graphics: variables: [t] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' plots: @@ -477,7 +477,7 @@ graphics: variables: [t] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1.png' plots: @@ -555,7 +555,7 @@ graphics: variables: [ua] figure: figure size: [20,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' plots: @@ -632,7 +632,7 @@ graphics: variables: [ua] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' plots: @@ -710,7 +710,7 @@ graphics: variables: [ua] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' plots: @@ -788,7 +788,7 @@ graphics: variables: [ua] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' plots: @@ -866,7 +866,7 @@ graphics: variables: [ua] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' plots: @@ -944,7 +944,7 @@ graphics: variables: [ua] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1.png' plots: @@ -1022,7 +1022,7 @@ graphics: variables: [va] figure: figure size: [20,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' plots: @@ -1100,7 +1100,7 @@ graphics: variables: [va] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' plots: @@ -1178,7 +1178,7 @@ graphics: variables: [va] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' plots: @@ -1256,7 +1256,7 @@ graphics: variables: [va] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' plots: @@ -1335,7 +1335,7 @@ graphics: variables: [va] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' plots: @@ -1413,7 +1413,7 @@ graphics: variables: [va] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1.png' plots: @@ -1492,7 +1492,7 @@ graphics: variables: [q] figure: figure size: [20,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' plots: @@ -1568,7 +1568,7 @@ graphics: variables: [q] figure: figure size: [20,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' plots: @@ -1646,7 +1646,7 @@ graphics: variables: [q] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' plots: @@ -1724,7 +1724,7 @@ graphics: variables: [q] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' plots: @@ -1802,7 +1802,7 @@ graphics: variables: [q] figure: figure size: [60,10] - layout: [1,1] + layout: [3,1] title: 'Increment from JEDI' output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' plots: From ff5960d26d06a9b49ffcaf6cddaa964012eaa4f4 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 26 Aug 2025 10:26:14 -0400 Subject: [PATCH 081/299] add examples --- docs/examples/soca/comparison_workflows.md | 21 +++++++++++++++++++ .../comparison_increment-geos_atmosphere.yaml | 6 +++--- .../compare_variational/suite_config.py | 2 -- src/swell/tasks/eva_comparison_jedi_log.py | 3 ++- src/swell/tasks/task_runtimes.py | 2 +- 5 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 docs/examples/soca/comparison_workflows.md diff --git a/docs/examples/soca/comparison_workflows.md b/docs/examples/soca/comparison_workflows.md new file mode 100644 index 000000000..f45c5db46 --- /dev/null +++ b/docs/examples/soca/comparison_workflows.md @@ -0,0 +1,21 @@ +# Running comparison workflows in Swell + +Comparison workflows are run on two experiments to compare their output. Current experiments available for comparison include `3dvar` and `3dvar_atmos` + +### Example workflow: 3dvar + +With two completed experiments, create a comparison workflow suite using an override yaml. This should contain the paths to the two experiments to be compared: + +```yaml +comparison_experiment_paths: + - /path/to/experiment1/experiment.yaml + - /path/to/experiment2/experiment.yaml +``` + +This will create a workflow properly configured to run tests on the two experiments. Launching the comparison experiment will run three tasks. One will generate plots using eva consisting of plots of the jedi output, and all increments avalailable for that suite. Each plot will contain the output from experiment 1, experiment 2, and the relative difference between the two. The other two tasks parse the jedi log output and retrieve information such as the residual norms. + +The output from these tasks are placed the the directory of the comparison suite. For the jedi log comparison, the information will be placed under `experiment_root/comparison_tests/` + +Comparison increment and jedi output plots are place under the cycle directory: +`experiment_root/run///eva` + diff --git a/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml b/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml index 5b0dd828f..981058d50 100644 --- a/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml +++ b/src/swell/suites/compare_variational/eva/comparison_increment-geos_atmosphere.yaml @@ -88,7 +88,7 @@ graphics: cmap: 'bwr' vmin: -100 vmax: 100 - + #map plot for temperature increment (lowest level) - batch figure: variables: [t] @@ -393,7 +393,7 @@ graphics: cmap: 'bwr' vmin: -1 vmax: 1 - + #map plot for temperature increment - batch figure: variables: [t] @@ -1540,7 +1540,7 @@ graphics: cmap: 'bwr' vmin: -0.001 vmax: 0.001 - # Experiment Diff + # Experiment Diff - mapping: projection: plcarr domain: global diff --git a/src/swell/suites/compare_variational/suite_config.py b/src/swell/suites/compare_variational/suite_config.py index 3821ddd26..d1168c582 100644 --- a/src/swell/suites/compare_variational/suite_config.py +++ b/src/swell/suites/compare_variational/suite_config.py @@ -7,8 +7,6 @@ # # -------------------------------------------------------------------------------------------------- -import importlib - from swell.utilities.swell_questions import QuestionContainer, QuestionList, WidgetType from swell.utilities.question_defaults import QuestionDefaults as qd from swell.suites.suite_questions import SuiteQuestions as sq diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 5cd7c726c..c258ee53a 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -64,7 +64,8 @@ def execute(self) -> None: # Write eva dictionary to file # ---------------------------- - conf_output = os.path.join(self.cycle_dir(), 'eva', 'jedi_log', 'comparison_jedi_log_eva.yaml') + conf_output = os.path.join(self.cycle_dir(), 'eva', 'jedi_log', + 'comparison_jedi_log_eva.yaml') os.makedirs(os.path.dirname(conf_output), exist_ok=True) with open(conf_output, 'w') as outfile: yaml.dump(eva_dict, outfile, default_flow_style=False) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 07f24d0bb..4054ba735 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -73,7 +73,7 @@ class EvaJediLog(Task): class EvaComparisonIncrement(Task): is_cycling: bool = True is_model: bool = True - + @dataclass class EvaComparisonJediLog(Task): is_cycling: bool = True From 04a5c37618eb9c7bb57ab33fcf2790e590fe5ea6 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 18 Sep 2025 14:08:52 -0400 Subject: [PATCH 082/299] remove flow.cylc --- src/swell/suites/3dfgat_atmos/flow.cylc | 188 -------------- src/swell/suites/3dfgat_cycle/flow.cylc | 269 --------------------- src/swell/suites/3dvar/flow.cylc | 178 -------------- src/swell/suites/3dvar_atmos/flow.cylc | 187 -------------- src/swell/suites/3dvar_cycle/flow.cylc | 269 --------------------- src/swell/suites/build_geos/flow.cylc | 56 ----- src/swell/suites/build_jedi/flow.cylc | 56 ----- src/swell/suites/convert_ncdiags/flow.cylc | 102 -------- src/swell/suites/forecast_geos/flow.cylc | 116 --------- src/swell/suites/geosadas/flow.cylc | 121 --------- src/swell/suites/hofx/flow.cylc | 164 ------------- src/swell/suites/localensembleda/flow.cylc | 228 ----------------- src/swell/suites/ufo_testing/flow.cylc | 147 ----------- 13 files changed, 2081 deletions(-) delete mode 100644 src/swell/suites/3dfgat_atmos/flow.cylc delete mode 100644 src/swell/suites/3dfgat_cycle/flow.cylc delete mode 100644 src/swell/suites/3dvar/flow.cylc delete mode 100644 src/swell/suites/3dvar_atmos/flow.cylc delete mode 100644 src/swell/suites/3dvar_cycle/flow.cylc delete mode 100644 src/swell/suites/build_geos/flow.cylc delete mode 100644 src/swell/suites/build_jedi/flow.cylc delete mode 100644 src/swell/suites/convert_ncdiags/flow.cylc delete mode 100644 src/swell/suites/forecast_geos/flow.cylc delete mode 100644 src/swell/suites/geosadas/flow.cylc delete mode 100644 src/swell/suites/hofx/flow.cylc delete mode 100644 src/swell/suites/localensembleda/flow.cylc delete mode 100644 src/swell/suites/ufo_testing/flow.cylc diff --git a/src/swell/suites/3dfgat_atmos/flow.cylc b/src/swell/suites/3dfgat_atmos/flow.cylc deleted file mode 100644 index 984445c17..000000000 --- a/src/swell/suites/3dfgat_atmos/flow.cylc +++ /dev/null @@ -1,188 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing JEDI-based non-cycling variational data assimilation - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - runahead limit = {{runahead_limit}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - {% for model_component in model_components %} - # Clone geos ana for generating observing system records - CloneGeosMksi-{{model_component}} - {% endfor %} - """ - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - {% for model_component in model_components %} - {% if cycle_time[model_component] %} - - # Task triggers for: {{model_component}} - # ------------------ - # Generate satellite channel records - CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} - - # Get background, provide a way to get background directly from GEOS experiment - GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} - - # Get observations - {% if cycling_varbc %} - # Cycling VarBC is active, biases from the previous cycle will be used - - RunJediVariationalExecutable-{{model_component}}[-PT6H] => GetObservations-{{model_component}} - {% else %} - - # Cycling VarBC is inactive, static bias files will be used - GetObservations-{{model_component}} - {% endif %} - - # Perform staging that is cycle dependent - StageJediCycle-{{model_component}} - - # Run Jedi variational executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} - CloneJedi[^] => StageJediCycle-{{model_component}} - StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} - GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GenerateObservingSystemRecords-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - - # EvaObservations - RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} - - # EvaJediLog - RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} - - # EvaIncrement - RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - - # Save observations - RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - - # Clean up large files - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} - - {% endif %} - {% endfor %} - """ - {% endfor %} - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% for model_component in model_components %} - - [[CloneGeosMksi-{{model_component}}]] - script = "swell task CloneGeosMksi $config -m {{model_component}}" - - [[GetObsNotInR2d2-{{model_component}}]] - script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" - - [[GenerateObservingSystemRecords-{{model_component}}]] - script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" - - [[StageJediCycle-{{model_component}}]] - script = "swell task StageJedi $config -d $datetime -m {{model_component}}" - - [[GetBackground-{{model_component}} ]] - script = "swell task GetBackground $config -d $datetime -m {{model_component}}" - - [[GetBackgroundGeosExperiment-{{model_component}} ]] - script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}" - - [[GetObservations-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - - [[RunJediVariationalExecutable-{{model_component}}]] - script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" - - [[RunJediVariationalExecutable-{{model_component}}]] - script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediVariationalExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[EvaJediLog-{{model_component}}]] - script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" - - [[EvaIncrement-{{model_component}}]] - script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" - - [[EvaObservations-{{model_component}}]] - script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[SaveObsDiags-{{model_component}}]] - script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" - - [[CleanCycle-{{model_component}}]] - script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" - {% endfor %} - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/flow.cylc b/src/swell/suites/3dfgat_cycle/flow.cylc deleted file mode 100644 index 9dbdf1b89..000000000 --- a/src/swell/suites/3dfgat_cycle/flow.cylc +++ /dev/null @@ -1,269 +0,0 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing Geos forecast - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - runahead limit = {{runahead_limit}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone Geos source code - CloneGeos - - # Clone JEDI source code - CloneJedi - - # Build Geos source code by linking - CloneGeos => BuildGeosByLinking? - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildGeosByLinking:fail? => BuildGeos - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - # Need first set of restarts to run model - GetGeosRestart => PrepGeosRunDir - - # Model cannot run without code - BuildGeosByLinking? | BuildGeos => RunGeosExecutable - - {% for model_component in model_components %} - - # JEDI cannot run without code - BuildJediByLinking? | BuildJedi => RunJediFgatExecutable-{{model_component}} - - # Stage JEDI static files - CloneJedi => StageJedi-{{model_component}} => RunJediFgatExecutable-{{model_component}} - - {% endfor %} - """ - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - {% for model_component in model_components %} - - # Model preperation - # Run the forecast through two windows (need to output restarts at the end of the - # first window and backgrounds for the second window) - MoveDaRestart-{{model_component}}[-{{models[model_component]["window_length"]}}] => PrepGeosRunDir - PrepGeosRunDir => RunGeosExecutable - - # Run the analysis - # RunGeosExecutable => StageJediCycle-{{model_component}} - RunGeosExecutable => LinkGeosOutput-{{model_component}} - LinkGeosOutput-{{model_component}} => GenerateBClimatology-{{model_component}} - - # Data assimilation preperation - GetObservations-{{model_component}} - GenerateBClimatologyByLinking-{{model_component}} :fail? => GenerateBClimatology-{{model_component}} - - LinkGeosOutput-{{model_component}} => RunJediFgatExecutable-{{model_component}} - StageJediCycle-{{model_component}} => RunJediFgatExecutable-{{model_component}} - GenerateBClimatologyByLinking-{{model_component}}? | GenerateBClimatology-{{model_component}} => RunJediFgatExecutable-{{model_component}} - GetObservations-{{model_component}} => RunJediFgatExecutable-{{model_component}} - - # Run analysis diagnostics - RunJediFgatExecutable-{{model_component}} => EvaObservations-{{model_component}} - RunJediFgatExecutable-{{model_component}} => EvaJediLog-{{model_component}} - EvaIncrement-{{model_component}} => PrepareAnalysis-{{model_component}} - - # Prepare analysis for next forecast - RunJediFgatExecutable-{{model_component}} => EvaIncrement-{{model_component}} - {% if 'cice6' in models[model_component]["marine_models"] %} - PrepareAnalysis-{{model_component}} => RunJediConvertStateSoca2ciceExecutable-{{model_component}} - RunJediConvertStateSoca2ciceExecutable-{{model_component}} => SaveRestart-{{model_component}} - RunJediConvertStateSoca2ciceExecutable-{{model_component}} => CleanCycle-{{model_component}} - {% else %} - PrepareAnalysis-{{model_component}} => SaveRestart-{{model_component}} - {% endif %} - - # Move restart to next cycle - SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} - - # Save analysis output - # RunJediFgatExecutable-{{model_component}} => SaveAnalysis-{{model_component}} - RunJediFgatExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - - # Save model output - # MoveBackground-{{model_component}} => StoreBackground-{{model_component}} - - # Remove Run Directory - # MoveDaRestart-{{model_component}} & MoveBackground-{{model_component}} => RemoveForecastDir - MoveDaRestart-{{model_component}} => RemoveForecastDir - - # Clean up large files - EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} - {% endfor %} - """ - {% endfor %} -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneGeos]] - script = "swell task CloneGeos $config" - - [[BuildGeosByLinking]] - script = "swell task BuildGeosByLinking $config" - - [[BuildGeos]] - script = "swell task BuildGeos $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildGeos"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[RunGeosExecutable]] - script = "swell task RunGeosExecutable $config -d $datetime" - platform = {{platform}} - execution time limit = {{scheduling["RunGeosExecutable"]["execution_time_limit"]}} - execution retry delays = 2*PT1M - [[[directives]]] - {%- for key, value in scheduling["RunGeosExecutable"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[PrepGeosRunDir]] - script = "swell task PrepGeosRunDir $config -d $datetime" - - [[RemoveForecastDir]] - script = "swell task RemoveForecastDir $config -d $datetime" - - [[GetGeosRestart]] - script = "swell task GetGeosRestart $config -d $datetime" - - {% for model_component in model_components %} - - [[LinkGeosOutput-{{model_component}}]] - script = "swell task LinkGeosOutput $config -d $datetime -m {{model_component}}" - - [[MoveDaRestart-{{model_component}}]] - script = "swell task MoveDaRestart $config -d $datetime -m {{model_component}}" - - [[SaveRestart-{{model_component}}]] - script = "swell task SaveRestart $config -d $datetime -m {{model_component}}" - - [[StageJedi-{{model_component}}]] - script = "swell task StageJedi $config -m {{model_component}}" - - [[StageJediCycle-{{model_component}}]] - script = "swell task StageJedi $config -d $datetime -m {{model_component}}" - - [[GetObservations-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - - [[GenerateBClimatology-{{model_component}}]] - script = "swell task GenerateBClimatology $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["GenerateBClimatology"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["GenerateBClimatology"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[GenerateBClimatologyByLinking-{{model_component}}]] - script = "swell task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" - - {% if 'cice6' in models["geos_marine"]["marine_models"] %} - - [[RunJediConvertStateSoca2ciceExecutable-{{model_component}}]] - script = "swell task RunJediConvertStateSoca2ciceExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediConvertStateSoca2ciceExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediConvertStateSoca2ciceExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% endif %} - - [[RunJediFgatExecutable-{{model_component}}]] - script = "swell task RunJediFgatExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediFgatExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediFgatExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[EvaJediLog-{{model_component}}]] - script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" - - [[EvaIncrement-{{model_component}}]] - script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" - - [[EvaObservations-{{model_component}}]] - script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[SaveRestart-{{model_component}}]] - script = "swell task SaveRestart $config -d $datetime -m {{model_component}}" - - [[SaveObsDiags-{{model_component}}]] - script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" - - [[PrepareAnalysis-{{model_component}}]] - script = "swell task PrepareAnalysis $config -d $datetime -m {{model_component}}" - - [[CleanCycle-{{model_component}}]] - script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" - {% endfor %} - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc deleted file mode 100644 index 3e598bfaa..000000000 --- a/src/swell/suites/3dvar/flow.cylc +++ /dev/null @@ -1,178 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing JEDI-based non-cycling variational data assimilation - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - runahead limit = {{runahead_limit}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - {% for model_component in model_components %} - # Stage JEDI static files - CloneJedi => StageJedi-{{model_component}} - {% endfor %} - """ - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - {% for model_component in model_components %} - {% if cycle_time[model_component] %} - # Task triggers for: {{model_component}} - # ------------------ - # Get background - GetBackground-{{model_component}} - - # Get observations - GetObservations-{{model_component}} - - # GenerateBClimatology, for ocean it is cycle dependent - GenerateBClimatologyByLinking-{{model_component}} :fail? => GenerateBClimatology-{{model_component}} - GetBackground-{{model_component}} => GenerateBClimatology-{{model_component}} - - # Perform staging that is cycle dependent - StageJediCycle-{{model_component}} - - # Run Jedi variational executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} - StageJedi-{{model_component}}[^] => RunJediVariationalExecutable-{{model_component}} - StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GenerateBClimatologyByLinking-{{model_component}}? | GenerateBClimatology-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - - # EvaObservations - RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} - - # EvaJediLog - RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} - - # EvaIncrement - RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - - # Save observations - RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - - # Clean up large files - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} - - {% endif %} - {% endfor %} - """ - {% endfor %} - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% for model_component in model_components %} - [[StageJedi-{{model_component}}]] - script = "swell task StageJedi $config -m {{model_component}}" - - [[StageJediCycle-{{model_component}}]] - script = "swell task StageJedi $config -d $datetime -m {{model_component}}" - - [[ GetBackground-{{model_component}} ]] - script = "swell task GetBackground $config -d $datetime -m {{model_component}}" - - [[GetObservations-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - - [[GenerateBClimatologyByLinking-{{model_component}}]] - script = "swell task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" - - [[GenerateBClimatology-{{model_component}}]] - script = "swell task GenerateBClimatology $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["GenerateBClimatology"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["GenerateBClimatology"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[RunJediVariationalExecutable-{{model_component}}]] - script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediVariationalExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[EvaJediLog-{{model_component}}]] - script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" - - [[EvaIncrement-{{model_component}}]] - script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" - - [[EvaObservations-{{model_component}}]] - script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[SaveObsDiags-{{model_component}}]] - script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" - - [[CleanCycle-{{model_component}}]] - script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" - {% endfor %} - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc deleted file mode 100644 index 19ffe8892..000000000 --- a/src/swell/suites/3dvar_atmos/flow.cylc +++ /dev/null @@ -1,187 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing JEDI-based non-cycling variational data assimilation - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - runahead limit = {{runahead_limit}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - {% for model_component in model_components %} - # Clone geos ana for generating observing system records - CloneGeosMksi-{{model_component}} - {% endfor %} - """ - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - {% for model_component in model_components %} - {% if cycle_time[model_component] %} - - # Task triggers for: {{model_component}} - # ------------------ - # Generate satellite channel records - CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} - - # Get background, provide a way to get background directly from GEOS experiment - GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} - - # Get observations - {% if cycling_varbc %} - # Cycling VarBC is active, biases from the previous cycle will be used - - RunJediVariationalExecutable-{{model_component}}[-PT6H] => GetObservations-{{model_component}} - {% else %} - - # Cycling VarBC is inactive, static bias files will be used - GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} - {% endif %} - - # Perform staging that is cycle dependent - StageJediCycle-{{model_component}} - - # Run Jedi variational executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} - CloneJedi[^] => StageJediCycle-{{model_component}} - StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GenerateObservingSystemRecords-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - - # EvaObservations - RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} - - # EvaJediLog - RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} - - # EvaIncrement - RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - - # Save observations - RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - - # Clean up large files - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} - - {% endif %} - {% endfor %} - """ - {% endfor %} - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% for model_component in model_components %} - - [[CloneGeosMksi-{{model_component}}]] - script = "swell task CloneGeosMksi $config -m {{model_component}}" - - [[GenerateObservingSystemRecords-{{model_component}}]] - script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" - - [[GetObsNotInR2d2-{{model_component}}]] - script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" - - [[StageJediCycle-{{model_component}}]] - script = "swell task StageJedi $config -d $datetime -m {{model_component}}" - - [[GetBackground-{{model_component}} ]] - script = "swell task GetBackground $config -d $datetime -m {{model_component}}" - - [[GetBackgroundGeosExperiment-{{model_component}} ]] - script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}" - - [[GetObservations-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - - [[RunJediVariationalExecutable-{{model_component}}]] - script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" - - [[RunJediVariationalExecutable-{{model_component}}]] - script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediVariationalExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[EvaJediLog-{{model_component}}]] - script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" - - [[EvaIncrement-{{model_component}}]] - script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" - - [[EvaObservations-{{model_component}}]] - script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[SaveObsDiags-{{model_component}}]] - script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" - - [[CleanCycle-{{model_component}}]] - script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" - {% endfor %} - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/flow.cylc b/src/swell/suites/3dvar_cycle/flow.cylc deleted file mode 100644 index 8630756b0..000000000 --- a/src/swell/suites/3dvar_cycle/flow.cylc +++ /dev/null @@ -1,269 +0,0 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing Geos forecast - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone Geos source code - CloneGeos - - # Clone JEDI source code - CloneJedi - - # Build Geos source code by linking - CloneGeos => BuildGeosByLinking? - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildGeosByLinking:fail? => BuildGeos - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - # Need first set of restarts to run model - GetGeosRestart => PrepGeosRunDir - - # Model cannot run without code - BuildGeosByLinking? | BuildGeos => RunGeosExecutable - - {% for model_component in model_components %} - - # JEDI cannot run without code - BuildJediByLinking? | BuildJedi => RunJediVariationalExecutable-{{model_component}} - - # Stage JEDI static files - CloneJedi => StageJedi-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - - {% endfor %} - """ - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - {% for model_component in model_components %} - - # Model things - # Run the forecast through two windows (need to output restarts at the end of the - # first window and backgrounds for the second window) - MoveDaRestart-{{model_component}}[-{{models[model_component]["window_length"]}}] => PrepGeosRunDir - PrepGeosRunDir => RunGeosExecutable - - # Run the analysis - # RunGeosExecutable => StageJediCycle-{{model_component}} - RunGeosExecutable => LinkGeosOutput-{{model_component}} - LinkGeosOutput-{{model_component}} => GenerateBClimatology-{{model_component}} - - # Data assimilation things - GetObservations-{{model_component}} - GenerateBClimatologyByLinking-{{model_component}} :fail? => GenerateBClimatology-{{model_component}} - - LinkGeosOutput-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GenerateBClimatologyByLinking-{{model_component}}? | GenerateBClimatology-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - - # Run analysis diagnostics - RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} - RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} - RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - - # Prepare analysis for next forecast - EvaIncrement-{{model_component}} => PrepareAnalysis-{{model_component}} - {% if 'cice6' in models[model_component]["marine_models"] %} - PrepareAnalysis-{{model_component}} => RunJediConvertStateSoca2ciceExecutable-{{model_component}} - RunJediConvertStateSoca2ciceExecutable-{{model_component}} => SaveRestart-{{model_component}} - RunJediConvertStateSoca2ciceExecutable-{{model_component}} => CleanCycle-{{model_component}} - {% else %} - PrepareAnalysis-{{model_component}} => SaveRestart-{{model_component}} - {% endif %} - - # Move restart to next cycle - SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} - - # Save analysis output - # RunJediVariationalExecutable-{{model_component}} => SaveAnalysis-{{model_component}} - RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - - # Save model output - # MoveBackground-{{model_component}} => StoreBackground-{{model_component}} - - # Remove Run Directory - # MoveDaRestart-{{model_component}} & MoveBackground-{{model_component}} => RemoveForecastDir - MoveDaRestart-{{model_component}} => RemoveForecastDir - - # Clean up large files - # EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & SaveObsDiags-{{model_component}} & RemoveForecastDir => - EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} - {% endfor %} - """ - {% endfor %} -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneGeos]] - script = "swell task CloneGeos $config" - - [[BuildGeosByLinking]] - script = "swell task BuildGeosByLinking $config" - - [[BuildGeos]] - script = "swell task BuildGeos $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildGeos"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[RunGeosExecutable]] - script = "swell task RunGeosExecutable $config -d $datetime" - platform = {{platform}} - execution time limit = {{scheduling["RunGeosExecutable"]["execution_time_limit"]}} - execution retry delays = 2*PT1M - [[[directives]]] - {%- for key, value in scheduling["RunGeosExecutable"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[PrepGeosRunDir]] - script = "swell task PrepGeosRunDir $config -d $datetime" - - [[RemoveForecastDir]] - script = "swell task RemoveForecastDir $config -d $datetime" - - [[GetGeosRestart]] - script = "swell task GetGeosRestart $config -d $datetime" - - {% for model_component in model_components %} - - [[LinkGeosOutput-{{model_component}}]] - script = "swell task LinkGeosOutput $config -d $datetime -m {{model_component}}" - - [[MoveDaRestart-{{model_component}}]] - script = "swell task MoveDaRestart $config -d $datetime -m {{model_component}}" - - [[SaveRestart-{{model_component}}]] - script = "swell task SaveRestart $config -d $datetime -m {{model_component}}" - - [[StageJedi-{{model_component}}]] - script = "swell task StageJedi $config -m {{model_component}}" - - [[StageJediCycle-{{model_component}}]] - script = "swell task StageJedi $config -d $datetime -m {{model_component}}" - - [[GetObservations-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - - [[GenerateBClimatology-{{model_component}}]] - script = "swell task GenerateBClimatology $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["GenerateBClimatology"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["GenerateBClimatology"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[GenerateBClimatologyByLinking-{{model_component}}]] - script = "swell task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" - - {% if 'cice6' in models["geos_marine"]["marine_models"] %} - - [[RunJediConvertStateSoca2ciceExecutable-{{model_component}}]] - script = "swell task RunJediConvertStateSoca2ciceExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediConvertStateSoca2ciceExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediConvertStateSoca2ciceExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% endif %} - - [[RunJediVariationalExecutable-{{model_component}}]] - script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediVariationalExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[EvaJediLog-{{model_component}}]] - script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" - - [[EvaIncrement-{{model_component}}]] - script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" - - [[EvaObservations-{{model_component}}]] - script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[SaveRestart-{{model_component}}]] - script = "swell task SaveRestart $config -d $datetime -m {{model_component}}" - - [[SaveObsDiags-{{model_component}}]] - script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" - - [[PrepareAnalysis-{{model_component}}]] - script = "swell task PrepareAnalysis $config -d $datetime -m {{model_component}}" - - [[CleanCycle-{{model_component}}]] - script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" - {% endfor %} - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/flow.cylc b/src/swell/suites/build_geos/flow.cylc deleted file mode 100644 index dce2ce3d2..000000000 --- a/src/swell/suites/build_geos/flow.cylc +++ /dev/null @@ -1,56 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for building the GEOS model - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - [[graph]] - R1 = """ - CloneGeos => BuildGeosByLinking? - - BuildGeosByLinking:fail? => BuildGeos - """ - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneGeos]] - script = "swell task CloneGeos $config" - - [[BuildGeosByLinking]] - script = "swell task BuildGeosByLinking $config" - - [[BuildGeos]] - script = "swell task BuildGeos $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildGeos"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/flow.cylc b/src/swell/suites/build_jedi/flow.cylc deleted file mode 100644 index b7e347dee..000000000 --- a/src/swell/suites/build_jedi/flow.cylc +++ /dev/null @@ -1,56 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for building the JEDI code - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - [[graph]] - R1 = """ - CloneJedi => BuildJediByLinking? - - BuildJediByLinking:fail? => BuildJedi - """ - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/flow.cylc b/src/swell/suites/convert_ncdiags/flow.cylc deleted file mode 100644 index c43986dcd..000000000 --- a/src/swell/suites/convert_ncdiags/flow.cylc +++ /dev/null @@ -1,102 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing geos_atmosphere ObsFilters tests - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - runahead limit = {{runahead_limit}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - """ - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - - # Convert bias correction to ioda - GetGsiBc - GetGsiBc => GsiBcToIoda - BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda - - # Convert ncdiags to ioda - GetGsiNcdiag - GetGsiNcdiag => GsiNcdiagToIoda - BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda - - # Clean up - GsiNcdiagToIoda => CleanCycle - """ - {% endfor %} - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[ GetGsiBc ]] - script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" - - [[ GsiBcToIoda ]] - script = "swell task GsiBcToIoda $config -d $datetime -m geos_atmosphere" - - [[ GetGsiNcdiag ]] - script = "swell task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" - - [[ GsiNcdiagToIoda ]] - script = "swell task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" - - [[CleanCycle]] - script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/flow.cylc b/src/swell/suites/forecast_geos/flow.cylc deleted file mode 100644 index cc8d74dea..000000000 --- a/src/swell/suites/forecast_geos/flow.cylc +++ /dev/null @@ -1,116 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for Geos forecast without DA - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone Geos source code - CloneGeos - - # Build Geos source code by linking - CloneGeos => BuildGeosByLinking? - - # If not able to link to build create the build - BuildGeosByLinking:fail? => BuildGeos - - # Need first set of restarts to run model - GetGeosRestart => PrepGeosRunDir - - # Get first set of restarts - BuildGeosByLinking? | BuildGeos => RunGeosExecutable - """ - - {% for cycle_time in cycle_times %} - {{cycle_time}} = """ - - # Run Geos Executable - PrepGeosRunDir => RunGeosExecutable - MoveForecastRestart[-PT6H] => PrepGeosRunDir - - # Move restart to next cycle - RunGeosExecutable => MoveForecastRestart - - # Save restarts if requested - # MoveForecastRestart[-PT6H] => SaveRestart - - # Remove Run Directory - MoveForecastRestart => RemoveForecastDir - """ - {% endfor %} - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneGeos]] - script = "swell task CloneGeos $config" - - [[BuildGeosByLinking]] - script = "swell task BuildGeosByLinking $config" - - [[BuildGeos]] - script = "swell task BuildGeos $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildGeos"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[PrepGeosRunDir]] - script = "swell task PrepGeosRunDir $config -d $datetime" - - [[RemoveForecastDir]] - script = "swell task RemoveForecastDir $config -d $datetime" - - [[GetGeosRestart]] - script = "swell task GetGeosRestart $config -d $datetime" - - [[MoveForecastRestart]] - script = "swell task MoveForecastRestart $config -d $datetime" - - [[SaveRestart]] - script = "swell task SaveRestart $config -d $datetime" - - [[RunGeosExecutable]] - script = "swell task RunGeosExecutable $config -d $datetime" - platform = {{platform}} - execution time limit = {{scheduling["RunGeosExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunGeosExecutable"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/flow.cylc b/src/swell/suites/geosadas/flow.cylc deleted file mode 100644 index f96b3d58e..000000000 --- a/src/swell/suites/geosadas/flow.cylc +++ /dev/null @@ -1,121 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing JEDI-based non-cycling variational data assimilation - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = 2020-12-15T00:00:00Z - final cycle point = 2020-12-15T00:00:00Z - - [[graph]] - R1 = """ - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - BuildJediByLinking? - - # Stage JEDI static files - CloneJedi => StageJedi - - # Clone geos ana for generating observing system records - CloneGeosMksi - """ - - T00 = """ - # Generate satellite channel records - CloneGeosMksi[^] => GenerateObservingSystemRecords - - # Get and convert bias correction coefficients - GetGsiBc => GsiBcToIoda - - # Get and convert ncdiags - GetGsiNcdiag => GsiNcdiagToIoda - - # Get background - GetGeosAdasBackground - - # Run Jedi variational executable - GenerateObservingSystemRecords => RunJediVariationalExecutable - BuildJediByLinking[^] => RunJediVariationalExecutable - StageJedi[^] => RunJediVariationalExecutable - GsiBcToIoda => RunJediVariationalExecutable - GsiNcdiagToIoda => RunJediVariationalExecutable - GetGeosAdasBackground => RunJediVariationalExecutable - - # Clean cycle - RunJediVariationalExecutable => CleanCycle - """ - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneGeosMksi]] - script = "swell task CloneGeosMksi $config" - - [[GenerateObservingSystemRecords]] - script = "swell task GenerateObservingSystemRecords $config -d $datetime -m geos_atmosphere" - - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[StageJedi]] - script = "swell task StageJedi $config -m geos_atmosphere" - - [[ GetGsiBc ]] - script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" - - [[ GsiBcToIoda ]] - script = "swell task GsiBcToIoda $config -d $datetime -m geos_atmosphere" - - [[ GetGsiNcdiag ]] - script = "swell task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" - - [[ GsiNcdiagToIoda ]] - script = "swell task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" - - [[ GetGeosAdasBackground ]] - script = "swell task GetGeosAdasBackground $config -d $datetime -m geos_atmosphere" - - [[RunJediVariationalExecutable]] - script = "swell task RunJediVariationalExecutable $config -d $datetime -m geos_atmosphere" - platform = {{platform}} - execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediVariationalExecutable"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[CleanCycle]] - script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc deleted file mode 100644 index af199788d..000000000 --- a/src/swell/suites/hofx/flow.cylc +++ /dev/null @@ -1,164 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing JEDI-based h(x) - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - runahead limit = {{runahead_limit}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - {% for model_component in model_components %} - # Clone geos ana for generating observing system records - CloneGeosMksi-{{model_component}} - {% endfor %} - """ - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - {% for model_component in model_components %} - {% if cycle_time[model_component] %} - - # Task triggers for: {{model_component}} - # ------------------ - # Generate satellite channel records - CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} - - # Get background, provide a way to get background directly from GEOS experiment - GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} - - # Get observations - GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} - - # Perform staging that is cycle dependent - StageJediCycle-{{model_component}} - - # Run Jedi hofx executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediHofxExecutable-{{model_component}} - CloneJedi[^] => StageJediCycle-{{model_component}} - StageJediCycle-{{model_component}} => RunJediHofxExecutable-{{model_component}} - GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediHofxExecutable-{{model_component}} - GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediHofxExecutable-{{model_component}} - GenerateObservingSystemRecords-{{model_component}} => RunJediHofxExecutable-{{model_component}} - - # EvaObservations - RunJediHofxExecutable-{{model_component}} => EvaObservations-{{model_component}} - - # Save observations - RunJediHofxExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - - # Clean up large files - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} - - {% endif %} - {% endfor %} - """ - {% endfor %} - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% for model_component in model_components %} - - [[CloneGeosMksi-{{model_component}}]] - script = "swell task CloneGeosMksi $config -m {{model_component}}" - - [[GenerateObservingSystemRecords-{{model_component}}]] - script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" - - [[StageJediCycle-{{model_component}}]] - script = "swell task StageJedi $config -d $datetime -m {{model_component}}" - - [[GetBackground-{{model_component}}]] - script = "swell task GetBackground $config -d $datetime -m {{model_component}}" - - [[GetBackgroundGeosExperiment-{{model_component}} ]] - script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}" - - [[GetObservations-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - - [[GetObsNotInR2d2-{{model_component}}]] - script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" - - [[RunJediHofxExecutable-{{model_component}}]] - script = "swell task RunJediHofxExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediHofxExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediHofxExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[EvaObservations-{{model_component}}]] - script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[SaveObsDiags-{{model_component}}]] - script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" - - [[CleanCycle-{{model_component}}]] - script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" - {% endfor %} - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/flow.cylc b/src/swell/suites/localensembleda/flow.cylc deleted file mode 100644 index e130348bf..000000000 --- a/src/swell/suites/localensembleda/flow.cylc +++ /dev/null @@ -1,228 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing JEDI-based LocalEnsembleDA Algorithm - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - runahead limit = {{runahead_limit}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - {% for model_component in model_components %} - # Clone geos ana for generating observing system records - CloneGeosMksi-{{model_component}} - {% endfor %} - """ - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - {% for model_component in model_components %} - {% if cycle_time[model_component] %} - # Task triggers for: {{model_component}} - # ------------------ - - # Perform staging that is cycle dependent - BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{{model_component}} => sync_point - - GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} - - GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => sync_point - - CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} => sync_point - - GetEnsembleGeosExperiment-{{model_component}} => sync_point - - sync_point => ThinObs - - {% if skip_ensemble_hofx %} - sync_point => ThinObs => RunJediLocalEnsembleDaExecutable-{{model_component}} - {% else %} - # Run hofx for ensemble members according to strategy - {% if ensemble_hofx_strategy == 'serial' %} - sync_point => RunJediEnsembleMeanVariance-{{model_component}} => RunJediHofxEnsembleExecutable-{{model_component}} - RunJediHofxEnsembleExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} - - {% elif ensemble_hofx_strategy == 'parallel' %} - {% for packet in range(ensemble_hofx_packets) %} - # When strategy is parallel, only proceed if all RunJediHofxEnsembleExecutable completes successfully for each packet - - # There is a need for a task to combine all hofx observations together, compute node preferred, put here as placeholder - # RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunEnsembleHofxCombiner-{{model_component}} - # RunEnsembleHofxCombiner-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} - - sync_point => RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} - RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunJediLocalEnsembleDaExecutable-{{model_component}} - {% endfor %} - {% endif %} - {% endif %} - - - # EvaObservations - RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaObservations-{{model_component}} - - # Save observations - RunJediLocalEnsembleDaExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - - # Clean up large files - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} - - {% endif %} - {% endfor %} - """ - {% endfor %} - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% for model_component in model_components %} - - [[CloneGeosMksi-{{model_component}}]] - script = "swell task CloneGeosMksi $config -m {{model_component}}" - - [[GenerateObservingSystemRecords-{{model_component}}]] - script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" - - [[StageJediCycle-{{model_component}}]] - script = "swell task StageJedi $config -d $datetime -m {{model_component}}" - - [[ GetBackground-{{model_component}} ]] - script = "swell task GetBackground $config -d $datetime -m {{model_component}}" - - [[GetEnsembleGeosExperiment-{{model_component}}]] - script = "swell task GetEnsembleGeosExperiment $config -d $datetime -m {{model_component}}" - - [[ThinObs]] - script = "swell task RunJediObsfiltersExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediObsfiltersExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediObsfiltersExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[RunJediEnsembleMeanVariance-{{model_component}}]] - script = "swell task RunJediEnsembleMeanVariance $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediEnsembleMeanVariance"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediEnsembleMeanVariance"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[GetObservations-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - - [[GetObsNotInR2d2-{{model_component}}]] - script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" - - {% if not skip_ensemble_hofx %} - {% if ensemble_hofx_strategy == 'serial' %} - [[RunJediHofxEnsembleExecutable-{{model_component}}]] - script = "swell task RunJediHofxEnsembleExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediHofxEnsembleExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediHofxEnsembleExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - {% elif ensemble_hofx_strategy == 'parallel' %} - {% for packet in range(ensemble_hofx_packets) %} - [[RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}}]] - script = "swell task RunJediHofxEnsembleExecutable $config -d $datetime -m {{model_component}} -p {{packet}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediHofxEnsembleExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediHofxEnsembleExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% endfor %} - {% endif %} - {% endif %} - - [[RunJediLocalEnsembleDaExecutable-{{model_component}}]] - script = "swell task RunJediLocalEnsembleDaExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediLocalEnsembleDaExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediLocalEnsembleDaExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[EvaObservations-{{model_component}}]] - script = true -# EnKF not ready to use Eva -# script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" -# platform = {{platform}} -# execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} -# [[[directives]]] -# {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} -# --{{key}} = {{value}} -# {%- endfor %} - - [[SaveObsDiags-{{model_component}}]] - script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" - - [[CleanCycle-{{model_component}}]] - script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" - {% endfor %} - - - [[sync_point]] - script = true -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc deleted file mode 100644 index 66e8fb888..000000000 --- a/src/swell/suites/ufo_testing/flow.cylc +++ /dev/null @@ -1,147 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing geos_atmosphere ObsFilters tests - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - runahead limit = {{runahead_limit}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - # Clone geos ana for generating observing system records - CloneGeosMksi - """ - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - - # Generate satellite channel records - CloneGeosMksi[^] => GenerateObservingSystemRecords - - # Convert bias correction to ioda - GetGsiBc - GetGsiBc => GsiBcToIoda - BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda - - # Convert ncdiags to ioda - GetGsiNcdiag - GetGsiNcdiag => GsiNcdiagToIoda - BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda - - GetGeovals - - # Run Jedi hofx executable - GenerateObservingSystemRecords => RunJediUfoTestsExecutable - GsiNcdiagToIoda => RunJediUfoTestsExecutable - GsiBcToIoda => RunJediUfoTestsExecutable - GetGeovals => RunJediUfoTestsExecutable - - # EvaObservations - RunJediUfoTestsExecutable => EvaObservations - - # Clean up large files - EvaObservations => CleanCycle - - """ - {% endfor %} - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[CloneGeosMksi]] - script = "swell task CloneGeosMksi $config -m geos_atmosphere" - - [[GenerateObservingSystemRecords]] - script = "swell task GenerateObservingSystemRecords $config -d $datetime -m geos_atmosphere" - - [[ GetGsiBc ]] - script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" - - [[ GsiBcToIoda ]] - script = "swell task GsiBcToIoda $config -d $datetime -m geos_atmosphere" - - [[ GetGsiNcdiag ]] - script = "swell task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" - - [[ GsiNcdiagToIoda ]] - script = "swell task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" - - [[ GetGeovals ]] - script = "swell task GetGeovals $config -d $datetime -m geos_atmosphere" - - [[RunJediUfoTestsExecutable]] - script = "swell task RunJediUfoTestsExecutable $config -d $datetime -m geos_atmosphere" - platform = {{platform}} - execution time limit = {{scheduling["RunJediUfoTestsExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediUfoTestsExecutable"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[EvaObservations]] - script = "swell task EvaObservations $config -d $datetime -m geos_atmosphere" - platform = {{platform}} - execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["EvaObservations"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[CleanCycle]] - script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" - -# -------------------------------------------------------------------------------------------------- From d1888ec93553ff4b2541abf196e599678c17a201 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 19 Sep 2025 09:53:55 -0400 Subject: [PATCH 083/299] add example to docs --- docs/examples/templating_workflows.md | 106 ++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 docs/examples/templating_workflows.md diff --git a/docs/examples/templating_workflows.md b/docs/examples/templating_workflows.md new file mode 100644 index 000000000..a59228f9a --- /dev/null +++ b/docs/examples/templating_workflows.md @@ -0,0 +1,106 @@ +# Templating cylc workflows within swell +The `flow.cylc` file informs the `Cylc` workflow engine on how to run an experiment. This includes the order in which tasks should be run, and the scripts and environment variables necessary for each task. Templating a workflow within Swell previously used `jinja2` templating on a file named `flow.cylc` under each suite. This has been replaced with an approach that uses a python class to manipulate strings to generated the `flow.cylc` used in the experiment. This allows for more complex logic to be performed in generating the workflow, but also may be confusing to users. This documentation serves to explain the new method of templating workflows under these changes. + +## Cylc sections + +The `flow.cylc` that is generated under this method is not much different from the one generated before, and users shouldn't notice a difference when it comes to creating an experiment, using overrides, etc. When creating an experiment, `swell` consults a file `src/swell/suites//workflow.py` on how to construct the suite. This file should be an extension of the `CylcWorkflow` class (defined in `src/swell/utilities/cylc_workflow.py`). The method `get_workflow_str` is called to return a string which fills the contents of the `flow.cylc` file. Overriding this method can be used to manually specify the contents of the file, but the intended method of using this class is to override methods which comprise the individual sections of the file. Since every suite within Swell contains roughly the same sections, some of which share the same content, it is only necessary to override a few of the methods, most notably the graph section. + +```python +def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + r1 = r1_template + + for model_component in self.experiment_dict['model_components']: + r1 += r1_model.format(model_component=model_component) + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Format the string for each cycle + for model_component in self.experiment_dict['model_components']: + if 'cycle_times' in self.experiment_dict['models'][model_component]: + for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: + cycle_str = cycle_template.format(model_component=model_component) + + # Add the cycle string to the graph string + graph_str += self.format_cycle(cycle_time, cycle_str) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section +``` + +The `define_graph_section` task is used to set the graph. There are a few built-in methods used to format the strings into cylc syntax. The `format_cycle` method is used to construct properly indented blocks for cycling intervals. `create_new_section` creates a section object that tracks indentation levels, and can be added to other sections, with properly handled indentation and spacing. Here the `graph` is constructed as one of the sections, since it itself is a sub-member of the `scheduling` block in a cylc graph. Any suite questions in the config can be referenced from `self.experiment_dict`. For example, this is often used to format the model component. + +## Tasks and the runtime section + +Swell will parse the graph section, which is constructed first, to obtain the tasks which are used by the experiment. It will then build the runtime section by consulting `src/swell/tasks/task_runtimes.py`. Since swell tasks broadly fall into only a few categories (model-dependent or independent, cycling or non-cycling) that do not differ much between suites, they are easily abstracted into a `Task` class. + +```python +@dataclass +class CloneJedi(Task): + pass + +@dataclass +class CloneGeosMksi(Task): + is_model: bool = True + +@dataclass +class EvaJediLog(Task): + is_cycling: bool = True + is_model: bool = True + +@dataclass +class EvaObservations(Task): + time_limit: bool = True + is_cycling: bool = True + is_model: bool = True + slurm: dict = mutable_field({}) + +``` + +Here, the tags `is_cycling` and `is_model` are used to specify what tags the task needs to be appended with in the runtime section. These are set to `False` by default. Tasks with a specified `slurm` dictionary (rather than set to null, as by default) will use their contents to build the `directives` section. + +``` +[[EvaObservations-geos_marine]] + script = "swell task EvaObservations $config -d $datetime -m geos_marine" + platform = nccs_discover_sles15 + execution time limit = PT30M + [[[directives]]] + --job-name = EvaObservations-geos_marine + --qos = allnccs + --nodes = 1 + --ntasks-per-node = 64 + --constraint = mil + --no-requeue = + --account = +``` + +This can be used to set task-specific defaults in `task_runtimes.py`, rather than being set in `slurm.py`: + +```python +@dataclass +class RunJediConvertStateSoca2ciceExecutable(Task): + is_cycling: bool = True + is_model: bool = True + time_limit: bool = True + slurm: dict = mutable_field({'nodes': 1}) +``` + +This supports setting platform-specific overrides, for example: + +```python +@dataclass +class RunJediConvertStateSoca2ciceExecutable(Task): + is_cycling: bool = True + is_model: bool = True + time_limit: bool = True + slurm: dict = mutable_field({'nodes': {'all': 1, + 'nccs_discover_cascade': 2}}) +``` + +On the `nccs_discover_cascade` platform, `nodes` will be set as 2, but on any other platform it will be 1. User overrides will still work as they did previously. From 66e1f6cb7196ef3ec31961298fd652209dd22b33 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 19 Sep 2025 10:19:43 -0400 Subject: [PATCH 084/299] add eva_capabilities workflow --- src/swell/suites/eva_capabilities/workflow.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 src/swell/suites/eva_capabilities/workflow.py diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py new file mode 100644 index 000000000..001b97a9a --- /dev/null +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -0,0 +1,73 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.cylc_workflow import CylcWorkflow + +# -------------------------------------------------------------------------------------------------- + +r1_model = """ +# Clone geos ana for generating observing system records +CloneGeosMksi-{model_component} +""" + +cycle_template = """ +GetNcdiags-{model_component} +CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} +""" + +r_final = """ +GetNcdiags-{model_component} => EvaTimeseries-{model_component} +GenerateObservingSystemRecords-{model_component} => EvaTimeseries-{model_component} +EvaTimeseries-{model_component} => CleanCycle-{model_component} +""" + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_eva_capabilities(CylcWorkflow): + def define_description(self): + description = self.comment_block(""" + # Cylc suite for executing geos_atmosphere ObsFilters tests + """) + + return description + + # -------------------------------------------------------------------------------------------------- + + def define_graph_section(self): + # Define the string of the graph section + graph_str = '' + + # Define the string for the R1 (first non-cycling) section + for model in self.experiment_dict['models'].keys(): + r1 += r1_model.format(model_component=model) + + # Format the R1 cycle and add it to the graph + graph_str += self.format_cycle('R1', r1) + + # Format the string for each cycle + for model in self.experiment_dict['models'].keys(): + if 'cycle_times' in self.experiment_dict['models'][model].keys(): + for cycle_time in self.experiment_dict['models'][model]['cycle_times']: + cycle_str = cycle_template.format(model_component = model) + graph_str += self.format_cycle(cycle_time, cycle_str) + + cycle_str = '' + + # Format the final cycle + for model in self.experiment_dict['models'].keys(): + cycle_str += r_final.format(model_component=model) + graph_str += self.format_cycle('R1/$', cycle_str) + + # Create the graph section + graph_section = self.create_new_section('graph', graph_str) + + return graph_section + +# -------------------------------------------------------------------------------------------------- From 607bddbe547a013a36b0110769ad266c22153800 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 19 Sep 2025 10:33:40 -0400 Subject: [PATCH 085/299] correct eva_capabilities workflow --- src/swell/suites/compare_variational/workflow.py | 1 - src/swell/suites/eva_capabilities/workflow.py | 4 +++- src/swell/suites/localensembleda/workflow.py | 3 ++- src/swell/tasks/task_questions.py | 12 ++++++------ 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 7097ba6f8..8a438ffc1 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -41,7 +41,6 @@ def define_scheduling(self): if 'models' in self.experiment_dict: for model in self.experiment_dict['models'].keys(): - print(model) cycle_times = self.experiment_dict['models'][model]['cycle_times'] start_cycle_point, final_cycle_point, cycle_times = check_da_params( diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index 001b97a9a..d6d069609 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -44,6 +44,8 @@ def define_graph_section(self): # Define the string of the graph section graph_str = '' + r1 = '' + # Define the string for the R1 (first non-cycling) section for model in self.experiment_dict['models'].keys(): r1 += r1_model.format(model_component=model) @@ -55,7 +57,7 @@ def define_graph_section(self): for model in self.experiment_dict['models'].keys(): if 'cycle_times' in self.experiment_dict['models'][model].keys(): for cycle_time in self.experiment_dict['models'][model]['cycle_times']: - cycle_str = cycle_template.format(model_component = model) + cycle_str = cycle_template.format(model_component=model) graph_str += self.format_cycle(cycle_time, cycle_str) cycle_str = '' diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 4656cf8f1..738afdbad 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -47,7 +47,8 @@ """ cycle_template_2 = """ -sync_point => RunJediObsfiltersExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{model_component} +sync_point => RunJediObsfiltersExecutable-{{model_component}} => +RunJediLocalEnsembleDaExecutable-{model_component} """ cycle_template_3 = """ diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 7c335ee60..25b6e0d22 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -174,21 +174,21 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- - EvaComparisonIncrement = QuestionList( + EvaIncrement = QuestionList( list_name="EvaIncrement", questions=[ - EvaIncrement + qd.marine_models(), + qd.window_offset(), + qd.window_type() ] ) # -------------------------------------------------------------------------------------------------- - EvaIncrement = QuestionList( + EvaComparisonIncrement = QuestionList( list_name="EvaIncrement", questions=[ - qd.marine_models(), - qd.window_offset(), - qd.window_type() + EvaIncrement ] ) From 3f7337c880aba31e9851df62622e5bd2488cea19 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 19 Sep 2025 12:41:05 -0400 Subject: [PATCH 086/299] fix sstagejedicycle --- src/swell/tasks/task_questions.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 25b6e0d22..02e478bf4 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -691,6 +691,15 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + StageJediCycle = QuestionList( + list_name="StageJediCycle", + questions=[ + StageJedi + ] + ) + + # -------------------------------------------------------------------------------------------------- + StoreBackground = QuestionList( list_name="StoreBackground", questions=[ From ceff12d7e6c6fd77a02bd115aec68669530fda7f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 19 Sep 2025 14:47:11 -0400 Subject: [PATCH 087/299] replace defer_to_platform --- .../prepare_config_and_suite.py | 13 ++++++------- src/swell/tasks/task_questions.py | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 2c9397568..d8cca3654 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -256,9 +256,9 @@ def override_with_defaults(self, suite_task: QuestionType) -> None: for question_name, question in self.question_dictionary_model_ind.items(): if question['question_type'] == suite_task: if question_name in platform_defaults.keys(): - for key, val in platform_defaults[question_name].items(): - if key not in question.keys() or question[key] == 'defer_to_platform': - question[key] = val + for platform_key, platform_val in platform_defaults[question_name].items(): + if platform_key not in question.keys() or question[platform_key] == 'defer_to_platform': + question[platform_key] = platform_val # Perform a model override on the model_dep dictionary # ---------------------------------------------------- @@ -295,10 +295,9 @@ def override_with_defaults(self, suite_task: QuestionType) -> None: question_name][key] if question_name in platform_defaults.keys(): - for key, val in platform_defaults[question_name].items(): - if val == 'defer_to_platform': - model_dict[question_name][key] = platform_defaults[ - question_name][key] + for platform_key, platform_val in platform_defaults[question_name].items(): + if question[platform_key] == 'defer_to_platform': + model_dict[question_name][platform_key] = platform_val # Look for defer_to_code in the model_ind dictionary # -------------------------------------------------- diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 02e478bf4..b78efa1b4 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -696,8 +696,8 @@ class TaskQuestions(QuestionContainer, Enum): questions=[ StageJedi ] - ) - + ) + # -------------------------------------------------------------------------------------------------- StoreBackground = QuestionList( From 6ac3415e91450e09ea6ecd973c5d447d185c89d6 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 19 Sep 2025 14:53:42 -0400 Subject: [PATCH 088/299] test fixes --- .../prepare_config_and_suite/prepare_config_and_suite.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index d8cca3654..d136d0f5e 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -257,7 +257,8 @@ def override_with_defaults(self, suite_task: QuestionType) -> None: if question['question_type'] == suite_task: if question_name in platform_defaults.keys(): for platform_key, platform_val in platform_defaults[question_name].items(): - if platform_key not in question.keys() or question[platform_key] == 'defer_to_platform': + if platform_key not in question.keys() or \ + question[platform_key] == 'defer_to_platform': question[platform_key] = platform_val # Perform a model override on the model_dep dictionary @@ -295,7 +296,8 @@ def override_with_defaults(self, suite_task: QuestionType) -> None: question_name][key] if question_name in platform_defaults.keys(): - for platform_key, platform_val in platform_defaults[question_name].items(): + for platform_key, platform_val in \ + platform_defaults[question_name].items(): if question[platform_key] == 'defer_to_platform': model_dict[question_name][platform_key] = platform_val From 66f7597a7cfa98520173f640a785faf80d8c3889 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 22 Sep 2025 15:41:51 -0400 Subject: [PATCH 089/299] simplify comparison suite --- .../suites/compare_variational/workflow.py | 7 +----- src/swell/swell.py | 6 +---- src/swell/tasks/base/task_base.py | 25 +++---------------- src/swell/tasks/jedi_oops_log_parser.py | 20 +++++---------- 4 files changed, 12 insertions(+), 46 deletions(-) diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 8a438ffc1..7bd656a6b 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -83,11 +83,6 @@ def define_scheduling(self): def define_runtime_task_overrides(self): overrides = {} - exp_root = os.path.expandvars(self.experiment_dict['experiment_root']) - exp_id = self.experiment_dict['experiment_id'] - - output_dir = os.path.join(exp_root, exp_id) - paths = self.experiment_dict['comparison_experiment_paths'] for i, path in enumerate(paths): @@ -100,7 +95,7 @@ def define_runtime_task_overrides(self): base_name='JediOopsLogParser', scheduling_name=(f'JediOopsLogParser-{model}-{i}'), script=(f'swell task JediOopsLogParser {config_file} -d $datetime' - f' -m {model} -i {i} -o {output_dir}')) + f' -m {model}')) return overrides diff --git a/src/swell/swell.py b/src/swell/swell.py index efb9b8888..a1f49a5ee 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -195,16 +195,12 @@ def launch( @click.option('-d', '--datetime', 'datetime', default=None, help=datetime_help) @click.option('-m', '--model', 'model', default=None, help=model_help) @click.option('-p', '--ensemblePacket', 'ensemblePacket', default=None, help=ensemble_help) -@click.option('-i', '--testIteration', 'testIteration', default=None, help=test_iteration_help) -@click.option('-o', '--testOutput', 'testOutput', default=None, help=test_output_help) def task( task: str, config: str, datetime: Optional[str], model: Optional[str], ensemblePacket: Optional[str], - testIteration: Optional[int], - testOutput: Optional[str] ) -> None: """ Run a workflow task @@ -216,7 +212,7 @@ def task( config (str): Path to the configuration file for the task.\n """ - task_wrapper(task, config, datetime, model, ensemblePacket, testIteration, testOutput) + task_wrapper(task, config, datetime, model, ensemblePacket) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 4c3d164fb..a7e46b45b 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -42,8 +42,6 @@ def __init__( model: str, ensemblePacket: Optional[str], task_name: str, - test_iteration: Optional[int], - test_output: Optional[str], ) -> None: # Create message logger @@ -123,17 +121,6 @@ def __init__( self.da_window_params = DataAssimilationWindowParams(self.logger, self.__datetime__.string_iso()) - # Set the test iteration number - # ----------------------------- - self.test_iteration = test_iteration - - # Set the output directory for test results - # ----------------------------------------- - if test_output is not None: - self.test_output = test_output - else: - self.test_output = self.experiment_path() - # ---------------------------------------------------------------------------------------------- # Execute is the place where a task does its work. It's defined as abstract in the base class @@ -289,8 +276,6 @@ def create_task( datetime: Union[str, dt, None], model: str, ensemblePacket: Optional[str], - test_iteration: Optional[int], - test_output: Optional[str] ) -> taskBase: # Convert camel case string to snake case @@ -301,7 +286,7 @@ def create_task( # Return task object return task_class(config, datetime, model, ensemblePacket, - task, test_iteration, test_output) + task) # -------------------------------------------------------------------------------------------------- @@ -332,16 +317,14 @@ def task_wrapper( config: str, datetime: Union[str, dt, None], model: Optional[str], - ensemblePacket: Optional[str], - test_iteration: Optional[int], - test_output: Optional[str] + ensemblePacket: Optional[str] ) -> None: # Create the object constrc_start = time.perf_counter() creator = taskFactory() - task_object = creator.create_task(task, config, datetime, model, ensemblePacket, - test_iteration, test_output) + task_object = creator.create_task(task, config, datetime, model, ensemblePacket) + constrc_final = time.perf_counter() constrc_time = f'Constructed in {constrc_final - constrc_start:0.4f} seconds' diff --git a/src/swell/tasks/jedi_oops_log_parser.py b/src/swell/tasks/jedi_oops_log_parser.py index 4174fe171..b513aec69 100644 --- a/src/swell/tasks/jedi_oops_log_parser.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -18,12 +18,12 @@ class JediOopsLogParser(taskBase): - def fgrep_residual_norm(self): + def fgrep_residual_norm(self, output_file): cycle_dir = self.cycle_dir() cycle_time = self.__datetime__.string_directory() - model = self.__model__ + model = self.get_model() # Build the command command = ['fgrep', '"Residual norm"'] + [ @@ -34,18 +34,8 @@ def fgrep_residual_norm(self): output = subprocess.run(command, capture_output=True, text=True, shell=True, check=True) results = output.stdout - # Create the output directory - out_dir = os.path.join(self.test_output, 'comparison_tests') - os.makedirs(out_dir, exist_ok=True) - - out_name = f'residual_norms_{model}_{cycle_time}' - if self.test_iteration is not None: - out_name += f'_{self.test_iteration}' - out_name += '.txt' - # Write the output file - out_file = os.path.join(out_dir, out_name) - with open(out_file, 'w') as f: + with open(output_file, 'w') as f: f.write(f'Residual norms output for experiment: {self.experiment_path()}:\n') f.write(f'Model: {model}\n') f.write(f'Cycle: {cycle_time}\n\n') @@ -54,9 +44,11 @@ def fgrep_residual_norm(self): def execute(self) -> None: + output_file = os.path.join(self.cycle_dir(), 'jedi_log_analysis.txt') + for parser_option in self.config.parser_options(['fgrep_residual_norm']): if parser_option == 'fgrep_residual_norm': - self.fgrep_residual_norm() + self.fgrep_residual_norm(output_file) # -------------------------------------------------------------------------------------------------- From d7e7ce952938d6c7169ecdebac3a6b55c2fa60bc Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 22 Sep 2025 16:02:19 -0400 Subject: [PATCH 090/299] add tier test output --- src/swell/tasks/tier_test_output.py | 93 +++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 src/swell/tasks/tier_test_output.py diff --git a/src/swell/tasks/tier_test_output.py b/src/swell/tasks/tier_test_output.py new file mode 100644 index 000000000..9f3afff66 --- /dev/null +++ b/src/swell/tasks/tier_test_output.py @@ -0,0 +1,93 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import re +import subprocess +import numpy as np + +from swell.tasks.base.task_base import taskBase + +# -------------------------------------------------------------------------------------------------- + +class TierTestOutput(taskBase): + + def execute(self): + + all_results = {} + + experiment_paths = self.config.comparison_experiment_paths() + + for i, experiment_path in enumerate(experiment_paths): + experiment_results = {} + + cycles_path = os.path.join(os.path.basename(experiment_path), '..', 'run') + + experiment_cycles = [dirname for dirname in os.listdir(cycles_path) + if re.match('[0-9]*T[0-9]*Z', dirname)] + + for cycle in experiment_cycles: + cycle_results = experiment_results[cycle] = {} + + with open(os.path.join(cycles_path, cycle, 'jedi_log_analysis.txt'), 'r') as f: + lines = f.readlines() + + for line in lines: + delim_chars = [':', '='] + for char in delim_chars: + if char in line: + key = line.split(char)[0].strip() + val = line.split(char)[1].strip() + + cycle_results[key] = val + + all_results[str(i)] = experiment_results + + formatted_results = {} + + for exp_num, experiment_results in all_results.items(): + cycles = experiment_results.keys() + + for cycle in cycles: + if cycle not in formatted_results: + formatted_results[cycle] = {} + + for key, val in experiment_results[cycle]: + if key not in formatted_results[cycle].keys(): + formatted_results[cycle][key] = np.empty(len(all_results.keys())) + formatted_results[cycle][key] = np.nan + else: + formatted_results[cycle][key][exp_num] = val + + output_string = '' + + for cycle in formatted_results.keys(): + header = [cycle].extend(['exp%d'%(exp_num) for exp_num in range(len(experiment_paths))]) + + widths = [''] + np.zeros(len(formatted_results[cycle].keys())) + + rows = [header] + for key, val in formatted_results[cycle].keys(): + row = [key].extend(val) + + for i, item in enumerate(row): + widths[i] = max(len(item), widths(i)) + + rows.append(row) + + for row in rows: + for item, width in zip(row, widths): + add_len = width - len(item) + 2 + output_string += item + add_len + output_string += '\n' + + print(output_string) + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file From d9c718e5e9536ba573edfd56129387d4ede50199 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 22 Sep 2025 17:31:29 -0400 Subject: [PATCH 091/299] clean up tier test output --- .../suites/compare_variational/workflow.py | 2 +- src/swell/tasks/base/task_base.py | 2 +- src/swell/tasks/task_runtimes.py | 4 ++ src/swell/tasks/tier_test_output.py | 57 ++++++++++++------- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 7bd656a6b..38718c001 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -65,7 +65,7 @@ def define_scheduling(self): cycle_str += f"EvaComparisonJediLog-{model}\n" for i in range(len(config_list)): - cycle_str += f"JediOopsLogParser-{model}-{i}\n" + cycle_str += f"JediOopsLogParser-{model}-{i} => TierTestOutput-{model}\n" graph_str += self.format_cycle(cycle_time, cycle_str) diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index a7e46b45b..6cec7eb52 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -324,7 +324,7 @@ def task_wrapper( constrc_start = time.perf_counter() creator = taskFactory() task_object = creator.create_task(task, config, datetime, model, ensemblePacket) - + constrc_final = time.perf_counter() constrc_time = f'Constructed in {constrc_final - constrc_start:0.4f} seconds' diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index cb43389d3..ceb4eb06c 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -286,6 +286,10 @@ class StageJediCycle(Task): class sync_point(Task): script: str = "true" + @dataclass + class TierTestOutput(Task): + is_model: bool = True + @dataclass class RunJediObsfiltersExecutable(Task): script: str = ("swell task RunJediObsfiltersExecutable $config" diff --git a/src/swell/tasks/tier_test_output.py b/src/swell/tasks/tier_test_output.py index 9f3afff66..cc9a174cf 100644 --- a/src/swell/tasks/tier_test_output.py +++ b/src/swell/tasks/tier_test_output.py @@ -10,25 +10,28 @@ import os import re -import subprocess import numpy as np from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- + class TierTestOutput(taskBase): def execute(self): + # Construct dictionary for all results from analysis file all_results = {} experiment_paths = self.config.comparison_experiment_paths() + # Iterate through experiments and collect results for i, experiment_path in enumerate(experiment_paths): experiment_results = {} - cycles_path = os.path.join(os.path.basename(experiment_path), '..', 'run') + # Paths to cycle dirs + cycles_path = os.path.join(os.path.dirname(experiment_path), '..', 'run') experiment_cycles = [dirname for dirname in os.listdir(cycles_path) if re.match('[0-9]*T[0-9]*Z', dirname)] @@ -36,58 +39,70 @@ def execute(self): for cycle in experiment_cycles: cycle_results = experiment_results[cycle] = {} - with open(os.path.join(cycles_path, cycle, 'jedi_log_analysis.txt'), 'r') as f: + with open(os.path.join(cycles_path, cycle, self.get_model(), + 'jedi_log_analysis.txt'), 'r') as f: lines = f.readlines() + # Collect cycle results into dictionary for line in lines: - delim_chars = [':', '='] + delim_chars = ['='] for char in delim_chars: if char in line: key = line.split(char)[0].strip() val = line.split(char)[1].strip() cycle_results[key] = val - + all_results[str(i)] = experiment_results + # Rearrange and group results together by cycle for each experiment formatted_results = {} for exp_num, experiment_results in all_results.items(): + exp_num = int(exp_num) cycles = experiment_results.keys() for cycle in cycles: if cycle not in formatted_results: formatted_results[cycle] = {} - for key, val in experiment_results[cycle]: + # Store each set of results in an array indexed by experiment number + for key, val in experiment_results[cycle].items(): if key not in formatted_results[cycle].keys(): - formatted_results[cycle][key] = np.empty(len(all_results.keys())) - formatted_results[cycle][key] = np.nan - else: - formatted_results[cycle][key][exp_num] = val + formatted_results[cycle][key] = np.empty(len(all_results.keys()), + dtype=object) + formatted_results[cycle][key][:] = 'N/A' + + formatted_results[cycle][key][exp_num] = val + + output_string = '\n' + for i, experiment_path in enumerate(experiment_paths): + output_string += f'exp{i}: {experiment_path}\n' + output_string += '\n' - output_string = '' - + # Format the output string for cycle in formatted_results.keys(): - header = [cycle].extend(['exp%d'%(exp_num) for exp_num in range(len(experiment_paths))]) + header = [cycle] + header.extend([f'exp{exp_num}' for exp_num in range(len(experiment_paths))]) - widths = [''] + np.zeros(len(formatted_results[cycle].keys())) + widths = [len(cycle)] + widths.extend(np.zeros(len(formatted_results[cycle].keys()))) rows = [header] - for key, val in formatted_results[cycle].keys(): - row = [key].extend(val) - + for key, val in formatted_results[cycle].items(): + row = [key] + row.extend(val) for i, item in enumerate(row): - widths[i] = max(len(item), widths(i)) + widths[i] = max(len(item), widths[i]) rows.append(row) - + for row in rows: for item, width in zip(row, widths): add_len = width - len(item) + 2 - output_string += item + add_len + output_string += item + add_len * ' ' output_string += '\n' print(output_string) -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- From 94520ddd8f15209401569ec7f47531f657125444 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 22 Sep 2025 17:34:45 -0400 Subject: [PATCH 092/299] write to file --- src/swell/tasks/tier_test_output.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/swell/tasks/tier_test_output.py b/src/swell/tasks/tier_test_output.py index cc9a174cf..c1b057319 100644 --- a/src/swell/tasks/tier_test_output.py +++ b/src/swell/tasks/tier_test_output.py @@ -103,6 +103,11 @@ def execute(self): output_string += item + add_len * ' ' output_string += '\n' + output_file = os.path.join(self.experiment_path(), 'jedi_log_comparison.txt') + + with open(output_file, 'w') as f: + f.write(output_string) + print(output_string) # -------------------------------------------------------------------------------------------------- From fd1ac21e467f5064440d89decc38cd24a71bb4f5 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 23 Sep 2025 11:52:43 -0400 Subject: [PATCH 093/299] add new documentation --- docs/examples/soca/comparison_workflows.md | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/docs/examples/soca/comparison_workflows.md b/docs/examples/soca/comparison_workflows.md index f45c5db46..9f02b99ec 100644 --- a/docs/examples/soca/comparison_workflows.md +++ b/docs/examples/soca/comparison_workflows.md @@ -1,21 +1,14 @@ -# Running comparison workflows in Swell +# Running comparison experiments within Swell -Comparison workflows are run on two experiments to compare their output. Current experiments available for comparison include `3dvar` and `3dvar_atmos` +Swell currently supports comparing variational experiments. -### Example workflow: 3dvar - -With two completed experiments, create a comparison workflow suite using an override yaml. This should contain the paths to the two experiments to be compared: +Creating a comparison experiment within swell requires an override yaml with two experiments to be compared (default paths are not set automatically). These filepaths should point to the experiment.yaml for each suite. ```yaml comparison_experiment_paths: - /path/to/experiment1/experiment.yaml - /path/to/experiment2/experiment.yaml ``` +These experiments should have matching assimilation window parameters. By default in this suite, start and end cycle points are not specified, in which case Swell will parse the two experiments to find the matching cycle times between the two. Alternatively, start and end cycle points can be set manually. -This will create a workflow properly configured to run tests on the two experiments. Launching the comparison experiment will run three tasks. One will generate plots using eva consisting of plots of the jedi output, and all increments avalailable for that suite. Each plot will contain the output from experiment 1, experiment 2, and the relative difference between the two. The other two tasks parse the jedi log output and retrieve information such as the residual norms. - -The output from these tasks are placed the the directory of the comparison suite. For the jedi log comparison, the information will be placed under `experiment_root/comparison_tests/` - -Comparison increment and jedi output plots are place under the cycle directory: -`experiment_root/run///eva` - +The experiment can then be created using `swell create compare_variational_marine -o override.yaml` or `swell create compare_variational_atmosphere -o override`, depending on the type of experiments being compared. Launching the experiment will run tasks analyzing the jedi log and generating plots using Eva for increments. Comparison of the log analysis will be placed under the comparison suite's directory in a file named `jedi_log_comparison.txt`, while the eva plots will be located under the cycle directory for each cycle. From 8040b97cdcf4fbf36493f3dca99c1428ecd47cd6 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 24 Sep 2025 16:50:36 -0400 Subject: [PATCH 094/299] Add diff and tolerances for residual norms --- src/swell/deployment/launch_experiment.py | 2 +- .../suites/compare_variational/workflow.py | 2 +- src/swell/tasks/task_runtimes.py | 2 +- src/swell/tasks/tier_test_output.py | 113 ------------------ 4 files changed, 3 insertions(+), 116 deletions(-) delete mode 100644 src/swell/tasks/tier_test_output.py diff --git a/src/swell/deployment/launch_experiment.py b/src/swell/deployment/launch_experiment.py index e4bf5d16f..25280b276 100644 --- a/src/swell/deployment/launch_experiment.py +++ b/src/swell/deployment/launch_experiment.py @@ -91,7 +91,7 @@ def cylc_run_experiment(self) -> None: # NB: Could be a factory based on workfl if send_messages == '1': self.logger.info(' Workflow will pause on tasks configured to do so. To unpause:') self.logger.info(' \u001b[32mcylc play ' + self.experiment_name + '\033[0m') - self.logger.info(' ', False) + self.logger.info(' ') # Launch the job monitor self.logger.critical('Launching the TUI, press \'q\' at any time to exit the TUI') diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 38718c001..5bff432c4 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -65,7 +65,7 @@ def define_scheduling(self): cycle_str += f"EvaComparisonJediLog-{model}\n" for i in range(len(config_list)): - cycle_str += f"JediOopsLogParser-{model}-{i} => TierTestOutput-{model}\n" + cycle_str += f"JediOopsLogParser-{model}-{i} => JediLogComparison-{model}\n" graph_str += self.format_cycle(cycle_time, cycle_str) diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index ceb4eb06c..9f7b939d4 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -287,7 +287,7 @@ class sync_point(Task): script: str = "true" @dataclass - class TierTestOutput(Task): + class JediLogComparison(Task): is_model: bool = True @dataclass diff --git a/src/swell/tasks/tier_test_output.py b/src/swell/tasks/tier_test_output.py deleted file mode 100644 index c1b057319..000000000 --- a/src/swell/tasks/tier_test_output.py +++ /dev/null @@ -1,113 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import os -import re -import numpy as np - -from swell.tasks.base.task_base import taskBase - -# -------------------------------------------------------------------------------------------------- - - -class TierTestOutput(taskBase): - - def execute(self): - - # Construct dictionary for all results from analysis file - all_results = {} - - experiment_paths = self.config.comparison_experiment_paths() - - # Iterate through experiments and collect results - for i, experiment_path in enumerate(experiment_paths): - experiment_results = {} - - # Paths to cycle dirs - cycles_path = os.path.join(os.path.dirname(experiment_path), '..', 'run') - - experiment_cycles = [dirname for dirname in os.listdir(cycles_path) - if re.match('[0-9]*T[0-9]*Z', dirname)] - - for cycle in experiment_cycles: - cycle_results = experiment_results[cycle] = {} - - with open(os.path.join(cycles_path, cycle, self.get_model(), - 'jedi_log_analysis.txt'), 'r') as f: - lines = f.readlines() - - # Collect cycle results into dictionary - for line in lines: - delim_chars = ['='] - for char in delim_chars: - if char in line: - key = line.split(char)[0].strip() - val = line.split(char)[1].strip() - - cycle_results[key] = val - - all_results[str(i)] = experiment_results - - # Rearrange and group results together by cycle for each experiment - formatted_results = {} - - for exp_num, experiment_results in all_results.items(): - exp_num = int(exp_num) - cycles = experiment_results.keys() - - for cycle in cycles: - if cycle not in formatted_results: - formatted_results[cycle] = {} - - # Store each set of results in an array indexed by experiment number - for key, val in experiment_results[cycle].items(): - if key not in formatted_results[cycle].keys(): - formatted_results[cycle][key] = np.empty(len(all_results.keys()), - dtype=object) - formatted_results[cycle][key][:] = 'N/A' - - formatted_results[cycle][key][exp_num] = val - - output_string = '\n' - for i, experiment_path in enumerate(experiment_paths): - output_string += f'exp{i}: {experiment_path}\n' - output_string += '\n' - - # Format the output string - for cycle in formatted_results.keys(): - header = [cycle] - header.extend([f'exp{exp_num}' for exp_num in range(len(experiment_paths))]) - - widths = [len(cycle)] - widths.extend(np.zeros(len(formatted_results[cycle].keys()))) - - rows = [header] - for key, val in formatted_results[cycle].items(): - row = [key] - row.extend(val) - for i, item in enumerate(row): - widths[i] = max(len(item), widths[i]) - - rows.append(row) - - for row in rows: - for item, width in zip(row, widths): - add_len = width - len(item) + 2 - output_string += item + add_len * ' ' - output_string += '\n' - - output_file = os.path.join(self.experiment_path(), 'jedi_log_comparison.txt') - - with open(output_file, 'w') as f: - f.write(output_string) - - print(output_string) - -# -------------------------------------------------------------------------------------------------- From c4d17ccc582d420c7fe2b523e7e073f32a3b8565 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 24 Sep 2025 17:06:17 -0400 Subject: [PATCH 095/299] clean up --- src/swell/suites/compare_jedi/suite_config.py | 30 -------- src/swell/suites/compare_jedi/workflow.py | 76 ------------------- src/swell/swell.py | 28 ------- src/swell/tasks/compare_jedi_c_test_output.py | 67 ---------------- src/swell/tasks/task_runtimes.py | 4 - src/swell/utilities/cylc_workflow.py | 2 +- src/swell/utilities/slurm.py | 7 +- 7 files changed, 2 insertions(+), 212 deletions(-) delete mode 100644 src/swell/suites/compare_jedi/suite_config.py delete mode 100644 src/swell/suites/compare_jedi/workflow.py delete mode 100644 src/swell/tasks/compare_jedi_c_test_output.py diff --git a/src/swell/suites/compare_jedi/suite_config.py b/src/swell/suites/compare_jedi/suite_config.py deleted file mode 100644 index 23950be55..000000000 --- a/src/swell/suites/compare_jedi/suite_config.py +++ /dev/null @@ -1,30 +0,0 @@ -# -------------------------------------------------------------------------------------------------- -# @package configuration -# -# Class containing the configuration. This is a dictionary that is converted from -# an input yaml configuration file. Various function are included for interacting with the -# dictionary. -# -# -------------------------------------------------------------------------------------------------- - - -from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.suites.suite_questions import SuiteQuestions as sq - -from enum import Enum - - -# -------------------------------------------------------------------------------------------------- - -class SuiteConfig(QuestionContainer, Enum): - - # -------------------------------------------------------------------------------------------------- - - compare_jedi = QuestionList( - list_name="compare_jedi", - questions=[ - sq.compare, - ] - ) - - # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_jedi/workflow.py b/src/swell/suites/compare_jedi/workflow.py deleted file mode 100644 index ab3b91de1..000000000 --- a/src/swell/suites/compare_jedi/workflow.py +++ /dev/null @@ -1,76 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - -import os - -from swell.utilities.cylc_workflow import CylcWorkflow -from swell.utilities.cylc_runtime import Task - -# -------------------------------------------------------------------------------------------------- - - -class Workflow_compare_jedi(CylcWorkflow): - - # -------------------------------------------------------------------------------------------------- - - def define_description(self): - description = self.comment_block(""" - # Cylc suite for running comparison tests on completed experiments - """) - - return description - - # -------------------------------------------------------------------------------------------------- - - def define_scheduling(self): - - paths = self.experiment_dict['comparison_experiment_paths'] - - if len(paths) < 2: - self.logger.info('Please specify two experiments') - - scheduling_section = self.create_new_section('scheduling') - - r1_str = '' - dependency_str = '' - for i in range(len(paths)): - r1_str += f"JediCTest-{i}\n" - dependency_str += f"JediCTest-{i} & " - - dependency_str = dependency_str[:-3] - dependency_str += ' => CompareJediCTestOutput\n' - - r1_str += dependency_str - - graph_str = self.format_cycle('R1', r1_str) - - graph_section = self.create_new_section('graph', graph_str) - scheduling_section.add_subsection(graph_section) - - return scheduling_section.get_section_str() - - # -------------------------------------------------------------------------------------------------- - - def define_runtime_task_overrides(self): - overrides = {} - - exp_root = os.path.expandvars(self.experiment_dict['experiment_root']) - exp_id = os.path.expandvars(self.experiment_dict['experiment_id']) - - output_dir = os.path.join(exp_root, exp_id) - - for i, path in enumerate(self.experiment_dict['comparison_experiment_paths']): - config_file = os.path.join(os.path.dirname(path), 'experiment.yaml') - overrides[f'JediCTest-{i}'] = Task( - base_name=f'JediCTest', - scheduling_name=f'JediCTest-{i}', - script=f'swell task JediCTest {config_file} -i {i} -o {output_dir}', - slurm={}) - - return overrides diff --git a/src/swell/swell.py b/src/swell/swell.py index a1f49a5ee..5a6b658a6 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -233,34 +233,6 @@ def utility(utility: str) -> None: utility_wrapper(utility) -# -------------------------------------------------------------------------------------------------- - -@swell_driver.command() -@click.argument('comparison_type', type=click.Choice(['variational', 'jedi'])) -@click.argument('experiments', type=click.Path(), nargs=-1) -@click.option('-p', '--platform', 'platform', default='nccs_discover_sles15', - type=click.Choice(get_platforms()), help=platform_help) -@click.option('-o', '--override', 'override', default=None, help=override_help) -@click.option('-a', '--advanced', 'advanced', default=False, help=advanced_help) -@click.option('-s', '--slurm', 'slurm', default=None, help=slurm_help) -def compare(comparison_type, experiments: Tuple[str], platform, override, advanced, slurm) -> None: - if override is not None: - if isinstance(override, str): - with open(override, 'r') as f: - override = yaml.safe_load(f) - else: - override = {} - - if 'comparison_experiment_paths' in override.keys(): - override['comparison_experiment_paths'] = ( - override['comparison_experiment_paths'].extend(list(experiments))) - else: - override['comparison_experiment_paths'] = list(experiments) - - create_experiment_directory(f'compare_{comparison_type}', 'defaults', - platform, override, advanced, slurm) - - # -------------------------------------------------------------------------------------------------- @swell_driver.command() diff --git a/src/swell/tasks/compare_jedi_c_test_output.py b/src/swell/tasks/compare_jedi_c_test_output.py deleted file mode 100644 index c7690d47b..000000000 --- a/src/swell/tasks/compare_jedi_c_test_output.py +++ /dev/null @@ -1,67 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import os -import re -import glob -import re - -from swell.tasks.base.task_base import taskBase - -# -------------------------------------------------------------------------------------------------- - - -class CompareJediCTestOutput(taskBase): - - def execute(self) -> None: - - # Create dictionaries to track the ctest results - comparison_dict_0 = {} - comparison_dict_1 = {} - - # Find the results files - ctest_output_files = sorted(glob.glob(os.path.join( - self.experiment_path(), 'comparison_tests', 'jedi_ctest_results*txt'))) - - # Get the results from each file - for i, file in enumerate(ctest_output_files): - with open(file, 'r') as f: - lines = f.readlines() - - comparison_dict[str(i)] = {} - - for line in lines: - if re.match('[0-9]*/[0-9]* Test'): - test = int(line.split('/')[0].strip()) - if 'Passed' in line: - status = True - else: - status = False - - comparison_dict[str(i)][str(test)] = status - - # Make sure that the number of tests match - if len(comparison_dict_0) != len(comparison_dict_1): - self.logger.info('The two JEDI versions specified do not' - ' share the same number of ctests') - - else: - - # Match the tests - for test in comparison_dict_0.keys(): - if comparison_dict_0[test] and comparison_dict_1[test]: - test_0 = 'Passed' if comparison_dict_0[test] else 'Failed' - test_1 = 'Passed' if comparison_dict_1[test] else 'Failed' - - self.logger.info(f'Mismatch in test #{test}: Build 0' - f' {test_0}, Build 1 {test_1}') - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index 9f7b939d4..f4c3770d9 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -60,10 +60,6 @@ class CloneJedi(Task): class CloneGeosMksi(Task): is_model: bool = True - @dataclass - class CompareJediCTestOutput(Task): - pass - @dataclass class EvaJediLog(Task): is_cycling: bool = True diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 10bd11d9d..45047143b 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -71,7 +71,7 @@ def reset_indentation(self, string: str) -> str: start = True if start: - out_string += line + out_string += line + '\n' return out_string diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 7c21adfed..18ef337ec 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -118,12 +118,7 @@ def validate_directives(directive_dict: dict) -> None: if isinstance(item, Mapping): validate_directives(item) else: - # Make sure that everything in `directive_dict` is in `directive_list`; - # i.e., that all entries are valid slurm directives. - invalid_directives = set(directive_dict.keys()).difference(directive_list) - assert \ - len(invalid_directives) == 0, \ - f"The following are invalid SLURM directives: {invalid_directives}" + assert key in directive_list # -------------------------------------------------------------------------------------------------- From ac7551e6c6b1f035374fa77cfc85cba676de86fb Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 23 Sep 2025 15:25:13 -0400 Subject: [PATCH 096/299] Add docs for csh interactive install (#631) * Initial commit * Update csh install instructions * Update installing_swell_csh_interactive.md * Update installing_swell_csh_interactive.md * fix order of steps * Update installing_swell_uv_venv.md --- .../installing_swell_csh_interactive.md | 51 +++++++++++++++++++ .../discover/installing_swell_uv_venv.md | 6 +-- 2 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 docs/platforms/discover/installing_swell_csh_interactive.md diff --git a/docs/platforms/discover/installing_swell_csh_interactive.md b/docs/platforms/discover/installing_swell_csh_interactive.md new file mode 100644 index 000000000..60f16f2a0 --- /dev/null +++ b/docs/platforms/discover/installing_swell_csh_interactive.md @@ -0,0 +1,51 @@ +# Installing swell in pip interactive mode on csh + +This is an alternative method to installing swell using `pip`, without requiring `uv`. This method is more likely to work for users who have `csh` as their shell. + +### Preliminary steps (need only be done once) + +1) Link cache directory to nobackup (Recommended): + +```csh +mv ~/.cache $NOBACKUP/.cache +``` + +and then create a symlink to your home: + +```csh +ln -s $NOBACKUP/.cache ~/.cache +``` + +2) Create an alias to load spack-stack modules in `~/.cshrc`: + +```csh +alias mod_swell 'module purge; source /discover/nobackup/projects/gmao/advda/swell/jedi_modules/spackstack_1.9_intel' +``` + +After creating this alias, reload your shell or run `source ~/.cshrc` for it to be available. + +### First time installing swell +Clone Swell to wherever you want it to live, for example: +```csh +cd $NOBACKUP +mkdir swell-project +cd swell-project +git clone https://github.com/GEOS-ESM/swell.git swell-develop +``` + +1) Switch to your folder where Swell is cloned: `cd $NOBACKUP/swell-project/swell-develop`. +2) (Optional) Checkout a new branch in a new git worktree: e.g., `git worktree add ../mybranch -b mybranch` will create a folder specifically for the `mybranch` branch (to create a workspace for an existing branch, leave out the `-b` flag). +3) (Optional) `cd ../mybranch' +4) Load all the modules that Swell needs: `mod_swell` (this is the `csh` alias function created in the preliminary steps) +5) Create a Python virtual environment: `python3 -m venv .venv` +6) Activate the virtual environment: `source .venv/bin/activate.csh` +7) Install Swell dependencies: `pip install -r --ignore-installed requirements.txt` +8) Install swell in editable mode: `pip install -e .` (note: make sure you run this while the `venv` is active) + +9) Now, work on SWELL. Any changes you make to the SWELL source code will be automatically applied to the install (because it's an editable install); no need to manually reinstall. + +#### Resuming work from a previous SWELL installation: +1) Switch to your folder where SWELL is installed: `cd $NOBACKUP/swell-project/mybranch`. +2) Load all the modules that SWELL needs: `mod_swell` +3) Activate the virtual environment: `source .venv/bin/activate.csh`. You may also use the full path: `source $NOBACKUP/swell-project/mybranch/.venv/bin/activate.csh`. You may consider adding this command to your `mod_swell` alias for future use. +4) SWELL is ready! See [examples here](../../examples/description.md) on how to run SWELL. diff --git a/docs/platforms/discover/installing_swell_uv_venv.md b/docs/platforms/discover/installing_swell_uv_venv.md index 7d0508954..37326491d 100644 --- a/docs/platforms/discover/installing_swell_uv_venv.md +++ b/docs/platforms/discover/installing_swell_uv_venv.md @@ -28,15 +28,15 @@ Clone Swell to wherever you want to live, for example: cd $NOBACKUP mkdir swell-project cd swell-project -git clone git@github.com:geos-esm/swell swell-develop +git clone https://github.com/GEOS-ESM/swell.git swell-develop ``` 1) Switch to your folder where SWELL is cloned: `cd $NOBACKUP/swell-project/swell-develop`. 2) (Optional) Checkout a new branch in a new git worktree: e.g., `git worktree add ../mybranch -b mybranch` will create a folder specifically for the `mybranch` branch. 3) (Optional) `cd ../mybranch` 4) Load all the modules that SWELL needs: `mod_swell` (this is the `bash` function created in the preliminary steps) 5) Create a Python virtual environment: `uv venv` -6) Install SWELL dependencies `uv pip install -r requirements.txt` -7) Activate the virtual environment: `source .venv/bin/activate` +6) Activate the virtual environment: `source .venv/bin/activate` +7) Install SWELL dependencies `uv pip install -r requirements.txt` 8) Install SWELL in editable mode: `python -m pip install -e .`. (note: make sure you run this while the `venv` is active) 9) Now, work on SWELL. Any changes you make to the SWELL source code will be automatically applied to the install (because it's an editable install); no need to manually reinstall. From de3ed0d133c3daa25c7d78dfec37a5ff40ca4247 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 24 Sep 2025 17:13:15 -0400 Subject: [PATCH 097/299] test fixes --- src/swell/swell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/swell.py b/src/swell/swell.py index 5a6b658a6..8854e2e8b 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -9,7 +9,7 @@ import click -from typing import Union, Optional, Literal, Tuple +from typing import Union, Optional, Literal from swell.deployment.platforms.platforms import get_platforms from swell.deployment.create_experiment import clone_config, create_experiment_directory From eee3ff754b879a0276158319a0f9e4305390cabf Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 26 Sep 2025 15:27:14 -0400 Subject: [PATCH 098/299] add log comparison --- src/swell/tasks/jedi_log_comparison.py | 154 +++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 src/swell/tasks/jedi_log_comparison.py diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py new file mode 100644 index 000000000..8823bceeb --- /dev/null +++ b/src/swell/tasks/jedi_log_comparison.py @@ -0,0 +1,154 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import re +import numpy as np + +from swell.tasks.base.task_base import taskBase + +# -------------------------------------------------------------------------------------------------- + +comparison_fields = {'Residual norm': {'delimiter': '=', 'dtype': float}} + +tolerances = {'Residual norm ( 0)': 0.01, + 'Residual norm ( 1)': 0.01, + 'Residual norm ( 2)': 0.01, + 'Residual norm ( 3)': 0.01, + 'Residual norm ( 4)': 0.01, + 'Residual norm ( 5)': 0.01} + +# -------------------------------------------------------------------------------------------------- + + +class JediLogComparison(taskBase): + + def execute(self): + + # Construct dictionary for all results from log file + all_results = {} + + # Boolean for whether fields fall within tolerances + passed = True + + experiment_paths = self.config.comparison_experiment_paths() + + for exp_num, experiment_path in enumerate(experiment_paths): + + # Paths to cycle dirs + cycles_path = os.path.join(os.path.dirname(experiment_path), '..', 'run') + + experiment_cycles = [dirname for dirname in os.listdir(cycles_path) + if re.match('[0-9]*T[0-9]*Z', dirname)] + + for cycle in experiment_cycles: + if cycle not in all_results.keys(): + cycle_results = all_results[cycle] = {} + + cycle_log_file = os.path.join(cycles_path, cycle, self.get_model(), + 'jedi_variational_log.log') + with open(cycle_log_file, 'r') as f: + lines = f.readlines() + + for line in lines: + for field_name in comparison_fields.keys(): + field = comparison_fields[field_name] + if field_name in line: + key_val = line.split(field['delimiter']) + key = key_val[0].strip() + val = key_val[1].strip() + + # val = field['dtype'](val) + + if key not in cycle_results.keys(): + cycle_results[key] = {f'exp{exp_num}': val} + else: + cycle_results[key][f'exp{exp_num}'] = val + + for key in cycle_results.keys(): + for field_name in comparison_fields.keys(): + if field_name in key: + dtype = comparison_fields[field_name]['dtype'] + + if dtype == float: + if 'exp0' in cycle_results[key] and 'exp1' in cycle_results[key]: + diff = float(cycle_results[key]['exp0']) - \ + float(cycle_results[key]['exp1']) + cycle_results[key]['diff'] = str(diff) + if key in tolerances: + key_passed = np.abs(diff) < tolerances[key] + cycle_results[key]['pass'] = key_passed + else: + cycle_results[key]['pass'] = '' + + if not key_passed: + passed = False + else: + cycle_results[key]['diff'] = 'N/A' + cycle_results[key]['pass'] = '' + + widths = {'key': len(cycle), 'exp0': 0, 'exp1': 0, 'diff': 0, 'pass': 0} + + for cycle, cycle_results in all_results.items(): + for key in cycle_results: + if 'exp0' not in cycle_results[key]: + cycle_results[key]['exp0'] = 'N/A' + if 'exp1' not in cycle_results[key]: + cycle_results[key]['exp1'] = 'N/A' + if 'diff' not in cycle_results[key]: + cycle_results[key]['diff'] = 'N/A' + if 'pass' not in cycle_results[key]: + cycle_results[key]['pass'] = False + passed = False + + widths['key'] = max(widths['key'], len(key)) + widths['exp0'] = max(widths['exp0'], len(cycle_results[key]['exp0'])) + widths['exp1'] = max(widths['exp1'], len(cycle_results[key]['exp1'])) + widths['diff'] = max(widths['diff'], len(str(cycle_results[key]['diff']))) + + for key, val in widths.items(): + widths[key] = val + 2 + + out_string = '\n' + out_string += f'exp0: {experiment_paths[0]}\n' + out_string += f'exp1: {experiment_paths[1]}\n' + out_string += '\n' + + for cycle, cycle_results in all_results.items(): + out_string += cycle + ' ' * (widths['key'] - len(cycle)) + out_string += 'exp0' + ' ' * (widths['exp0'] - len('exp0')) + out_string += 'exp1' + ' ' * (widths['exp1'] - len('exp1')) + out_string += 'diff' + ' ' * (widths['diff'] - len('diff')) + out_string += 'pass' + ' ' * (widths['pass'] - len('pass')) + + out_string += '\n' + + for key in tolerances.keys(): + val_dict = cycle_results[key] + + out_string += key + ' ' * (widths['key'] - len(key)) + out_string += val_dict['exp0'] + ' ' * (widths['exp0'] - len(val_dict['exp0'])) + out_string += val_dict['exp1'] + ' ' * (widths['exp1'] - len(val_dict['exp1'])) + out_string += val_dict['diff'] + ' ' * (widths['diff'] - len(val_dict['diff'])) + out_string += str(val_dict['pass']) + ' ' * ( + widths['pass'] - len(str(val_dict['pass']))) + out_string += '\n' + + out_string += '\n' + + self.logger.info(out_string) + + with open(os.path.join(self.experiment_path(), 'jedi_log_comparison.txt'), 'w') as f: + f.write(out_string) + + if not passed: + self.logger.abort(f'Parameters not within tolerance') + +# -------------------------------------------------------------------------------------------------- From 8a307bce8f885ccfb8f588926e63912e21dae52d Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 30 Sep 2025 15:54:51 -0400 Subject: [PATCH 099/299] revise number of iterations --- src/swell/tasks/jedi_log_comparison.py | 11 ++++------- src/swell/tasks/task_questions.py | 9 +++++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py index 8823bceeb..13f3e1714 100644 --- a/src/swell/tasks/jedi_log_comparison.py +++ b/src/swell/tasks/jedi_log_comparison.py @@ -18,13 +18,6 @@ comparison_fields = {'Residual norm': {'delimiter': '=', 'dtype': float}} -tolerances = {'Residual norm ( 0)': 0.01, - 'Residual norm ( 1)': 0.01, - 'Residual norm ( 2)': 0.01, - 'Residual norm ( 3)': 0.01, - 'Residual norm ( 4)': 0.01, - 'Residual norm ( 5)': 0.01} - # -------------------------------------------------------------------------------------------------- @@ -32,6 +25,10 @@ class JediLogComparison(taskBase): def execute(self): + tolerances = {} + for number in range(self.config.number_of_iterations()): + tolerances['Residual norm ( {number})'] = 0.01 + # Construct dictionary for all results from log file all_results = {} diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index b78efa1b4..c828287ea 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -440,6 +440,15 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + JediLogComparison = QuestionList( + list_name="JediLogComparison", + questions=[ + qd.number_of_iterations(), + ] + ) + + # -------------------------------------------------------------------------------------------------- + LinkGeosOutput = QuestionList( list_name="LinkGeosOutput", questions=[ From b76b1b24a6149cad5f4148ff7631cc9f0f82f98f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 30 Sep 2025 16:07:10 -0400 Subject: [PATCH 100/299] add window_length --- src/swell/suites/3dfgat_cycle/workflow.py | 4 ++-- src/swell/suites/3dvar_cycle/workflow.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index c14ad1447..bd747bf8d 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -52,7 +52,7 @@ # Run the forecast through two windows (need to output restarts at the end of the # first window and backgrounds for the second window) # MoveDaRestart-{model_component}[-P1D] => PrepGeosRunDir -MoveDaRestart-{model_component}[-PT6H] => PrepGeosRunDir +MoveDaRestart-{model_component}[-{models[model_component]["window_length"]}] => PrepGeosRunDir PrepGeosRunDir => RunGeosExecutable # Run the analysis @@ -151,7 +151,7 @@ def define_graph_section(self): for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: cycle_str = cycle_template_1.format(model_component=model_component) - if 'cice6' in self.experiment_dict['models']['geos_marine']['marine_models']: + if 'cice6' in self.experiment_dict['models'][model_component]['marine_models']: cycle_str += cycle_template_2.format(model_component=model_component) else: cycle_str += cycle_template_3.format(model_component=model_component) diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index def9de928..f1c22a261 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -53,7 +53,7 @@ # Run the forecast through two windows (need to output restarts at the end of the # first window and backgrounds for the second window) # MoveDaRestart-{model_component}[-P1D] => PrepGeosRunDir -MoveDaRestart-{model_component}[-PT6H] => PrepGeosRunDir +MoveDaRestart-{model_component}[-{models[model_component]["window_length"]}] => PrepGeosRunDir PrepGeosRunDir => RunGeosExecutable # Run the analysis @@ -146,7 +146,7 @@ def define_graph_section(self): for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: cycle_str = cycle_template_1.format(model_component=model_component) - if 'cice6' in self.experiment_dict['models']['geos_marine']['marine_models']: + if 'cice6' in self.experiment_dict['models'][model_component]['marine_models']: cycle_str += cycle_template_2.format(model_component=model_component) else: cycle_str += cycle_template_3.format(model_component=model_component) From a08c1fe10e36bc5e5c2116dd1775ba49c78e2fad Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 9 Oct 2025 14:50:12 -0400 Subject: [PATCH 101/299] simplify 3dvar: --- src/swell/suites/3dvar/workflow.py | 154 +++++++++++++------------- src/swell/utilities/cylc_workflow.py | 156 ++++++--------------------- 2 files changed, 107 insertions(+), 203 deletions(-) diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 489e3e7b8..3fceb183d 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -7,113 +7,113 @@ # -------------------------------------------------------------------------------------------------- +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow # -------------------------------------------------------------------------------------------------- -r1_template = """ -# Triggers for non cycle time dependent tasks -# ------------------------------------------- -# Clone JEDI source code -CloneJedi - -# Build JEDI source code by linking -CloneJedi => BuildJediByLinking? - -# If not able to link to build create the build -BuildJediByLinking:fail? => BuildJedi -""" +template_str = ''' +# -------------------------------------------------------------------------------------------------- -r1_model = """ -# Stage JEDI static files -CloneJedi => StageJedi-{model_component} -""" +# Cylc suite for executing JEDI-based non-cycling variational data assimilation -cycle_template = """ -# Task triggers for: {model_component} -# ------------------ -# Get background -GetBackground-{model_component} +# -------------------------------------------------------------------------------------------------- -# Get observations -GetObservations-{model_component} +[scheduler] + UTC mode = True + allow implicit tasks = False -# GenerateBClimatology, for ocean it is cycle dependent -GenerateBClimatologyByLinking-{model_component}:fail? => -GenerateBClimatology-{model_component} +# -------------------------------------------------------------------------------------------------- -GetBackground-{model_component} => GenerateBClimatology-{model_component} +[scheduling] -# Perform staging that is cycle dependent -StageJediCycle-{model_component} + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} -# Run Jedi variational executable -BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} -StageJedi-{model_component}[^] => RunJediVariationalExecutable-{model_component} -StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} -GetBackground-{model_component} => RunJediVariationalExecutable-{model_component} + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi -GenerateBClimatologyByLinking-{model_component}? | -GenerateBClimatology-{model_component} => -RunJediVariationalExecutable-{model_component} + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? -GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi -# EvaObservations -RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} + {% for model_component in model_components %} + # Stage JEDI static files + CloneJedi => StageJedi-{{model_component}} + {% endfor %} + """ -# EvaJediLog -RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + # Task triggers for: {{model_component}} + # ------------------ + # Get background + GetBackground-{{model_component}} -# EvaIncrement -RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} + # Get observations + GetObservations-{{model_component}} -# Save observations -RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} + # GenerateBClimatology, for ocean it is cycle dependent + GenerateBClimatologyByLinking-{{model_component}} :fail? => GenerateBClimatology-{{model_component}} + GetBackground-{{model_component}} => GenerateBClimatology-{{model_component}} -# Clean up large files -EvaObservations-{model_component} & SaveObsDiags-{model_component} => -CleanCycle-{model_component} -""" + # Perform staging that is cycle dependent + StageJediCycle-{{model_component}} -# -------------------------------------------------------------------------------------------------- + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} + StageJedi-{{model_component}}[^] => RunJediVariationalExecutable-{{model_component}} + StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GenerateBClimatologyByLinking-{{model_component}}? | GenerateBClimatology-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + # EvaObservations + RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} -class Workflow_3dvar(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for executing JEDI-based non-cycling variational data assimilation - """) + # EvaJediLog + RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} - return description + # EvaIncrement + RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - # -------------------------------------------------------------------------------------------------- + # Save observations + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' + # Clean up large files + EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} - # Define the string for the R1 (first non-cycling) section - r1 = r1_template + {% endif %} + {% endfor %} + """ + {% endfor %} - for model_component in self.experiment_dict['model_components']: - r1 += r1_model.format(model_component=model_component) +# -------------------------------------------------------------------------------------------------- +''' - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) +# -------------------------------------------------------------------------------------------------- - # Format the string for each cycle - for model_component in self.experiment_dict['model_components']: - if 'cycle_times' in self.experiment_dict['models'][model_component]: - for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = cycle_template.format(model_component=model_component) - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) +class Workflow_3dvar(CylcWorkflow): - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) + def define_initial_workflow(self): + workflow_str = self.default_header() - return graph_section + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 45047143b..4728682ba 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -8,6 +8,7 @@ # -------------------------------------------------------------------------------------------------- from typing import Union, Optional, Tuple +from collections.abc import abstractmethod import os import yaml @@ -15,10 +16,23 @@ from swell.tasks.task_runtimes import TaskRuntimes from swell.utilities.dictionary import update_dict from swell.utilities.logger import get_logger +from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- +header_str = ''' +#jinja2 +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- +''' + class CylcWorkflow(): ''' @@ -35,7 +49,12 @@ def __init__(self, experiment_dict, slurm_external) -> None: self.logger = get_logger(self.__class__.__name__) - self.setup_workflow() + self.initial_workflow_str = self.define_initial_workflow() + + # -------------------------------------------------------------------------------------------------- + + def default_header(self) -> str: + return header_str # -------------------------------------------------------------------------------------------------- @@ -77,122 +96,9 @@ def reset_indentation(self, string: str) -> str: # -------------------------------------------------------------------------------------------------- - def setup_workflow(self) -> None: - # Initial setup for the workflow, includes everything but the runtime section - - self.header = self.define_header() - self.description = self.define_description() - self.scheduler = self.define_scheduler() - self.scheduling = self.define_scheduling() - - self.tasks = self.parse_graph_for_tasks() - - # -------------------------------------------------------------------------------------------------- - - def define_header(self) -> str: - # Define a 'header', usually this includes any copyright information - - header = '#!jinja2\n' - header += self.comment_block(string=""" - # (C) Copyright 2021- United States Government as represented by the Administrator of the - # National Aeronautics and Space Administration. All Rights Reserved. - # - # This software is licensed under the terms of the Apache Licence Version 2.0 - # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.""") - - return header - - # -------------------------------------------------------------------------------------------------- - - def define_description(self) -> str: - description = self.comment_block( - """# Cylc workflow auto-generated for suite {suite_to_run} by Swell.""" - .format(**self.experiment_dict)) - return description - - # -------------------------------------------------------------------------------------------------- - - def comment_block(self, string, level: int = 0, section_break: bool = True): - # Format a comment block with proper indentation - - out_string = '' - - string = indent_lines(string, level, reset=True) - - start = False - for line in string.split('\n'): - if len(line.strip()) > 0: - start = True - - if start: - out_string += f'{line}\n' - - if section_break: - out_string += f'\n# {"-" * 98}\n\n' - - return out_string - - # -------------------------------------------------------------------------------------------------- - - def define_scheduler(self) -> str: - # Define a scheduler section that includes email infrastructure - - scheduler_str = 'UTC mode = True\nallow implicit tasks = False\n' - - settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) - if os.path.exists(settings_file): - with open(settings_file, 'r') as f: - settings_dict = yaml.safe_load(f) - if 'email_address' in settings_dict.keys(): - email_address = settings_dict['email_address'] - - message_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" - message_str += '[[events]]\n' - message_str += indent_lines('mail events = startup, shutdown\n', 1) - message_str += '[[mail]]\n' - message_str += indent_lines(f'to = {email_address}\n', 1) - message_str += '{% endif %}\n' - - scheduler_str += message_str - - scheduler = self.create_new_section('scheduler', scheduler_str) - - return scheduler.get_section_str() - - # -------------------------------------------------------------------------------------------------- - - def define_scheduling(self) -> str: - # Get the string for the entire scheduling section - - scheduling = self.define_scheduling_section() - graph = self.define_graph_section() - - scheduling.add_subsection(graph) - - return scheduling.get_section_str() - - # -------------------------------------------------------------------------------------------------- - - def define_scheduling_section(self) -> CylcSection: - # Define a CylcSection object that comprises the initial contents of the scheduling section - # This will be appended to the graph section to create the overall section - scheduling_dict = {'initial cycle point': self.experiment_dict['start_cycle_point'], - 'final cycle point': self.experiment_dict['final_cycle_point']} - - if 'runahead_limit' in self.experiment_dict: - scheduling_dict = update_dict( - scheduling_dict, - {'runahead limit': self.experiment_dict['runahead_limit']}) - - scheduling_section = self.create_new_section('scheduling', scheduling_dict) - - return scheduling_section - - # -------------------------------------------------------------------------------------------------- - - def define_graph_section(self) -> CylcSection: - # Define a CylcSection object that will be used to build the scheduling section - return self.create_new_section('graph') + @abstractmethod + def define_initial_workflow(self) -> str: + return '' # -------------------------------------------------------------------------------------------------- @@ -206,7 +112,7 @@ def parse_graph_for_tasks(self) -> CylcSection: in_graph = False in_cycle = False - for line in self.scheduling.split('\n'): + for line in self.initial_workflow_str.split('\n'): comment = False sub_strings = line.split(' ') @@ -320,15 +226,13 @@ def define_runtime(self) -> str: def get_workflow_str(self) -> str: # Get the whole string to go into the flow.cylc file - workflow_str = '' - - workflow_str += self.header - workflow_str += self.description - workflow_str += self.scheduler - workflow_str += self.scheduling + workflow_str = self.initial_workflow_str + workflow_str += self.define_runtime() - runtime = self.define_runtime() - workflow_str += runtime + workflow_str = template_string_jinja2(logger=self.logger, + templated_string=workflow_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=False) return workflow_str From 26a51a12e8017d91da57ebfc5a052ff569f855cf Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 9 Oct 2025 15:11:11 -0400 Subject: [PATCH 102/299] fixes --- src/swell/suites/3dvar/workflow.py | 10 ++++------ src/swell/utilities/cylc_workflow.py | 8 ++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 3fceb183d..a698d8939 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -50,10 +50,10 @@ {% endfor %} """ - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ {% for model_component in model_components %} - {% if cycle_time[model_component] %} + {% if models[model_component]['cycle_times'] %} + {% for cycle_time in models[model_component]['cycle_times'] %} + {{cycle_time}} = """ # Task triggers for: {{model_component}} # ------------------ # Get background @@ -92,9 +92,8 @@ # Clean up large files EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} - - {% endif %} {% endfor %} + {% endif %} """ {% endfor %} @@ -108,7 +107,6 @@ class Workflow_3dvar(CylcWorkflow): def define_initial_workflow(self): workflow_str = self.default_header() - workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 4728682ba..719cbd2ee 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from typing import Union, Optional, Tuple -from collections.abc import abstractmethod +from abc import abstractmethod import os import yaml @@ -50,6 +50,7 @@ def __init__(self, experiment_dict, slurm_external) -> None: self.logger = get_logger(self.__class__.__name__) self.initial_workflow_str = self.define_initial_workflow() + self.tasks = self.parse_graph_for_tasks() # -------------------------------------------------------------------------------------------------- @@ -186,7 +187,6 @@ def create_new_section(self, name: Optional[str] = None, content: Union[str, dic def define_runtime(self) -> str: # Handle adding runtime sections for all tasks - runtime_section = self.create_new_section('runtime', '\n# Task defaults\n# -------------\n') # Grab any overrides for certain tasks @@ -227,13 +227,13 @@ def get_workflow_str(self) -> str: # Get the whole string to go into the flow.cylc file workflow_str = self.initial_workflow_str - workflow_str += self.define_runtime() - workflow_str = template_string_jinja2(logger=self.logger, templated_string=workflow_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=False) + workflow_str += '\n\n' + self.define_runtime() + return workflow_str # -------------------------------------------------------------------------------------------------- From 768572002000a28188d5151a54e87b0d466fb5fa Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 9 Oct 2025 15:22:29 -0400 Subject: [PATCH 103/299] runtime --- src/swell/utilities/cylc_runtime.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 19859cc0a..a2cbba57e 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -43,6 +43,9 @@ class Task: is_cycling: bool = False is_model: bool = False + additional_sections: list = [] + mail_events: list = ['failed', 'submit-failed'] + # -------------------------------------------------------------------------------------------------- def __post_init__(self): @@ -207,12 +210,16 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): runtime_section.add_subsection(directive_section) + # Append additional sections to runtime + for section in self.additional_sections: + runtime_section.add_subsection(section) + # Check slurm messaging parameters events = [] if 'task_email_parameters' in experiment_dict.keys(): if experiment_dict['task_email_parameters'] == 'auto': # Set message status to fail or event fail - events = ['failed', 'submit-failed'] + events = self.message_events elif self.scheduling_name in experiment_dict['task_email_parameters'].keys(): events = experiment_dict['task_email_parameters'][self.scheduling_name] elif self.base_name in experiment_dict['task_email_parameters'].keys(): From 2b314968c0ab9d0c63a0a1d93eb3e312671181ca Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 9 Oct 2025 15:27:34 -0400 Subject: [PATCH 104/299] fixes --- src/swell/utilities/cylc_runtime.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index a2cbba57e..6b60ddf2f 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -16,6 +16,7 @@ from swell.utilities.cylc_formatting import CylcSection, indent_lines from swell.utilities.suite_utils import get_model_components from swell.utilities.dictionary import update_dict +from swell.utilities.dataclass_utils import mutable_field # -------------------------------------------------------------------------------------------------- @@ -43,8 +44,8 @@ class Task: is_cycling: bool = False is_model: bool = False - additional_sections: list = [] - mail_events: list = ['failed', 'submit-failed'] + additional_sections: list = mutable_field([]) + mail_events: list = mutable_field(['failed', 'submit-failed']) # -------------------------------------------------------------------------------------------------- @@ -219,7 +220,7 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): if 'task_email_parameters' in experiment_dict.keys(): if experiment_dict['task_email_parameters'] == 'auto': # Set message status to fail or event fail - events = self.message_events + events = self.mail_events elif self.scheduling_name in experiment_dict['task_email_parameters'].keys(): events = experiment_dict['task_email_parameters'][self.scheduling_name] elif self.base_name in experiment_dict['task_email_parameters'].keys(): From fa6827c65c6c87d9db9a06c93ec86ef5e8f0073b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 9 Oct 2025 15:37:40 -0400 Subject: [PATCH 105/299] add flow.cylc --- src/swell/suites/3dfgat_atmos/flow.cylc | 188 ++++++++++++ src/swell/suites/3dfgat_cycle/flow.cylc | 269 ++++++++++++++++++ src/swell/suites/3dvar/flow.cylc | 178 ++++++++++++ src/swell/suites/3dvar_atmos/flow.cylc | 187 ++++++++++++ src/swell/suites/3dvar_cycle/flow.cylc | 269 ++++++++++++++++++ src/swell/suites/build_geos/flow.cylc | 56 ++++ src/swell/suites/build_jedi/flow.cylc | 56 ++++ .../suites/compare_variational/flow.cylc | 75 +++++ src/swell/suites/convert_ncdiags/flow.cylc | 102 +++++++ src/swell/suites/forecast_geos/flow.cylc | 116 ++++++++ src/swell/suites/geosadas/flow.cylc | 121 ++++++++ src/swell/suites/hofx/flow.cylc | 164 +++++++++++ src/swell/suites/localensembleda/flow.cylc | 233 +++++++++++++++ src/swell/suites/ufo_testing/flow.cylc | 147 ++++++++++ 14 files changed, 2161 insertions(+) create mode 100644 src/swell/suites/3dfgat_atmos/flow.cylc create mode 100644 src/swell/suites/3dfgat_cycle/flow.cylc create mode 100644 src/swell/suites/3dvar/flow.cylc create mode 100644 src/swell/suites/3dvar_atmos/flow.cylc create mode 100644 src/swell/suites/3dvar_cycle/flow.cylc create mode 100644 src/swell/suites/build_geos/flow.cylc create mode 100644 src/swell/suites/build_jedi/flow.cylc create mode 100644 src/swell/suites/compare_variational/flow.cylc create mode 100644 src/swell/suites/convert_ncdiags/flow.cylc create mode 100644 src/swell/suites/forecast_geos/flow.cylc create mode 100644 src/swell/suites/geosadas/flow.cylc create mode 100644 src/swell/suites/hofx/flow.cylc create mode 100644 src/swell/suites/localensembleda/flow.cylc create mode 100644 src/swell/suites/ufo_testing/flow.cylc diff --git a/src/swell/suites/3dfgat_atmos/flow.cylc b/src/swell/suites/3dfgat_atmos/flow.cylc new file mode 100644 index 000000000..984445c17 --- /dev/null +++ b/src/swell/suites/3dfgat_atmos/flow.cylc @@ -0,0 +1,188 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing JEDI-based non-cycling variational data assimilation + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + {% for model_component in model_components %} + # Clone geos ana for generating observing system records + CloneGeosMksi-{{model_component}} + {% endfor %} + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + + # Task triggers for: {{model_component}} + # ------------------ + # Generate satellite channel records + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} + + # Get background, provide a way to get background directly from GEOS experiment + GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} + + # Get observations + {% if cycling_varbc %} + # Cycling VarBC is active, biases from the previous cycle will be used + + RunJediVariationalExecutable-{{model_component}}[-PT6H] => GetObservations-{{model_component}} + {% else %} + + # Cycling VarBC is inactive, static bias files will be used + GetObservations-{{model_component}} + {% endif %} + + # Perform staging that is cycle dependent + StageJediCycle-{{model_component}} + + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} + CloneJedi[^] => StageJediCycle-{{model_component}} + StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GenerateObservingSystemRecords-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + + # EvaObservations + RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} + + # EvaJediLog + RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} + + # EvaIncrement + RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} + + # Save observations + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + + # Clean up large files + EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} + + {% endif %} + {% endfor %} + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + {% for model_component in model_components %} + + [[CloneGeosMksi-{{model_component}}]] + script = "swell task CloneGeosMksi $config -m {{model_component}}" + + [[GetObsNotInR2d2-{{model_component}}]] + script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" + + [[GenerateObservingSystemRecords-{{model_component}}]] + script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" + + [[StageJediCycle-{{model_component}}]] + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" + + [[GetBackground-{{model_component}} ]] + script = "swell task GetBackground $config -d $datetime -m {{model_component}}" + + [[GetBackgroundGeosExperiment-{{model_component}} ]] + script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[RunJediVariationalExecutable-{{model_component}}]] + script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" + + [[RunJediVariationalExecutable-{{model_component}}]] + script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediVariationalExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[EvaJediLog-{{model_component}}]] + script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" + + [[EvaIncrement-{{model_component}}]] + script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" + + [[EvaObservations-{{model_component}}]] + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[SaveObsDiags-{{model_component}}]] + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" + + [[CleanCycle-{{model_component}}]] + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" + {% endfor %} + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/flow.cylc b/src/swell/suites/3dfgat_cycle/flow.cylc new file mode 100644 index 000000000..9dbdf1b89 --- /dev/null +++ b/src/swell/suites/3dfgat_cycle/flow.cylc @@ -0,0 +1,269 @@ +# (C) Copyright 2023 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing Geos forecast + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone Geos source code + CloneGeos + + # Clone JEDI source code + CloneJedi + + # Build Geos source code by linking + CloneGeos => BuildGeosByLinking? + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildGeosByLinking:fail? => BuildGeos + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + # Need first set of restarts to run model + GetGeosRestart => PrepGeosRunDir + + # Model cannot run without code + BuildGeosByLinking? | BuildGeos => RunGeosExecutable + + {% for model_component in model_components %} + + # JEDI cannot run without code + BuildJediByLinking? | BuildJedi => RunJediFgatExecutable-{{model_component}} + + # Stage JEDI static files + CloneJedi => StageJedi-{{model_component}} => RunJediFgatExecutable-{{model_component}} + + {% endfor %} + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + + # Model preperation + # Run the forecast through two windows (need to output restarts at the end of the + # first window and backgrounds for the second window) + MoveDaRestart-{{model_component}}[-{{models[model_component]["window_length"]}}] => PrepGeosRunDir + PrepGeosRunDir => RunGeosExecutable + + # Run the analysis + # RunGeosExecutable => StageJediCycle-{{model_component}} + RunGeosExecutable => LinkGeosOutput-{{model_component}} + LinkGeosOutput-{{model_component}} => GenerateBClimatology-{{model_component}} + + # Data assimilation preperation + GetObservations-{{model_component}} + GenerateBClimatologyByLinking-{{model_component}} :fail? => GenerateBClimatology-{{model_component}} + + LinkGeosOutput-{{model_component}} => RunJediFgatExecutable-{{model_component}} + StageJediCycle-{{model_component}} => RunJediFgatExecutable-{{model_component}} + GenerateBClimatologyByLinking-{{model_component}}? | GenerateBClimatology-{{model_component}} => RunJediFgatExecutable-{{model_component}} + GetObservations-{{model_component}} => RunJediFgatExecutable-{{model_component}} + + # Run analysis diagnostics + RunJediFgatExecutable-{{model_component}} => EvaObservations-{{model_component}} + RunJediFgatExecutable-{{model_component}} => EvaJediLog-{{model_component}} + EvaIncrement-{{model_component}} => PrepareAnalysis-{{model_component}} + + # Prepare analysis for next forecast + RunJediFgatExecutable-{{model_component}} => EvaIncrement-{{model_component}} + {% if 'cice6' in models[model_component]["marine_models"] %} + PrepareAnalysis-{{model_component}} => RunJediConvertStateSoca2ciceExecutable-{{model_component}} + RunJediConvertStateSoca2ciceExecutable-{{model_component}} => SaveRestart-{{model_component}} + RunJediConvertStateSoca2ciceExecutable-{{model_component}} => CleanCycle-{{model_component}} + {% else %} + PrepareAnalysis-{{model_component}} => SaveRestart-{{model_component}} + {% endif %} + + # Move restart to next cycle + SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} + + # Save analysis output + # RunJediFgatExecutable-{{model_component}} => SaveAnalysis-{{model_component}} + RunJediFgatExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + + # Save model output + # MoveBackground-{{model_component}} => StoreBackground-{{model_component}} + + # Remove Run Directory + # MoveDaRestart-{{model_component}} & MoveBackground-{{model_component}} => RemoveForecastDir + MoveDaRestart-{{model_component}} => RemoveForecastDir + + # Clean up large files + EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} + {% endfor %} + """ + {% endfor %} +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneGeos]] + script = "swell task CloneGeos $config" + + [[BuildGeosByLinking]] + script = "swell task BuildGeosByLinking $config" + + [[BuildGeos]] + script = "swell task BuildGeos $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildGeos"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[RunGeosExecutable]] + script = "swell task RunGeosExecutable $config -d $datetime" + platform = {{platform}} + execution time limit = {{scheduling["RunGeosExecutable"]["execution_time_limit"]}} + execution retry delays = 2*PT1M + [[[directives]]] + {%- for key, value in scheduling["RunGeosExecutable"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[PrepGeosRunDir]] + script = "swell task PrepGeosRunDir $config -d $datetime" + + [[RemoveForecastDir]] + script = "swell task RemoveForecastDir $config -d $datetime" + + [[GetGeosRestart]] + script = "swell task GetGeosRestart $config -d $datetime" + + {% for model_component in model_components %} + + [[LinkGeosOutput-{{model_component}}]] + script = "swell task LinkGeosOutput $config -d $datetime -m {{model_component}}" + + [[MoveDaRestart-{{model_component}}]] + script = "swell task MoveDaRestart $config -d $datetime -m {{model_component}}" + + [[SaveRestart-{{model_component}}]] + script = "swell task SaveRestart $config -d $datetime -m {{model_component}}" + + [[StageJedi-{{model_component}}]] + script = "swell task StageJedi $config -m {{model_component}}" + + [[StageJediCycle-{{model_component}}]] + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[GenerateBClimatology-{{model_component}}]] + script = "swell task GenerateBClimatology $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["GenerateBClimatology"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["GenerateBClimatology"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[GenerateBClimatologyByLinking-{{model_component}}]] + script = "swell task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" + + {% if 'cice6' in models["geos_marine"]["marine_models"] %} + + [[RunJediConvertStateSoca2ciceExecutable-{{model_component}}]] + script = "swell task RunJediConvertStateSoca2ciceExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediConvertStateSoca2ciceExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediConvertStateSoca2ciceExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + {% endif %} + + [[RunJediFgatExecutable-{{model_component}}]] + script = "swell task RunJediFgatExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediFgatExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediFgatExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[EvaJediLog-{{model_component}}]] + script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" + + [[EvaIncrement-{{model_component}}]] + script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" + + [[EvaObservations-{{model_component}}]] + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[SaveRestart-{{model_component}}]] + script = "swell task SaveRestart $config -d $datetime -m {{model_component}}" + + [[SaveObsDiags-{{model_component}}]] + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" + + [[PrepareAnalysis-{{model_component}}]] + script = "swell task PrepareAnalysis $config -d $datetime -m {{model_component}}" + + [[CleanCycle-{{model_component}}]] + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" + {% endfor %} + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc new file mode 100644 index 000000000..3e598bfaa --- /dev/null +++ b/src/swell/suites/3dvar/flow.cylc @@ -0,0 +1,178 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing JEDI-based non-cycling variational data assimilation + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + {% for model_component in model_components %} + # Stage JEDI static files + CloneJedi => StageJedi-{{model_component}} + {% endfor %} + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + # Task triggers for: {{model_component}} + # ------------------ + # Get background + GetBackground-{{model_component}} + + # Get observations + GetObservations-{{model_component}} + + # GenerateBClimatology, for ocean it is cycle dependent + GenerateBClimatologyByLinking-{{model_component}} :fail? => GenerateBClimatology-{{model_component}} + GetBackground-{{model_component}} => GenerateBClimatology-{{model_component}} + + # Perform staging that is cycle dependent + StageJediCycle-{{model_component}} + + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} + StageJedi-{{model_component}}[^] => RunJediVariationalExecutable-{{model_component}} + StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GenerateBClimatologyByLinking-{{model_component}}? | GenerateBClimatology-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + + # EvaObservations + RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} + + # EvaJediLog + RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} + + # EvaIncrement + RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} + + # Save observations + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + + # Clean up large files + EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} + + {% endif %} + {% endfor %} + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + {% for model_component in model_components %} + [[StageJedi-{{model_component}}]] + script = "swell task StageJedi $config -m {{model_component}}" + + [[StageJediCycle-{{model_component}}]] + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" + + [[ GetBackground-{{model_component}} ]] + script = "swell task GetBackground $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[GenerateBClimatologyByLinking-{{model_component}}]] + script = "swell task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" + + [[GenerateBClimatology-{{model_component}}]] + script = "swell task GenerateBClimatology $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["GenerateBClimatology"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["GenerateBClimatology"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[RunJediVariationalExecutable-{{model_component}}]] + script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediVariationalExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[EvaJediLog-{{model_component}}]] + script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" + + [[EvaIncrement-{{model_component}}]] + script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" + + [[EvaObservations-{{model_component}}]] + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[SaveObsDiags-{{model_component}}]] + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" + + [[CleanCycle-{{model_component}}]] + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" + {% endfor %} + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc new file mode 100644 index 000000000..19ffe8892 --- /dev/null +++ b/src/swell/suites/3dvar_atmos/flow.cylc @@ -0,0 +1,187 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing JEDI-based non-cycling variational data assimilation + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + {% for model_component in model_components %} + # Clone geos ana for generating observing system records + CloneGeosMksi-{{model_component}} + {% endfor %} + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + + # Task triggers for: {{model_component}} + # ------------------ + # Generate satellite channel records + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} + + # Get background, provide a way to get background directly from GEOS experiment + GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} + + # Get observations + {% if cycling_varbc %} + # Cycling VarBC is active, biases from the previous cycle will be used + + RunJediVariationalExecutable-{{model_component}}[-PT6H] => GetObservations-{{model_component}} + {% else %} + + # Cycling VarBC is inactive, static bias files will be used + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + {% endif %} + + # Perform staging that is cycle dependent + StageJediCycle-{{model_component}} + + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} + CloneJedi[^] => StageJediCycle-{{model_component}} + StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GenerateObservingSystemRecords-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + + # EvaObservations + RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} + + # EvaJediLog + RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} + + # EvaIncrement + RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} + + # Save observations + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + + # Clean up large files + EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} + + {% endif %} + {% endfor %} + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + {% for model_component in model_components %} + + [[CloneGeosMksi-{{model_component}}]] + script = "swell task CloneGeosMksi $config -m {{model_component}}" + + [[GenerateObservingSystemRecords-{{model_component}}]] + script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" + + [[GetObsNotInR2d2-{{model_component}}]] + script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" + + [[StageJediCycle-{{model_component}}]] + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" + + [[GetBackground-{{model_component}} ]] + script = "swell task GetBackground $config -d $datetime -m {{model_component}}" + + [[GetBackgroundGeosExperiment-{{model_component}} ]] + script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[RunJediVariationalExecutable-{{model_component}}]] + script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" + + [[RunJediVariationalExecutable-{{model_component}}]] + script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediVariationalExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[EvaJediLog-{{model_component}}]] + script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" + + [[EvaIncrement-{{model_component}}]] + script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" + + [[EvaObservations-{{model_component}}]] + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[SaveObsDiags-{{model_component}}]] + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" + + [[CleanCycle-{{model_component}}]] + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" + {% endfor %} + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/flow.cylc b/src/swell/suites/3dvar_cycle/flow.cylc new file mode 100644 index 000000000..8630756b0 --- /dev/null +++ b/src/swell/suites/3dvar_cycle/flow.cylc @@ -0,0 +1,269 @@ +# (C) Copyright 2023 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing Geos forecast + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone Geos source code + CloneGeos + + # Clone JEDI source code + CloneJedi + + # Build Geos source code by linking + CloneGeos => BuildGeosByLinking? + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildGeosByLinking:fail? => BuildGeos + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + # Need first set of restarts to run model + GetGeosRestart => PrepGeosRunDir + + # Model cannot run without code + BuildGeosByLinking? | BuildGeos => RunGeosExecutable + + {% for model_component in model_components %} + + # JEDI cannot run without code + BuildJediByLinking? | BuildJedi => RunJediVariationalExecutable-{{model_component}} + + # Stage JEDI static files + CloneJedi => StageJedi-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + + {% endfor %} + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + + # Model things + # Run the forecast through two windows (need to output restarts at the end of the + # first window and backgrounds for the second window) + MoveDaRestart-{{model_component}}[-{{models[model_component]["window_length"]}}] => PrepGeosRunDir + PrepGeosRunDir => RunGeosExecutable + + # Run the analysis + # RunGeosExecutable => StageJediCycle-{{model_component}} + RunGeosExecutable => LinkGeosOutput-{{model_component}} + LinkGeosOutput-{{model_component}} => GenerateBClimatology-{{model_component}} + + # Data assimilation things + GetObservations-{{model_component}} + GenerateBClimatologyByLinking-{{model_component}} :fail? => GenerateBClimatology-{{model_component}} + + LinkGeosOutput-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GenerateBClimatologyByLinking-{{model_component}}? | GenerateBClimatology-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + + # Run analysis diagnostics + RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} + RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} + RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} + + # Prepare analysis for next forecast + EvaIncrement-{{model_component}} => PrepareAnalysis-{{model_component}} + {% if 'cice6' in models[model_component]["marine_models"] %} + PrepareAnalysis-{{model_component}} => RunJediConvertStateSoca2ciceExecutable-{{model_component}} + RunJediConvertStateSoca2ciceExecutable-{{model_component}} => SaveRestart-{{model_component}} + RunJediConvertStateSoca2ciceExecutable-{{model_component}} => CleanCycle-{{model_component}} + {% else %} + PrepareAnalysis-{{model_component}} => SaveRestart-{{model_component}} + {% endif %} + + # Move restart to next cycle + SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} + + # Save analysis output + # RunJediVariationalExecutable-{{model_component}} => SaveAnalysis-{{model_component}} + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + + # Save model output + # MoveBackground-{{model_component}} => StoreBackground-{{model_component}} + + # Remove Run Directory + # MoveDaRestart-{{model_component}} & MoveBackground-{{model_component}} => RemoveForecastDir + MoveDaRestart-{{model_component}} => RemoveForecastDir + + # Clean up large files + # EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & SaveObsDiags-{{model_component}} & RemoveForecastDir => + EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} + {% endfor %} + """ + {% endfor %} +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneGeos]] + script = "swell task CloneGeos $config" + + [[BuildGeosByLinking]] + script = "swell task BuildGeosByLinking $config" + + [[BuildGeos]] + script = "swell task BuildGeos $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildGeos"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[RunGeosExecutable]] + script = "swell task RunGeosExecutable $config -d $datetime" + platform = {{platform}} + execution time limit = {{scheduling["RunGeosExecutable"]["execution_time_limit"]}} + execution retry delays = 2*PT1M + [[[directives]]] + {%- for key, value in scheduling["RunGeosExecutable"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[PrepGeosRunDir]] + script = "swell task PrepGeosRunDir $config -d $datetime" + + [[RemoveForecastDir]] + script = "swell task RemoveForecastDir $config -d $datetime" + + [[GetGeosRestart]] + script = "swell task GetGeosRestart $config -d $datetime" + + {% for model_component in model_components %} + + [[LinkGeosOutput-{{model_component}}]] + script = "swell task LinkGeosOutput $config -d $datetime -m {{model_component}}" + + [[MoveDaRestart-{{model_component}}]] + script = "swell task MoveDaRestart $config -d $datetime -m {{model_component}}" + + [[SaveRestart-{{model_component}}]] + script = "swell task SaveRestart $config -d $datetime -m {{model_component}}" + + [[StageJedi-{{model_component}}]] + script = "swell task StageJedi $config -m {{model_component}}" + + [[StageJediCycle-{{model_component}}]] + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[GenerateBClimatology-{{model_component}}]] + script = "swell task GenerateBClimatology $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["GenerateBClimatology"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["GenerateBClimatology"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[GenerateBClimatologyByLinking-{{model_component}}]] + script = "swell task GenerateBClimatologyByLinking $config -d $datetime -m {{model_component}}" + + {% if 'cice6' in models["geos_marine"]["marine_models"] %} + + [[RunJediConvertStateSoca2ciceExecutable-{{model_component}}]] + script = "swell task RunJediConvertStateSoca2ciceExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediConvertStateSoca2ciceExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediConvertStateSoca2ciceExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + {% endif %} + + [[RunJediVariationalExecutable-{{model_component}}]] + script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediVariationalExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[EvaJediLog-{{model_component}}]] + script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" + + [[EvaIncrement-{{model_component}}]] + script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" + + [[EvaObservations-{{model_component}}]] + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[SaveRestart-{{model_component}}]] + script = "swell task SaveRestart $config -d $datetime -m {{model_component}}" + + [[SaveObsDiags-{{model_component}}]] + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" + + [[PrepareAnalysis-{{model_component}}]] + script = "swell task PrepareAnalysis $config -d $datetime -m {{model_component}}" + + [[CleanCycle-{{model_component}}]] + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" + {% endfor %} + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/flow.cylc b/src/swell/suites/build_geos/flow.cylc new file mode 100644 index 000000000..dce2ce3d2 --- /dev/null +++ b/src/swell/suites/build_geos/flow.cylc @@ -0,0 +1,56 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for building the GEOS model + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + [[graph]] + R1 = """ + CloneGeos => BuildGeosByLinking? + + BuildGeosByLinking:fail? => BuildGeos + """ + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneGeos]] + script = "swell task CloneGeos $config" + + [[BuildGeosByLinking]] + script = "swell task BuildGeosByLinking $config" + + [[BuildGeos]] + script = "swell task BuildGeos $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildGeos"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/flow.cylc b/src/swell/suites/build_jedi/flow.cylc new file mode 100644 index 000000000..b7e347dee --- /dev/null +++ b/src/swell/suites/build_jedi/flow.cylc @@ -0,0 +1,56 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for building the JEDI code + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + [[graph]] + R1 = """ + CloneJedi => BuildJediByLinking? + + BuildJediByLinking:fail? => BuildJedi + """ + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_variational/flow.cylc b/src/swell/suites/compare_variational/flow.cylc new file mode 100644 index 000000000..f90bf4716 --- /dev/null +++ b/src/swell/suites/compare_variational/flow.cylc @@ -0,0 +1,75 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for running comparison tests on completed experiments + + + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + EvaComparisonIncrement-{{model_component}} + EvaComparisonJediLog-{{model_component}} + {% for path in comparison_experiment_paths %} + JediOopsLogParser-{{model_component}}-{{ loop.index0 }} => JediLogComparison-{{model_component}} + {% endfor %} + {% endif %} + {% endfor %} + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + + [[root]] + pre-script = """ + source $CYLC_SUITE_DEF_PATH/modules + """ + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + {% for model_component in model_components %} + [[EvaComparisonIncrement-{{model_component}}]] + script = "swell task EvaComparisonIncrement $config -d $datetime -m {{model_component}}" + + [[EvaComparisonJediLog-{{model_component}}]] + script = "swell task EvaComparisonJediLog $config -d $datetime -m {{model_component}}" + + {% for path in comparison_experiment_paths %} + [[JediOopsLogParser-{{model_component}}-{{ loop.index0 }}]] + script = "swell task JediOopsLogParser {{path}} -d $datetime -m {{model_component}}" + {% endfor %} + + [[JediLogComparison-{{model_component}}]] + script = "swell task JediLogComparison $config -m {{model_component}}" + {% endfor %} + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/flow.cylc b/src/swell/suites/convert_ncdiags/flow.cylc new file mode 100644 index 000000000..c43986dcd --- /dev/null +++ b/src/swell/suites/convert_ncdiags/flow.cylc @@ -0,0 +1,102 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing geos_atmosphere ObsFilters tests + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + + # Convert bias correction to ioda + GetGsiBc + GetGsiBc => GsiBcToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda + + # Convert ncdiags to ioda + GetGsiNcdiag + GetGsiNcdiag => GsiNcdiagToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda + + # Clean up + GsiNcdiagToIoda => CleanCycle + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[ GetGsiBc ]] + script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" + + [[ GsiBcToIoda ]] + script = "swell task GsiBcToIoda $config -d $datetime -m geos_atmosphere" + + [[ GetGsiNcdiag ]] + script = "swell task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + + [[ GsiNcdiagToIoda ]] + script = "swell task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + + [[CleanCycle]] + script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/flow.cylc b/src/swell/suites/forecast_geos/flow.cylc new file mode 100644 index 000000000..cc8d74dea --- /dev/null +++ b/src/swell/suites/forecast_geos/flow.cylc @@ -0,0 +1,116 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for Geos forecast without DA + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone Geos source code + CloneGeos + + # Build Geos source code by linking + CloneGeos => BuildGeosByLinking? + + # If not able to link to build create the build + BuildGeosByLinking:fail? => BuildGeos + + # Need first set of restarts to run model + GetGeosRestart => PrepGeosRunDir + + # Get first set of restarts + BuildGeosByLinking? | BuildGeos => RunGeosExecutable + """ + + {% for cycle_time in cycle_times %} + {{cycle_time}} = """ + + # Run Geos Executable + PrepGeosRunDir => RunGeosExecutable + MoveForecastRestart[-PT6H] => PrepGeosRunDir + + # Move restart to next cycle + RunGeosExecutable => MoveForecastRestart + + # Save restarts if requested + # MoveForecastRestart[-PT6H] => SaveRestart + + # Remove Run Directory + MoveForecastRestart => RemoveForecastDir + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneGeos]] + script = "swell task CloneGeos $config" + + [[BuildGeosByLinking]] + script = "swell task BuildGeosByLinking $config" + + [[BuildGeos]] + script = "swell task BuildGeos $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildGeos"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[PrepGeosRunDir]] + script = "swell task PrepGeosRunDir $config -d $datetime" + + [[RemoveForecastDir]] + script = "swell task RemoveForecastDir $config -d $datetime" + + [[GetGeosRestart]] + script = "swell task GetGeosRestart $config -d $datetime" + + [[MoveForecastRestart]] + script = "swell task MoveForecastRestart $config -d $datetime" + + [[SaveRestart]] + script = "swell task SaveRestart $config -d $datetime" + + [[RunGeosExecutable]] + script = "swell task RunGeosExecutable $config -d $datetime" + platform = {{platform}} + execution time limit = {{scheduling["RunGeosExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunGeosExecutable"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/flow.cylc b/src/swell/suites/geosadas/flow.cylc new file mode 100644 index 000000000..f96b3d58e --- /dev/null +++ b/src/swell/suites/geosadas/flow.cylc @@ -0,0 +1,121 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing JEDI-based non-cycling variational data assimilation + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = 2020-12-15T00:00:00Z + final cycle point = 2020-12-15T00:00:00Z + + [[graph]] + R1 = """ + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + BuildJediByLinking? + + # Stage JEDI static files + CloneJedi => StageJedi + + # Clone geos ana for generating observing system records + CloneGeosMksi + """ + + T00 = """ + # Generate satellite channel records + CloneGeosMksi[^] => GenerateObservingSystemRecords + + # Get and convert bias correction coefficients + GetGsiBc => GsiBcToIoda + + # Get and convert ncdiags + GetGsiNcdiag => GsiNcdiagToIoda + + # Get background + GetGeosAdasBackground + + # Run Jedi variational executable + GenerateObservingSystemRecords => RunJediVariationalExecutable + BuildJediByLinking[^] => RunJediVariationalExecutable + StageJedi[^] => RunJediVariationalExecutable + GsiBcToIoda => RunJediVariationalExecutable + GsiNcdiagToIoda => RunJediVariationalExecutable + GetGeosAdasBackground => RunJediVariationalExecutable + + # Clean cycle + RunJediVariationalExecutable => CleanCycle + """ + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneGeosMksi]] + script = "swell task CloneGeosMksi $config" + + [[GenerateObservingSystemRecords]] + script = "swell task GenerateObservingSystemRecords $config -d $datetime -m geos_atmosphere" + + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[StageJedi]] + script = "swell task StageJedi $config -m geos_atmosphere" + + [[ GetGsiBc ]] + script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" + + [[ GsiBcToIoda ]] + script = "swell task GsiBcToIoda $config -d $datetime -m geos_atmosphere" + + [[ GetGsiNcdiag ]] + script = "swell task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + + [[ GsiNcdiagToIoda ]] + script = "swell task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + + [[ GetGeosAdasBackground ]] + script = "swell task GetGeosAdasBackground $config -d $datetime -m geos_atmosphere" + + [[RunJediVariationalExecutable]] + script = "swell task RunJediVariationalExecutable $config -d $datetime -m geos_atmosphere" + platform = {{platform}} + execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediVariationalExecutable"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[CleanCycle]] + script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc new file mode 100644 index 000000000..af199788d --- /dev/null +++ b/src/swell/suites/hofx/flow.cylc @@ -0,0 +1,164 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing JEDI-based h(x) + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + {% for model_component in model_components %} + # Clone geos ana for generating observing system records + CloneGeosMksi-{{model_component}} + {% endfor %} + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + + # Task triggers for: {{model_component}} + # ------------------ + # Generate satellite channel records + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} + + # Get background, provide a way to get background directly from GEOS experiment + GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} + + # Get observations + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + + # Perform staging that is cycle dependent + StageJediCycle-{{model_component}} + + # Run Jedi hofx executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediHofxExecutable-{{model_component}} + CloneJedi[^] => StageJediCycle-{{model_component}} + StageJediCycle-{{model_component}} => RunJediHofxExecutable-{{model_component}} + GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediHofxExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediHofxExecutable-{{model_component}} + GenerateObservingSystemRecords-{{model_component}} => RunJediHofxExecutable-{{model_component}} + + # EvaObservations + RunJediHofxExecutable-{{model_component}} => EvaObservations-{{model_component}} + + # Save observations + RunJediHofxExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + + # Clean up large files + EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} + + {% endif %} + {% endfor %} + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + {% for model_component in model_components %} + + [[CloneGeosMksi-{{model_component}}]] + script = "swell task CloneGeosMksi $config -m {{model_component}}" + + [[GenerateObservingSystemRecords-{{model_component}}]] + script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" + + [[StageJediCycle-{{model_component}}]] + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" + + [[GetBackground-{{model_component}}]] + script = "swell task GetBackground $config -d $datetime -m {{model_component}}" + + [[GetBackgroundGeosExperiment-{{model_component}} ]] + script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[GetObsNotInR2d2-{{model_component}}]] + script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" + + [[RunJediHofxExecutable-{{model_component}}]] + script = "swell task RunJediHofxExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediHofxExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediHofxExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[EvaObservations-{{model_component}}]] + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[SaveObsDiags-{{model_component}}]] + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" + + [[CleanCycle-{{model_component}}]] + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" + {% endfor %} + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/flow.cylc b/src/swell/suites/localensembleda/flow.cylc new file mode 100644 index 000000000..8126c1d9a --- /dev/null +++ b/src/swell/suites/localensembleda/flow.cylc @@ -0,0 +1,233 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing JEDI-based LocalEnsembleDA Algorithm + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + {% for model_component in model_components %} + # Clone geos ana for generating observing system records + CloneGeosMksi-{{model_component}} + {% endfor %} + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + # Task triggers for: {{model_component}} + # ------------------ + + # Perform staging that is cycle dependent + BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{{model_component}} => sync_point + + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => sync_point + + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} => sync_point + + GetEnsembleGeosExperiment-{{model_component}} => sync_point + + sync_point => RunJediObsfiltersExecutable-{{model_component}} + {% if skip_ensemble_hofx %} + sync_point => RunJediObsfiltersExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + {% else %} + # Run hofx for ensemble members according to strategy + {% if ensemble_hofx_strategy == 'serial' %} + sync_point => RunJediEnsembleMeanVariance-{{model_component}} => RunJediHofxEnsembleExecutable-{{model_component}} + RunJediHofxEnsembleExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + + {% elif ensemble_hofx_strategy == 'parallel' %} + {% for packet in range(ensemble_hofx_packets) %} + # When strategy is parallel, only proceed if all RunJediHofxEnsembleExecutable completes successfully for each packet + + # There is a need for a task to combine all hofx observations together, compute node preferred, put here as placeholder + # RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunEnsembleHofxCombiner-{{model_component}} + # RunEnsembleHofxCombiner-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + + sync_point => RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} + RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + {% endfor %} + {% endif %} + {% endif %} + + + # EvaIncrement + RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaIncrement-{{model_component}} + + # EvaObservations + # RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaObservations-{{model_component}} + + # Save observations + # RunJediLocalEnsembleDaExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + + # Clean up large files + # EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} & + EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} + + {% endif %} + {% endfor %} + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + {% for model_component in model_components %} + + [[CloneGeosMksi-{{model_component}}]] + script = "swell task CloneGeosMksi $config -m {{model_component}}" + + [[GenerateObservingSystemRecords-{{model_component}}]] + script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" + + [[StageJediCycle-{{model_component}}]] + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" + + [[ GetBackground-{{model_component}} ]] + script = "swell task GetBackground $config -d $datetime -m {{model_component}}" + + [[GetEnsembleGeosExperiment-{{model_component}}]] + script = "swell task GetEnsembleGeosExperiment $config -d $datetime -m {{model_component}}" + + [[RunJediObsfiltersExecutable-{{model_component}}]] + script = "swell task RunJediObsfiltersExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediObsfiltersExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediObsfiltersExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[RunJediEnsembleMeanVariance-{{model_component}}]] + script = "swell task RunJediEnsembleMeanVariance $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediEnsembleMeanVariance"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediEnsembleMeanVariance"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[GetObsNotInR2d2-{{model_component}}]] + script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" + + {% if not skip_ensemble_hofx %} + {% if ensemble_hofx_strategy == 'serial' %} + [[RunJediHofxEnsembleExecutable-{{model_component}}]] + script = "swell task RunJediHofxEnsembleExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediHofxEnsembleExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediHofxEnsembleExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + {% elif ensemble_hofx_strategy == 'parallel' %} + {% for packet in range(ensemble_hofx_packets) %} + [[RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}}]] + script = "swell task RunJediHofxEnsembleExecutable $config -d $datetime -m {{model_component}} -p {{packet}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediHofxEnsembleExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediHofxEnsembleExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + {% endfor %} + {% endif %} + {% endif %} + + [[RunJediLocalEnsembleDaExecutable-{{model_component}}]] + script = "swell task RunJediLocalEnsembleDaExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediLocalEnsembleDaExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediLocalEnsembleDaExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[EvaIncrement-{{model_component}}]] + script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" + + [[EvaObservations-{{model_component}}]] + script = true +# EnKF not ready to use Eva +# script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" +# platform = {{platform}} +# execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} +# [[[directives]]] +# {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} +# --{{key}} = {{value}} +# {%- endfor %} + + [[SaveObsDiags-{{model_component}}]] + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" + + [[CleanCycle-{{model_component}}]] + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" + {% endfor %} + + + [[sync_point]] + script = true +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc new file mode 100644 index 000000000..66e8fb888 --- /dev/null +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -0,0 +1,147 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing geos_atmosphere ObsFilters tests + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + # Clone geos ana for generating observing system records + CloneGeosMksi + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + + # Generate satellite channel records + CloneGeosMksi[^] => GenerateObservingSystemRecords + + # Convert bias correction to ioda + GetGsiBc + GetGsiBc => GsiBcToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda + + # Convert ncdiags to ioda + GetGsiNcdiag + GetGsiNcdiag => GsiNcdiagToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda + + GetGeovals + + # Run Jedi hofx executable + GenerateObservingSystemRecords => RunJediUfoTestsExecutable + GsiNcdiagToIoda => RunJediUfoTestsExecutable + GsiBcToIoda => RunJediUfoTestsExecutable + GetGeovals => RunJediUfoTestsExecutable + + # EvaObservations + RunJediUfoTestsExecutable => EvaObservations + + # Clean up large files + EvaObservations => CleanCycle + + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[CloneGeosMksi]] + script = "swell task CloneGeosMksi $config -m geos_atmosphere" + + [[GenerateObservingSystemRecords]] + script = "swell task GenerateObservingSystemRecords $config -d $datetime -m geos_atmosphere" + + [[ GetGsiBc ]] + script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" + + [[ GsiBcToIoda ]] + script = "swell task GsiBcToIoda $config -d $datetime -m geos_atmosphere" + + [[ GetGsiNcdiag ]] + script = "swell task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + + [[ GsiNcdiagToIoda ]] + script = "swell task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + + [[ GetGeovals ]] + script = "swell task GetGeovals $config -d $datetime -m geos_atmosphere" + + [[RunJediUfoTestsExecutable]] + script = "swell task RunJediUfoTestsExecutable $config -d $datetime -m geos_atmosphere" + platform = {{platform}} + execution time limit = {{scheduling["RunJediUfoTestsExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediUfoTestsExecutable"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[EvaObservations]] + script = "swell task EvaObservations $config -d $datetime -m geos_atmosphere" + platform = {{platform}} + execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["EvaObservations"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[CleanCycle]] + script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere" + +# -------------------------------------------------------------------------------------------------- From 0ba365e0baef0f26cbdd073655b1d4ec6f46e201 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 10 Oct 2025 10:34:25 -0400 Subject: [PATCH 106/299] add cycle_times --- src/swell/deployment/create_experiment.py | 163 +++-------- src/swell/suites/3dvar_atmos/workflow.py | 173 ++++++------ src/swell/suites/3dvar_cycle/workflow.py | 260 +++++++++--------- src/swell/suites/build_geos/workflow.py | 79 ++---- src/swell/suites/build_jedi/workflow.py | 79 ++---- .../suites/compare_variational/workflow.py | 118 ++++---- src/swell/suites/convert_ncdiags/workflow.py | 98 +++---- src/swell/suites/eva_capabilities/workflow.py | 102 +++---- src/swell/suites/forecast_geos/workflow.py | 101 +++---- src/swell/suites/geosadas/workflow.py | 113 ++++---- src/swell/suites/hofx/workflow.py | 141 +++++----- src/swell/suites/localensembleda/workflow.py | 187 ++++++------- src/swell/suites/ufo_testing/workflow.py | 125 +++++---- 13 files changed, 781 insertions(+), 958 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index acee17a07..0a75ce561 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -117,15 +117,53 @@ def prepare_config( # ---------------------- experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() - # Update the workflow with the answered task questions - # ---------------------------------------------------- - - workflow.experiment_dict = experiment_dict - # Finalize the workflow by adding the runtime section, and get the contents # ------------------------------------------------------------------------- workflow_string = workflow.get_workflow_str() + if 'model_components' in experiment_dict: + model_components = experiment_dict['model_components'] + + # Since cycle times are used, the render_dictionary will need to include cycle_times + # If there are different model components then process each to gather cycle times + if len(model_components) > 0 and all('cycle_times' in experiment_dict['models'][model] \ + for model in model_components): + cycle_times = [] + for model_component in model_components: + cycle_times_mc = experiment_dict['models'][model_component]['cycle_times'] + cycle_times = list(set(cycle_times + cycle_times_mc)) + cycle_times.sort() + + cycle_times_dict_list = [] + for cycle_time in cycle_times: + cycle_time_dict = {} + cycle_time_dict['cycle_time'] = cycle_time + for model_component in model_components: + cycle_time_dict[model_component] = False + if cycle_time in experiment_dict['models'][model_component]['cycle_times']: + cycle_time_dict[model_component] = True + cycle_times_dict_list.append(cycle_time_dict) + + experiment_dict['cycle_times'] = cycle_times_dict_list + + # Otherwise check that experiment_dict has cycle_times + elif 'cycle_times' in experiment_dict: + + cycle_times = list(set(experiment_dict['cycle_times'])) + cycle_times.sort() + experiment_dict['cycle_times'] = cycle_times + + else: + + # Otherwise use logger to abort + logger.abort('The suite file required cycle_times but there are no model components ' + + 'to gather them from or they are not provided in the experiment ' + + 'dictionary.') + + # Update the workflow with the answered task questions + # ---------------------------------------------------- + workflow.experiment_dict = experiment_dict + # Expand all environment vars in the dictionary # --------------------------------------------- experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) @@ -382,117 +420,4 @@ def create_modules_csh( modules_file_open.write(modules_file_line) -# -------------------------------------------------------------------------------------------------- - - -def prepare_cylc_suite_jinja2( - logger: Logger, - swell_suite_path: str, - exp_suite_path: str, - experiment_dict: dict, - platform: str -) -> None: - - # Open suite file from swell - # -------------------------- - with open(os.path.join(swell_suite_path, 'flow.cylc'), 'r') as file: - suite_file = file.read() - - # Copy the experiment dictionary to the rendering dictionary - # ---------------------------------------------------------- - render_dictionary = copy.deepcopy(experiment_dict) - - # Get unique list of cycle times with model flags to render dictionary - # -------------------------------------------------------------------- - - # Convenience - fetch model_components prior to search for 'cycle_times' and 'ensemble_*' - model_components = dict_get(logger, experiment_dict, 'model_components', []) - - # Check if 'cycle_times' appears anywhere in the suite_file - if 'cycle_times' in suite_file: - - # Since cycle times are used, the render_dictionary will need to include cycle_times - # If there are different model components then process each to gather cycle times - if len(model_components) > 0: - cycle_times = [] - for model_component in model_components: - cycle_times_mc = experiment_dict['models'][model_component]['cycle_times'] - cycle_times = list(set(cycle_times + cycle_times_mc)) - cycle_times.sort() - - cycle_times_dict_list = [] - for cycle_time in cycle_times: - cycle_time_dict = {} - cycle_time_dict['cycle_time'] = cycle_time - for model_component in model_components: - cycle_time_dict[model_component] = False - if cycle_time in experiment_dict['models'][model_component]['cycle_times']: - cycle_time_dict[model_component] = True - cycle_times_dict_list.append(cycle_time_dict) - - render_dictionary['cycle_times'] = cycle_times_dict_list - - # Otherwise check that experiment_dict has cycle_times - elif 'cycle_times' in experiment_dict: - - cycle_times = list(set(experiment_dict['cycle_times'])) - cycle_times.sort() - render_dictionary['cycle_times'] = cycle_times - - else: - - # Otherwise use logger to abort - logger.abort('The suite file required cycle_times but there are no model components ' + - 'to gather them from or they are not provided in the experiment ' + - 'dictionary.') - - # Check if 'ensemble_hofx_strategy' appears anywhere in suite_file - ensemble_list = ['ensemble_'+s for s in ['num_members', 'hofx_strategy', 'hofx_packets']] - ensemble_list = ensemble_list + ['skip_ensemble_hofx'] - for ensemble_aspect in ensemble_list: - if ensemble_aspect in suite_file: - if len(model_components) == 0: - logger.abort(f'The suite file required {ensemble_aspect} ' + - 'there are no model components to gather them from or ' + - 'they are not provided in the experiment dictionary.') - for model_component in model_components: - render_dictionary[ensemble_aspect] = \ - experiment_dict['models'][model_component][ensemble_aspect] - - # Cycling VarBC exception (only for geos_atmosphere) - if 'cycling_varbc' in suite_file: - if 'geos_atmosphere' in model_components: - render_dictionary['cycling_varbc'] = \ - experiment_dict['models']['geos_atmosphere']['cycling_varbc'] - else: - logger.abort('The suite file required cycling_varbc but ' + - 'geos_atmosphere is not in the model components.') - - # Marine model toggles (only for geos_marine) - if 'marine_models' in suite_file: - if 'geos_marine' in model_components: - render_dictionary['marine_models'] = \ - experiment_dict['models']['geos_marine']['marine_models'] - else: - logger.abort('The suite file required marine_models but ' + - 'geos_marine is not in the model components.') - - render_dictionary['scheduling'] = prepare_scheduling_dict(logger, experiment_dict, platform) - - # Set some specific values for: - # ------------------------------ - # run time (note: these overwrite defaults above) - render_dictionary['scheduling']['BuildJedi']['execution_time_limit'] = 'PT3H' - render_dictionary['scheduling']['EvaObservations']['execution_time_limit'] = 'PT30M' - - # Render the template - # ------------------- - new_suite_file = template_string_jinja2(logger, suite_file, render_dictionary, False) - - # Write suite file to experiment - # ------------------------------ - with open(os.path.join(exp_suite_path, 'flow.cylc'), 'w') as file: - file.write(new_suite_file) - - -# -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index d555d4314..e791439bb 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -7,128 +7,119 @@ # -------------------------------------------------------------------------------------------------- +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow # -------------------------------------------------------------------------------------------------- -r1_template = """ -# Triggers for non cycle time dependent tasks -# ------------------------------------------- -# Clone JEDI source code -CloneJedi - -# Build JEDI source code by linking -CloneJedi => BuildJediByLinking? - -# If not able to link to build create the build -BuildJediByLinking:fail? => BuildJedi -""" - -r1_model = """ - -# Stage JEDI static files -CloneGeosMksi-{model_component} -""" - -cycle_template_1 = """ -# Task triggers for: {model_component} -# ------------------ -# Generate satellite channel records -CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} +template_str = ''' +# -------------------------------------------------------------------------------------------------- -# Get background, provide a way to get background directly from GEOS experiment -GetBackgroundGeosExperiment-{model_component} :fail? => GetBackground-{model_component} +# Cylc suite for executing JEDI-based non-cycling variational data assimilation -# Get observations -""" +# -------------------------------------------------------------------------------------------------- -cycle_template_2 = """ -# Cycling VarBC is active, biases from the previous cycle will be used +[scheduler] + UTC mode = True + allow implicit tasks = False -RunJediVariationalExecutable-{model_component}[-PT6H] => GetObservations-{model_component} -""" +# -------------------------------------------------------------------------------------------------- -cycle_template_3 = """ -# Cycling VarBC is inactive, static bias files will be used -GetObservations-{model_component} -""" +[scheduling] -cycle_template_4 = """ -# Perform staging that is cycle dependent -StageJediCycle-{model_component} + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} -# Run Jedi variational executable -BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} -CloneJedi[^] => StageJediCycle-{model_component} -StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} -GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => -RunJediVariationalExecutable-{model_component} + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi -GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} -GenerateObservingSystemRecords-{model_component} => RunJediVariationalExecutable-{model_component} + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? -# EvaObservations -RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi -# EvaJediLog -RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} + {% for model_component in model_components %} + # Clone geos ana for generating observing system records + CloneGeosMksi-{{model_component}} + {% endfor %} + """ -# EvaIncrement -RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} + {% for model_component in model_components %} + {% if models[model_component]['cycle_times'] %} + {% for cycle_time in models[model_component]['cycle_times'] %} + {{cycle_time}} = """ + # Task triggers for: {{model_component}} + # ------------------ + # Generate satellite channel records + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} -# Save observations -RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} + # Get background, provide a way to get background directly from GEOS experiment + GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} -# Clean up large files -EvaObservations-{model_component} & SaveObsDiags-{model_component} => -CleanCycle-{model_component} -""" + # Get observations + {% if cycling_varbc %} + # Cycling VarBC is active, biases from the previous cycle will be used -# -------------------------------------------------------------------------------------------------- + RunJediVariationalExecutable-{{model_component}}[-PT6H] => GetObservations-{{model_component}} + {% else %} + # Cycling VarBC is inactive, static bias files will be used + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + {% endif %} -class Workflow_3dvar_atmos(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for executing JEDI-based non-cycling variational data assimilation - """) + # Perform staging that is cycle dependent + StageJediCycle-{{model_component}} - return description + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} + CloneJedi[^] => StageJediCycle-{{model_component}} + StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GenerateObservingSystemRecords-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - # -------------------------------------------------------------------------------------------------- + # EvaObservations + RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' + # EvaJediLog + RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} - # Define the string for the R1 (first non-cycling) section - r1 = r1_template + # EvaIncrement + RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - for model_component in self.experiment_dict['model_components']: - r1 += r1_model.format(model_component=model_component) + # Save observations + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) + # Clean up large files + EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} - # Format the string for each cycle - for model_component in self.experiment_dict['model_components']: - if 'cycle_times' in self.experiment_dict['models'][model_component]: - for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = cycle_template_1.format(model_component=model_component) + {% endfor %} + {% endif %} + """ + {% endfor %} - if self.experiment_dict['models'][model_component]['cycling_varbc']: - cycle_str += cycle_template_2.format(model_component=model_component) - else: - cycle_str += cycle_template_3.format(model_component=model_component) +# -------------------------------------------------------------------------------------------------- +''' - cycle_str += cycle_template_4.format(model_component=model_component) +# -------------------------------------------------------------------------------------------------- - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) +class Workflow_3dvar_atmos(CylcWorkflow): - return graph_section + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index f1c22a261..797669c57 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -7,158 +7,144 @@ # -------------------------------------------------------------------------------------------------- +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow # -------------------------------------------------------------------------------------------------- -r1_template = """ -# Triggers for non cycle time dependent tasks -# ------------------------------------------- -# Clone Geos source code -CloneGeos - -# Clone JEDI source code -CloneJedi - -# Build Geos source code by linking -CloneGeos => BuildGeosByLinking? - -# Build JEDI source code by linking -CloneJedi => BuildJediByLinking? - -# If not able to link to build create the build -BuildGeosByLinking:fail? => BuildGeos - -# If not able to link to build create the build -BuildJediByLinking:fail? => BuildJedi - -# Need first set of restarts to run model -GetGeosRestart => PrepGeosRunDir - -# Model cannot run without code -BuildGeosByLinking? | BuildGeos => RunGeosExecutable -""" - -r1_model = """ - -# JEDI cannot run without code -BuildJediByLinking? | BuildJedi => RunJediVariationalExecutable-{model_component} - -# Stage JEDI static files -CloneJedi => StageJedi-{model_component} => RunJediVariationalExecutable-{model_component} -""" - -cycle_template_1 = """ -# Model things -# Run the forecast through two windows (need to output restarts at the end of the -# first window and backgrounds for the second window) -# MoveDaRestart-{model_component}[-P1D] => PrepGeosRunDir -MoveDaRestart-{model_component}[-{models[model_component]["window_length"]}] => PrepGeosRunDir -PrepGeosRunDir => RunGeosExecutable - -# Run the analysis -# RunGeosExecutable => StageJediCycle-{model_component} -RunGeosExecutable => LinkGeosOutput-{model_component} -LinkGeosOutput-{model_component} => GenerateBClimatology-{model_component} - -# Data assimilation things -GetObservations-{model_component} -GenerateBClimatologyByLinking-{model_component} :fail? => GenerateBClimatology-{model_component} - -LinkGeosOutput-{model_component} => RunJediVariationalExecutable-{model_component} -StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} -GenerateBClimatologyByLinking-{model_component}? | GenerateBClimatology-{model_component} => -RunJediVariationalExecutable-{model_component} - -GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} - -# Run analysis diagnostics -RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} -RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} -RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} - -# Prepare analysis for next forecast -EvaIncrement-{model_component} => PrepareAnalysis-{model_component} -""" - -cycle_template_2 = """ -PrepareAnalysis-{model_component} => RunJediConvertStateSoca2ciceExecutable-{model_component} -RunJediConvertStateSoca2ciceExecutable-{model_component} => SaveRestart-{model_component} -RunJediConvertStateSoca2ciceExecutable-{model_component} => CleanCycle-{model_component} -""" +template_str = ''' +# -------------------------------------------------------------------------------------------------- -cycle_template_3 = """ -PrepareAnalysis-{model_component} => SaveRestart-{model_component} -""" +# Cylc suite for executing Geos forecast -cycle_template_4 = """ -# Move restart to next cycle -SaveRestart-{model_component} => MoveDaRestart-{model_component} +# -------------------------------------------------------------------------------------------------- -# Save analysis output -# RunJediVariationalExecutable-{model_component} => SaveAnalysis-{model_component} -RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} +[scheduler] + UTC mode = True + allow implicit tasks = False -# Save model output -# MoveBackground-{model_component} => StoreBackground-{model_component} +# -------------------------------------------------------------------------------------------------- -# Remove Run Directory -# MoveDaRestart-{model_component} & MoveBackground-{model_component} => RemoveForecastDir -MoveDaRestart-{model_component} => RemoveForecastDir +[scheduling] -# Clean up large files -# EvaObservations-{model_component} & EvaJediLog-{model_component} & -# SaveObsDiags-{model_component} & RemoveForecastDir => -EvaObservations-{model_component} & EvaJediLog-{model_component} & -EvaIncrement-{model_component} & SaveObsDiags-{model_component} => -CleanCycle-{model_component} -""" + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone Geos source code + CloneGeos + + # Clone JEDI source code + CloneJedi + + # Build Geos source code by linking + CloneGeos => BuildGeosByLinking? + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildGeosByLinking:fail? => BuildGeos + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + # Need first set of restarts to run model + GetGeosRestart => PrepGeosRunDir + + # Model cannot run without code + BuildGeosByLinking? | BuildGeos => RunGeosExecutable + + {% for model_component in model_components %} + + # JEDI cannot run without code + BuildJediByLinking? | BuildJedi => RunJediVariationalExecutable-{{model_component}} + + # Stage JEDI static files + CloneJedi => StageJedi-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + + {% endfor %} + """ + + {% for model_component in model_components %} + {% if models[model_component]['cycle_times'] %} + {% for cycle_time in models[model_component]['cycle_times'] %} + {{cycle_time}} = """ + # Model things + # Run the forecast through two windows (need to output restarts at the end of the + # first window and backgrounds for the second window) + MoveDaRestart-{{model_component}}[-{{models[model_component]["window_length"]}}] => PrepGeosRunDir + PrepGeosRunDir => RunGeosExecutable + + # Run the analysis + # RunGeosExecutable => StageJediCycle-{{model_component}} + RunGeosExecutable => LinkGeosOutput-{{model_component}} + LinkGeosOutput-{{model_component}} => GenerateBClimatology-{{model_component}} + + # Data assimilation things + GetObservations-{{model_component}} + GenerateBClimatologyByLinking-{{model_component}} :fail? => GenerateBClimatology-{{model_component}} + + LinkGeosOutput-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GenerateBClimatologyByLinking-{{model_component}}? | GenerateBClimatology-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + + # Run analysis diagnostics + RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} + RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} + RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} + + # Prepare analysis for next forecast + EvaIncrement-{{model_component}} => PrepareAnalysis-{{model_component}} + {% if 'cice6' in models[model_component]["marine_models"] %} + PrepareAnalysis-{{model_component}} => RunJediConvertStateSoca2ciceExecutable-{{model_component}} + RunJediConvertStateSoca2ciceExecutable-{{model_component}} => SaveRestart-{{model_component}} + RunJediConvertStateSoca2ciceExecutable-{{model_component}} => CleanCycle-{{model_component}} + {% else %} + PrepareAnalysis-{{model_component}} => SaveRestart-{{model_component}} + {% endif %} + + # Move restart to next cycle + SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} + + # Save analysis output + # RunJediVariationalExecutable-{{model_component}} => SaveAnalysis-{{model_component}} + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + + # Save model output + # MoveBackground-{{model_component}} => StoreBackground-{{model_component}} + + # Remove Run Directory + # MoveDaRestart-{{model_component}} & MoveBackground-{{model_component}} => RemoveForecastDir + MoveDaRestart-{{model_component}} => RemoveForecastDir + + # Clean up large files + # EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & SaveObsDiags-{{model_component}} & RemoveForecastDir => + EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} + {% endfor %} + {% endif %} + """ + {% endfor %} +# -------------------------------------------------------------------------------------------------- +''' # -------------------------------------------------------------------------------------------------- class Workflow_3dvar_cycle(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for executing JEDI-based non-cycling variational data assimilation - """) - - return description - - # -------------------------------------------------------------------------------------------------- - - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' - - # Define the string for the R1 (first non-cycling) section - r1 = r1_template - - for model_component in self.experiment_dict['model_components']: - r1 += r1_model.format(model_component=model_component) - - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) - - # Format the string for each cycle - for model_component in self.experiment_dict['model_components']: - if 'cycle_times' in self.experiment_dict['models'][model_component]: - for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = cycle_template_1.format(model_component=model_component) - - if 'cice6' in self.experiment_dict['models'][model_component]['marine_models']: - cycle_str += cycle_template_2.format(model_component=model_component) - else: - cycle_str += cycle_template_3.format(model_component=model_component) - - cycle_str += cycle_template_4.format(model_component=model_component) - - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) - - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) - return graph_section + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py index 15386f253..c4f36eafe 100644 --- a/src/swell/suites/build_geos/workflow.py +++ b/src/swell/suites/build_geos/workflow.py @@ -7,74 +7,47 @@ # -------------------------------------------------------------------------------------------------- -import os -import yaml - +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.utilities.cylc_formatting import indent_lines # -------------------------------------------------------------------------------------------------- +template_str = ''' +# -------------------------------------------------------------------------------------------------- -class Workflow_build_geos(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for building the GEOS model - """) - - return description - - # -------------------------------------------------------------------------------------------------- - - def define_scheduler(self) -> str: - scheduler_str = 'allow implicit tasks = False\n' - - settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) - if os.path.exists(settings_file): - with open(settings_file, 'r') as f: - settings_dict = yaml.safe_load(f) - if 'email_address' in settings_dict.keys(): - email_address = settings_dict['email_address'] - - message_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" - message_str += '[[events]]\n' - message_str += indent_lines('mail events = startup, shutdown\n', 1) - message_str += '[[mail]]\n' - message_str += indent_lines(f'to = {email_address}\n', 1) - message_str += '{% endif %}\n' - - scheduler_str += message_str - - scheduler = self.create_new_section('scheduler', scheduler_str) +# Cylc suite for building the GEOS model - return scheduler.get_section_str() +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +[scheduler] + allow implicit tasks = False - def define_scheduling_section(self): - scheduling_section = self.create_new_section('scheduling', '') +# -------------------------------------------------------------------------------------------------- - return scheduling_section +[scheduling] - # -------------------------------------------------------------------------------------------------- + [[graph]] + R1 = """ + CloneGeos => BuildGeosByLinking? - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' + BuildGeosByLinking:fail? => BuildGeos + """ - # Define the string for the R1 (first non-cycling) section - r1 = """ - CloneGeos => BuildGeosByLinking? +# -------------------------------------------------------------------------------------------------- +''' - BuildGeosByLinking:fail? => BuildGeos - """ +# -------------------------------------------------------------------------------------------------- - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) +class Workflow_build_geos(CylcWorkflow): - return graph_section + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index 1f066cacf..0ebd0bdec 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -7,74 +7,47 @@ # -------------------------------------------------------------------------------------------------- -import os -import yaml - +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.utilities.cylc_formatting import indent_lines # -------------------------------------------------------------------------------------------------- +template_str = ''' +# -------------------------------------------------------------------------------------------------- -class Workflow_build_jedi(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for building the JEDI code - """) - - return description - - # -------------------------------------------------------------------------------------------------- - - def define_scheduler(self) -> str: - scheduler_str = 'allow implicit tasks = False\n' - - settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) - if os.path.exists(settings_file): - with open(settings_file, 'r') as f: - settings_dict = yaml.safe_load(f) - if 'email_address' in settings_dict.keys(): - email_address = settings_dict['email_address'] - - message_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" - message_str += '[[events]]\n' - message_str += indent_lines('mail events = startup, shutdown\n', 1) - message_str += '[[mail]]\n' - message_str += indent_lines(f'to = {email_address}\n', 1) - message_str += '{% endif %}\n' - - scheduler_str += message_str - - scheduler = self.create_new_section('scheduler', scheduler_str) +# Cylc suite for building the JEDI code - return scheduler.get_section_str() +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +[scheduler] + allow implicit tasks = False - def define_scheduling_section(self): - scheduling_section = self.create_new_section('scheduling', '') +# -------------------------------------------------------------------------------------------------- - return scheduling_section +[scheduling] - # -------------------------------------------------------------------------------------------------- + [[graph]] + R1 = """ + CloneJedi => BuildJediByLinking? - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' + BuildJediByLinking:fail? => BuildJedi + """ - # Define the string for the R1 (first non-cycling) section - r1 = """ - CloneJedi => BuildJediByLinking? +# -------------------------------------------------------------------------------------------------- +''' - BuildJediByLinking:fail? => BuildJedi - """ +# -------------------------------------------------------------------------------------------------- - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) +class Workflow_build_jedi(CylcWorkflow): - return graph_section + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 5bff432c4..f93434166 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -7,96 +7,80 @@ # -------------------------------------------------------------------------------------------------- -import yaml -import os - +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.utilities.cylc_runtime import Task from swell.utilities.check_da_params import check_da_params # -------------------------------------------------------------------------------------------------- +template_str = ''' +# -------------------------------------------------------------------------------------------------- -class Workflow_compare_variational(CylcWorkflow): +# Cylc suite for running comparison tests on completed experiments - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- - def define_description(self): - description = self.comment_block(""" - # Cylc suite for running comparison tests on completed experiments - """) +[scheduler] + UTC mode = True + allow implicit tasks = False - return description +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + EvaComparisonIncrement-{{model_component}} + EvaComparisonJediLog-{{model_component}} + {% for path in comparison_experiment_paths %} + JediOopsLogParser-{{model_component}}-{{ loop.index0 }} => JediLogComparison-{{model_component}} + {% endfor %} + {% endif %} + {% endfor %} + """ + {% endfor %} - def define_scheduling(self): +# -------------------------------------------------------------------------------------------------- +''' - config_list = self.experiment_dict['comparison_experiment_paths'] +# -------------------------------------------------------------------------------------------------- - start_cycle_point = self.experiment_dict['start_cycle_point'] - final_cycle_point = self.experiment_dict['final_cycle_point'] - graph_str = '' +class Workflow_compare_variational(CylcWorkflow): - if 'models' in self.experiment_dict: - for model in self.experiment_dict['models'].keys(): - cycle_times = self.experiment_dict['models'][model]['cycle_times'] + def define_initial_workflow(self): + workflow_str = self.default_header() + # Overrides for comparison suites + start_cycle_point = self.experiment_dict['start_cycle_point'] + final_cycle_point = self.experiment_dict['final_cycle_point'] + if self.experiment_dict['start_cycle_point'] is None: + config_list = self.experiment_dict['comparison_experiment_paths'] + for model in self.experiment_dict['model_components']: + cycle_times = self.experiment_dict['models'][model]['cycle_times'] start_cycle_point, final_cycle_point, cycle_times = check_da_params( config_list, model, start_cycle_point, final_cycle_point, cycle_times) - - # Set the values in the config + self.experiment_dict['start_cycle_point'] = start_cycle_point self.experiment_dict['final_cycle_point'] = final_cycle_point - - scheduling_section = self.create_new_section( - 'scheduling', {'initial cycle point': start_cycle_point, - 'final cycle point': final_cycle_point}) - - for cycle_time in cycle_times: - cycle_str = '' - - cycle_str += f"EvaComparisonIncrement-{model}\n" - cycle_str += f"EvaComparisonJediLog-{model}\n" - - for i in range(len(config_list)): - cycle_str += f"JediOopsLogParser-{model}-{i} => JediLogComparison-{model}\n" - - graph_str += self.format_cycle(cycle_time, cycle_str) - - else: - scheduling_section = self.create_new_section('scheduling', {}) - - graph_section = self.create_new_section('graph', graph_str) - - scheduling_section.add_subsection(graph_section) - - return scheduling_section.get_section_str() - - # -------------------------------------------------------------------------------------------------- - - def define_runtime_task_overrides(self): - overrides = {} - - paths = self.experiment_dict['comparison_experiment_paths'] - - for i, path in enumerate(paths): - config_file = os.path.join(os.path.dirname(path), 'experiment.yaml') - with open(config_file, 'r') as f: - exp_dict = yaml.safe_load(f) - - for model in exp_dict['model_components']: - overrides[f'JediOopsLogParser-{model}-{i}'] = Task( - base_name='JediOopsLogParser', - scheduling_name=(f'JediOopsLogParser-{model}-{i}'), - script=(f'swell task JediOopsLogParser {config_file} -d $datetime' - f' -m {model}')) - - return overrides + self.experiment_dict['models'][model]['cycle_times'] = cycle_times + + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index 2cdd11ee2..9286951b7 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -7,71 +7,77 @@ # -------------------------------------------------------------------------------------------------- +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow # -------------------------------------------------------------------------------------------------- -r1_template = """ -# Triggers for non cycle time dependent tasks -# ------------------------------------------- -# Clone JEDI source code -CloneJedi +template_str = ''' +# -------------------------------------------------------------------------------------------------- -# Build JEDI source code by linking -CloneJedi => BuildJediByLinking? +# Cylc suite for executing geos_atmosphere ObsFilters tests -# If not able to link to build create the build -BuildJediByLinking:fail? => BuildJedi -""" +# -------------------------------------------------------------------------------------------------- -cycle_template = """ -# Convert bias correction to ioda -GetGsiBc -GetGsiBc => GsiBcToIoda -BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda +[scheduler] + UTC mode = True + allow implicit tasks = False -# Convert ncdiags to ioda -GetGsiNcdiag -GetGsiNcdiag => GsiNcdiagToIoda -BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda +# -------------------------------------------------------------------------------------------------- -# Clean up -GsiNcdiagToIoda => CleanCycle -""" +[scheduling] -# -------------------------------------------------------------------------------------------------- + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi -class Workflow_convert_ncdiags(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for executing geos_atmosphere ObsFilters tests - """) + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? - return description + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + """ - # -------------------------------------------------------------------------------------------------- + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' + # Convert bias correction to ioda + GetGsiBc + GetGsiBc => GsiBcToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda - # Define the string for the R1 (first non-cycling) section - r1 = r1_template + # Convert ncdiags to ioda + GetGsiNcdiag + GetGsiNcdiag => GsiNcdiagToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) + # Clean up + GsiNcdiagToIoda => CleanCycle + """ + {% endfor %} - # Format the string for each cycle - for model in self.experiment_dict['models'].keys(): - if 'cycle_times' in self.experiment_dict['models'][model].keys(): - for cycle_time in self.experiment_dict['models'][model]['cycle_times']: - cycle_str = cycle_template - graph_str += self.format_cycle(cycle_time, cycle_str) +# -------------------------------------------------------------------------------------------------- +''' + +# -------------------------------------------------------------------------------------------------- - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) - return graph_section +class Workflow_convert_ncdiags(CylcWorkflow): + + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index d6d069609..2173b96f1 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -7,69 +7,73 @@ # -------------------------------------------------------------------------------------------------- +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow # -------------------------------------------------------------------------------------------------- -r1_model = """ -# Clone geos ana for generating observing system records -CloneGeosMksi-{model_component} -""" - -cycle_template = """ -GetNcdiags-{model_component} -CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} -""" - -r_final = """ -GetNcdiags-{model_component} => EvaTimeseries-{model_component} -GenerateObservingSystemRecords-{model_component} => EvaTimeseries-{model_component} -EvaTimeseries-{model_component} => CleanCycle-{model_component} -""" - +template_str = ''' # -------------------------------------------------------------------------------------------------- +# Cylc suite for executing geos_atmosphere ObsFilters tests -class Workflow_eva_capabilities(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for executing geos_atmosphere ObsFilters tests - """) - - return description - - # -------------------------------------------------------------------------------------------------- - - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' +# -------------------------------------------------------------------------------------------------- - r1 = '' +[scheduler] + UTC mode = True + allow implicit tasks = False - # Define the string for the R1 (first non-cycling) section - for model in self.experiment_dict['models'].keys(): - r1 += r1_model.format(model_component=model) +# -------------------------------------------------------------------------------------------------- - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + {% for model_component in model_components %} + # Clone geos ana for generating observing system records + CloneGeosMksi-{{model_component}} + {% endfor %} + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + GetNcdiags-{{model_component}} + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} + {% endif %} + {% endfor %} + """ + {% endfor %} + + # Run once at the final cycle point + {% for model_component in model_components %} + R1/$ = """ + GetNcdiags-{{model_component}} => EvaTimeseries-{{model_component}} + GenerateObservingSystemRecords-{{model_component}} => EvaTimeseries-{{model_component}} + EvaTimeseries-{{model_component}} => CleanCycle-{{model_component}} + """ + {% endfor %} - # Format the string for each cycle - for model in self.experiment_dict['models'].keys(): - if 'cycle_times' in self.experiment_dict['models'][model].keys(): - for cycle_time in self.experiment_dict['models'][model]['cycle_times']: - cycle_str = cycle_template.format(model_component=model) - graph_str += self.format_cycle(cycle_time, cycle_str) +# -------------------------------------------------------------------------------------------------- +''' - cycle_str = '' +# -------------------------------------------------------------------------------------------------- - # Format the final cycle - for model in self.experiment_dict['models'].keys(): - cycle_str += r_final.format(model_component=model) - graph_str += self.format_cycle('R1/$', cycle_str) - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) +class Workflow_eva_capabilities(CylcWorkflow): - return graph_section + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index ddf024c81..a70914e96 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -7,75 +7,82 @@ # -------------------------------------------------------------------------------------------------- +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow # -------------------------------------------------------------------------------------------------- -r1_template = """ -# Triggers for non cycle time dependent tasks -# ------------------------------------------- -# Clone Geos source code -CloneGeos +template_str = ''' +# -------------------------------------------------------------------------------------------------- -# Build Geos source code by linking -CloneGeos => BuildGeosByLinking? +# Cylc suite for Geos forecast without DA -# If not able to link to build create the build -BuildGeosByLinking:fail? => BuildGeos +# -------------------------------------------------------------------------------------------------- -# Need first set of restarts to run model -GetGeosRestart => PrepGeosRunDir +[scheduler] + UTC mode = True + allow implicit tasks = False -# Get first set of restarts -BuildGeosByLinking? | BuildGeos => RunGeosExecutable -""" +# -------------------------------------------------------------------------------------------------- -cycle_template = """ -# Run Geos Executable -PrepGeosRunDir => RunGeosExecutable -MoveForecastRestart[-PT6H] => PrepGeosRunDir +[scheduling] -# Move restart to next cycle -RunGeosExecutable => MoveForecastRestart + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} -# Save restarts if requested -# MoveForecastRestart[-PT6H] => SaveRestart + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone Geos source code + CloneGeos -# Remove Run Directory -MoveForecastRestart => RemoveForecastDir -""" + # Build Geos source code by linking + CloneGeos => BuildGeosByLinking? -# -------------------------------------------------------------------------------------------------- + # If not able to link to build create the build + BuildGeosByLinking:fail? => BuildGeos + # Need first set of restarts to run model + GetGeosRestart => PrepGeosRunDir -class Workflow_forecast_geos(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for executing geos_atmosphere ObsFilters tests - """) + # Get first set of restarts + BuildGeosByLinking? | BuildGeos => RunGeosExecutable + """ - return description + {% for cycle_time in cycle_times %} + {{cycle_time}} = """ - # -------------------------------------------------------------------------------------------------- + # Run Geos Executable + PrepGeosRunDir => RunGeosExecutable + MoveForecastRestart[-PT6H] => PrepGeosRunDir - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' + # Move restart to next cycle + RunGeosExecutable => MoveForecastRestart - # Define the string for the R1 (first non-cycling) section - r1 = r1_template + # Save restarts if requested + # MoveForecastRestart[-PT6H] => SaveRestart - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) + # Remove Run Directory + MoveForecastRestart => RemoveForecastDir + """ + {% endfor %} - # Format the string for each cycle - for cycle_time in self.experiment_dict['cycle_times']: - cycle_str = cycle_template - graph_str += self.format_cycle(cycle_time, cycle_str) +# -------------------------------------------------------------------------------------------------- +''' + +# -------------------------------------------------------------------------------------------------- - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) - return graph_section +class Workflow_forecast_geos(CylcWorkflow): + + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index 2b5154212..f5395e3d3 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -7,89 +7,80 @@ # -------------------------------------------------------------------------------------------------- +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow # -------------------------------------------------------------------------------------------------- -r1_template = """ -# Clone JEDI source code -CloneJedi - -# Build JEDI source code by linking -BuildJediByLinking? - -# Stage JEDI static files -CloneJedi => StageJedi +template_str = ''' +# -------------------------------------------------------------------------------------------------- -# Clone geos ana for generating observing system records -CloneGeosMksi -""" +[scheduler] + UTC mode = True + allow implicit tasks = False -cycle_template = """ -# Generate satellite channel records -CloneGeosMksi[^] => GenerateObservingSystemRecords +# -------------------------------------------------------------------------------------------------- -# Get and convert bias correction coefficients -GetGsiBc => GsiBcToIoda +[scheduling] -# Get and convert ncdiags -GetGsiNcdiag => GsiNcdiagToIoda + initial cycle point = 2020-12-15T00:00:00Z + final cycle point = 2020-12-15T00:00:00Z -# Get background -GetGeosAdasBackground + [[graph]] + R1 = """ + # Clone JEDI source code + CloneJedi -# Run Jedi variational executable -GenerateObservingSystemRecords => RunJediVariationalExecutable -BuildJediByLinking[^] => RunJediVariationalExecutable -StageJedi[^] => RunJediVariationalExecutable -GsiBcToIoda => RunJediVariationalExecutable -GsiNcdiagToIoda => RunJediVariationalExecutable -GetGeosAdasBackground => RunJediVariationalExecutable + # Build JEDI source code by linking + BuildJediByLinking? -# Clean cycle -RunJediVariationalExecutable => CleanCycle -""" + # Stage JEDI static files + CloneJedi => StageJedi -# -------------------------------------------------------------------------------------------------- + # Clone geos ana for generating observing system records + CloneGeosMksi + """ + T00 = """ + # Generate satellite channel records + CloneGeosMksi[^] => GenerateObservingSystemRecords -class Workflow_geosadas(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for executing JEDI-based non-cycling variational data assimilation - """) + # Get and convert bias correction coefficients + GetGsiBc => GsiBcToIoda - return description + # Get and convert ncdiags + GetGsiNcdiag => GsiNcdiagToIoda - # -------------------------------------------------------------------------------------------------- + # Get background + GetGeosAdasBackground - def define_scheduling_section(self): - scheduling_dict = {'initial cycle point': '2020-12-15T00:00:00Z', - 'final cycle point': '2020-12-15T00:00:00Z'} + # Run Jedi variational executable + GenerateObservingSystemRecords => RunJediVariationalExecutable + BuildJediByLinking[^] => RunJediVariationalExecutable + StageJedi[^] => RunJediVariationalExecutable + GsiBcToIoda => RunJediVariationalExecutable + GsiNcdiagToIoda => RunJediVariationalExecutable + GetGeosAdasBackground => RunJediVariationalExecutable - scheduling_section = self.create_new_section('scheduling', scheduling_dict) - return scheduling_section + # Clean cycle + RunJediVariationalExecutable => CleanCycle + """ - # -------------------------------------------------------------------------------------------------- - - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' - - # Define the string for the R1 (first non-cycling) section - r1 = r1_template +# -------------------------------------------------------------------------------------------------- +''' - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) +# -------------------------------------------------------------------------------------------------- - # Format the string for each cycle - for cycle_time in ['T00']: - cycle_str = cycle_template - graph_str += self.format_cycle(cycle_time, cycle_str) - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) +class Workflow_geosadas(CylcWorkflow): - return graph_section + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index 9dd458faf..7d0847f32 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -7,103 +7,106 @@ # -------------------------------------------------------------------------------------------------- +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow # -------------------------------------------------------------------------------------------------- -r1_template = """ -# Triggers for non cycle time dependent tasks -# ------------------------------------------- -# Clone JEDI source code -CloneJedi +template_str = ''' +# -------------------------------------------------------------------------------------------------- -# Build JEDI source code by linking -CloneJedi => BuildJediByLinking? +# Cylc suite for executing JEDI-based h(x) -# If not able to link to build create the build -BuildJediByLinking:fail? => BuildJedi -""" +# -------------------------------------------------------------------------------------------------- -r1_model = """ +[scheduler] + UTC mode = True + allow implicit tasks = False -# Clone geos ana for generating observing system records -CloneGeosMksi-{model_component} -""" +# -------------------------------------------------------------------------------------------------- -cycle_template_1 = """ -# Task triggers for: {model_component} -# ------------------ -# Generate satellite channel records -CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} +[scheduling] -# Get background, provide a way to get background directly from GEOS experiment -GetBackgroundGeosExperiment-{model_component} :fail? => GetBackground-{model_component} + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} -# Get observations -GetObservations-{model_component} + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi -# Perform staging that is cycle dependent -StageJediCycle-{model_component} + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? -# Run Jedi hofx executable -BuildJediByLinking[^]? | BuildJedi[^] => RunJediHofxExecutable-{model_component} -CloneJedi[^] => StageJediCycle-{model_component} -StageJediCycle-{model_component} => RunJediHofxExecutable-{model_component} -GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => -RunJediHofxExecutable-{model_component} + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi -GetObservations-{model_component} => RunJediHofxExecutable-{model_component} -GenerateObservingSystemRecords-{model_component} => RunJediHofxExecutable-{model_component} + {% for model_component in model_components %} + # Clone geos ana for generating observing system records + CloneGeosMksi-{{model_component}} + {% endfor %} + """ -# EvaObservations -RunJediHofxExecutable-{model_component} => EvaObservations-{model_component} + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} -# Save observations -RunJediHofxExecutable-{model_component} => SaveObsDiags-{model_component} + # Task triggers for: {{model_component}} + # ------------------ + # Generate satellite channel records + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} -# Clean up large files -EvaObservations-{model_component} & SaveObsDiags-{model_component} => -CleanCycle-{model_component} -""" + # Get background, provide a way to get background directly from GEOS experiment + GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} -# -------------------------------------------------------------------------------------------------- + # Get observations + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + # Perform staging that is cycle dependent + StageJediCycle-{{model_component}} -class Workflow_hofx(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for executing JEDI-based h(x) - """) + # Run Jedi hofx executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediHofxExecutable-{{model_component}} + CloneJedi[^] => StageJediCycle-{{model_component}} + StageJediCycle-{{model_component}} => RunJediHofxExecutable-{{model_component}} + GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediHofxExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediHofxExecutable-{{model_component}} + GenerateObservingSystemRecords-{{model_component}} => RunJediHofxExecutable-{{model_component}} - return description + # EvaObservations + RunJediHofxExecutable-{{model_component}} => EvaObservations-{{model_component}} - # -------------------------------------------------------------------------------------------------- + # Save observations + RunJediHofxExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' + # Clean up large files + EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} - # Define the string for the R1 (first non-cycling) section - r1 = r1_template + {% endif %} + {% endfor %} + """ + {% endfor %} - for model_component in self.experiment_dict['model_components']: - r1 += r1_model.format(model_component=model_component) - - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) +# -------------------------------------------------------------------------------------------------- +''' - # Format the string for each cycle - for model_component in self.experiment_dict['model_components']: - if 'cycle_times' in self.experiment_dict['models'][model_component]: - for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = cycle_template_1.format(model_component=model_component) +# -------------------------------------------------------------------------------------------------- - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) +class Workflow_hofx(CylcWorkflow): - return graph_section + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 738afdbad..909425cac 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -7,142 +7,123 @@ # -------------------------------------------------------------------------------------------------- +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow # -------------------------------------------------------------------------------------------------- -r1_template = """ -# Triggers for non cycle time dependent tasks -# ------------------------------------------- -# Clone JEDI source code -CloneJedi +template_str = ''' -# Build JEDI source code by linking -CloneJedi => BuildJediByLinking? - -# If not able to link to build create the build -BuildJediByLinking:fail? => BuildJedi -""" - -r1_model = """ - -# Clone geos ana for generating observing system records -CloneGeosMksi-{model_component} -""" - -cycle_template_1 = """ -# Task triggers for: {model_component} -# ------------------ - -# Perform staging that is cycle dependent -BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{model_component} => sync_point - -GetObservations-{model_component} => sync_point - -CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} => sync_point +# -------------------------------------------------------------------------------------------------- -GetEnsembleGeosExperiment-{model_component} => sync_point +# Cylc suite for executing JEDI-based LocalEnsembleDA Algorithm -sync_point => RunJediObsfiltersExecutable-{{model_component}} -""" +# -------------------------------------------------------------------------------------------------- -cycle_template_2 = """ -sync_point => RunJediObsfiltersExecutable-{{model_component}} => -RunJediLocalEnsembleDaExecutable-{model_component} -""" +[scheduler] + UTC mode = True + allow implicit tasks = False -cycle_template_3 = """ -sync_point => RunJediEnsembleMeanVariance-{model_component} => -RunJediHofxEnsembleExecutable-{model_component} +# -------------------------------------------------------------------------------------------------- -RunJediHofxEnsembleExecutable-{model_component} => -RunJediLocalEnsembleDaExecutable-{model_component} -""" +[scheduling] -cycle_template_4 = """ -# When strategy is parallel, only proceed if all RunJediHofxEnsembleExecutable completes -# successfully for each packet + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} -# There is a need for a task to combine all hofx observations together, compute node preferred, -# put here as placeholder + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi -# RunJediHofxEnsembleExecutable-{model_component}_pack{packet} => -# RunEnsembleHofxCombiner-{model_component} + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? -# RunEnsembleHofxCombiner-{model_component} => -# RunJediLocalEnsembleDaExecutable-{model_component} + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi -sync_point => RunJediHofxEnsembleExecutable-{model_component}_pack{packet} -RunJediHofxEnsembleExecutable-{model_component}_pack{packet} => -RunJediLocalEnsembleDaExecutable-{model_component} -""" + {% for model_component in model_components %} + # Clone geos ana for generating observing system records + CloneGeosMksi-{{model_component}} + {% endfor %} + """ -cycle_template_5 = """ -# EvaIncrement -RunJediLocalEnsembleDaExecutable-{model_component} => EvaIncrement-{model_component} + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + # Task triggers for: {{model_component}} + # ------------------ -# EvaObservations -# RunJediLocalEnsembleDaExecutable-{model_component} => EvaObservations-{model_component} + # Perform staging that is cycle dependent + BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{{model_component}} => sync_point -# Save observations -# RunJediLocalEnsembleDaExecutable-{model_component} => SaveObsDiags-{model_component} + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} -# Clean up large files -# EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} & -EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => sync_point -""" + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} => sync_point -# -------------------------------------------------------------------------------------------------- + GetEnsembleGeosExperiment-{{model_component}} => sync_point + sync_point => RunJediObsfiltersExecutable-{{model_component}} + {% if skip_ensemble_hofx %} + sync_point => RunJediObsfiltersExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + {% else %} + # Run hofx for ensemble members according to strategy + {% if ensemble_hofx_strategy == 'serial' %} + sync_point => RunJediEnsembleMeanVariance-{{model_component}} => RunJediHofxEnsembleExecutable-{{model_component}} + RunJediHofxEnsembleExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} -class Workflow_localensembleda(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for executing JEDI-based h(x) - """) + {% elif ensemble_hofx_strategy == 'parallel' %} + {% for packet in range(ensemble_hofx_packets) %} + # When strategy is parallel, only proceed if all RunJediHofxEnsembleExecutable completes successfully for each packet - return description + # There is a need for a task to combine all hofx observations together, compute node preferred, put here as placeholder + # RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunEnsembleHofxCombiner-{{model_component}} + # RunEnsembleHofxCombiner-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} - # -------------------------------------------------------------------------------------------------- + sync_point => RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} + RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunJediLocalEnsembleDaExecutable-{{model_component}} + {% endfor %} + {% endif %} + {% endif %} - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' - # Define the string for the R1 (first non-cycling) section - r1 = r1_template + # EvaIncrement + RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaIncrement-{{model_component}} - for model_component in self.experiment_dict['model_components']: - r1 += r1_model.format(model_component=model_component) + # EvaObservations + # RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaObservations-{{model_component}} - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) + # Save observations + # RunJediLocalEnsembleDaExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - # Format the string for each cycle - for model_component in self.experiment_dict['model_components']: - if 'cycle_times' in self.experiment_dict['models'][model_component]: - for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = cycle_template_1.format(model_component=model_component) + # Clean up large files + # EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} & + EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} - if self.experiment_dict['models'][model_component]['skip_ensemble_hofx']: - cycle_str += cycle_template_2.format(model_component=model_component) - else: - if self.experiment_dict['ensemble_hofx_strategy'] == 'serial': - cycle_str += cycle_template_3.format(model_component=model_component) - elif self.experiment_dict['ensemble_hofx_strategy'] == 'parallel': - for packet in range(self.experiment_dict['ensemble_hofx_packets']): - cycle_str += cycle_template_4.format( - model_component=model_component, packet=packet) + {% endif %} + {% endfor %} + """ + {% endfor %} +''' - cycle_str += cycle_template_5.format(model_component=model_component) +# -------------------------------------------------------------------------------------------------- - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) +class Workflow_localensembleda(CylcWorkflow): - return graph_section + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str -# -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index 1de3395e2..1ae4d99b9 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -7,96 +7,95 @@ # -------------------------------------------------------------------------------------------------- +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow # -------------------------------------------------------------------------------------------------- -r1_template = """ -# Triggers for non cycle time dependent tasks -# ------------------------------------------- -# Clone JEDI source code -CloneJedi +template_str = ''' +# -------------------------------------------------------------------------------------------------- -# Build JEDI source code by linking -CloneJedi => BuildJediByLinking? +# Cylc suite for executing geos_atmosphere ObsFilters tests -# If not able to link to build create the build -BuildJediByLinking:fail? => BuildJedi -""" +# -------------------------------------------------------------------------------------------------- -r1_model = """ +[scheduler] + UTC mode = True + allow implicit tasks = False -# Clone geos ana for generating observing system records -CloneGeosMksi-{model_component} -""" +# -------------------------------------------------------------------------------------------------- -cycle_template_1 = """ -# Generate satellite channel records -CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} +[scheduling] -# Convert bias correction to ioda -GetGsiBc-{model_component} -GetGsiBc-{model_component} => GsiBcToIoda-{model_component} -BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda-{model_component} + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} -# Convert ncdiags to ioda -GetGsiNcdiag-{model_component} -GetGsiNcdiag-{model_component} => GsiNcdiagToIoda-{model_component} -BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda-{model_component} + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi -GetGeovals-{model_component} + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? -# Run Jedi hofx executable -GenerateObservingSystemRecords-{model_component} => RunJediUfoTestsExecutable-{model_component} -GsiNcdiagToIoda-{model_component} => RunJediUfoTestsExecutable-{model_component} -GsiBcToIoda-{model_component} => RunJediUfoTestsExecutable-{model_component} -GetGeovals-{model_component} => RunJediUfoTestsExecutable-{model_component} + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi -# EvaObservations -RunJediUfoTestsExecutable-{model_component} => EvaObservations-{model_component} + # Clone geos ana for generating observing system records + CloneGeosMksi + """ -# Clean up large files -EvaObservations-{model_component} => CleanCycle-{model_component} -""" + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ -# -------------------------------------------------------------------------------------------------- + # Generate satellite channel records + CloneGeosMksi[^] => GenerateObservingSystemRecords + # Convert bias correction to ioda + GetGsiBc + GetGsiBc => GsiBcToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda -class Workflow_ufo_testing(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for executing geos_atmosphere ObsFilters tests - """) + # Convert ncdiags to ioda + GetGsiNcdiag + GetGsiNcdiag => GsiNcdiagToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda - return description + GetGeovals - # -------------------------------------------------------------------------------------------------- + # Run Jedi hofx executable + GenerateObservingSystemRecords => RunJediUfoTestsExecutable + GsiNcdiagToIoda => RunJediUfoTestsExecutable + GsiBcToIoda => RunJediUfoTestsExecutable + GetGeovals => RunJediUfoTestsExecutable - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' + # EvaObservations + RunJediUfoTestsExecutable => EvaObservations - # Define the string for the R1 (first non-cycling) section - r1 = r1_template + # Clean up large files + EvaObservations => CleanCycle - for model_component in self.experiment_dict['models']: - r1 += r1_model.format(model_component=model_component) + """ + {% endfor %} - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) +# -------------------------------------------------------------------------------------------------- +''' - # Format the string for each cycle - for model_component in self.experiment_dict['models']: - if 'cycle_times' in self.experiment_dict['models'][model_component].keys(): - for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = cycle_template_1.format(model_component=model_component) +# -------------------------------------------------------------------------------------------------- - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) +class Workflow_ufo_testing(CylcWorkflow): - return graph_section + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- From 76ead50f36f004e266bfa00a84febfda2931ed8d Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 10 Oct 2025 10:47:19 -0400 Subject: [PATCH 107/299] fix cycle_times --- src/swell/deployment/create_experiment.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 0a75ce561..6324d3d50 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -121,6 +121,12 @@ def prepare_config( # ------------------------------------------------------------------------- workflow_string = workflow.get_workflow_str() + # Separate dictionary for rendering + # --------------------------------- + render_template = experiment_dict.copy() + + # Resolve cycle times for models + # ------------------------------ if 'model_components' in experiment_dict: model_components = experiment_dict['model_components'] @@ -144,14 +150,14 @@ def prepare_config( cycle_time_dict[model_component] = True cycle_times_dict_list.append(cycle_time_dict) - experiment_dict['cycle_times'] = cycle_times_dict_list + render_template['cycle_times'] = cycle_times_dict_list # Otherwise check that experiment_dict has cycle_times elif 'cycle_times' in experiment_dict: cycle_times = list(set(experiment_dict['cycle_times'])) cycle_times.sort() - experiment_dict['cycle_times'] = cycle_times + render_template['cycle_times'] = cycle_times else: @@ -162,7 +168,7 @@ def prepare_config( # Update the workflow with the answered task questions # ---------------------------------------------------- - workflow.experiment_dict = experiment_dict + workflow.experiment_dict = render_template # Expand all environment vars in the dictionary # --------------------------------------------- @@ -420,4 +426,4 @@ def create_modules_csh( modules_file_open.write(modules_file_line) -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- From f29416bb2228d23830f53ef4bfbdf60c4642c818 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 10 Oct 2025 11:04:36 -0400 Subject: [PATCH 108/299] revert workflows --- src/swell/suites/3dfgat_atmos/workflow.py | 172 +++++++++--------- src/swell/suites/3dfgat_cycle/workflow.py | 204 +++++++++------------- src/swell/suites/3dvar/workflow.py | 9 +- src/swell/suites/3dvar_atmos/workflow.py | 9 +- src/swell/suites/3dvar_cycle/workflow.py | 7 +- 5 files changed, 177 insertions(+), 224 deletions(-) diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index efe0c408c..4cf9229a4 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -7,127 +7,121 @@ # -------------------------------------------------------------------------------------------------- +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow # -------------------------------------------------------------------------------------------------- -r1_template = """ -# Triggers for non cycle time dependent tasks -# ------------------------------------------- -# Clone JEDI source code -CloneJedi - -# Build JEDI source code by linking -CloneJedi => BuildJediByLinking? +template_str = ''' +# -------------------------------------------------------------------------------------------------- -# If not able to link to build create the build -BuildJediByLinking:fail? => BuildJedi -""" +# Cylc suite for executing JEDI-based non-cycling variational data assimilation -r1_model = """ -# Clone geos ana for generating observing system records -CloneGeosMksi-{model_component} -""" +# -------------------------------------------------------------------------------------------------- -cycle_template_1 = """ -# Task triggers for: {model_component} -# ------------------ -# Generate satellite channel records -CloneGeosMksi-{model_component}[^] => GenerateObservingSystemRecords-{model_component} +[scheduler] + UTC mode = True + allow implicit tasks = False -# Get background, provide a way to get background directly from GEOS experiment -GetBackgroundGeosExperiment-{model_component} :fail? => GetBackground-{model_component} +# -------------------------------------------------------------------------------------------------- -# Get observations -""" +[scheduling] -cycle_template_2 = """ -# Cycling VarBC is active, biases from the previous cycle will be used + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} -RunJediVariationalExecutable-{model_component}[-PT6H] => GetObservations-{model_component} -""" + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi -cycle_template_3 = """ -# Cycling VarBC is inactive, static bias files will be used -GetObservations-{model_component} -""" + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? -cycling_template_4 = """ -# Perform staging that is cycle dependent -StageJediCycle-{model_component} + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi -# Run Jedi variational executable -BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{model_component} -CloneJedi[^] => StageJediCycle-{model_component} -StageJediCycle-{model_component} => RunJediVariationalExecutable-{model_component} -GetBackgroundGeosExperiment-{model_component}? | GetBackground-{model_component} => -RunJediVariationalExecutable-{model_component} + {% for model_component in model_components %} + # Clone geos ana for generating observing system records + CloneGeosMksi-{{model_component}} + {% endfor %} + """ -GetObservations-{model_component} => RunJediVariationalExecutable-{model_component} -GenerateObservingSystemRecords-{model_component} => RunJediVariationalExecutable-{model_component} + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} -# EvaObservations -RunJediVariationalExecutable-{model_component} => EvaObservations-{model_component} + # Task triggers for: {{model_component}} + # ------------------ + # Generate satellite channel records + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} -# EvaJediLog -RunJediVariationalExecutable-{model_component} => EvaJediLog-{model_component} + # Get background, provide a way to get background directly from GEOS experiment + GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} -# EvaIncrement -RunJediVariationalExecutable-{model_component} => EvaIncrement-{model_component} + # Get observations + {% if cycling_varbc %} + # Cycling VarBC is active, biases from the previous cycle will be used -# Save observations -RunJediVariationalExecutable-{model_component} => SaveObsDiags-{model_component} + RunJediVariationalExecutable-{{model_component}}[-PT6H] => GetObservations-{{model_component}} + {% else %} -# Clean up large files -EvaObservations-{model_component} & SaveObsDiags-{model_component} => -CleanCycle-{model_component} -""" + # Cycling VarBC is inactive, static bias files will be used + GetObservations-{{model_component}} + {% endif %} -# -------------------------------------------------------------------------------------------------- + # Perform staging that is cycle dependent + StageJediCycle-{{model_component}} + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} + CloneJedi[^] => StageJediCycle-{{model_component}} + StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GenerateObservingSystemRecords-{{model_component}} => RunJediVariationalExecutable-{{model_component}} -class Workflow_3dfgat_atmos(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for executing JEDI-based non-cycling variational data assimilation - """) + # EvaObservations + RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} - return description + # EvaJediLog + RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} - # -------------------------------------------------------------------------------------------------- + # EvaIncrement + RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' + # Save observations + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - # Define the string for the R1 (first non-cycling) section - r1 = r1_template + # Clean up large files + EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} - for model_component in self.experiment_dict['model_components']: - r1 += r1_model.format(model_component=model_component) + {% endif %} + {% endfor %} + """ + {% endfor %} - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) - - # Format the string for each cycle - for model_component in self.experiment_dict['model_components']: - if 'cycle_times' in self.experiment_dict['models'][model_component]: - for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = cycle_template_1.format(model_component=model_component) - - if self.experiment_dict['models'][model_component]['cycling_varbc']: - cycle_str += cycle_template_2.format(model_component=model_component) - else: - cycle_str += cycle_template_3.format(model_component=model_component) +# -------------------------------------------------------------------------------------------------- +''' - cycle_str += cycling_template_4.format(model_component=model_component) +# -------------------------------------------------------------------------------------------------- - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) +class Workflow_3dfgat_atmos(CylcWorkflow): - return graph_section + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index bd747bf8d..5c22c9088 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -7,163 +7,121 @@ # -------------------------------------------------------------------------------------------------- +from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow # -------------------------------------------------------------------------------------------------- -r1_template = """ -# Triggers for non cycle time dependent tasks -# ------------------------------------------- -# Clone Geos source code -CloneGeos - -# Clone JEDI source code -CloneJedi +template_str = ''' +# -------------------------------------------------------------------------------------------------- -# Build Geos source code by linking -CloneGeos => BuildGeosByLinking? +# Cylc suite for executing JEDI-based non-cycling variational data assimilation -# Build JEDI source code by linking -CloneJedi => BuildJediByLinking? +# -------------------------------------------------------------------------------------------------- -# If not able to link to build create the build -BuildGeosByLinking:fail? => BuildGeos +[scheduler] + UTC mode = True + allow implicit tasks = False -# If not able to link to build create the build -BuildJediByLinking:fail? => BuildJedi +# -------------------------------------------------------------------------------------------------- -# Need first set of restarts to run model -GetGeosRestart => PrepGeosRunDir +[scheduling] -# Model cannot run without code -BuildGeosByLinking? | BuildGeos => RunGeosExecutable -""" + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} -r1_model = """ -# JEDI cannot run without code -BuildJediByLinking? | BuildJedi => RunJediFgatExecutable-{model_component} + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi -# Stage JEDI static files -CloneJedi => StageJedi-{model_component} => RunJediFgatExecutable-{model_component} -""" + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? -cycle_template_1 = """ -# Model preperation -# Run the forecast through two windows (need to output restarts at the end of the -# first window and backgrounds for the second window) -# MoveDaRestart-{model_component}[-P1D] => PrepGeosRunDir -MoveDaRestart-{model_component}[-{models[model_component]["window_length"]}] => PrepGeosRunDir -PrepGeosRunDir => RunGeosExecutable + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi -# Run the analysis -# RunGeosExecutable => StageJediCycle-{model_component} -RunGeosExecutable => LinkGeosOutput-{model_component} -LinkGeosOutput-{model_component} => GenerateBClimatology-{model_component} + {% for model_component in model_components %} + # Clone geos ana for generating observing system records + CloneGeosMksi-{{model_component}} + {% endfor %} + """ -# Data assimilation preperation -GetObservations-{model_component} -GenerateBClimatologyByLinking-{model_component} :fail? => -GenerateBClimatology-{model_component} + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} -LinkGeosOutput-{model_component} => RunJediFgatExecutable-{model_component} -StageJediCycle-{model_component} => RunJediFgatExecutable-{model_component} -GenerateBClimatologyByLinking-{model_component}? | -GenerateBClimatology-{model_component} => RunJediFgatExecutable-{model_component} + # Task triggers for: {{model_component}} + # ------------------ + # Generate satellite channel records + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} -GetObservations-{model_component} => RunJediFgatExecutable-{model_component} + # Get background, provide a way to get background directly from GEOS experiment + GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} -# Run analysis diagnostics -RunJediFgatExecutable-{model_component} => EvaObservations-{model_component} -RunJediFgatExecutable-{model_component} => EvaJediLog-{model_component} -EvaIncrement-{model_component} => PrepareAnalysis-{model_component} + # Get observations + {% if cycling_varbc %} + # Cycling VarBC is active, biases from the previous cycle will be used -# Prepare analysis for next forecast -RunJediFgatExecutable-{model_component} => EvaIncrement-{model_component} -""" + RunJediVariationalExecutable-{{model_component}}[-PT6H] => GetObservations-{{model_component}} + {% else %} -cycle_template_2 = """ -PrepareAnalysis-{model_component} => -RunJediConvertStateSoca2ciceExecutable-{model_component} + # Cycling VarBC is inactive, static bias files will be used + GetObservations-{{model_component}} + {% endif %} -RunJediConvertStateSoca2ciceExecutable-{model_component} => -SaveRestart-{model_component} + # Perform staging that is cycle dependent + StageJediCycle-{{model_component}} -RunJediConvertStateSoca2ciceExecutable-{model_component} => -CleanCycle-{model_component} -""" + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} + CloneJedi[^] => StageJediCycle-{{model_component}} + StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GenerateObservingSystemRecords-{{model_component}} => RunJediVariationalExecutable-{{model_component}} -cycle_template_3 = """ -PrepareAnalysis-{model_component} => SaveRestart-{model_component} -""" + # EvaObservations + RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} -cycle_template_4 = """ -# Move restart to next cycle -SaveRestart-{model_component} => MoveDaRestart-{model_component} + # EvaJediLog + RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} -# Save analysis output -# RunJediFgatExecutable-{model_component} => SaveAnalysis-{model_component} -RunJediFgatExecutable-{model_component} => SaveObsDiags-{model_component} + # EvaIncrement + RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} -# Save model output -# MoveBackground-{model_component} => StoreBackground-{model_component} + # Save observations + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} -# Remove Run Directory -# MoveDaRestart-{model_component} & MoveBackground-{model_component} => -RemoveForecastDir + # Clean up large files + EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} -MoveDaRestart-{model_component} => RemoveForecastDir + {% endif %} + {% endfor %} + """ + {% endfor %} -# Clean up large files -EvaObservations-{model_component} & EvaJediLog-{model_component} & -EvaIncrement-{model_component} & SaveObsDiags-{model_component} => -CleanCycle-{model_component} -""" +# -------------------------------------------------------------------------------------------------- +''' # -------------------------------------------------------------------------------------------------- class Workflow_3dfgat_cycle(CylcWorkflow): - def define_description(self): - description = self.comment_block(""" - # Cylc suite for executing Geos forecast - """) - - return description - - # -------------------------------------------------------------------------------------------------- - - def define_graph_section(self): - # Define the string of the graph section - graph_str = '' - - # Define the string for the R1 (first non-cycling) section - r1 = r1_template - - for model_component in self.experiment_dict['model_components']: - r1 += r1_model.format(model_component=model_component) - - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) - - # Format the string for each cycle - for model_component in self.experiment_dict['model_components']: - if 'cycle_times' in self.experiment_dict['models'][model_component]: - for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - - cycle_str = cycle_template_1.format(model_component=model_component) - if 'cice6' in self.experiment_dict['models'][model_component]['marine_models']: - cycle_str += cycle_template_2.format(model_component=model_component) - else: - cycle_str += cycle_template_3.format(model_component=model_component) - - cycle_str += cycle_template_4.format(model_component=model_component) - - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) - - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) - return graph_section + def define_initial_workflow(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index a698d8939..37df877d9 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -50,10 +50,10 @@ {% endfor %} """ + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ {% for model_component in model_components %} - {% if models[model_component]['cycle_times'] %} - {% for cycle_time in models[model_component]['cycle_times'] %} - {{cycle_time}} = """ + {% if cycle_time[model_component] %} # Task triggers for: {{model_component}} # ------------------ # Get background @@ -92,8 +92,9 @@ # Clean up large files EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} - {% endfor %} + {% endif %} + {% endfor %} """ {% endfor %} diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index e791439bb..34d9756df 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -50,10 +50,11 @@ {% endfor %} """ + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ {% for model_component in model_components %} - {% if models[model_component]['cycle_times'] %} - {% for cycle_time in models[model_component]['cycle_times'] %} - {{cycle_time}} = """ + {% if cycle_time[model_component] %} + # Task triggers for: {{model_component}} # ------------------ # Generate satellite channel records @@ -100,8 +101,8 @@ EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} - {% endfor %} {% endif %} + {% endfor %} """ {% endfor %} diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index 797669c57..5deef2d42 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -69,10 +69,10 @@ {% endfor %} """ + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ {% for model_component in model_components %} - {% if models[model_component]['cycle_times'] %} - {% for cycle_time in models[model_component]['cycle_times'] %} - {{cycle_time}} = """ + # Model things # Run the forecast through two windows (need to output restarts at the end of the # first window and backgrounds for the second window) @@ -127,7 +127,6 @@ EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} {% endfor %} - {% endif %} """ {% endfor %} # -------------------------------------------------------------------------------------------------- From e9f8fcf2ac2a094d4342b4d7cc94519d8ed3b598 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 10 Oct 2025 12:29:41 -0400 Subject: [PATCH 109/299] create experiment --- src/swell/deployment/create_experiment.py | 7 ------- src/swell/tasks/task_runtimes.py | 5 +++++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 6324d3d50..bdcbdb9f4 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -159,13 +159,6 @@ def prepare_config( cycle_times.sort() render_template['cycle_times'] = cycle_times - else: - - # Otherwise use logger to abort - logger.abort('The suite file required cycle_times but there are no model components ' + - 'to gather them from or they are not provided in the experiment ' + - 'dictionary.') - # Update the workflow with the answered task questions # ---------------------------------------------------- workflow.experiment_dict = render_template diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index f4c3770d9..e876ef960 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -143,6 +143,11 @@ class GsiNcdiagToIoda(Task): is_cycling: bool = True is_model: bool = True + @dataclass + class GetNcdiags(Task): + is_cycling: bool = True + is_model: bool = True + @dataclass class GetGeosAdasBackground(Task): is_cycling: bool = True From 2b5d3859012cbeef36cc61c0aa5ae2a5b777561a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 10 Oct 2025 12:32:50 -0400 Subject: [PATCH 110/299] fix 3dfgat_cycle --- src/swell/suites/3dfgat_cycle/workflow.py | 106 +++++++++++++--------- 1 file changed, 64 insertions(+), 42 deletions(-) diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index 5c22c9088..da8c7b32a 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -15,7 +15,7 @@ template_str = ''' # -------------------------------------------------------------------------------------------------- -# Cylc suite for executing JEDI-based non-cycling variational data assimilation +# Cylc suite for executing Geos forecast # -------------------------------------------------------------------------------------------------- @@ -35,78 +35,100 @@ R1 = """ # Triggers for non cycle time dependent tasks # ------------------------------------------- + # Clone Geos source code + CloneGeos + # Clone JEDI source code CloneJedi + # Build Geos source code by linking + CloneGeos => BuildGeosByLinking? + # Build JEDI source code by linking CloneJedi => BuildJediByLinking? + # If not able to link to build create the build + BuildGeosByLinking:fail? => BuildGeos + # If not able to link to build create the build BuildJediByLinking:fail? => BuildJedi + # Need first set of restarts to run model + GetGeosRestart => PrepGeosRunDir + + # Model cannot run without code + BuildGeosByLinking? | BuildGeos => RunGeosExecutable + {% for model_component in model_components %} - # Clone geos ana for generating observing system records - CloneGeosMksi-{{model_component}} + + # JEDI cannot run without code + BuildJediByLinking? | BuildJedi => RunJediFgatExecutable-{{model_component}} + + # Stage JEDI static files + CloneJedi => StageJedi-{{model_component}} => RunJediFgatExecutable-{{model_component}} + {% endfor %} """ {% for cycle_time in cycle_times %} {{cycle_time.cycle_time}} = """ {% for model_component in model_components %} - {% if cycle_time[model_component] %} - # Task triggers for: {{model_component}} - # ------------------ - # Generate satellite channel records - CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} + # Model preperation + # Run the forecast through two windows (need to output restarts at the end of the + # first window and backgrounds for the second window) + MoveDaRestart-{{model_component}}[-{{models[model_component]["window_length"]}}] => PrepGeosRunDir + PrepGeosRunDir => RunGeosExecutable - # Get background, provide a way to get background directly from GEOS experiment - GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}} + # Run the analysis + # RunGeosExecutable => StageJediCycle-{{model_component}} + RunGeosExecutable => LinkGeosOutput-{{model_component}} + LinkGeosOutput-{{model_component}} => GenerateBClimatology-{{model_component}} - # Get observations - {% if cycling_varbc %} - # Cycling VarBC is active, biases from the previous cycle will be used - - RunJediVariationalExecutable-{{model_component}}[-PT6H] => GetObservations-{{model_component}} - {% else %} - - # Cycling VarBC is inactive, static bias files will be used + # Data assimilation preperation GetObservations-{{model_component}} + GenerateBClimatologyByLinking-{{model_component}} :fail? => GenerateBClimatology-{{model_component}} + + LinkGeosOutput-{{model_component}} => RunJediFgatExecutable-{{model_component}} + StageJediCycle-{{model_component}} => RunJediFgatExecutable-{{model_component}} + GenerateBClimatologyByLinking-{{model_component}}? | GenerateBClimatology-{{model_component}} => RunJediFgatExecutable-{{model_component}} + GetObservations-{{model_component}} => RunJediFgatExecutable-{{model_component}} + + # Run analysis diagnostics + RunJediFgatExecutable-{{model_component}} => EvaObservations-{{model_component}} + RunJediFgatExecutable-{{model_component}} => EvaJediLog-{{model_component}} + EvaIncrement-{{model_component}} => PrepareAnalysis-{{model_component}} + + # Prepare analysis for next forecast + RunJediFgatExecutable-{{model_component}} => EvaIncrement-{{model_component}} + {% if 'cice6' in models[model_component]["marine_models"] %} + PrepareAnalysis-{{model_component}} => RunJediConvertStateSoca2ciceExecutable-{{model_component}} + RunJediConvertStateSoca2ciceExecutable-{{model_component}} => SaveRestart-{{model_component}} + RunJediConvertStateSoca2ciceExecutable-{{model_component}} => CleanCycle-{{model_component}} + {% else %} + PrepareAnalysis-{{model_component}} => SaveRestart-{{model_component}} {% endif %} - # Perform staging that is cycle dependent - StageJediCycle-{{model_component}} + # Move restart to next cycle + SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} - # Run Jedi variational executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} - CloneJedi[^] => StageJediCycle-{{model_component}} - StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} - GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GenerateObservingSystemRecords-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + # Save analysis output + # RunJediFgatExecutable-{{model_component}} => SaveAnalysis-{{model_component}} + RunJediFgatExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - # EvaObservations - RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} + # Save model output + # MoveBackground-{{model_component}} => StoreBackground-{{model_component}} - # EvaJediLog - RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} - - # EvaIncrement - RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - - # Save observations - RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + # Remove Run Directory + # MoveDaRestart-{{model_component}} & MoveBackground-{{model_component}} => RemoveForecastDir + MoveDaRestart-{{model_component}} => RemoveForecastDir # Clean up large files - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} - - {% endif %} {% endfor %} """ {% endfor %} - # -------------------------------------------------------------------------------------------------- ''' From b1ea5e1d84233140f57c1ee6813545c2a515d036 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 10 Oct 2025 13:05:35 -0400 Subject: [PATCH 111/299] code test fixes --- src/swell/deployment/create_experiment.py | 2 +- src/swell/suites/3dfgat_atmos/workflow.py | 15 ++++++++++----- src/swell/suites/3dfgat_cycle/workflow.py | 4 ++-- src/swell/suites/3dvar/workflow.py | 4 ++-- src/swell/suites/3dvar_atmos/workflow.py | 4 ++-- src/swell/suites/3dvar_cycle/workflow.py | 4 ++-- src/swell/suites/build_geos/workflow.py | 2 +- src/swell/suites/build_jedi/workflow.py | 2 +- src/swell/suites/compare_variational/workflow.py | 6 +++--- src/swell/suites/convert_ncdiags/workflow.py | 2 +- src/swell/suites/eva_capabilities/workflow.py | 4 ++-- src/swell/suites/forecast_geos/workflow.py | 2 +- src/swell/suites/geosadas/workflow.py | 2 +- src/swell/suites/hofx/workflow.py | 4 ++-- src/swell/suites/localensembleda/workflow.py | 6 +++--- src/swell/suites/ufo_testing/workflow.py | 2 +- src/swell/utilities/cylc_workflow.py | 4 +--- src/swell/utilities/scripts/compare_questions.py | 2 +- 18 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index bdcbdb9f4..bb293c646 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -132,7 +132,7 @@ def prepare_config( # Since cycle times are used, the render_dictionary will need to include cycle_times # If there are different model components then process each to gather cycle times - if len(model_components) > 0 and all('cycle_times' in experiment_dict['models'][model] \ + if len(model_components) > 0 and all('cycle_times' in experiment_dict['models'][model] for model in model_components): cycle_times = [] for model_component in model_components: diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 4cf9229a4..9ce9c2f62 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -81,10 +81,15 @@ BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} CloneJedi[^] => StageJediCycle-{{model_component}} StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => + RunJediVariationalExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} - GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GenerateObservingSystemRecords-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => + RunJediVariationalExecutable-{{model_component}} + + GenerateObservingSystemRecords-{{model_component}} => + RunJediVariationalExecutable-{{model_component}} # EvaObservations RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} @@ -108,7 +113,7 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- -''' +''' # noqa # -------------------------------------------------------------------------------------------------- @@ -121,7 +126,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index da8c7b32a..d7bdcf3d7 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -130,7 +130,7 @@ """ {% endfor %} # -------------------------------------------------------------------------------------------------- -''' +''' # noqa # -------------------------------------------------------------------------------------------------- @@ -143,7 +143,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 37df877d9..c277eef0d 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -99,7 +99,7 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- -''' +''' # noqa # -------------------------------------------------------------------------------------------------- @@ -112,7 +112,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index 34d9756df..64b0c49b2 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -107,7 +107,7 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- -''' +''' # noqa # -------------------------------------------------------------------------------------------------- @@ -120,7 +120,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index 5deef2d42..5c578cabc 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -130,7 +130,7 @@ """ {% endfor %} # -------------------------------------------------------------------------------------------------- -''' +''' # noqa # -------------------------------------------------------------------------------------------------- @@ -143,7 +143,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py index c4f36eafe..9ab68c776 100644 --- a/src/swell/suites/build_geos/workflow.py +++ b/src/swell/suites/build_geos/workflow.py @@ -47,7 +47,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index 0ebd0bdec..9b0a6f4dd 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -47,7 +47,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index f93434166..741839b92 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -48,7 +48,7 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- -''' +''' # noqa # -------------------------------------------------------------------------------------------------- @@ -71,7 +71,7 @@ def define_initial_workflow(self): start_cycle_point, final_cycle_point, cycle_times) - + self.experiment_dict['start_cycle_point'] = start_cycle_point self.experiment_dict['final_cycle_point'] = final_cycle_point self.experiment_dict['models'][model]['cycle_times'] = cycle_times @@ -80,7 +80,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index 9286951b7..9a2e49620 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -77,7 +77,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index 2173b96f1..bf6a30933 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -60,7 +60,7 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- -''' +''' # noqa # -------------------------------------------------------------------------------------------------- @@ -73,7 +73,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index a70914e96..998890b23 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -82,7 +82,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index f5395e3d3..ffbc50946 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -80,7 +80,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index 7d0847f32..1dfd099b9 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -93,7 +93,7 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- -''' +''' # noqa # -------------------------------------------------------------------------------------------------- @@ -106,7 +106,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 909425cac..ebb8b0fc6 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -110,7 +110,7 @@ {% endfor %} """ {% endfor %} -''' +''' # noqa # -------------------------------------------------------------------------------------------------- @@ -123,7 +123,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index 1ae4d99b9..b8ffa19dc 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -95,7 +95,7 @@ def define_initial_workflow(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + return workflow_str # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 719cbd2ee..504fba994 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -9,12 +9,9 @@ from typing import Union, Optional, Tuple from abc import abstractmethod -import os -import yaml from swell.utilities.cylc_formatting import CylcSection, indent_lines from swell.tasks.task_runtimes import TaskRuntimes -from swell.utilities.dictionary import update_dict from swell.utilities.logger import get_logger from swell.utilities.jinja2 import template_string_jinja2 @@ -33,6 +30,7 @@ # -------------------------------------------------------------------------------------------------- ''' + class CylcWorkflow(): ''' diff --git a/src/swell/utilities/scripts/compare_questions.py b/src/swell/utilities/scripts/compare_questions.py index a046a54ec..246462ffb 100644 --- a/src/swell/utilities/scripts/compare_questions.py +++ b/src/swell/utilities/scripts/compare_questions.py @@ -66,7 +66,7 @@ def get_scheduling(suite: str): workflow_obj = get_workflow(suite) - scheduling_section = workflow_obj.define_scheduling() + scheduling_section = workflow_obj.define_initial_workflow() return scheduling_section From 36af1d5b1fdc41e4af80bcfe7d2bb8f30229366b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 10 Oct 2025 14:33:57 -0400 Subject: [PATCH 112/299] fix cycles --- src/swell/deployment/create_experiment.py | 78 +++++++++++------------ src/swell/utilities/cylc_workflow.py | 3 +- 2 files changed, 39 insertions(+), 42 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index bb293c646..4e0ce53e2 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -95,6 +95,40 @@ def prepare_config( # -------------------------------------- suite_dict = prepare_config_and_suite.get_experiment_dict() + # Resolve cycle times for models + # ------------------------------ + if 'model_components' in suite_dict: + model_components = suite_dict['model_components'] + + # Since cycle times are used, the render_dictionary will need to include cycle_times + # If there are different model components then process each to gather cycle times + if len(model_components) > 0 and all('cycle_times' in suite_dict['models'][model] + for model in model_components): + cycle_times = [] + for model_component in model_components: + cycle_times_mc = suite_dict['models'][model_component]['cycle_times'] + cycle_times = list(set(cycle_times + cycle_times_mc)) + cycle_times.sort() + + cycle_times_dict_list = [] + for cycle_time in cycle_times: + cycle_time_dict = {} + cycle_time_dict['cycle_time'] = cycle_time + for model_component in model_components: + cycle_time_dict[model_component] = False + if cycle_time in suite_dict['models'][model_component]['cycle_times']: + cycle_time_dict[model_component] = True + cycle_times_dict_list.append(cycle_time_dict) + + suite_dict['cycle_times'] = cycle_times_dict_list + + # Otherwise check that experiment_dict has cycle_times + elif 'cycle_times' in suite_dict: + + cycle_times = list(set(suite_dict['cycle_times'])) + cycle_times.sort() + suite_dict['cycle_times'] = cycle_times + # Get the slurm defaults from the user and platform # ------------------------------------------------- slurm_dict = prepare_slurm_defaults_and_overrides(logger, platform, slurm) @@ -117,51 +151,13 @@ def prepare_config( # ---------------------- experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() - # Finalize the workflow by adding the runtime section, and get the contents - # ------------------------------------------------------------------------- - workflow_string = workflow.get_workflow_str() - # Separate dictionary for rendering # --------------------------------- render_template = experiment_dict.copy() - # Resolve cycle times for models - # ------------------------------ - if 'model_components' in experiment_dict: - model_components = experiment_dict['model_components'] - - # Since cycle times are used, the render_dictionary will need to include cycle_times - # If there are different model components then process each to gather cycle times - if len(model_components) > 0 and all('cycle_times' in experiment_dict['models'][model] - for model in model_components): - cycle_times = [] - for model_component in model_components: - cycle_times_mc = experiment_dict['models'][model_component]['cycle_times'] - cycle_times = list(set(cycle_times + cycle_times_mc)) - cycle_times.sort() - - cycle_times_dict_list = [] - for cycle_time in cycle_times: - cycle_time_dict = {} - cycle_time_dict['cycle_time'] = cycle_time - for model_component in model_components: - cycle_time_dict[model_component] = False - if cycle_time in experiment_dict['models'][model_component]['cycle_times']: - cycle_time_dict[model_component] = True - cycle_times_dict_list.append(cycle_time_dict) - - render_template['cycle_times'] = cycle_times_dict_list - - # Otherwise check that experiment_dict has cycle_times - elif 'cycle_times' in experiment_dict: - - cycle_times = list(set(experiment_dict['cycle_times'])) - cycle_times.sort() - render_template['cycle_times'] = cycle_times - - # Update the workflow with the answered task questions - # ---------------------------------------------------- - workflow.experiment_dict = render_template + # Finalize the workflow by adding the runtime section, and get the contents + # ------------------------------------------------------------------------- + workflow_string = workflow.get_workflow_str() # Expand all environment vars in the dictionary # --------------------------------------------- diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 816d11123..f02790057 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -134,6 +134,8 @@ def parse_graph_for_tasks(self) -> CylcSection: if '[graph]' in line: in_graph = True + tasks = list(set(tasks) - set(['fail'])) + return tasks # -------------------------------------------------------------------------------------------------- @@ -220,7 +222,6 @@ def define_runtime(self) -> str: def get_workflow_str(self) -> str: # Get the whole string to go into the flow.cylc file - workflow_str = self.initial_workflow_str workflow_str = template_string_jinja2(logger=self.logger, templated_string=workflow_str, From c12d2ccbc70278e94c279bfc77ecd22dc4e6ce24 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 10 Oct 2025 15:01:11 -0400 Subject: [PATCH 113/299] fix cycle_times --- src/swell/deployment/create_experiment.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 4e0ce53e2..fd3fcda03 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -95,6 +95,8 @@ def prepare_config( # -------------------------------------- suite_dict = prepare_config_and_suite.get_experiment_dict() + suite_dict = suite_dict.copy() + # Resolve cycle times for models # ------------------------------ if 'model_components' in suite_dict: @@ -122,7 +124,7 @@ def prepare_config( suite_dict['cycle_times'] = cycle_times_dict_list - # Otherwise check that experiment_dict has cycle_times + # Otherwise check that suite_dict has cycle_times elif 'cycle_times' in suite_dict: cycle_times = list(set(suite_dict['cycle_times'])) @@ -151,10 +153,6 @@ def prepare_config( # ---------------------- experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() - # Separate dictionary for rendering - # --------------------------------- - render_template = experiment_dict.copy() - # Finalize the workflow by adding the runtime section, and get the contents # ------------------------------------------------------------------------- workflow_string = workflow.get_workflow_str() From 9baf275980e90347142f67eb61208d11d946b5a0 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 15 Oct 2025 15:41:10 -0400 Subject: [PATCH 114/299] refactor 3dvar --- src/swell/deployment/create_experiment.py | 2 +- src/swell/suites/3dvar/workflow.py | 37 ++++- src/swell/utilities/cylc_runtime.py | 6 +- src/swell/utilities/cylc_workflow.py | 164 +--------------------- 4 files changed, 47 insertions(+), 162 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index fd3fcda03..58e5befe3 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -155,7 +155,7 @@ def prepare_config( # Finalize the workflow by adding the runtime section, and get the contents # ------------------------------------------------------------------------- - workflow_string = workflow.get_workflow_str() + workflow_string = workflow.get_workflow_string() # Expand all environment vars in the dictionary # --------------------------------------------- diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index c277eef0d..3acef638b 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -99,6 +100,12 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # noqa # -------------------------------------------------------------------------------------------------- @@ -106,13 +113,41 @@ class Workflow_3dvar(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = {} + tasks['root'] = tr.root() + tasks['CloneJedi'] = tr.CloneJedi() + tasks['BuildJediByLinking'] = tr.BuildJediByLinking() + tasks['BuildJedi'] = tr.BuildJedi() + + for model in self.experiment_dict['model_components']: + tasks[f'StageJedi-{model}'] = tr.StageJedi(model=model) + tasks[f'GetBackground-{model}'] = tr.GetBackground(model=model) + tasks[f'GetObservations-{model}'] = tr.GetObservations(model=model) + tasks[f'GenerateBClimatologyByLinking-{model}'] = tr.GenerateBClimatologyByLinking(model=model) + tasks[f'GenerateBClimatology-{model}'] = tr.GenerateBClimatology(model=model) + tasks[f'StageJediCycle-{model}'] = tr.StageJedi(model=model) + tasks[f'GetBackground-{model}'] = tr.GetBackground(model=model) + tasks[f'RunJediVariational-{model}'] = tr.RunJediVariationalExecutable(model=model) + tasks[f'EvaObservations-{model}'] = tr.EvaObservations(model=model) + tasks[f'EvaJediLog-{model}'] = tr.EvaJediLog(model=model) + tasks[f'EvaIncrement-{model}'] = tr.EvaIncrement(model=model) + tasks[f'SaveObsDiags-{model}'] = tr.SaveObsDiags(model=model) + tasks[f'CleanCycle-{model}'] = tr.CleanCycle(model=model) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 6b60ddf2f..9fa7cac3f 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -150,7 +150,7 @@ def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Ma # -------------------------------------------------------------------------------------------------- - def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): + def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): ''' Return the runtime section for the given task. ''' platform = experiment_dict['platform'] @@ -243,6 +243,8 @@ def get_section(self, experiment_dict: Mapping, slurm_external: Mapping): event_section = self.create_new_section('events', event_str) runtime_section.add_subsection(event_section) - return runtime_section + runtime_string = runtime_section.get_section_str() + + return runtime_string # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index f02790057..472ad7b82 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -44,9 +44,6 @@ def __init__(self, experiment_dict, slurm_external) -> None: self.logger = get_logger(self.__class__.__name__) - self.initial_workflow_str = self.define_initial_workflow() - self.tasks = self.parse_graph_for_tasks() - # -------------------------------------------------------------------------------------------------- def default_header(self) -> str: @@ -54,92 +51,6 @@ def default_header(self) -> str: # -------------------------------------------------------------------------------------------------- - def format_string_block(self, string) -> str: - # Format a string block with proper indentation - - out_string = '"""\n' - out_string += indent_lines(string, 1, True) - out_string += '"""\n' - - return out_string - - # -------------------------------------------------------------------------------------------------- - - def format_cycle(self, name: str, cycle: str) -> str: - # Format a cycle in the graph section - - cycle_string = f'{name} = ' - cycle_string += self.format_string_block(cycle) - return cycle_string - - # -------------------------------------------------------------------------------------------------- - - def reset_indentation(self, string: str) -> str: - - out_string = '' - - start = False - for line in string.split('\n'): - line = line.strip() - - if len(line) > 0: - start = True - - if start: - out_string += line + '\n' - - return out_string - - # -------------------------------------------------------------------------------------------------- - - @abstractmethod - def define_initial_workflow(self) -> str: - return '' - - # -------------------------------------------------------------------------------------------------- - - def parse_graph_for_tasks(self) -> CylcSection: - # Iterate through the graph section and determine all the tasks used by the suite - - tasks = [] - - cylc_characters = [':', '[', ']', '?'] - - in_graph = False - in_cycle = False - - for line in self.initial_workflow_str.split('\n'): - comment = False - sub_strings = line.split(' ') - - for sub_string in sub_strings: - sub_string = sub_string.strip() - - if sub_string.startswith('#'): - comment = True - - if not comment and in_graph and in_cycle: - if len(sub_string) > 0 and sub_string not in ['=>', '&', '|', '"""']: - task = sub_string - for i, char in enumerate(task): - if char in cylc_characters: - task = task.split(char)[0] - - if len(task) > 0 and task not in tasks: - tasks.append(task) - - if '"""' in sub_string: - in_cycle = not in_cycle - - if '[graph]' in line: - in_graph = True - - tasks = list(set(tasks) - set(['fail'])) - - return tasks - - # -------------------------------------------------------------------------------------------------- - def get_independent_and_model_tasks(self) -> Tuple[list, dict]: # Separate the tasks into model independent and dependent @@ -156,80 +67,17 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: model_tasks[model] = [] for task in self.tasks: - if '-' in task: - task_name = task.split('-')[0] - - for entry in task.split('-'): - if entry in models: - model_tasks[model].append(task_name) - + if task.model is not None: + model_tasks[task.model] = task.base_name else: - ind_tasks.append(task) + ind_tasks.append(task.base_name) return ind_tasks, model_tasks # -------------------------------------------------------------------------------------------------- - def define_runtime_task_overrides(self) -> dict: - # Override in suite file to set any custom runtimes as needed by the suite - return {} - - # -------------------------------------------------------------------------------------------------- - - def create_new_section(self, name: Optional[str] = None, content: Union[str, dict] = ''): - # Create a new section with indentation and content - return CylcSection(name, content) - - # -------------------------------------------------------------------------------------------------- - - def define_runtime(self) -> str: - # Handle adding runtime sections for all tasks - runtime_section = self.create_new_section('runtime', '\n# Task defaults\n# -------------\n') - - # Grab any overrides for certain tasks - runtime_overrides = self.define_runtime_task_overrides() - - for task in ['root'] + self.tasks: - if task in runtime_overrides.keys(): - task_section = runtime_overrides[task].get_section( - self.experiment_dict, self.slurm_external) - - runtime_section.add_subsection(task_section) - - else: - if '-' in task: - task_name = task.split('-')[0] - model = task.split('-')[1] - if 'model_components' not in self.experiment_dict or ( - model not in self.experiment_dict['model_components']): - task_name = task - model = None - else: - task_name = task - model = None - - task_class = TaskRuntimes.get(task_name) - task_section = task_class(model=model).get_section( - self.experiment_dict, self.slurm_external) - - runtime_section.add_subsection(task_section) - - runtime_str = runtime_section.get_section_str() - - return runtime_str - - # -------------------------------------------------------------------------------------------------- - - def get_workflow_str(self) -> str: - # Get the whole string to go into the flow.cylc file - workflow_str = self.initial_workflow_str - workflow_str = template_string_jinja2(logger=self.logger, - templated_string=workflow_str, - dictionary_of_templates=self.experiment_dict, - allow_unresolved=False) - - workflow_str += '\n\n' + self.define_runtime() - - return workflow_str + @abstractmethod + def get_workflow_string(self) -> str: + return '' # -------------------------------------------------------------------------------------------------- From a10fe652255b74cc03011046e261cccd9425843a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 15 Oct 2025 16:15:12 -0400 Subject: [PATCH 115/299] fixes --- src/swell/suites/3dvar/workflow.py | 2 +- src/swell/utilities/cylc_runtime.py | 2 +- src/swell/utilities/cylc_workflow.py | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 3acef638b..f078874a6 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -120,7 +120,7 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task_name, task in self.tasks().items(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 9fa7cac3f..2e6fbb67f 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -243,7 +243,7 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): event_section = self.create_new_section('events', event_str) runtime_section.add_subsection(event_section) - runtime_string = runtime_section.get_section_str() + runtime_string = runtime_section.get_section_str(1) return runtime_string diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 472ad7b82..4f8b8b189 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -66,11 +66,13 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: for model in models: model_tasks[model] = [] - for task in self.tasks: + task_dict = self.tasks() + + for task_name, task in task_dict.items(): if task.model is not None: - model_tasks[task.model] = task.base_name + model_tasks[task.model] = task_name else: - ind_tasks.append(task.base_name) + ind_tasks.append(task_name) return ind_tasks, model_tasks From ca80be79e31579753053f9230bb8c3a4075b05ad Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 16 Oct 2025 10:41:54 -0400 Subject: [PATCH 116/299] updates --- src/swell/suites/3dvar/workflow.py | 2 +- src/swell/utilities/cylc_runtime.py | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index f078874a6..b64cbcbde 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -139,7 +139,7 @@ def tasks(self) -> list: tasks[f'GetObservations-{model}'] = tr.GetObservations(model=model) tasks[f'GenerateBClimatologyByLinking-{model}'] = tr.GenerateBClimatologyByLinking(model=model) tasks[f'GenerateBClimatology-{model}'] = tr.GenerateBClimatology(model=model) - tasks[f'StageJediCycle-{model}'] = tr.StageJedi(model=model) + tasks[f'StageJediCycle-{model}'] = tr.StageJediCycle(model=model) tasks[f'GetBackground-{model}'] = tr.GetBackground(model=model) tasks[f'RunJediVariational-{model}'] = tr.RunJediVariationalExecutable(model=model) tasks[f'EvaObservations-{model}'] = tr.EvaObservations(model=model) diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index 2e6fbb67f..d74284ce1 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -192,7 +192,7 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): runtime_dict['execution retry delays'] = retry runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) - + print(self.scheduling_name) # Set the environment dictionary if self.environment is not None: environment_section = self.create_new_section('environment', self.environment) @@ -217,14 +217,7 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): # Check slurm messaging parameters events = [] - if 'task_email_parameters' in experiment_dict.keys(): - if experiment_dict['task_email_parameters'] == 'auto': - # Set message status to fail or event fail - events = self.mail_events - elif self.scheduling_name in experiment_dict['task_email_parameters'].keys(): - events = experiment_dict['task_email_parameters'][self.scheduling_name] - elif self.base_name in experiment_dict['task_email_parameters'].keys(): - events = experiment_dict['task_email_parameters'][self.base_name] + events = self.mail_events # Add messaging section settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) From a345f53ac56fd0d3f27f780948a4a22d1321957f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 24 Oct 2025 12:11:07 -0400 Subject: [PATCH 117/299] rework --- src/swell/suites/3dvar/workflow.py | 38 ++++++++++++++-------------- src/swell/utilities/cylc_runtime.py | 2 +- src/swell/utilities/cylc_workflow.py | 8 +++--- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index b64cbcbde..ef00dd9bd 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -120,33 +120,33 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task_name, task in self.tasks().items(): + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str def tasks(self) -> list: - tasks = {} - tasks['root'] = tr.root() - tasks['CloneJedi'] = tr.CloneJedi() - tasks['BuildJediByLinking'] = tr.BuildJediByLinking() - tasks['BuildJedi'] = tr.BuildJedi() + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneJedi()) + tasks.append(tr.BuildJediByLinking()) + tasks.append(tr.BuildJedi()) for model in self.experiment_dict['model_components']: - tasks[f'StageJedi-{model}'] = tr.StageJedi(model=model) - tasks[f'GetBackground-{model}'] = tr.GetBackground(model=model) - tasks[f'GetObservations-{model}'] = tr.GetObservations(model=model) - tasks[f'GenerateBClimatologyByLinking-{model}'] = tr.GenerateBClimatologyByLinking(model=model) - tasks[f'GenerateBClimatology-{model}'] = tr.GenerateBClimatology(model=model) - tasks[f'StageJediCycle-{model}'] = tr.StageJediCycle(model=model) - tasks[f'GetBackground-{model}'] = tr.GetBackground(model=model) - tasks[f'RunJediVariational-{model}'] = tr.RunJediVariationalExecutable(model=model) - tasks[f'EvaObservations-{model}'] = tr.EvaObservations(model=model) - tasks[f'EvaJediLog-{model}'] = tr.EvaJediLog(model=model) - tasks[f'EvaIncrement-{model}'] = tr.EvaIncrement(model=model) - tasks[f'SaveObsDiags-{model}'] = tr.SaveObsDiags(model=model) - tasks[f'CleanCycle-{model}'] = tr.CleanCycle(model=model) + tasks.append(tr.StageJedi(model=model)) + tasks.append(tr.GetBackground(model=model)) + tasks.append(tr.GetObservations(model=model)) + tasks.append(tr.GenerateBClimatologyByLinking(model=model)) + tasks.append(tr.GenerateBClimatology(model=model)) + tasks.append(tr.StageJediCycle(model=model)) + tasks.append(tr.GetBackground(model=model)) + tasks.append(tr.RunJediVariationalExecutable(model=model)) + tasks.append(tr.EvaObservations(model=model)) + tasks.append(tr.EvaJediLog(model=model)) + tasks.append(tr.EvaIncrement(model=model)) + tasks.append(tr.SaveObsDiags(model=model)) + tasks.append(tr.CleanCycle(model=model)) return tasks diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py index d74284ce1..7bb3ada50 100644 --- a/src/swell/utilities/cylc_runtime.py +++ b/src/swell/utilities/cylc_runtime.py @@ -192,7 +192,7 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): runtime_dict['execution retry delays'] = retry runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) - print(self.scheduling_name) + # Set the environment dictionary if self.environment is not None: environment_section = self.create_new_section('environment', self.environment) diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 4f8b8b189..7694836db 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -66,13 +66,11 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: for model in models: model_tasks[model] = [] - task_dict = self.tasks() - - for task_name, task in task_dict.items(): + for task in self.tasks(): if task.model is not None: - model_tasks[task.model] = task_name + model_tasks[task.model].append(task.base_name) else: - ind_tasks.append(task_name) + ind_tasks.append(task.base_name) return ind_tasks, model_tasks From 64f6d07afc8d480228efdc5b70a886a654b8a165 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 28 Oct 2025 10:24:33 -0400 Subject: [PATCH 118/299] migrate all files --- src/swell/suites/3dfgat_atmos/workflow.py | 34 +++++++++++++- src/swell/suites/3dfgat_cycle/workflow.py | 44 ++++++++++++++++++- src/swell/suites/3dvar/workflow.py | 1 - src/swell/suites/3dvar_atmos/workflow.py | 32 +++++++++++++- src/swell/suites/3dvar_cycle/workflow.py | 41 ++++++++++++++++- src/swell/suites/build_geos/workflow.py | 16 ++++++- src/swell/suites/build_jedi/workflow.py | 16 ++++++- .../suites/compare_variational/workflow.py | 36 +++++++-------- src/swell/suites/convert_bufr/workflow.py | 24 +++++++++- src/swell/suites/convert_ncdiags/workflow.py | 21 ++++++++- src/swell/suites/eva_capabilities/workflow.py | 20 ++++++++- src/swell/suites/forecast_geos/workflow.py | 22 +++++++++- src/swell/suites/geosadas/workflow.py | 27 +++++++++++- src/swell/suites/hofx/workflow.py | 30 ++++++++++++- src/swell/suites/localensembleda/workflow.py | 33 +++++++++++++- src/swell/suites/ufo_testing/workflow.py | 28 +++++++++++- src/swell/tasks/task_runtimes.py | 14 ++++++ 17 files changed, 404 insertions(+), 35 deletions(-) diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 9ce9c2f62..8440014bd 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -120,13 +121,44 @@ class Workflow_3dfgat_atmos(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneJedi()) + tasks.append(tr.BuildJediByLinking()) + tasks.append(tr.BuildJedi()) + + for model in self.experiment_dict['model_components']: + tasks.append(tr.CloneGeosMksi(model=model)) + tasks.append(tr.StageJedi(model=model)) + tasks.append(tr.GetBackground(model=model)) + tasks.append(tr.GetObservations(model=model)) + tasks.append(tr.GenerateBClimatologyByLinking(model=model)) + tasks.append(tr.GenerateBClimatology(model=model)) + tasks.append(tr.GetObsNotInR2d2(model=model)) + tasks.append(tr.GetBackgroundGeosExperiment(model=model)) + tasks.append(tr.GenerateObservingSystemRecords(model=model)) + tasks.append(tr.StageJediCycle(model=model)) + tasks.append(tr.RunJediVariationalExecutable(model=model)) + tasks.append(tr.EvaObservations(model=model)) + tasks.append(tr.EvaJediLog(model=model)) + tasks.append(tr.EvaIncrement(model=model)) + tasks.append(tr.SaveObsDiags(model=model)) + tasks.append(tr.CleanCycle(model=model)) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index d7bdcf3d7..9b29eff09 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -137,13 +138,54 @@ class Workflow_3dfgat_cycle(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneJedi()) + tasks.append(tr.CloneGeos()) + tasks.append(tr.BuildJediByLinking()) + tasks.append(tr.BuildJedi()) + tasks.append(tr.BuildGeos()) + tasks.append(tr.BuildGeosByLinking()) + + tasks.append(tr.GetGeosRestart()) + tasks.append(tr.PrepGeosRunDir()) + tasks.append(tr.RunGeosExecutable()) + + for model in self.experiment_dict['model_components']: + tasks.append(tr.RunJediFgatExecutable(model=model)) + tasks.append(tr.StageJedi(model=model)) + tasks.append(tr.StageJediCycle(model=model)) + tasks.append(tr.MoveDaRestart(model=model)) + tasks.append(tr.LinkGeosOutput(model=model)) + tasks.append(tr.GenerateBClimatology(model=model)) + tasks.append(tr.GenerateBClimatologyByLinking(model=model)) + tasks.append(tr.GetObservations(model=model)) + tasks.append(tr.EvaObservations(model=model)) + tasks.append(tr.EvaJediLog(model=model)) + tasks.append(tr.EvaIncrement(model=model)) + tasks.append(tr.PrepareAnalysis(model=model)) + tasks.append(tr.RunJediConvertStateSoca2ciceExecutable(model=model)) + tasks.append(tr.SaveRestart(model=model)) + tasks.append(tr.CleanCycle(model=model)) + tasks.append(tr.PrepareAnalysis(model=model)) + tasks.append(tr.RemoveForecastDir(model=model)) + tasks.append(tr.SaveObsDiags(model=model)) + + return tasks + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index ef00dd9bd..5ca2f5a06 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -135,7 +135,6 @@ def tasks(self) -> list: for model in self.experiment_dict['model_components']: tasks.append(tr.StageJedi(model=model)) - tasks.append(tr.GetBackground(model=model)) tasks.append(tr.GetObservations(model=model)) tasks.append(tr.GenerateBClimatologyByLinking(model=model)) tasks.append(tr.GenerateBClimatology(model=model)) diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index 64b0c49b2..a35a37d1a 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -114,13 +115,42 @@ class Workflow_3dvar_atmos(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneJedi()) + tasks.append(tr.BuildJediByLinking()) + tasks.append(tr.BuildJedi()) + + for model in self.experiment_dict['model_components']: + tasks.append(tr.CloneGeosMksi(model=model)) + tasks.append(tr.StageJedi(model=model)) + tasks.append(tr.GetObservations(model=model)) + tasks.append(tr.GenerateBClimatologyByLinking(model=model)) + tasks.append(tr.GenerateBClimatology(model=model)) + tasks.append(tr.GenerateObservingSystemRecords(model=model)) + tasks.append(tr.GetObsNotInR2d2(model=model)) + tasks.append(tr.StageJediCycle(model=model)) + tasks.append(tr.RunJediVariationalExecutable(model=model)) + tasks.append(tr.EvaObservations(model=model)) + tasks.append(tr.EvaJediLog(model=model)) + tasks.append(tr.EvaIncrement(model=model)) + tasks.append(tr.SaveObsDiags(model=model)) + tasks.append(tr.CleanCycle(model=model)) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index 5c578cabc..81c6df20d 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -137,13 +138,51 @@ class Workflow_3dvar_cycle(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneJedi()) + tasks.append(tr.CloneGeos()) + tasks.append(tr.BuildJediByLinking()) + tasks.append(tr.BuildGeosByLinking()) + tasks.append(tr.BuildJedi()) + tasks.append(tr.BuildGeos()) + tasks.append(tr.GetGeosRestart()) + tasks.append(tr.PrepGeosRunDir) + tasks.append(tr.RunGeosExecutable()) + + for model in self.experiment_dict['model_components']: + tasks.append(tr.StageJedi(model=model)) + tasks.append(tr.StageJediCycle(model=model)) + tasks.append(tr.RunJediVariationalExecutable(model=model)) + tasks.append(tr.MoveDaRestart(model=model)) + tasks.append(tr.LinkGeosOutput(model=model)) + tasks.append(tr.GenerateBClimatology(model=model)) + tasks.append(tr.GenerateBClimatologyByLinking(model=model)) + tasks.append(tr.GetObservations(model=model)) + tasks.append(tr.PrepareAnalysis(model=model)) + tasks.append(tr.RunJediConvertStateSoca2ciceExecutable(model=model)) + tasks.append(tr.SaveRestart(model=model)) + tasks.append(tr.RemoveForecastDir(model=model)) + tasks.append(tr.EvaObservations(model=model)) + tasks.append(tr.EvaJediLog(model=model)) + tasks.append(tr.EvaIncrement(model=model)) + tasks.append(tr.SaveObsDiags(model=model)) + tasks.append(tr.CleanCycle(model=model)) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py index 9ab68c776..57ffe01ef 100644 --- a/src/swell/suites/build_geos/workflow.py +++ b/src/swell/suites/build_geos/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -41,13 +42,26 @@ class Workflow_build_geos(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneGeos()) + tasks.append(tr.BuildGeos()) + tasks.append(tr.BuildGeosByLinking()) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index 9b0a6f4dd..0e97cc61d 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -41,13 +42,26 @@ class Workflow_build_jedi(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneJedi()) + tasks.append(tr.BuildJedi()) + tasks.append(tr.BuildJediByLinking()) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 741839b92..1d05b2ddf 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow from swell.utilities.check_da_params import check_da_params +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -55,32 +56,27 @@ class Workflow_compare_variational(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() - - # Overrides for comparison suites - start_cycle_point = self.experiment_dict['start_cycle_point'] - final_cycle_point = self.experiment_dict['final_cycle_point'] - if self.experiment_dict['start_cycle_point'] is None: - config_list = self.experiment_dict['comparison_experiment_paths'] - for model in self.experiment_dict['model_components']: - cycle_times = self.experiment_dict['models'][model]['cycle_times'] - start_cycle_point, final_cycle_point, cycle_times = check_da_params( - config_list, - model, - start_cycle_point, - final_cycle_point, - cycle_times) - - self.experiment_dict['start_cycle_point'] = start_cycle_point - self.experiment_dict['final_cycle_point'] = final_cycle_point - self.experiment_dict['models'][model]['cycle_times'] = cycle_times - workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + + for model in self.experiment_dict['model_components']: + tasks.append(tr.EvaComparisonIncrement(model=model)) + tasks.append(tr.EvaComparisonJediLog(model=model)) + tasks.append(tr.JediOopsLogParser(model=model)) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_bufr/workflow.py b/src/swell/suites/convert_bufr/workflow.py index 5e9bc1ee8..1e38b0f31 100644 --- a/src/swell/suites/convert_bufr/workflow.py +++ b/src/swell/suites/convert_bufr/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -82,13 +83,34 @@ class Workflow_convert_bufr(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneJedi()) + tasks.append(tr.BuildJediByLinking()) + tasks.append(tr.BuildJedi()) + tasks.append(tr.CloneGmaoPerllib()) + + + for model in self.experiment_dict['model_components']: + tasks.append(tr.CloneGeosMksi(model=model)) + tasks.append(tr.GetBufr(model=model)) + tasks.append(tr.BufrToIoda(model=model)) + tasks.append(tr.CleanCycle(model=model)) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index 9a2e49620..636679df5 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -71,13 +72,31 @@ class Workflow_convert_ncdiags(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneJedi()) + tasks.append(tr.BuildJediByLinking()) + tasks.append(tr.BuildJedi()) + tasks.append(tr.GetGsiBc()) + tasks.append(tr.GsiBcToIoda()) + tasks.append(tr.GetGsiNcdiag()) + tasks.append(tr.GsiNcdiagToIoda()) + tasks.append(tr.CleanCycle()) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index bf6a30933..3e8b532f0 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -67,13 +68,30 @@ class Workflow_eva_capabilities(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + + for model in self.experiment_dict['model_components']: + tasks.append(tr.CloneGeosMksi(model=model)) + tasks.append(tr.GetNcdiags(model=model)) + tasks.append(tr.GenerateObservingSystemRecords(model=model)) + tasks.append(tr.EvaTimeseries(model=model)) + tasks.append(tr.CleanCycle(model=model)) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index 998890b23..328e38b79 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -76,13 +77,32 @@ class Workflow_forecast_geos(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneGeos()) + tasks.append(tr.BuildGeosByLinking()) + tasks.append(tr.BuildGeos()) + tasks.append(tr.GetGeosRestart()) + tasks.append(tr.PrepGeosRunDir()) + tasks.append(tr.RunGeosExecutable()) + tasks.append(tr.MoveForecastRestart()) + tasks.append(tr.SaveRestart()) + tasks.append(tr.RemoveForecastDir()) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index ffbc50946..06745ceb0 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -74,13 +75,37 @@ class Workflow_geosadas(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneJedi()) + tasks.append(tr.BuildJediByLinking()) + tasks.append(tr.CloneGeosMksi) + + for model in self.experiment_dict['model_components']: + tasks.append(tr.GenerateObservingSystemRecords(model=model)) + tasks.append(tr.StageJedi(model=model)) + tasks.append(tr.GetGsiBc(model=model)) + tasks.append(tr.GsiBcToIoda(model=model)) + tasks.append(tr.GetGsiNcdiag(model=model)) + tasks.append(tr.GsiNcdiagToIoda(model=model)) + tasks.append(tr.GetGeosAdasBackground(model=model)) + tasks.append(tr.RunJediVariationalExecutable(model=model)) + tasks.append(tr.CleanCycle(model=model)) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index 1dfd099b9..12c671570 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -100,13 +101,40 @@ class Workflow_hofx(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneJedi()) + tasks.append(tr.BuildJedi()) + tasks.append(tr.BuildJediByLinking()) + tasks.append(tr.CloneGeosMksi()) + + for model in self.experiment_dict['model_components']: + tasks.append(tr.CloneGeosMksi(model=model)) + tasks.append(tr.GenerateObservingSystemRecords(model=model)) + tasks.append(tr.GetBackgroundGeosExperiment(model=model)) + tasks.append(tr.GetBackground(model=model)) + tasks.append(tr.GetObservations(model=model)) + tasks.append(tr.GetObsNotInR2d2(model=model)) + tasks.append(tr.StageJediCycle(model=model)) + tasks.append(tr.RunJediHofxExecutable(model=model)) + tasks.append(tr.EvaObservations(model=model)) + tasks.append(tr.SaveObsDiags(model=model)) + tasks.append(tr.CleanCycle(model=model)) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index ebb8b0fc6..2301e2200 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -117,13 +118,43 @@ class Workflow_localensembleda(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneJedi()) + tasks.append(tr.BuildJedi()) + tasks.append(tr.BuildJediByLinking()) + + for model in self.experiment_dict['model_components']: + tasks.append(tr.CloneGeosMksi(model=model)) + tasks.append(tr.StageJediCycle(model=model)) + tasks.append(tr.GetObsNotInR2d2(model=model)) + tasks.append(tr.GetObservations(model=model)) + tasks.append(tr.GenerateObservingSystemRecords(model=model)) + tasks.append(tr.GetEnsembleGeosExperiment(model=model)) + tasks.append(tr.sync_point(model=model)) + tasks.append(tr.RunJediObsfiltersExecutable(model=model)) + tasks.append(tr.RunJediLocalEnsembleDaExecutable(model=model)) + tasks.append(tr.RunJediHofxEnsembleMeanVariance(model=model)) + tasks.append(tr.RunJediHofxEnsembleExecutable(model=model)) + tasks.append(tr.EvaIncrement(model=model)) + tasks.append(tr.EvaObservations(model=model)) + tasks.append(tr.SaveObsDiags(model=model)) + tasks.append(tr.CleanCycle(model=model)) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index b8ffa19dc..f4bfdfd77 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -9,6 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow +from swell.tasks.task_runtimes import TaskRuntimes as tr # -------------------------------------------------------------------------------------------------- @@ -89,13 +90,38 @@ class Workflow_ufo_testing(CylcWorkflow): - def define_initial_workflow(self): + def get_workflow_string(self): workflow_str = self.default_header() workflow_str += template_string_jinja2(logger=self.logger, templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) + + for task in self.tasks(): + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) return workflow_str + + def tasks(self) -> list: + tasks = [] + tasks.append(tr.root()) + tasks.append(tr.CloneJedi()) + tasks.append(tr.BuildJedi()) + tasks.append(tr.BuildJediByLinking()) + + for model in self.experiment_dict['model_components']: + tasks.append(tr.CloneGeosMksi(model=model)) + tasks.append(tr.GenerateObservingSystemRecords(model=model)) + tasks.append(tr.GetGsiBc(model=model)) + tasks.append(tr.GsiBcToIoda(model=model)) + tasks.append(tr.GetGsiNcdiag(model=model)) + tasks.append(tr.GsiNcdiagToIoda(model=model)) + tasks.append(tr.RunJediUfoTestsExecutable(model=model)) + tasks.append(tr.GetGeovals(model=model)) + tasks.append(tr.EvaObservations(model=model)) + tasks.append(tr.CleanCycle(model=model)) + + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py index bf5194f87..77da90131 100644 --- a/src/swell/tasks/task_runtimes.py +++ b/src/swell/tasks/task_runtimes.py @@ -226,6 +226,20 @@ class RunJediFgatExecutable(Task): time_limit: bool = True slurm: dict = mutable_field({}) + @dataclass + class RunJediHofxEnsembleMeanVariance(Task): + is_cycling: bool = True + is_model: bool = True + time_limit: bool = True + slurm: dict = mutable_field({}) + + @dataclass + class RunJediHofxEnsembleExecutable(Task): + is_cycling: bool = True + is_model: bool = True + time_limit: bool = True + slurm: dict = mutable_field({}) + @dataclass class RunJediHofxExecutable(Task): is_cycling: bool = True From 3e740197e83c65b787ae5d4fde67e4c5304a652d Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 28 Oct 2025 10:35:21 -0400 Subject: [PATCH 119/299] fix task runtime section --- src/swell/suites/3dfgat_atmos/workflow.py | 6 ++++++ src/swell/suites/3dfgat_cycle/workflow.py | 6 ++++++ src/swell/suites/3dvar_atmos/workflow.py | 6 ++++++ src/swell/suites/3dvar_cycle/workflow.py | 6 ++++++ src/swell/suites/build_jedi/workflow.py | 6 ++++++ src/swell/suites/compare_variational/workflow.py | 6 ++++++ src/swell/suites/convert_bufr/workflow.py | 6 ++++++ src/swell/suites/convert_ncdiags/workflow.py | 6 ++++++ src/swell/suites/eva_capabilities/workflow.py | 6 ++++++ src/swell/suites/forecast_geos/workflow.py | 6 ++++++ src/swell/suites/geosadas/workflow.py | 6 ++++++ src/swell/suites/hofx/workflow.py | 6 ++++++ src/swell/suites/localensembleda/workflow.py | 6 ++++++ src/swell/suites/ufo_testing/workflow.py | 6 ++++++ 14 files changed, 84 insertions(+) diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 8440014bd..c7b11bd68 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -114,6 +114,12 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # noqa # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index 9b29eff09..9f2916157 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -131,6 +131,12 @@ """ {% endfor %} # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # noqa # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index a35a37d1a..1e2fa7820 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -108,6 +108,12 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # noqa # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index 81c6df20d..107ed35a1 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -131,6 +131,12 @@ """ {% endfor %} # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # noqa # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index 0e97cc61d..ca78b32b0 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -35,6 +35,12 @@ """ # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 1d05b2ddf..5f8e6d4ab 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -49,6 +49,12 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # noqa # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_bufr/workflow.py b/src/swell/suites/convert_bufr/workflow.py index 1e38b0f31..9726ef9d1 100644 --- a/src/swell/suites/convert_bufr/workflow.py +++ b/src/swell/suites/convert_bufr/workflow.py @@ -76,6 +76,12 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # noqa # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index 636679df5..542f3e942 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -65,6 +65,12 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index 3e8b532f0..d82773471 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -61,6 +61,12 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # noqa # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index 328e38b79..02967aaa9 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -70,6 +70,12 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index 06745ceb0..edb583fde 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -68,6 +68,12 @@ """ # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index 12c671570..757ce1fc9 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -94,6 +94,12 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # noqa # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 2301e2200..83bf97d24 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -111,6 +111,12 @@ {% endfor %} """ {% endfor %} + +[runtime] + + # Task defaults + # ------------- + ''' # noqa # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index f4bfdfd77..79260986c 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -83,6 +83,12 @@ {% endfor %} # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # -------------------------------------------------------------------------------------------------- From f3da5e3b0e3e3ec7b2d249d8091e74d4c7420766 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 30 Oct 2025 13:18:15 -0400 Subject: [PATCH 120/299] Refactor into task_attributes class --- .../prepare_config_and_suite.py | 61 +- src/swell/suites/3dfgat_atmos/workflow.py | 42 +- src/swell/suites/3dfgat_cycle/workflow.py | 60 +- src/swell/suites/3dvar/workflow.py | 34 +- src/swell/suites/3dvar_atmos/workflow.py | 38 +- src/swell/suites/3dvar_cycle/workflow.py | 56 +- src/swell/suites/build_geos/workflow.py | 16 +- src/swell/suites/build_jedi/workflow.py | 10 +- .../suites/compare_variational/workflow.py | 8 +- src/swell/suites/convert_bufr/workflow.py | 20 +- src/swell/suites/convert_ncdiags/workflow.py | 20 +- src/swell/suites/eva_capabilities/workflow.py | 14 +- src/swell/suites/forecast_geos/workflow.py | 22 +- src/swell/suites/geosadas/workflow.py | 28 +- src/swell/suites/hofx/workflow.py | 34 +- src/swell/suites/localensembleda/workflow.py | 40 +- src/swell/suites/ufo_testing/workflow.py | 30 +- src/swell/tasks/task_attributes.py | 866 ++++++++++++++++++ src/swell/tasks/task_runtimes.py | 342 ------- src/swell/utilities/config.py | 6 +- src/swell/utilities/swell_questions.py | 13 +- src/swell/utilities/task_spec.py | 253 +++++ 22 files changed, 1400 insertions(+), 613 deletions(-) create mode 100644 src/swell/tasks/task_attributes.py delete mode 100644 src/swell/tasks/task_runtimes.py create mode 100644 src/swell/utilities/task_spec.py diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index d136d0f5e..2d3578aaf 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -22,7 +22,7 @@ from swell.utilities.dictionary import dict_get from swell.utilities.logger import Logger from swell.utilities.dictionary import update_dict, add_dict -from swell.tasks.task_questions import TaskQuestions as task_questions +from swell.tasks.task_attributes import TaskAttributes as task_attributes from swell.suites.all_suites import suite_configs from swell.utilities.swell_questions import QuestionType from swell.utilities.question_defaults import QuestionDefaults as qd @@ -186,43 +186,46 @@ def prepare_task_question_dictionary(self): # Iterate through model independent tasks and update with defaults if not already set for task in self.model_independent_tasks: task_options.append(task) - if task in task_questions.get_all(): - question_list = task_questions[task].value.expand_question_list() - for question in question_list: - question_dict = {question['question_name']: question} - if question['models'] is not None: - model_dict = {} + task_class = getattr(task_attributes, task) - for question_model in question['models']: - if question_model == 'all_models': - for model in model_components: - model_dict[model] = question_dict - elif question_model in model_components: - model_dict[question_model] = question_dict + question_list = task_class().question_list.expand_question_list() + for question in question_list: + question_dict = {question['question_name']: question} - self.question_dictionary_model_dep = add_dict( - self.question_dictionary_model_dep, model_dict) + if question['models'] is not None: + model_dict = {} - else: - self.question_dictionary_model_ind = add_dict( - self.question_dictionary_model_ind, question_dict) + for question_model in question['models']: + if question_model == 'all_models': + for model in model_components: + model_dict[model] = question_dict + elif question_model in model_components: + model_dict[question_model] = question_dict + + self.question_dictionary_model_dep = add_dict( + self.question_dictionary_model_dep, model_dict) + + else: + self.question_dictionary_model_ind = add_dict( + self.question_dictionary_model_ind, question_dict) # Iterate through model dependent tasks and update if not already set for model, task_list in self.model_dependent_tasks.items(): for task in task_list: task_options.append(task) - if task in task_questions.get_all(): - question_list = task_questions[task].value.expand_question_list() - - for question in question_list: - question_dict = {question['question_name']: question} - if question['models'] is None: - self.question_dictionary_model_ind = add_dict( - self.question_dictionary_model_ind, question_dict) - elif model in question['models'] or 'all_models' in question['models']: - self.question_dictionary_model_dep = add_dict( - self.question_dictionary_model_dep, {model: question_dict}) + + task_class = getattr(task_attributes, task) + question_list = task_class(model=model).question_list.expand_question_list() + + for question in question_list: + question_dict = {question['question_name']: question} + if question['models'] is None: + self.question_dictionary_model_ind = add_dict( + self.question_dictionary_model_ind, question_dict) + elif model in question['models'] or 'all_models' in question['models']: + self.question_dictionary_model_dep = add_dict( + self.question_dictionary_model_dep, {model: question_dict}) # Set options for task email parameters message_question_dict = {'task_email_parameters': diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index c7b11bd68..1d6f0727e 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -142,28 +142,28 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneJedi()) - tasks.append(tr.BuildJediByLinking()) - tasks.append(tr.BuildJedi()) + tasks.append(ta.root()) + tasks.append(ta.CloneJedi()) + tasks.append(ta.BuildJediByLinking()) + tasks.append(ta.BuildJedi()) for model in self.experiment_dict['model_components']: - tasks.append(tr.CloneGeosMksi(model=model)) - tasks.append(tr.StageJedi(model=model)) - tasks.append(tr.GetBackground(model=model)) - tasks.append(tr.GetObservations(model=model)) - tasks.append(tr.GenerateBClimatologyByLinking(model=model)) - tasks.append(tr.GenerateBClimatology(model=model)) - tasks.append(tr.GetObsNotInR2d2(model=model)) - tasks.append(tr.GetBackgroundGeosExperiment(model=model)) - tasks.append(tr.GenerateObservingSystemRecords(model=model)) - tasks.append(tr.StageJediCycle(model=model)) - tasks.append(tr.RunJediVariationalExecutable(model=model)) - tasks.append(tr.EvaObservations(model=model)) - tasks.append(tr.EvaJediLog(model=model)) - tasks.append(tr.EvaIncrement(model=model)) - tasks.append(tr.SaveObsDiags(model=model)) - tasks.append(tr.CleanCycle(model=model)) + tasks.append(ta.CloneGeosMksi(model=model)) + tasks.append(ta.StageJedi(model=model)) + tasks.append(ta.GetBackground(model=model)) + tasks.append(ta.GetObservations(model=model)) + tasks.append(ta.GenerateBClimatologyByLinking(model=model)) + tasks.append(ta.GenerateBClimatology(model=model)) + tasks.append(ta.GetObsNotInR2d2(model=model)) + tasks.append(ta.GetBackgroundGeosExperiment(model=model)) + tasks.append(ta.GenerateObservingSystemRecords(model=model)) + tasks.append(ta.StageJediCycle(model=model)) + tasks.append(ta.RunJediVariationalExecutable(model=model)) + tasks.append(ta.EvaObservations(model=model)) + tasks.append(ta.EvaJediLog(model=model)) + tasks.append(ta.EvaIncrement(model=model)) + tasks.append(ta.SaveObsDiags(model=model)) + tasks.append(ta.CleanCycle(model=model)) return tasks diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index 9f2916157..9cd2149d5 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -159,37 +159,37 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneJedi()) - tasks.append(tr.CloneGeos()) - tasks.append(tr.BuildJediByLinking()) - tasks.append(tr.BuildJedi()) - tasks.append(tr.BuildGeos()) - tasks.append(tr.BuildGeosByLinking()) - - tasks.append(tr.GetGeosRestart()) - tasks.append(tr.PrepGeosRunDir()) - tasks.append(tr.RunGeosExecutable()) + tasks.append(ta.root()) + tasks.append(ta.CloneJedi()) + tasks.append(ta.CloneGeos()) + tasks.append(ta.BuildJediByLinking()) + tasks.append(ta.BuildJedi()) + tasks.append(ta.BuildGeos()) + tasks.append(ta.BuildGeosByLinking()) + + tasks.append(ta.GetGeosRestart()) + tasks.append(ta.PrepGeosRunDir()) + tasks.append(ta.RunGeosExecutable()) for model in self.experiment_dict['model_components']: - tasks.append(tr.RunJediFgatExecutable(model=model)) - tasks.append(tr.StageJedi(model=model)) - tasks.append(tr.StageJediCycle(model=model)) - tasks.append(tr.MoveDaRestart(model=model)) - tasks.append(tr.LinkGeosOutput(model=model)) - tasks.append(tr.GenerateBClimatology(model=model)) - tasks.append(tr.GenerateBClimatologyByLinking(model=model)) - tasks.append(tr.GetObservations(model=model)) - tasks.append(tr.EvaObservations(model=model)) - tasks.append(tr.EvaJediLog(model=model)) - tasks.append(tr.EvaIncrement(model=model)) - tasks.append(tr.PrepareAnalysis(model=model)) - tasks.append(tr.RunJediConvertStateSoca2ciceExecutable(model=model)) - tasks.append(tr.SaveRestart(model=model)) - tasks.append(tr.CleanCycle(model=model)) - tasks.append(tr.PrepareAnalysis(model=model)) - tasks.append(tr.RemoveForecastDir(model=model)) - tasks.append(tr.SaveObsDiags(model=model)) + tasks.append(ta.RunJediFgatExecutable(model=model)) + tasks.append(ta.StageJedi(model=model)) + tasks.append(ta.StageJediCycle(model=model)) + tasks.append(ta.MoveDaRestart(model=model)) + tasks.append(ta.LinkGeosOutput(model=model)) + tasks.append(ta.GenerateBClimatology(model=model)) + tasks.append(ta.GenerateBClimatologyByLinking(model=model)) + tasks.append(ta.GetObservations(model=model)) + tasks.append(ta.EvaObservations(model=model)) + tasks.append(ta.EvaJediLog(model=model)) + tasks.append(ta.EvaIncrement(model=model)) + tasks.append(ta.PrepareAnalysis(model=model)) + tasks.append(ta.RunJediConvertStateSoca2ciceExecutable(model=model)) + tasks.append(ta.SaveRestart(model=model)) + tasks.append(ta.CleanCycle(model=model)) + tasks.append(ta.PrepareAnalysis(model=model)) + tasks.append(ta.RemoveForecastDir(model=model)) + tasks.append(ta.SaveObsDiags(model=model)) return tasks diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 5ca2f5a06..7c7dbacf5 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -128,24 +128,24 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneJedi()) - tasks.append(tr.BuildJediByLinking()) - tasks.append(tr.BuildJedi()) + tasks.append(ta.root()) + tasks.append(ta.CloneJedi()) + tasks.append(ta.BuildJediByLinking()) + tasks.append(ta.BuildJedi()) for model in self.experiment_dict['model_components']: - tasks.append(tr.StageJedi(model=model)) - tasks.append(tr.GetObservations(model=model)) - tasks.append(tr.GenerateBClimatologyByLinking(model=model)) - tasks.append(tr.GenerateBClimatology(model=model)) - tasks.append(tr.StageJediCycle(model=model)) - tasks.append(tr.GetBackground(model=model)) - tasks.append(tr.RunJediVariationalExecutable(model=model)) - tasks.append(tr.EvaObservations(model=model)) - tasks.append(tr.EvaJediLog(model=model)) - tasks.append(tr.EvaIncrement(model=model)) - tasks.append(tr.SaveObsDiags(model=model)) - tasks.append(tr.CleanCycle(model=model)) + tasks.append(ta.StageJedi(model=model)) + tasks.append(ta.GetObservations(model=model)) + tasks.append(ta.GenerateBClimatologyByLinking(model=model)) + tasks.append(ta.GenerateBClimatology(model=model)) + tasks.append(ta.StageJediCycle(model=model)) + tasks.append(ta.GetBackground(model=model)) + tasks.append(ta.RunJediVariationalExecutable(model=model)) + tasks.append(ta.EvaObservations(model=model)) + tasks.append(ta.EvaJediLog(model=model)) + tasks.append(ta.EvaIncrement(model=model)) + tasks.append(ta.SaveObsDiags(model=model)) + tasks.append(ta.CleanCycle(model=model)) return tasks diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index 1e2fa7820..bf1acc790 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -136,26 +136,26 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneJedi()) - tasks.append(tr.BuildJediByLinking()) - tasks.append(tr.BuildJedi()) + tasks.append(ta.root()) + tasks.append(ta.CloneJedi()) + tasks.append(ta.BuildJediByLinking()) + tasks.append(ta.BuildJedi()) for model in self.experiment_dict['model_components']: - tasks.append(tr.CloneGeosMksi(model=model)) - tasks.append(tr.StageJedi(model=model)) - tasks.append(tr.GetObservations(model=model)) - tasks.append(tr.GenerateBClimatologyByLinking(model=model)) - tasks.append(tr.GenerateBClimatology(model=model)) - tasks.append(tr.GenerateObservingSystemRecords(model=model)) - tasks.append(tr.GetObsNotInR2d2(model=model)) - tasks.append(tr.StageJediCycle(model=model)) - tasks.append(tr.RunJediVariationalExecutable(model=model)) - tasks.append(tr.EvaObservations(model=model)) - tasks.append(tr.EvaJediLog(model=model)) - tasks.append(tr.EvaIncrement(model=model)) - tasks.append(tr.SaveObsDiags(model=model)) - tasks.append(tr.CleanCycle(model=model)) + tasks.append(ta.CloneGeosMksi(model=model)) + tasks.append(ta.StageJedi(model=model)) + tasks.append(ta.GetObservations(model=model)) + tasks.append(ta.GenerateBClimatologyByLinking(model=model)) + tasks.append(ta.GenerateBClimatology(model=model)) + tasks.append(ta.GenerateObservingSystemRecords(model=model)) + tasks.append(ta.GetObsNotInR2d2(model=model)) + tasks.append(ta.StageJediCycle(model=model)) + tasks.append(ta.RunJediVariationalExecutable(model=model)) + tasks.append(ta.EvaObservations(model=model)) + tasks.append(ta.EvaJediLog(model=model)) + tasks.append(ta.EvaIncrement(model=model)) + tasks.append(ta.SaveObsDiags(model=model)) + tasks.append(ta.CleanCycle(model=model)) return tasks diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index 107ed35a1..efacbb538 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -159,35 +159,35 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneJedi()) - tasks.append(tr.CloneGeos()) - tasks.append(tr.BuildJediByLinking()) - tasks.append(tr.BuildGeosByLinking()) - tasks.append(tr.BuildJedi()) - tasks.append(tr.BuildGeos()) - tasks.append(tr.GetGeosRestart()) - tasks.append(tr.PrepGeosRunDir) - tasks.append(tr.RunGeosExecutable()) + tasks.append(ta.root()) + tasks.append(ta.CloneJedi()) + tasks.append(ta.CloneGeos()) + tasks.append(ta.BuildJediByLinking()) + tasks.append(ta.BuildGeosByLinking()) + tasks.append(ta.BuildJedi()) + tasks.append(ta.BuildGeos()) + tasks.append(ta.GetGeosRestart()) + tasks.append(ta.PrepGeosRunDir) + tasks.append(ta.RunGeosExecutable()) for model in self.experiment_dict['model_components']: - tasks.append(tr.StageJedi(model=model)) - tasks.append(tr.StageJediCycle(model=model)) - tasks.append(tr.RunJediVariationalExecutable(model=model)) - tasks.append(tr.MoveDaRestart(model=model)) - tasks.append(tr.LinkGeosOutput(model=model)) - tasks.append(tr.GenerateBClimatology(model=model)) - tasks.append(tr.GenerateBClimatologyByLinking(model=model)) - tasks.append(tr.GetObservations(model=model)) - tasks.append(tr.PrepareAnalysis(model=model)) - tasks.append(tr.RunJediConvertStateSoca2ciceExecutable(model=model)) - tasks.append(tr.SaveRestart(model=model)) - tasks.append(tr.RemoveForecastDir(model=model)) - tasks.append(tr.EvaObservations(model=model)) - tasks.append(tr.EvaJediLog(model=model)) - tasks.append(tr.EvaIncrement(model=model)) - tasks.append(tr.SaveObsDiags(model=model)) - tasks.append(tr.CleanCycle(model=model)) + tasks.append(ta.StageJedi(model=model)) + tasks.append(ta.StageJediCycle(model=model)) + tasks.append(ta.RunJediVariationalExecutable(model=model)) + tasks.append(ta.MoveDaRestart(model=model)) + tasks.append(ta.LinkGeosOutput(model=model)) + tasks.append(ta.GenerateBClimatology(model=model)) + tasks.append(ta.GenerateBClimatologyByLinking(model=model)) + tasks.append(ta.GetObservations(model=model)) + tasks.append(ta.PrepareAnalysis(model=model)) + tasks.append(ta.RunJediConvertStateSoca2ciceExecutable(model=model)) + tasks.append(ta.SaveRestart(model=model)) + tasks.append(ta.RemoveForecastDir(model=model)) + tasks.append(ta.EvaObservations(model=model)) + tasks.append(ta.EvaJediLog(model=model)) + tasks.append(ta.EvaIncrement(model=model)) + tasks.append(ta.SaveObsDiags(model=model)) + tasks.append(ta.CleanCycle(model=model)) return tasks diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py index 57ffe01ef..52279fb6d 100644 --- a/src/swell/suites/build_geos/workflow.py +++ b/src/swell/suites/build_geos/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -35,6 +35,12 @@ """ # -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + ''' # -------------------------------------------------------------------------------------------------- @@ -57,10 +63,10 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneGeos()) - tasks.append(tr.BuildGeos()) - tasks.append(tr.BuildGeosByLinking()) + tasks.append(ta.root()) + tasks.append(ta.CloneGeos()) + tasks.append(ta.BuildGeos()) + tasks.append(ta.BuildGeosByLinking()) return tasks diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index ca78b32b0..32e5a84bb 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -63,10 +63,10 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneJedi()) - tasks.append(tr.BuildJedi()) - tasks.append(tr.BuildJediByLinking()) + tasks.append(ta.root()) + tasks.append(ta.CloneJedi()) + tasks.append(ta.BuildJedi()) + tasks.append(ta.BuildJediByLinking()) return tasks diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 5f8e6d4ab..4b944cf31 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow from swell.utilities.check_da_params import check_da_params -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -79,9 +79,9 @@ def tasks(self) -> list: tasks = [] for model in self.experiment_dict['model_components']: - tasks.append(tr.EvaComparisonIncrement(model=model)) - tasks.append(tr.EvaComparisonJediLog(model=model)) - tasks.append(tr.JediOopsLogParser(model=model)) + tasks.append(ta.EvaComparisonIncrement(model=model)) + tasks.append(ta.EvaComparisonJediLog(model=model)) + tasks.append(ta.JediOopsLogParser(model=model)) return tasks diff --git a/src/swell/suites/convert_bufr/workflow.py b/src/swell/suites/convert_bufr/workflow.py index 9726ef9d1..827b4c6a4 100644 --- a/src/swell/suites/convert_bufr/workflow.py +++ b/src/swell/suites/convert_bufr/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -104,18 +104,18 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneJedi()) - tasks.append(tr.BuildJediByLinking()) - tasks.append(tr.BuildJedi()) - tasks.append(tr.CloneGmaoPerllib()) + tasks.append(ta.root()) + tasks.append(ta.CloneJedi()) + tasks.append(ta.BuildJediByLinking()) + tasks.append(ta.BuildJedi()) + tasks.append(ta.CloneGmaoPerllib()) for model in self.experiment_dict['model_components']: - tasks.append(tr.CloneGeosMksi(model=model)) - tasks.append(tr.GetBufr(model=model)) - tasks.append(tr.BufrToIoda(model=model)) - tasks.append(tr.CleanCycle(model=model)) + tasks.append(ta.CloneGeosMksi(model=model)) + tasks.append(ta.GetBufr(model=model)) + tasks.append(ta.BufrToIoda(model=model)) + tasks.append(ta.CleanCycle(model=model)) return tasks diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index 542f3e942..dc64fef16 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -93,15 +93,15 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneJedi()) - tasks.append(tr.BuildJediByLinking()) - tasks.append(tr.BuildJedi()) - tasks.append(tr.GetGsiBc()) - tasks.append(tr.GsiBcToIoda()) - tasks.append(tr.GetGsiNcdiag()) - tasks.append(tr.GsiNcdiagToIoda()) - tasks.append(tr.CleanCycle()) + tasks.append(ta.root()) + tasks.append(ta.CloneJedi()) + tasks.append(ta.BuildJediByLinking()) + tasks.append(ta.BuildJedi()) + tasks.append(ta.GetGsiBc()) + tasks.append(ta.GsiBcToIoda()) + tasks.append(ta.GetGsiNcdiag()) + tasks.append(ta.GsiNcdiagToIoda()) + tasks.append(ta.CleanCycle()) return tasks diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index d82773471..88b1d7a9f 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -89,14 +89,14 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) + tasks.append(ta.root()) for model in self.experiment_dict['model_components']: - tasks.append(tr.CloneGeosMksi(model=model)) - tasks.append(tr.GetNcdiags(model=model)) - tasks.append(tr.GenerateObservingSystemRecords(model=model)) - tasks.append(tr.EvaTimeseries(model=model)) - tasks.append(tr.CleanCycle(model=model)) + tasks.append(ta.CloneGeosMksi(model=model)) + tasks.append(ta.GetNcdiags(model=model)) + tasks.append(ta.GenerateObservingSystemRecords(model=model)) + tasks.append(ta.EvaTimeseries(model=model)) + tasks.append(ta.CleanCycle(model=model)) return tasks diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index 02967aaa9..b806aaa04 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -98,16 +98,16 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneGeos()) - tasks.append(tr.BuildGeosByLinking()) - tasks.append(tr.BuildGeos()) - tasks.append(tr.GetGeosRestart()) - tasks.append(tr.PrepGeosRunDir()) - tasks.append(tr.RunGeosExecutable()) - tasks.append(tr.MoveForecastRestart()) - tasks.append(tr.SaveRestart()) - tasks.append(tr.RemoveForecastDir()) + tasks.append(ta.root()) + tasks.append(ta.CloneGeos()) + tasks.append(ta.BuildGeosByLinking()) + tasks.append(ta.BuildGeos()) + tasks.append(ta.GetGeosRestart()) + tasks.append(ta.PrepGeosRunDir()) + tasks.append(ta.RunGeosExecutable()) + tasks.append(ta.MoveForecastRestart()) + tasks.append(ta.SaveRestart()) + tasks.append(ta.RemoveForecastDir()) return tasks diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index edb583fde..5b8b7e4f5 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -96,21 +96,21 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneJedi()) - tasks.append(tr.BuildJediByLinking()) - tasks.append(tr.CloneGeosMksi) + tasks.append(ta.root()) + tasks.append(ta.CloneJedi()) + tasks.append(ta.BuildJediByLinking()) + tasks.append(ta.CloneGeosMksi) for model in self.experiment_dict['model_components']: - tasks.append(tr.GenerateObservingSystemRecords(model=model)) - tasks.append(tr.StageJedi(model=model)) - tasks.append(tr.GetGsiBc(model=model)) - tasks.append(tr.GsiBcToIoda(model=model)) - tasks.append(tr.GetGsiNcdiag(model=model)) - tasks.append(tr.GsiNcdiagToIoda(model=model)) - tasks.append(tr.GetGeosAdasBackground(model=model)) - tasks.append(tr.RunJediVariationalExecutable(model=model)) - tasks.append(tr.CleanCycle(model=model)) + tasks.append(ta.GenerateObservingSystemRecords(model=model)) + tasks.append(ta.StageJedi(model=model)) + tasks.append(ta.GetGsiBc(model=model)) + tasks.append(ta.GsiBcToIoda(model=model)) + tasks.append(ta.GetGsiNcdiag(model=model)) + tasks.append(ta.GsiNcdiagToIoda(model=model)) + tasks.append(ta.GetGeosAdasBackground(model=model)) + tasks.append(ta.RunJediVariationalExecutable(model=model)) + tasks.append(ta.CleanCycle(model=model)) return tasks diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index 757ce1fc9..bd4ba6477 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -122,24 +122,24 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneJedi()) - tasks.append(tr.BuildJedi()) - tasks.append(tr.BuildJediByLinking()) - tasks.append(tr.CloneGeosMksi()) + tasks.append(ta.root()) + tasks.append(ta.CloneJedi()) + tasks.append(ta.BuildJedi()) + tasks.append(ta.BuildJediByLinking()) + tasks.append(ta.CloneGeosMksi()) for model in self.experiment_dict['model_components']: - tasks.append(tr.CloneGeosMksi(model=model)) - tasks.append(tr.GenerateObservingSystemRecords(model=model)) - tasks.append(tr.GetBackgroundGeosExperiment(model=model)) - tasks.append(tr.GetBackground(model=model)) - tasks.append(tr.GetObservations(model=model)) - tasks.append(tr.GetObsNotInR2d2(model=model)) - tasks.append(tr.StageJediCycle(model=model)) - tasks.append(tr.RunJediHofxExecutable(model=model)) - tasks.append(tr.EvaObservations(model=model)) - tasks.append(tr.SaveObsDiags(model=model)) - tasks.append(tr.CleanCycle(model=model)) + tasks.append(ta.CloneGeosMksi(model=model)) + tasks.append(ta.GenerateObservingSystemRecords(model=model)) + tasks.append(ta.GetBackgroundGeosExperiment(model=model)) + tasks.append(ta.GetBackground(model=model)) + tasks.append(ta.GetObservations(model=model)) + tasks.append(ta.GetObsNotInR2d2(model=model)) + tasks.append(ta.StageJediCycle(model=model)) + tasks.append(ta.RunJediHofxExecutable(model=model)) + tasks.append(ta.EvaObservations(model=model)) + tasks.append(ta.SaveObsDiags(model=model)) + tasks.append(ta.CleanCycle(model=model)) return tasks diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 83bf97d24..3cb72a187 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -139,27 +139,27 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneJedi()) - tasks.append(tr.BuildJedi()) - tasks.append(tr.BuildJediByLinking()) + tasks.append(ta.root()) + tasks.append(ta.CloneJedi()) + tasks.append(ta.BuildJedi()) + tasks.append(ta.BuildJediByLinking()) for model in self.experiment_dict['model_components']: - tasks.append(tr.CloneGeosMksi(model=model)) - tasks.append(tr.StageJediCycle(model=model)) - tasks.append(tr.GetObsNotInR2d2(model=model)) - tasks.append(tr.GetObservations(model=model)) - tasks.append(tr.GenerateObservingSystemRecords(model=model)) - tasks.append(tr.GetEnsembleGeosExperiment(model=model)) - tasks.append(tr.sync_point(model=model)) - tasks.append(tr.RunJediObsfiltersExecutable(model=model)) - tasks.append(tr.RunJediLocalEnsembleDaExecutable(model=model)) - tasks.append(tr.RunJediHofxEnsembleMeanVariance(model=model)) - tasks.append(tr.RunJediHofxEnsembleExecutable(model=model)) - tasks.append(tr.EvaIncrement(model=model)) - tasks.append(tr.EvaObservations(model=model)) - tasks.append(tr.SaveObsDiags(model=model)) - tasks.append(tr.CleanCycle(model=model)) + tasks.append(ta.CloneGeosMksi(model=model)) + tasks.append(ta.StageJediCycle(model=model)) + tasks.append(ta.GetObsNotInR2d2(model=model)) + tasks.append(ta.GetObservations(model=model)) + tasks.append(ta.GenerateObservingSystemRecords(model=model)) + tasks.append(ta.GetEnsembleGeosExperiment(model=model)) + tasks.append(ta.sync_point(model=model)) + tasks.append(ta.RunJediObsfiltersExecutable(model=model)) + tasks.append(ta.RunJediLocalEnsembleDaExecutable(model=model)) + tasks.append(ta.RunJediEnsembleMeanVariance(model=model)) + tasks.append(ta.RunJediHofxEnsembleExecutable(model=model)) + tasks.append(ta.EvaIncrement(model=model)) + tasks.append(ta.EvaObservations(model=model)) + tasks.append(ta.SaveObsDiags(model=model)) + tasks.append(ta.CleanCycle(model=model)) return tasks diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index 79260986c..16cf54baa 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_runtimes import TaskRuntimes as tr +from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -111,22 +111,22 @@ def get_workflow_string(self): def tasks(self) -> list: tasks = [] - tasks.append(tr.root()) - tasks.append(tr.CloneJedi()) - tasks.append(tr.BuildJedi()) - tasks.append(tr.BuildJediByLinking()) + tasks.append(ta.root()) + tasks.append(ta.CloneJedi()) + tasks.append(ta.BuildJedi()) + tasks.append(ta.BuildJediByLinking()) for model in self.experiment_dict['model_components']: - tasks.append(tr.CloneGeosMksi(model=model)) - tasks.append(tr.GenerateObservingSystemRecords(model=model)) - tasks.append(tr.GetGsiBc(model=model)) - tasks.append(tr.GsiBcToIoda(model=model)) - tasks.append(tr.GetGsiNcdiag(model=model)) - tasks.append(tr.GsiNcdiagToIoda(model=model)) - tasks.append(tr.RunJediUfoTestsExecutable(model=model)) - tasks.append(tr.GetGeovals(model=model)) - tasks.append(tr.EvaObservations(model=model)) - tasks.append(tr.CleanCycle(model=model)) + tasks.append(ta.CloneGeosMksi(model=model)) + tasks.append(ta.GenerateObservingSystemRecords(model=model)) + tasks.append(ta.GetGsiBc(model=model)) + tasks.append(ta.GsiBcToIoda(model=model)) + tasks.append(ta.GetGsiNcdiag(model=model)) + tasks.append(ta.GsiNcdiagToIoda(model=model)) + tasks.append(ta.RunJediUfoTestsExecutable(model=model)) + tasks.append(ta.GetGeovals(model=model)) + tasks.append(ta.EvaObservations(model=model)) + tasks.append(ta.CleanCycle(model=model)) return tasks diff --git a/src/swell/tasks/task_attributes.py b/src/swell/tasks/task_attributes.py new file mode 100644 index 000000000..b51c1333e --- /dev/null +++ b/src/swell/tasks/task_attributes.py @@ -0,0 +1,866 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from typing import Union, Optional, Self +from collections.abc import Mapping + +from swell.utilities.task_spec import Task +from swell.utilities.swell_questions import QuestionList +from swell.utilities.question_defaults import QuestionDefaults as qd + +# -------------------------------------------------------------------------------------------------- + +background_crtm_obs = QuestionList( + list_name="background_crtm_obs", + questions=[ + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path() + ] +) + +# -------------------------------------------------------------------------------------------------- + +np_proc_resolution = QuestionList( + list_name="np_resolution", + questions=[ + qd.npx_proc(), + qd.npy_proc(), + qd.horizontal_resolution(), + qd.vertical_resolution() + ] +) + +# -------------------------------------------------------------------------------------------------- + +window_questions = QuestionList( + list_name="window_questions", + questions=[ + qd.window_length(), + qd.window_offset(), + qd.window_type() + ] +) + +# -------------------------------------------------------------------------------------------------- + +run_jedi_executable = QuestionList( + list_name="run_jedi_executable", + questions=[ + background_crtm_obs, + np_proc_resolution, + window_questions, + qd.analysis_variables(), + qd.background_frequency(), + qd.generate_yaml_and_exit(), + qd.gradient_norm_reduction(), + qd.gsibec_configuration(), + qd.jedi_forecast_model(), + qd.minimizer(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.number_of_iterations(), + qd.total_processors(), + ] +) + +# -------------------------------------------------------------------------------------------------- + +swell_static_file_questions = QuestionList( + list_name="swell_static_file_questions", + questions=[ + qd.swell_static_files(), + qd.swell_static_files_user() + ] +) + +# -------------------------------------------------------------------------------------------------- + +class TaskAttributes(): + + # -------------------------------------------------------------------------------------------------- + + class root(Task): + + def set_attributes(self): + self.script = False + self.pre_script = "source $CYLC_SUITE_DEF_PATH/modules" + self.addional_sections = [self.create_new_section('environment', {'datetime': '$CYLC_TASK_CYCLE_POINT', + 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'})] + + # -------------------------------------------------------------------------------------------------- + + class BuildGeos(Task): + def set_attributes(self): + self.question_list = QuestionList([ + qd.geos_build_method() + ]) + + # -------------------------------------------------------------------------------------------------- + + class BuildGeosByLinking(Task): + def set_attributes(self): + self.mail_events = ['submit-failed'] + self.question_list = QuestionList([ + qd.existing_geos_gcm_build_path(), + qd.geos_build_method() + ]) + + # -------------------------------------------------------------------------------------------------- + + class BuildJediByLinking(Task): + def set_attributes(self): + self.mail_events = ['submit-failed'] + self.question_list = QuestionList([ + qd.existing_jedi_build_directory(), + qd.existing_jedi_build_directory_pinned(), + qd.jedi_build_method() + ]) + + # -------------------------------------------------------------------------------------------------- + + class BuildJedi(Task): + def set_attributes(self): + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + qd.bundles(), + qd.jedi_build_method() + ]) + + # -------------------------------------------------------------------------------------------------- + + class CleanCycle(Task): + def set_attributes(self): + + self.is_cycling = True + self.is_model = True + + self.question_list = QuestionList([ + qd.clean_patterns() + ]) + + # -------------------------------------------------------------------------------------------------- + + class CloneGeos(Task): + def set_attributes(self): + self.question_list = QuestionList([ + qd.existing_geos_gcm_source_path(), + qd.geos_build_method(), + qd.geos_gcm_tag() + ]) + + # -------------------------------------------------------------------------------------------------- + + class CloneJedi(Task): + def set_attributes(self): + self.self.question_list = QuestionList([ + qd.bundles(), + qd.existing_jedi_source_directory(), + qd.existing_jedi_source_directory_pinned(), + qd.jedi_build_method() + ]) + + # -------------------------------------------------------------------------------------------------- + + class CloneGeosMksi(Task): + def set_attributes(self): + self.is_model = True + self.question_list = QuestionList([ + qd.observing_system_records_mksi_path(), + qd.observing_system_records_mksi_path_tag() + ]) + + # -------------------------------------------------------------------------------------------------- + + class CloneGmaoPerllib(Task): + def set_attributes(self): + self.question_list = QuestionList([ + qd.existing_perllib_path(), + qd.gmao_perllib_tag() + ]) + + # -------------------------------------------------------------------------------------------------- + + class EvaJediLog(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + + # -------------------------------------------------------------------------------------------------- + + class EvaComparisonIncrement(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.marine_models(), + qd.window_offset(), + qd.window_type() + ]) + + # -------------------------------------------------------------------------------------------------- + + class EvaComparisonJediLog(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + + # -------------------------------------------------------------------------------------------------- + + class EvaIncrement(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.marine_models(), + qd.window_offset(), + qd.window_type() + ]) + + # -------------------------------------------------------------------------------------------------- + + class EvaObservations(Task): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {} + self.question_list = QuestionList([ + background_crtm_obs, + qd.marine_models(), + qd.observing_system_records_path(), + qd.window_offset(), + qd.marine_models(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class EvaTimeseries(Task): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {} + self.question_list = QuestionList([ + background_crtm_obs, + qd.window_length(), + qd.window_offset(), + qd.ncdiag_experiments(), + qd.marine_models(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class JediOopsLogParser(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.parser_options(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetBackground(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + window_questions, + qd.analysis_forecast_window_offset(), + qd.background_experiment(), + qd.background_frequency(), + qd.horizontal_resolution(), + qd.marine_models(), + qd.r2d2_local_path(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetBackgroundGeosExperiment(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.mail_events = ['submit-failed'] + self.question_list = QuestionList([ + qd.horizontal_resolution(), + qd.background_experiment(), + qd.background_time_offset(), + qd.geos_x_background_directory() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetBufr(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.bufr_obs_classes() + ]) + + # -------------------------------------------------------------------------------------------------- + + class BufrToIoda(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + + # -------------------------------------------------------------------------------------------------- + + class GetEnsembleGeosExperiment(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.background_experiment(), + qd.background_time_offset(), + qd.geos_x_ensemble_directory() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetGeosRestart(Task): + def set_attributes(self): + self.is_cycling = True + self.question_list = QuestionList([ + swell_static_file_questions, + qd.geos_restarts_directory() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetGeovals(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + background_crtm_obs, + qd.geovals_experiment(), + qd.geovals_provider(), + qd.r2d2_local_path(), + qd.window_length(), + qd.window_offset() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetGsiBc(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.path_to_gsi_bc_coefficients(), + qd.window_length() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GsiBcToIoda(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + background_crtm_obs, + qd.observing_system_records_path(), + qd.window_offset() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetGsiNcdiag(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.path_to_gsi_nc_diags() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GsiNcdiagToIoda(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.observations(), + qd.produce_geovals(), + qd.single_observations(), + qd.window_offset() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetNcdiags(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + background_crtm_obs, + qd.ncdiag_experiments(), + qd.marine_models(), + qd.r2d2_local_path(), + qd.window_length(), + qd.window_offset(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetGeosAdasBackground(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.path_to_geos_adas_background() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetObservations(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + background_crtm_obs, + qd.cycling_varbc(), + qd.obs_experiment(), + qd.obs_provider(), + qd.observing_system_records_path(), + qd.r2d2_local_path(), + qd.window_length(), + qd.window_offset(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetObsNotInR2d2(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.mail_events = ['submit-failed'] + self.question_list = QuestionList([ + qd.ioda_locations_not_in_r2d2(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class GenerateBClimatology(Task): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {} + self.question_list = QuestionList([ + np_proc_resolution, + swell_static_file_questions, + qd.analysis_variables(), + qd.background_error_model(), + qd.generate_yaml_and_exit(), + qd.gradient_norm_reduction(), + qd.gsibec_configuration(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.jedi_forecast_model(), + qd.marine_models(), + qd.minimizer(), + qd.number_of_iterations(), + qd.observing_system_records_path(), + qd.total_processors(), + qd.window_offset(), + qd.window_type() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GenerateBClimatologyByLinking(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + swell_static_file_questions, + qd.background_error_model(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.window_offset(), + qd.window_type() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GenerateObservingSystemRecords(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.observations(), + qd.observing_system_records_mksi_path(), + qd.observing_system_records_path() + ]) + + # -------------------------------------------------------------------------------------------------- + + class LinkGeosOutput(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + window_questions, + qd.background_frequency(), + qd.marine_models() + ]) + + # -------------------------------------------------------------------------------------------------- + + class MoveDaRestart(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.analysis_forecast_window_offset(), + qd.mom6_iau(), + qd.window_length() + ]) + + # -------------------------------------------------------------------------------------------------- + + class MoveForecastRestart(Task): + def set_attributes(self): + self.is_cycling = True + self.question_list = QuestionList([ + qd.forecast_duration() + ]) + + # -------------------------------------------------------------------------------------------------- + + class PrepGeosRunDir(Task): + def set_attributes(self): + self.is_cycling = True + self.question_list = QuestionList([ + swell_static_file_questions, + qd.existing_geos_gcm_build_path(), + qd.forecast_duration(), + qd.geos_experiment_directory(), + qd.mom6_iau_nhours() + ]) + + # -------------------------------------------------------------------------------------------------- + + class PrepareAnalysis(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.analysis_forecast_window_offset(), + qd.analysis_variables(), + qd.mom6_iau(), + qd.total_processors() + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediFgatExecutable(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + run_jedi_executable, + qd.marine_models() + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediEnsembleMeanVariance(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + np_proc_resolution, + window_questions, + qd.analysis_variables(), + qd.ensemble_num_members(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.observations(), + qd.observing_system_records_path(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediHofxEnsembleExecutable(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.background_frequency(), + qd.ensemble_hofx_packets(), + qd.ensemble_hofx_strategy(), + qd.ensemble_num_members(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.total_processors() + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediHofxExecutable(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.background_frequency(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.save_geovals(), + qd.total_processors() + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediLocalEnsembleDaExecutable(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.ensemble_hofx_packets(), + qd.ensemble_hofx_strategy(), + qd.ensemble_num_members(), + qd.ensmean_only(), + qd.ensmeanvariance_only(), + qd.generate_yaml_and_exit(), + qd.horizontal_localization_lengthscale(), + qd.horizontal_localization_max_nobs(), + qd.horizontal_localization_method(), + qd.jedi_forecast_model(), + qd.local_ensemble_inflation_mult(), + qd.local_ensemble_inflation_rtpp(), + qd.local_ensemble_inflation_rtps(), + qd.local_ensemble_save_posterior_ensemble(), + qd.local_ensemble_save_posterior_ensemble_increments(), + qd.local_ensemble_save_posterior_mean(), + qd.local_ensemble_save_posterior_mean_increment(), + qd.local_ensemble_solver(), + qd.local_ensemble_use_linear_observer(), + qd.skip_ensemble_hofx(), + qd.total_processors(), + qd.vertical_localization_apply_log_transform(), + qd.vertical_localization_function(), + qd.vertical_localization_ioda_vertical_coord(), + qd.vertical_localization_ioda_vertical_coord_group(), + qd.vertical_localization_lengthscale(), + qd.vertical_localization_method(), + qd.perhost() + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediVariationalExecutable(Task): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {'nodes': 3} + self.question_list = QuestionList([ + run_jedi_executable, + qd.perhost() + ]) + + # -------------------------------------------------------------------------------------------------- + + class RemoveForecastDir(Task): + def set_attributes(self): + self.is_cycling = True + + # -------------------------------------------------------------------------------------------------- + + class RunGeosExecutable(Task): + def set_attributes(self): + self.is_cycling = True + + # -------------------------------------------------------------------------------------------------- + + class RunJediUfoExecutable(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.slurm = {} + self.time_limit = True + + # -------------------------------------------------------------------------------------------------- + + class RunJediUfoTestsExecutable(Task): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {'ntasks-per-node': 1} + self.question_list = QuestionList([ + background_crtm_obs, + qd.generate_yaml_and_exit(), + qd.single_observations(), + qd.window_length(), + qd.window_offset() + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediConvertStateSoca2ciceExecutable(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {'nodes': 1} + self.question_list = QuestionList([ + qd.analysis_variables(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.marine_models(), + qd.observations(), + qd.total_processors(), + qd.window_offset(), + qd.window_type() + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediFgatExecutable(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + run_jedi_executable, + qd.marine_models() + ]) + + # -------------------------------------------------------------------------------------------------- + + class SaveObsDiags(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + background_crtm_obs, + qd.r2d2_local_path(), + qd.window_offset(), + qd.marine_models() + ]) + + # -------------------------------------------------------------------------------------------------- + + class SaveRestart(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + window_questions, + qd.background_time_offset(), + qd.forecast_duration(), + qd.horizontal_resolution(), + qd.marine_models(), + qd.r2d2_local_path() + ]) + + # -------------------------------------------------------------------------------------------------- + + class StageJedi(Task): + def set_attributes(self): + self.is_model = True + self.question_list = QuestionList([ + swell_static_file_questions, + qd.gsibec_configuration(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.horizontal_resolution(), + qd.vertical_resolution() + ]) + + # -------------------------------------------------------------------------------------------------- + + class StageJediCycle(Task): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.base_name = "StageJedi" + self.scheduling_name = "StageJediCycle-{self.model}" + self.question_list = QuestionList([ + swell_static_file_questions, + qd.gsibec_configuration(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.horizontal_resolution(), + qd.vertical_resolution() + ]) + + # -------------------------------------------------------------------------------------------------- + + class sync_point(Task): + def set_attributes(self): + self.script = "true" + + # -------------------------------------------------------------------------------------------------- + + class JediLogComparison(Task): + def set_attributes(self): + self.is_model = True + self.question_list = QuestionList([ + qd.number_of_iterations(), + qd.comparison_log_type(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediObsfiltersExecutable(Task): + def set_attributes(self): + self.script = ("swell task RunJediObsfiltersExecutable $config" + " -d $datetime -m geos_atmosphere") + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.background_frequency(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.observing_system_records_path(), + qd.total_processors(), + qd.obs_thinning_rej_fraction() + ]) + + # -------------------------------------------------------------------------------------------------- + + @classmethod + def get(cls, name: str) -> Task: + return getattr(cls, name) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_runtimes.py b/src/swell/tasks/task_runtimes.py deleted file mode 100644 index 77da90131..000000000 --- a/src/swell/tasks/task_runtimes.py +++ /dev/null @@ -1,342 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - -from typing import Union, Optional, Self -from collections.abc import Mapping -from dataclasses import dataclass - -from swell.utilities.dataclass_utils import mutable_field -from swell.utilities.cylc_runtime import Task - -# -------------------------------------------------------------------------------------------------- - - -class TaskRuntimes(): - - @dataclass - class root(Task): - script: bool = False - pre_script: str = "source $CYLC_SUITE_DEF_PATH/modules" - environment: dict = mutable_field({'datetime': '$CYLC_TASK_CYCLE_POINT', - 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'}) - - @dataclass - class BuildGeos(Task): - pass - - @dataclass - class BuildGeosByLinking(Task): - mail_events: list = mutable_field(['submit-failed']) - - @dataclass - class BuildJediByLinking(Task): - mail_events: list = mutable_field(['submit-failed']) - - @dataclass - class BuildJedi(Task): - time_limit: bool = True - slurm: dict = mutable_field({}) - - @dataclass - class CleanCycle(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class CloneGeos(Task): - pass - - @dataclass - class CloneJedi(Task): - pass - - @dataclass - class CloneGeosMksi(Task): - is_model: bool = True - - @dataclass - class CloneGmaoPerllib(Task): - pass - - @dataclass - class EvaJediLog(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class EvaComparisonIncrement(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class EvaComparisonJediLog(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class EvaIncrement(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class EvaObservations(Task): - time_limit: bool = True - is_cycling: bool = True - is_model: bool = True - slurm: dict = mutable_field({}) - - @dataclass - class EvaTimeseries(Task): - time_limit: bool = True - is_cycling: bool = True - is_model: bool = True - slurm: dict = mutable_field({}) - - @dataclass - class JediOopsLogParser(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GetBackground(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GetBackgroundGeosExperiment(Task): - is_cycling: bool = True - is_model: bool = True - mail_events: list = mutable_field(['submit-failed']) - - @dataclass - class GetBufr(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class BufrToIoda(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GetEnsembleGeosExperiment(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GetGeosRestart(Task): - is_cycling: bool = True - - @dataclass - class GetGeovals(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GetGsiBc(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GsiBcToIoda(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GetGsiNcdiag(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GsiNcdiagToIoda(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GetNcdiags(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GetGeosAdasBackground(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GetObservations(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GetObsNotInR2d2(Task): - is_cycling: bool = True - is_model: bool = True - mail_events: list = mutable_field(['submit-failed']) - - @dataclass - class GenerateBClimatology(Task): - time_limit: bool = True - is_cycling: bool = True - is_model: bool = True - slurm: dict = mutable_field({}) - - @dataclass - class GenerateBClimatologyByLinking(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class GenerateObservingSystemRecords(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class LinkGeosOutput(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class MoveDaRestart(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class MoveForecastRestart(Task): - is_cycling: bool = True - - @dataclass - class PrepGeosRunDir(Task): - is_cycling: bool = True - - @dataclass - class PrepareAnalysis(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class RunJediFgatExecutable(Task): - is_cycling: bool = True - is_model: bool = True - time_limit: bool = True - slurm: dict = mutable_field({}) - - @dataclass - class RunJediHofxEnsembleMeanVariance(Task): - is_cycling: bool = True - is_model: bool = True - time_limit: bool = True - slurm: dict = mutable_field({}) - - @dataclass - class RunJediHofxEnsembleExecutable(Task): - is_cycling: bool = True - is_model: bool = True - time_limit: bool = True - slurm: dict = mutable_field({}) - - @dataclass - class RunJediHofxExecutable(Task): - is_cycling: bool = True - is_model: bool = True - time_limit: bool = True - slurm: dict = mutable_field({}) - - @dataclass - class RunJediLocalEnsembleDaExecutable(Task): - is_cycling: bool = True - is_model: bool = True - time_limit: bool = True - slurm: dict = mutable_field({}) - - @dataclass - class RunJediVariationalExecutable(Task): - time_limit: bool = True - is_cycling: bool = True - is_model: bool = True - slurm: dict = mutable_field({'nodes': 3}) - - @dataclass - class RemoveForecastDir(Task): - is_cycling: bool = True - - @dataclass - class RunGeosExecutable(Task): - is_cycling: bool = True - - @dataclass - class RunJediUfoExecutable(Task): - is_cycling: bool = True - is_model: bool = True - slurm: dict = mutable_field({}) - time_limit: bool = True - - @dataclass - class RunJediUfoTestsExecutable(Task): - time_limit: bool = True - is_cycling: bool = True - is_model: bool = True - slurm: dict = mutable_field({'ntasks-per-node': 1}) - - @dataclass - class RunJediConvertStateSoca2ciceExecutable(Task): - is_cycling: bool = True - is_model: bool = True - time_limit: bool = True - slurm: dict = mutable_field({'nodes': 1}) - - @dataclass - class RunJediFgatExecutable(Task): - is_cycling: bool = True - is_model: bool = True - time_limit: bool = True - slurm: dict = mutable_field({}) - - @dataclass - class SaveObsDiags(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class SaveRestart(Task): - is_cycling: bool = True - is_model: bool = True - - @dataclass - class StageJedi(Task): - is_model: bool = True - - @dataclass - class StageJediCycle(Task): - is_cycling: bool = True - is_model: bool = True - base_name: str = "StageJedi" - scheduling_name: str = "StageJediCycle-{model}" - - @dataclass - class sync_point(Task): - script: str = "true" - - @dataclass - class JediLogComparison(Task): - is_model: bool = True - - @dataclass - class RunJediObsfiltersExecutable(Task): - script: str = ("swell task RunJediObsfiltersExecutable $config" - " -d $datetime -m geos_atmosphere") - is_cycling: bool = True - is_model: bool = True - time_limit: bool = True - slurm: dict = mutable_field({}) - - @classmethod - def get(cls, name: str) -> Task: - return getattr(cls, name) - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index b896f56ff..4a0d10fa2 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -10,7 +10,7 @@ import yaml from typing import Callable -from swell.tasks.task_questions import TaskQuestions as task_questions +from swell.tasks.task_attributes import TaskAttributes as task_attributes from swell.utilities.logger import Logger from swell.suites.all_suites import suite_configs @@ -122,8 +122,8 @@ def __init__(self, input_file: str, logger: Logger, task_name: str, model: str) question_list.append(question) # Find the questions associated with the task - if task_name in task_questions.get_all(): - question_list.extend(task_questions[task_name].value.get_all_question_names()) + task_class = getattr(task_attributes, task_name) + question_list.extend(task_class().question_list.get_all_question_names()) # Loop through the experiment dictionary for exp_key, exp_val in experiment_dict.items(): diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index d8420d5f6..cfe5a84da 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -15,6 +15,7 @@ from isodate import parse_datetime, parse_duration, ISO8601Error from swell.swell_path import get_swell_path +from swell.utilities.dataclass_utils import mutable_field # -------------------------------------------------------------------------------------------------- @@ -132,11 +133,11 @@ def get_all(cls): @dataclass class QuestionList: """Basic dataclass containing a list of questions for each model, suite, task""" - list_name: str - questions: List[Union[SwellQuestion, Self]] + questions: List[Union[SwellQuestion, Self]] = mutable_field([]) + list_name: str = '' - geos_atmosphere: list = field(default_factory=lambda: []) - geos_marine: list = field(default_factory=lambda: []) + geos_atmosphere: list = mutable_field([]) + geos_marine: list = mutable_field([]) # -------------------------------------------------------------------------------------------------- @@ -170,7 +171,7 @@ def expand_question_list(self, model: Optional[str] = None): question = asdict(question_obj) # If the item is a question list, expand its contents - if 'list_name' in question.keys(): + if 'questions' in question.keys(): question_list.extend(question_obj.expand_question_list(model)) elif model is None: # Add to the model_independent question list @@ -184,7 +185,7 @@ def expand_question_list(self, model: Optional[str] = None): question_obj = question_obj.value question = asdict(question_obj) - if 'list_name' in question.keys(): + if 'questions' in question.keys(): question_list.extend(question_obj.expand_question_list(model)) else: question_list.append(question) diff --git a/src/swell/utilities/task_spec.py b/src/swell/utilities/task_spec.py new file mode 100644 index 000000000..8a5a1bd07 --- /dev/null +++ b/src/swell/utilities/task_spec.py @@ -0,0 +1,253 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +import yaml +from typing import Union, Optional +from collections.abc import Mapping + +from swell.utilities.cylc_formatting import CylcSection, indent_lines +from swell.utilities.suite_utils import get_model_components +from swell.utilities.dictionary import update_dict +from swell.utilities.swell_questions import QuestionList + +# -------------------------------------------------------------------------------------------------- + + +class Task: + + ''' + Contains the basic properties and information needed to format the cylc [runtime] section. + ''' + + def __init__(self, model: Optional[str] = None, platform: Optional[str] = None) -> None: + + self.model = model + self.platform = platform + + self.base_name = None + self.scheduling_name = None + + self.is_cycling = False + self.is_model = False + + self.pre_script = False + self.script = None + + self.retry = None + self.time_limit = None + self.slurm = None + + self.subsections = [] + self.mail_events = ['failed', 'submit-failed'] + + self.questions = QuestionList([]) + + self.set_attributes() + self.post_init() + + # -------------------------------------------------------------------------------------------------- + + def set_attributes(self) -> None: + pass + + # -------------------------------------------------------------------------------------------------- + + def post_init(self): + + if self.base_name is None: + self.base_name = self.__class__.__name__ + + if self.scheduling_name is None: + self.scheduling_name = self.base_name + + if self.is_model and self.model is not None: + self.scheduling_name += f'-{self.model}' + + if self.script is None: + self.script = f'swell task {self.base_name} $config' + + if self.is_cycling: + self.script += ' -d $datetime' + + if self.is_model and self.model is not None: + self.script += ' -m {model}' + + if self.is_model and self.model is not None: + self.script = self.script.format(model=self.model) + self.scheduling_name = self.scheduling_name.format(model=self.model) + + # -------------------------------------------------------------------------------------------------- + + def format_string_block(self, string: str) -> str: + out_string = '"""\n' + out_string += indent_lines(string, 1) + out_string += '"""' + + return out_string + + # -------------------------------------------------------------------------------------------------- + + def match_platform(self, content: Union[str, dict], platform: str): + # Resolve platform-specific entries in the task object + + if isinstance(content, Mapping): + if platform in content.keys(): + content = content[platform] + elif 'all' in content.keys(): + content = content['all'] + + return content + + # -------------------------------------------------------------------------------------------------- + + def create_new_section(self, + name: Optional[str] = None, + content: Union[str, dict] = '' + ) -> CylcSection: + return CylcSection(name, content) + + # -------------------------------------------------------------------------------------------------- + + def resolve_model(self, slurm_dict: Mapping) -> dict: + ''' Resolve "all" and "model" entries in slurm dictionary ''' + if 'all' in slurm_dict.keys() and isinstance(slurm_dict['all'], Mapping): + slurm_dict = update_dict(slurm_dict, slurm_dict['all']) + del slurm_dict['all'] + if self.model in slurm_dict.keys() and isinstance(slurm_dict[self.model], Mapping): + slurm_dict = update_dict(slurm_dict, slurm_dict[self.model]) + + for model in get_model_components(): + if model in slurm_dict.keys(): + del slurm_dict[model] + + return slurm_dict + + # -------------------------------------------------------------------------------------------------- + + def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Mapping: + # Take the external slurm dictionary and merge it with the task's parameters + # to get the dict that will be output in the flow.cylc + + slurm_dict = {} + if self.slurm is not None: + for key, value in self.slurm.items(): + slurm_dict[key] = self.match_platform(value, platform) + + slurm_globals = slurm_external['slurm_directives_global'] + slurm_task = {} + + if 'slurm_directives_tasks' in slurm_external.keys(): + task_directives = slurm_external['slurm_directives_tasks'] + + if self.base_name in task_directives: + slurm_task = task_directives[self.base_name] + if self.scheduling_name in task_directives: + slurm_task = task_directives[self.scheduling_name] + + slurm_dict = {'job-name': self.scheduling_name, + **self.resolve_model(slurm_globals), + **self.resolve_model(slurm_dict), + **self.resolve_model(slurm_task)} + + return slurm_dict + + # -------------------------------------------------------------------------------------------------- + + def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): + ''' Return the runtime section for the given task. ''' + + platform = experiment_dict['platform'] + runtime_dict = {} + + # Set the pre_script only if it is specified + if self.pre_script: + runtime_dict['pre-script'] = self.format_string_block(self.pre_script) + + # Set the script + if self.script: + script_str = self.script + + if 'pause_on_tasks' in experiment_dict.keys(): + if len(set([self.base_name, self.scheduling_name]) + & set(experiment_dict['pause_on_tasks'])) > 0: + script_str += '\ncylc pause $CYLC_WORKFLOW_ID' + + runtime_dict['script'] = self.format_string_block(script_str) + + # Specify the platform if this is a slurm task + if self.slurm is not None: + runtime_dict['platform'] = platform + + # Set the time limit, default is 1 hour + if self.time_limit is True: + runtime_dict['execution time limit'] = 'PT1H' + elif self.time_limit: + time_limit = self.match_platform(self.time_limit, platform) + runtime_dict['execution time limit'] = time_limit + + # Set the retry if this task needs it + if self.retry: + if self.retry is True: + retry = '2*PT1M' + else: + retry = self.match_platform(self.retry, platform) + + runtime_dict['execution retry delays'] = retry + + runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) + + # Set the environment dictionary + if self.environment is not None: + environment_section = self.create_new_section('environment', self.environment) + runtime_section.add_subsection(environment_section) + + # Specify the slurm dictionary with defaults from user and global settings + if self.slurm is not None: + + slurm_dict = self.generate_task_slurm_dict(slurm_external, platform) + + slurm_section_dict = {} + for key, value in slurm_dict.items(): + slurm_section_dict[f'--{key}'] = value + + directive_section = self.create_new_section('directives', slurm_section_dict) + + runtime_section.add_subsection(directive_section) + + # Append additional sections to runtime + for section in self.additional_sections: + runtime_section.add_subsection(section) + + # Check slurm messaging parameters + events = [] + events = self.mail_events + + # Add messaging section + settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) + if os.path.exists(settings_file) and len(events) > 0: + with open(settings_file, 'r') as f: + settings_dict = yaml.safe_load(f) + if 'email_address' in settings_dict.keys(): + email_address = settings_dict['email_address'] + address_section = self.create_new_section('mail', f'to = {email_address}') + runtime_section.add_subsection(address_section) + + event_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" + event_str += "mail events = " + ', '.join(events) + event_str += "\n{% endif %}\n" + + event_section = self.create_new_section('events', event_str) + runtime_section.add_subsection(event_section) + + runtime_string = runtime_section.get_section_str(1) + + return runtime_string + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file From ad01ab69b705da87a1b7bd5ada8a038995545dd4 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 30 Oct 2025 14:45:47 -0400 Subject: [PATCH 121/299] fixes for all suites --- src/swell/deployment/create_experiment.py | 6 +++--- src/swell/suites/3dvar_cycle/workflow.py | 2 +- src/swell/suites/geosadas/workflow.py | 2 +- src/swell/tasks/task_attributes.py | 6 +++--- src/swell/utilities/cylc_workflow.py | 1 - src/swell/utilities/task_spec.py | 12 +++++------- 6 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 58e5befe3..b6d8896c3 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -96,11 +96,11 @@ def prepare_config( suite_dict = prepare_config_and_suite.get_experiment_dict() suite_dict = suite_dict.copy() - + print(suite_dict) # Resolve cycle times for models # ------------------------------ - if 'model_components' in suite_dict: - model_components = suite_dict['model_components'] + if 'models' in suite_dict: + model_components = suite_dict['models'] # Since cycle times are used, the render_dictionary will need to include cycle_times # If there are different model components then process each to gather cycle times diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index efacbb538..a68542bdb 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -167,7 +167,7 @@ def tasks(self) -> list: tasks.append(ta.BuildJedi()) tasks.append(ta.BuildGeos()) tasks.append(ta.GetGeosRestart()) - tasks.append(ta.PrepGeosRunDir) + tasks.append(ta.PrepGeosRunDir()) tasks.append(ta.RunGeosExecutable()) for model in self.experiment_dict['model_components']: diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index 5b8b7e4f5..a0399368b 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -99,7 +99,7 @@ def tasks(self) -> list: tasks.append(ta.root()) tasks.append(ta.CloneJedi()) tasks.append(ta.BuildJediByLinking()) - tasks.append(ta.CloneGeosMksi) + tasks.append(ta.CloneGeosMksi()) for model in self.experiment_dict['model_components']: tasks.append(ta.GenerateObservingSystemRecords(model=model)) diff --git a/src/swell/tasks/task_attributes.py b/src/swell/tasks/task_attributes.py index b51c1333e..d97485368 100644 --- a/src/swell/tasks/task_attributes.py +++ b/src/swell/tasks/task_attributes.py @@ -92,7 +92,7 @@ class root(Task): def set_attributes(self): self.script = False self.pre_script = "source $CYLC_SUITE_DEF_PATH/modules" - self.addional_sections = [self.create_new_section('environment', {'datetime': '$CYLC_TASK_CYCLE_POINT', + self.additional_sections = [self.create_new_section('environment', {'datetime': '$CYLC_TASK_CYCLE_POINT', 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'})] # -------------------------------------------------------------------------------------------------- @@ -161,7 +161,7 @@ def set_attributes(self): class CloneJedi(Task): def set_attributes(self): - self.self.question_list = QuestionList([ + self.question_list = QuestionList([ qd.bundles(), qd.existing_jedi_source_directory(), qd.existing_jedi_source_directory_pinned(), @@ -809,7 +809,7 @@ def set_attributes(self): self.is_cycling = True self.is_model = True self.base_name = "StageJedi" - self.scheduling_name = "StageJediCycle-{self.model}" + self.scheduling_name = "StageJediCycle-{model}" self.question_list = QuestionList([ swell_static_file_questions, qd.gsibec_configuration(), diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 7694836db..6528e9922 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -11,7 +11,6 @@ from abc import abstractmethod from swell.utilities.cylc_formatting import CylcSection, indent_lines -from swell.tasks.task_runtimes import TaskRuntimes from swell.utilities.logger import get_logger from swell.utilities.jinja2 import template_string_jinja2 diff --git a/src/swell/utilities/task_spec.py b/src/swell/utilities/task_spec.py index 8a5a1bd07..1f2f41686 100644 --- a/src/swell/utilities/task_spec.py +++ b/src/swell/utilities/task_spec.py @@ -47,7 +47,8 @@ def __init__(self, model: Optional[str] = None, platform: Optional[str] = None) self.subsections = [] self.mail_events = ['failed', 'submit-failed'] - self.questions = QuestionList([]) + self.question_list = QuestionList([]) + self.additional_sections = [] self.set_attributes() self.post_init() @@ -203,11 +204,6 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) - # Set the environment dictionary - if self.environment is not None: - environment_section = self.create_new_section('environment', self.environment) - runtime_section.add_subsection(environment_section) - # Specify the slurm dictionary with defaults from user and global settings if self.slurm is not None: @@ -248,6 +244,8 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): runtime_string = runtime_section.get_section_str(1) + runtime_string += ' # ' + '-' * 96 + '\n\n' + return runtime_string -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- From c4ba5591de52388872efaf92541d4a29ade7fa92 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 30 Oct 2025 16:08:43 -0400 Subject: [PATCH 122/299] change names of task_spec --- src/swell/tasks/task_attributes.py | 4 +- src/swell/tasks/task_questions.py | 744 ------------------ .../{task_spec.py => task_specification.py} | 0 3 files changed, 1 insertion(+), 747 deletions(-) delete mode 100644 src/swell/tasks/task_questions.py rename src/swell/utilities/{task_spec.py => task_specification.py} (100%) diff --git a/src/swell/tasks/task_attributes.py b/src/swell/tasks/task_attributes.py index d97485368..0965b06ce 100644 --- a/src/swell/tasks/task_attributes.py +++ b/src/swell/tasks/task_attributes.py @@ -10,7 +10,7 @@ from typing import Union, Optional, Self from collections.abc import Mapping -from swell.utilities.task_spec import Task +from swell.utilities.task_specification import Task from swell.utilities.swell_questions import QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd @@ -139,10 +139,8 @@ def set_attributes(self): class CleanCycle(Task): def set_attributes(self): - self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ qd.clean_patterns() ]) diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py deleted file mode 100644 index 9f053cc46..000000000 --- a/src/swell/tasks/task_questions.py +++ /dev/null @@ -1,744 +0,0 @@ -# -------------------------------------------------------------------------------------------------- -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -from enum import Enum - -from swell.utilities.swell_questions import QuestionList, QuestionContainer -from swell.utilities.question_defaults import QuestionDefaults as qd - - -# -------------------------------------------------------------------------------------------------- - -class TaskQuestions(QuestionContainer, Enum): - - # -------------------------------------------------------------------------------------------------- - # Helper question lists used by multiple tasks (in order of use) - # -------------------------------------------------------------------------------------------------- - - background_crtm_obs = QuestionList( - list_name="background_crtm_obs", - questions=[ - qd.background_time_offset(), - qd.crtm_coeff_dir(), - qd.observations(), - qd.observing_system_records_path() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - np_proc_resolution = QuestionList( - list_name="np_resolution", - questions=[ - qd.npx_proc(), - qd.npy_proc(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - window_questions = QuestionList( - list_name="window_questions", - questions=[ - qd.window_length(), - qd.window_offset(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - run_jedi_executable = QuestionList( - list_name="run_jedi_executable", - questions=[ - background_crtm_obs, - np_proc_resolution, - window_questions, - qd.analysis_variables(), - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.gradient_norm_reduction(), - qd.gsibec_configuration(), - qd.jedi_forecast_model(), - qd.minimizer(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.number_of_iterations(), - qd.total_processors(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - swell_static_file_questions = QuestionList( - list_name="swell_static_file_questions", - questions=[ - qd.swell_static_files(), - qd.swell_static_files_user() - ] - ) - - # -------------------------------------------------------------------------------------------------- - # Task-specific question lists (in alphabetical order) - # -------------------------------------------------------------------------------------------------- - - BuildGeos = QuestionList( - list_name="BuildGeos", - questions=[ - qd.geos_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - BuildGeosByLinking = QuestionList( - list_name="BuildGeosByLinking", - questions=[ - qd.existing_geos_gcm_build_path(), - qd.geos_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - BuildJedi = QuestionList( - list_name="BuildJedi", - questions=[ - qd.bundles(), - qd.jedi_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - BuildJediByLinking = QuestionList( - list_name="BuildJediByLinking", - questions=[ - qd.existing_jedi_build_directory(), - qd.existing_jedi_build_directory_pinned(), - qd.jedi_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CleanCycle = QuestionList( - list_name="CleanCycle", - questions=[ - qd.clean_patterns() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CloneGeos = QuestionList( - list_name="CloneGeos", - questions=[ - qd.existing_geos_gcm_source_path(), - qd.geos_build_method(), - qd.geos_gcm_tag() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CloneGeosMksi = QuestionList( - list_name="CloneGeosMksi", - questions=[ - qd.observing_system_records_mksi_path(), - qd.observing_system_records_mksi_path_tag() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CloneGmaoPerllib = QuestionList( - list_name="CloneGmaoPerllib", - questions=[ - qd.existing_perllib_path(), - qd.gmao_perllib_tag() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CloneJedi = QuestionList( - list_name="CloneJedi", - questions=[ - qd.bundles(), - qd.existing_jedi_source_directory(), - qd.existing_jedi_source_directory_pinned(), - qd.jedi_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaIncrement = QuestionList( - list_name="EvaIncrement", - questions=[ - qd.marine_models(), - qd.window_offset(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaComparisonIncrement = QuestionList( - list_name="EvaIncrement", - questions=[ - EvaIncrement - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaObservations = QuestionList( - list_name="EvaObservations", - questions=[ - background_crtm_obs, - qd.marine_models(), - qd.observing_system_records_path(), - qd.window_offset(), - qd.marine_models(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaTimeseries = QuestionList( - list_name="EvaTimeseries", - questions=[ - background_crtm_obs, - qd.window_length(), - qd.window_offset(), - qd.ncdiag_experiments(), - qd.marine_models(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GenerateBClimatology = QuestionList( - list_name="GenerateBClimatology", - questions=[ - np_proc_resolution, - swell_static_file_questions, - qd.analysis_variables(), - qd.background_error_model(), - qd.generate_yaml_and_exit(), - qd.gradient_norm_reduction(), - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.jedi_forecast_model(), - qd.marine_models(), - qd.minimizer(), - qd.number_of_iterations(), - qd.observing_system_records_path(), - qd.total_processors(), - qd.window_offset(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GenerateBClimatologyByLinking = QuestionList( - list_name="GenerateBClimatologyByLinking", - questions=[ - swell_static_file_questions, - qd.background_error_model(), - qd.horizontal_resolution(), - qd.vertical_resolution(), - qd.window_offset(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GenerateObservingSystemRecords = QuestionList( - list_name="GenerateObservingSystemRecords", - questions=[ - qd.observations(), - qd.observing_system_records_mksi_path(), - qd.observing_system_records_path() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetBackground = QuestionList( - list_name="GetBackground", - questions=[ - window_questions, - qd.analysis_forecast_window_offset(), - qd.background_experiment(), - qd.background_frequency(), - qd.horizontal_resolution(), - qd.marine_models(), - qd.r2d2_local_path(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetBackgroundGeosExperiment = QuestionList( - list_name="GetBackgroundGeosExperiment", - questions=[ - qd.horizontal_resolution(), - qd.background_experiment(), - qd.background_time_offset(), - qd.geos_x_background_directory() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetBufr = QuestionList( - list_name="GetBufr", - questions=[ - qd.bufr_obs_classes() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetEnsemble = QuestionList( - list_name="GetEnsemble", - questions=[ - qd.path_to_ensemble() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetEnsembleGeosExperiment = QuestionList( - list_name="GetEnsembleGeosExperiment", - questions=[ - qd.background_experiment(), - qd.background_time_offset(), - qd.geos_x_ensemble_directory() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGeovals = QuestionList( - list_name="GetGeovals", - questions=[ - background_crtm_obs, - qd.geovals_experiment(), - qd.geovals_provider(), - qd.r2d2_local_path(), - qd.window_length(), - qd.window_offset() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGeosAdasBackground = QuestionList( - list_name="GetGeosAdasBackground", - questions=[ - qd.path_to_geos_adas_background() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGeosRestart = QuestionList( - list_name="GetGeosRestart", - questions=[ - swell_static_file_questions, - qd.geos_restarts_directory() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGsiBc = QuestionList( - list_name="GetGsiBc", - questions=[ - qd.path_to_gsi_bc_coefficients(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGsiNcdiag = QuestionList( - list_name="GetGsiNcdiag", - questions=[ - qd.path_to_gsi_nc_diags() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetNcdiags = QuestionList( - list_name="GetNcdiags", - questions=[ - background_crtm_obs, - qd.ncdiag_experiments(), - qd.marine_models(), - qd.r2d2_local_path(), - qd.window_length(), - qd.window_offset(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetObservations = QuestionList( - list_name="GetObservations", - questions=[ - background_crtm_obs, - qd.cycling_varbc(), - qd.obs_experiment(), - qd.obs_provider(), - qd.observing_system_records_path(), - qd.r2d2_local_path(), - qd.window_length(), - qd.window_offset(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetObsNotInR2d2 = QuestionList( - list_name="GetExistingObservations", - questions=[ - qd.ioda_locations_not_in_r2d2(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GsiBcToIoda = QuestionList( - list_name="GsiBcToIoda", - questions=[ - background_crtm_obs, - qd.observing_system_records_path(), - qd.window_offset() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GsiNcdiagToIoda = QuestionList( - list_name="GsiNcdiagToIoda", - questions=[ - qd.observations(), - qd.produce_geovals(), - qd.single_observations(), - qd.window_offset() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - JediOopsLogParser = QuestionList( - list_name="JediOopsLogParser", - questions=[ - qd.parser_options(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - JediLogComparison = QuestionList( - list_name="JediLogComparison", - questions=[ - qd.number_of_iterations(), - qd.comparison_log_type(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - LinkGeosOutput = QuestionList( - list_name="LinkGeosOutput", - questions=[ - window_questions, - qd.background_frequency(), - qd.marine_models() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - MoveDaRestart = QuestionList( - list_name="MoveDaRestart", - questions=[ - qd.analysis_forecast_window_offset(), - qd.mom6_iau(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - MoveForecastRestart = QuestionList( - list_name="MoveForecastRestart", - questions=[ - qd.forecast_duration() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - PrepareAnalysis = QuestionList( - list_name="PrepareAnalysis", - questions=[ - qd.analysis_forecast_window_offset(), - qd.analysis_variables(), - qd.mom6_iau(), - qd.total_processors() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - PrepGeosRunDir = QuestionList( - list_name="PrepGeosRunDir", - questions=[ - swell_static_file_questions, - qd.existing_geos_gcm_build_path(), - qd.forecast_duration(), - qd.geos_experiment_directory(), - qd.mom6_iau_nhours() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediConvertStateSoca2ciceExecutable = QuestionList( - list_name="RunJediConvertStateSoca2ciceExecutable", - questions=[ - qd.analysis_variables(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.marine_models(), - qd.observations(), - qd.total_processors(), - qd.window_offset(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediEnsembleMeanVariance = QuestionList( - list_name="RunJediEnsembleMeanVariance", - questions=[ - np_proc_resolution, - window_questions, - qd.analysis_variables(), - qd.ensemble_num_members(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.observations(), - qd.observing_system_records_path(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediFgatExecutable = QuestionList( - list_name="RunJediFgatExecutable", - questions=[ - run_jedi_executable, - qd.marine_models() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediHofxEnsembleExecutable = QuestionList( - list_name="RunJediHofxEnsembleExecutable", - questions=[ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.ensemble_num_members(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.total_processors() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediHofxExecutable = QuestionList( - list_name="RunJediHofxExecutable", - questions=[ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.save_geovals(), - qd.total_processors() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediLocalEnsembleDaExecutable = QuestionList( - list_name="RunJediLocalEnsembleDaExecutable", - questions=[ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.ensemble_num_members(), - qd.ensmean_only(), - qd.ensmeanvariance_only(), - qd.generate_yaml_and_exit(), - qd.horizontal_localization_lengthscale(), - qd.horizontal_localization_max_nobs(), - qd.horizontal_localization_method(), - qd.jedi_forecast_model(), - qd.local_ensemble_inflation_mult(), - qd.local_ensemble_inflation_rtpp(), - qd.local_ensemble_inflation_rtps(), - qd.local_ensemble_save_posterior_ensemble(), - qd.local_ensemble_save_posterior_ensemble_increments(), - qd.local_ensemble_save_posterior_mean(), - qd.local_ensemble_save_posterior_mean_increment(), - qd.local_ensemble_solver(), - qd.local_ensemble_use_linear_observer(), - qd.skip_ensemble_hofx(), - qd.total_processors(), - qd.vertical_localization_apply_log_transform(), - qd.vertical_localization_function(), - qd.vertical_localization_ioda_vertical_coord(), - qd.vertical_localization_ioda_vertical_coord_group(), - qd.vertical_localization_lengthscale(), - qd.vertical_localization_method(), - qd.perhost() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediObsfiltersExecutable = QuestionList( - list_name="RunJediObsfiltersExecutable", - questions=[ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.observing_system_records_path(), - qd.total_processors(), - qd.obs_thinning_rej_fraction() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediUfoTestsExecutable = QuestionList( - list_name="RunJediUfoTestsExecutable", - questions=[ - background_crtm_obs, - qd.generate_yaml_and_exit(), - qd.single_observations(), - qd.window_length(), - qd.window_offset() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediVariationalExecutable = QuestionList( - list_name="RunJediVariationalExecutable", - questions=[ - run_jedi_executable, - qd.perhost() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - SaveObsDiags = QuestionList( - list_name="SaveObsDiags", - questions=[ - background_crtm_obs, - qd.r2d2_local_path(), - qd.window_offset(), - qd.marine_models() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - SaveRestart = QuestionList( - list_name="SaveRestart", - questions=[ - window_questions, - qd.background_time_offset(), - qd.forecast_duration(), - qd.horizontal_resolution(), - qd.marine_models(), - qd.r2d2_local_path() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - StageJedi = QuestionList( - list_name="StageJedi", - questions=[ - swell_static_file_questions, - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - StageJediCycle = QuestionList( - list_name="StageJediCycle", - questions=[ - StageJedi - ] - ) - - # -------------------------------------------------------------------------------------------------- - - StoreBackground = QuestionList( - list_name="StoreBackground", - questions=[ - window_questions, - qd.analysis_forecast_window_offset(), - qd.background_experiment(), - qd.background_frequency(), - qd.horizontal_resolution(), - qd.r2d2_local_path(), - ] - ) - - # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/task_spec.py b/src/swell/utilities/task_specification.py similarity index 100% rename from src/swell/utilities/task_spec.py rename to src/swell/utilities/task_specification.py From ee07db049abea60e2c8d985b644a278b32237de7 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 7 Nov 2025 16:42:26 -0500 Subject: [PATCH 123/299] clean up --- src/swell/deployment/create_experiment.py | 2 +- src/swell/suites/3dfgat_atmos/workflow.py | 6 +- src/swell/suites/3dfgat_cycle/workflow.py | 4 +- src/swell/suites/3dvar/workflow.py | 6 +- src/swell/suites/3dvar_atmos/workflow.py | 4 +- src/swell/suites/3dvar_cycle/workflow.py | 6 +- src/swell/suites/build_geos/workflow.py | 4 +- src/swell/suites/build_jedi/workflow.py | 4 +- .../suites/compare_variational/workflow.py | 7 +- src/swell/suites/convert_bufr/workflow.py | 7 +- src/swell/suites/convert_ncdiags/workflow.py | 6 +- src/swell/suites/eva_capabilities/workflow.py | 6 +- src/swell/suites/forecast_geos/workflow.py | 6 +- src/swell/suites/geosadas/workflow.py | 6 +- src/swell/suites/hofx/workflow.py | 6 +- src/swell/suites/localensembleda/workflow.py | 6 +- src/swell/suites/ufo_testing/workflow.py | 6 +- src/swell/tasks/task_attributes.py | 119 ++++--- src/swell/test/code_tests/code_tests.py | 4 - .../question_dictionary_comparison_test.py | 43 --- src/swell/test/code_tests/slurm_test.py | 20 +- src/swell/utilities/cylc_workflow.py | 4 +- .../utilities/scripts/compare_questions.py | 292 ------------------ src/swell/utilities/swell_questions.py | 2 +- 24 files changed, 116 insertions(+), 460 deletions(-) delete mode 100644 src/swell/test/code_tests/question_dictionary_comparison_test.py delete mode 100644 src/swell/utilities/scripts/compare_questions.py diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index b6d8896c3..300e4ad24 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -96,7 +96,7 @@ def prepare_config( suite_dict = prepare_config_and_suite.get_experiment_dict() suite_dict = suite_dict.copy() - print(suite_dict) + # Resolve cycle times for models # ------------------------------ if 'models' in suite_dict: diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 1d6f0727e..de4110c0c 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -133,13 +133,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) @@ -164,7 +164,7 @@ def tasks(self) -> list: tasks.append(ta.EvaIncrement(model=model)) tasks.append(ta.SaveObsDiags(model=model)) tasks.append(ta.CleanCycle(model=model)) - + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index 9cd2149d5..3da6922f2 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -150,13 +150,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 7c7dbacf5..0d3db308d 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -119,13 +119,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) @@ -146,7 +146,7 @@ def tasks(self) -> list: tasks.append(ta.EvaIncrement(model=model)) tasks.append(ta.SaveObsDiags(model=model)) tasks.append(ta.CleanCycle(model=model)) - + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index bf1acc790..929641635 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -127,13 +127,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index a68542bdb..4d35c1e35 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -150,13 +150,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) @@ -188,7 +188,7 @@ def tasks(self) -> list: tasks.append(ta.EvaIncrement(model=model)) tasks.append(ta.SaveObsDiags(model=model)) tasks.append(ta.CleanCycle(model=model)) - + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py index 52279fb6d..9c7b8c168 100644 --- a/src/swell/suites/build_geos/workflow.py +++ b/src/swell/suites/build_geos/workflow.py @@ -54,13 +54,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index 32e5a84bb..33c86e313 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -54,13 +54,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index 4b944cf31..c23405cd5 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -9,7 +9,6 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.utilities.check_da_params import check_da_params from swell.tasks.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- @@ -68,13 +67,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] @@ -82,7 +81,7 @@ def tasks(self) -> list: tasks.append(ta.EvaComparisonIncrement(model=model)) tasks.append(ta.EvaComparisonJediLog(model=model)) tasks.append(ta.JediOopsLogParser(model=model)) - + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_bufr/workflow.py b/src/swell/suites/convert_bufr/workflow.py index 827b4c6a4..5ffe2bf5b 100644 --- a/src/swell/suites/convert_bufr/workflow.py +++ b/src/swell/suites/convert_bufr/workflow.py @@ -95,13 +95,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) @@ -110,13 +110,12 @@ def tasks(self) -> list: tasks.append(ta.BuildJedi()) tasks.append(ta.CloneGmaoPerllib()) - for model in self.experiment_dict['model_components']: tasks.append(ta.CloneGeosMksi(model=model)) tasks.append(ta.GetBufr(model=model)) tasks.append(ta.BufrToIoda(model=model)) tasks.append(ta.CleanCycle(model=model)) - + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index dc64fef16..4ff88d64f 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -84,13 +84,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) @@ -102,7 +102,7 @@ def tasks(self) -> list: tasks.append(ta.GetGsiNcdiag()) tasks.append(ta.GsiNcdiagToIoda()) tasks.append(ta.CleanCycle()) - + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index 88b1d7a9f..1a923ee9d 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -80,13 +80,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) @@ -97,7 +97,7 @@ def tasks(self) -> list: tasks.append(ta.GenerateObservingSystemRecords(model=model)) tasks.append(ta.EvaTimeseries(model=model)) tasks.append(ta.CleanCycle(model=model)) - + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index b806aaa04..1ec662ae1 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -89,13 +89,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) @@ -108,7 +108,7 @@ def tasks(self) -> list: tasks.append(ta.MoveForecastRestart()) tasks.append(ta.SaveRestart()) tasks.append(ta.RemoveForecastDir()) - + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index a0399368b..cf87e11e3 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -87,13 +87,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) @@ -111,7 +111,7 @@ def tasks(self) -> list: tasks.append(ta.GetGeosAdasBackground(model=model)) tasks.append(ta.RunJediVariationalExecutable(model=model)) tasks.append(ta.CleanCycle(model=model)) - + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index bd4ba6477..6f58aa766 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -113,13 +113,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) @@ -140,7 +140,7 @@ def tasks(self) -> list: tasks.append(ta.EvaObservations(model=model)) tasks.append(ta.SaveObsDiags(model=model)) tasks.append(ta.CleanCycle(model=model)) - + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 3cb72a187..4622c51c4 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -130,13 +130,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) @@ -160,7 +160,7 @@ def tasks(self) -> list: tasks.append(ta.EvaObservations(model=model)) tasks.append(ta.SaveObsDiags(model=model)) tasks.append(ta.CleanCycle(model=model)) - + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index 16cf54baa..b6ae9b0a6 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -102,13 +102,13 @@ def get_workflow_string(self): templated_string=template_str, dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - + for task in self.tasks(): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - + def tasks(self) -> list: tasks = [] tasks.append(ta.root()) @@ -127,7 +127,7 @@ def tasks(self) -> list: tasks.append(ta.GetGeovals(model=model)) tasks.append(ta.EvaObservations(model=model)) tasks.append(ta.CleanCycle(model=model)) - + return tasks # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_attributes.py b/src/swell/tasks/task_attributes.py index 0965b06ce..51288ec71 100644 --- a/src/swell/tasks/task_attributes.py +++ b/src/swell/tasks/task_attributes.py @@ -7,9 +7,6 @@ # -------------------------------------------------------------------------------------------------- -from typing import Union, Optional, Self -from collections.abc import Mapping - from swell.utilities.task_specification import Task from swell.utilities.swell_questions import QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd @@ -83,6 +80,7 @@ # -------------------------------------------------------------------------------------------------- + class TaskAttributes(): # -------------------------------------------------------------------------------------------------- @@ -92,10 +90,11 @@ class root(Task): def set_attributes(self): self.script = False self.pre_script = "source $CYLC_SUITE_DEF_PATH/modules" - self.additional_sections = [self.create_new_section('environment', {'datetime': '$CYLC_TASK_CYCLE_POINT', - 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'})] + self.additional_sections = [self.create_new_section('environment', + {'datetime': '$CYLC_TASK_CYCLE_POINT', + 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'})] # noqa - # -------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------- class BuildGeos(Task): def set_attributes(self): @@ -104,7 +103,7 @@ def set_attributes(self): ]) # -------------------------------------------------------------------------------------------------- - + class BuildGeosByLinking(Task): def set_attributes(self): self.mail_events = ['submit-failed'] @@ -114,7 +113,7 @@ def set_attributes(self): ]) # -------------------------------------------------------------------------------------------------- - + class BuildJediByLinking(Task): def set_attributes(self): self.mail_events = ['submit-failed'] @@ -146,7 +145,7 @@ def set_attributes(self): ]) # -------------------------------------------------------------------------------------------------- - + class CloneGeos(Task): def set_attributes(self): self.question_list = QuestionList([ @@ -156,7 +155,7 @@ def set_attributes(self): ]) # -------------------------------------------------------------------------------------------------- - + class CloneJedi(Task): def set_attributes(self): self.question_list = QuestionList([ @@ -167,7 +166,7 @@ def set_attributes(self): ]) # -------------------------------------------------------------------------------------------------- - + class CloneGeosMksi(Task): def set_attributes(self): self.is_model = True @@ -175,7 +174,7 @@ def set_attributes(self): qd.observing_system_records_mksi_path(), qd.observing_system_records_mksi_path_tag() ]) - + # -------------------------------------------------------------------------------------------------- class CloneGmaoPerllib(Task): @@ -184,7 +183,7 @@ def set_attributes(self): qd.existing_perllib_path(), qd.gmao_perllib_tag() ]) - + # -------------------------------------------------------------------------------------------------- class EvaJediLog(Task): @@ -193,7 +192,7 @@ def set_attributes(self): self.is_model = True # -------------------------------------------------------------------------------------------------- - + class EvaComparisonIncrement(Task): def set_attributes(self): self.is_cycling = True @@ -205,14 +204,14 @@ def set_attributes(self): ]) # -------------------------------------------------------------------------------------------------- - + class EvaComparisonJediLog(Task): def set_attributes(self): self.is_cycling = True self.is_model = True # -------------------------------------------------------------------------------------------------- - + class EvaIncrement(Task): def set_attributes(self): self.is_cycling = True @@ -222,7 +221,7 @@ def set_attributes(self): qd.window_offset(), qd.window_type() ]) - + # -------------------------------------------------------------------------------------------------- class EvaObservations(Task): @@ -238,7 +237,7 @@ def set_attributes(self): qd.window_offset(), qd.marine_models(), ]) - + # -------------------------------------------------------------------------------------------------- class EvaTimeseries(Task): @@ -254,7 +253,7 @@ def set_attributes(self): qd.ncdiag_experiments(), qd.marine_models(), ]) - + # -------------------------------------------------------------------------------------------------- class JediOopsLogParser(Task): @@ -266,7 +265,7 @@ def set_attributes(self): ]) # -------------------------------------------------------------------------------------------------- - + class GetBackground(Task): def set_attributes(self): self.is_cycling = True @@ -280,7 +279,7 @@ def set_attributes(self): qd.marine_models(), qd.r2d2_local_path(), ]) - + # -------------------------------------------------------------------------------------------------- class GetBackgroundGeosExperiment(Task): @@ -294,7 +293,7 @@ def set_attributes(self): qd.background_time_offset(), qd.geos_x_background_directory() ]) - + # -------------------------------------------------------------------------------------------------- class GetBufr(Task): @@ -304,7 +303,7 @@ def set_attributes(self): self.question_list = QuestionList([ qd.bufr_obs_classes() ]) - + # -------------------------------------------------------------------------------------------------- class BufrToIoda(Task): @@ -313,7 +312,7 @@ def set_attributes(self): self.is_model = True # -------------------------------------------------------------------------------------------------- - + class GetEnsembleGeosExperiment(Task): def set_attributes(self): self.is_cycling = True @@ -325,7 +324,7 @@ def set_attributes(self): ]) # -------------------------------------------------------------------------------------------------- - + class GetGeosRestart(Task): def set_attributes(self): self.is_cycling = True @@ -333,7 +332,7 @@ def set_attributes(self): swell_static_file_questions, qd.geos_restarts_directory() ]) - + # -------------------------------------------------------------------------------------------------- class GetGeovals(Task): @@ -359,7 +358,7 @@ def set_attributes(self): qd.path_to_gsi_bc_coefficients(), qd.window_length() ]) - + # -------------------------------------------------------------------------------------------------- class GsiBcToIoda(Task): @@ -371,7 +370,7 @@ def set_attributes(self): qd.observing_system_records_path(), qd.window_offset() ]) - + # -------------------------------------------------------------------------------------------------- class GetGsiNcdiag(Task): @@ -383,7 +382,7 @@ def set_attributes(self): ]) # -------------------------------------------------------------------------------------------------- - + class GsiNcdiagToIoda(Task): def set_attributes(self): self.is_cycling = True @@ -394,7 +393,7 @@ def set_attributes(self): qd.single_observations(), qd.window_offset() ]) - + # -------------------------------------------------------------------------------------------------- class GetNcdiags(Task): @@ -411,7 +410,7 @@ def set_attributes(self): ]) # -------------------------------------------------------------------------------------------------- - + class GetGeosAdasBackground(Task): def set_attributes(self): self.is_cycling = True @@ -436,7 +435,7 @@ def set_attributes(self): qd.window_length(), qd.window_offset(), ]) - + # -------------------------------------------------------------------------------------------------- class GetObsNotInR2d2(Task): @@ -447,7 +446,7 @@ def set_attributes(self): self.question_list = QuestionList([ qd.ioda_locations_not_in_r2d2(), ]) - + # -------------------------------------------------------------------------------------------------- class GenerateBClimatology(Task): @@ -475,7 +474,7 @@ def set_attributes(self): qd.window_offset(), qd.window_type() ]) - + # -------------------------------------------------------------------------------------------------- class GenerateBClimatologyByLinking(Task): @@ -490,7 +489,7 @@ def set_attributes(self): qd.window_offset(), qd.window_type() ]) - + # -------------------------------------------------------------------------------------------------- class GenerateObservingSystemRecords(Task): @@ -502,7 +501,7 @@ def set_attributes(self): qd.observing_system_records_mksi_path(), qd.observing_system_records_path() ]) - + # -------------------------------------------------------------------------------------------------- class LinkGeosOutput(Task): @@ -514,7 +513,7 @@ def set_attributes(self): qd.background_frequency(), qd.marine_models() ]) - + # -------------------------------------------------------------------------------------------------- class MoveDaRestart(Task): @@ -526,7 +525,7 @@ def set_attributes(self): qd.mom6_iau(), qd.window_length() ]) - + # -------------------------------------------------------------------------------------------------- class MoveForecastRestart(Task): @@ -535,7 +534,7 @@ def set_attributes(self): self.question_list = QuestionList([ qd.forecast_duration() ]) - + # -------------------------------------------------------------------------------------------------- class PrepGeosRunDir(Task): @@ -548,7 +547,7 @@ def set_attributes(self): qd.geos_experiment_directory(), qd.mom6_iau_nhours() ]) - + # -------------------------------------------------------------------------------------------------- class PrepareAnalysis(Task): @@ -561,7 +560,7 @@ def set_attributes(self): qd.mom6_iau(), qd.total_processors() ]) - + # -------------------------------------------------------------------------------------------------- class RunJediFgatExecutable(Task): @@ -574,7 +573,7 @@ def set_attributes(self): run_jedi_executable, qd.marine_models() ]) - + # -------------------------------------------------------------------------------------------------- class RunJediEnsembleMeanVariance(Task): @@ -593,7 +592,7 @@ def set_attributes(self): qd.observations(), qd.observing_system_records_path(), ]) - + # -------------------------------------------------------------------------------------------------- class RunJediHofxEnsembleExecutable(Task): @@ -614,7 +613,7 @@ def set_attributes(self): qd.jedi_forecast_model(), qd.total_processors() ]) - + # -------------------------------------------------------------------------------------------------- class RunJediHofxExecutable(Task): @@ -633,7 +632,7 @@ def set_attributes(self): qd.save_geovals(), qd.total_processors() ]) - + # -------------------------------------------------------------------------------------------------- class RunJediLocalEnsembleDaExecutable(Task): @@ -675,7 +674,7 @@ def set_attributes(self): qd.vertical_localization_method(), qd.perhost() ]) - + # -------------------------------------------------------------------------------------------------- class RunJediVariationalExecutable(Task): @@ -688,7 +687,7 @@ def set_attributes(self): run_jedi_executable, qd.perhost() ]) - + # -------------------------------------------------------------------------------------------------- class RemoveForecastDir(Task): @@ -696,11 +695,11 @@ def set_attributes(self): self.is_cycling = True # -------------------------------------------------------------------------------------------------- - + class RunGeosExecutable(Task): def set_attributes(self): self.is_cycling = True - + # -------------------------------------------------------------------------------------------------- class RunJediUfoExecutable(Task): @@ -711,7 +710,7 @@ def set_attributes(self): self.time_limit = True # -------------------------------------------------------------------------------------------------- - + class RunJediUfoTestsExecutable(Task): def set_attributes(self): self.time_limit = True @@ -725,7 +724,7 @@ def set_attributes(self): qd.window_length(), qd.window_offset() ]) - + # -------------------------------------------------------------------------------------------------- class RunJediConvertStateSoca2ciceExecutable(Task): @@ -744,7 +743,7 @@ def set_attributes(self): qd.window_offset(), qd.window_type() ]) - + # -------------------------------------------------------------------------------------------------- class RunJediFgatExecutable(Task): @@ -757,7 +756,7 @@ def set_attributes(self): run_jedi_executable, qd.marine_models() ]) - + # -------------------------------------------------------------------------------------------------- class SaveObsDiags(Task): @@ -770,7 +769,7 @@ def set_attributes(self): qd.window_offset(), qd.marine_models() ]) - + # -------------------------------------------------------------------------------------------------- class SaveRestart(Task): @@ -785,7 +784,7 @@ def set_attributes(self): qd.marine_models(), qd.r2d2_local_path() ]) - + # -------------------------------------------------------------------------------------------------- class StageJedi(Task): @@ -799,7 +798,7 @@ def set_attributes(self): qd.horizontal_resolution(), qd.vertical_resolution() ]) - + # -------------------------------------------------------------------------------------------------- class StageJediCycle(Task): @@ -816,7 +815,7 @@ def set_attributes(self): qd.horizontal_resolution(), qd.vertical_resolution() ]) - + # -------------------------------------------------------------------------------------------------- class sync_point(Task): @@ -824,7 +823,7 @@ def set_attributes(self): self.script = "true" # -------------------------------------------------------------------------------------------------- - + class JediLogComparison(Task): def set_attributes(self): self.is_model = True @@ -832,13 +831,13 @@ def set_attributes(self): qd.number_of_iterations(), qd.comparison_log_type(), ]) - + # -------------------------------------------------------------------------------------------------- class RunJediObsfiltersExecutable(Task): def set_attributes(self): self.script = ("swell task RunJediObsfiltersExecutable $config" - " -d $datetime -m geos_atmosphere") + " -d $datetime -m geos_atmosphere") self.is_cycling = True self.is_model = True self.time_limit = True diff --git a/src/swell/test/code_tests/code_tests.py b/src/swell/test/code_tests/code_tests.py index c9ac63473..c77c6766d 100644 --- a/src/swell/test/code_tests/code_tests.py +++ b/src/swell/test/code_tests/code_tests.py @@ -15,7 +15,6 @@ from swell.test.code_tests.slurm_test import SLURMConfigTest from swell.test.code_tests.test_pinned_versions import PinnedVersionsTest from swell.test.code_tests.unused_variables_test import UnusedVariablesTest -from swell.test.code_tests.question_dictionary_comparison_test import QuestionDictionaryTest from swell.test.code_tests.test_generate_observing_system import GenerateObservingSystemTest from swell.test.code_tests.question_order_test import QuestionOrderTest @@ -41,9 +40,6 @@ def code_tests() -> None: # Load unused variable test test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(UnusedVariablesTest)) - # Load tests from UnusedVariablesTest - test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(QuestionDictionaryTest)) - # Load question order test test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(QuestionOrderTest)) diff --git a/src/swell/test/code_tests/question_dictionary_comparison_test.py b/src/swell/test/code_tests/question_dictionary_comparison_test.py deleted file mode 100644 index 850f7b0d2..000000000 --- a/src/swell/test/code_tests/question_dictionary_comparison_test.py +++ /dev/null @@ -1,43 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import unittest - -from swell.utilities.scripts.compare_questions import compare_used_and_set_questions - - -# -------------------------------------------------------------------------------------------------- - - -class QuestionDictionaryTest(unittest.TestCase): - - def test_dictionary_comparison(self): - - used_not_set, set_not_used = compare_used_and_set_questions() - - # Throw error if there are any unassigned variables used by the code - if len(used_not_set) > 0: - error_msg = ("Questions which are required by the code are missing from the question " - "configurations:\n\n") - - for suite in used_not_set.keys(): - for task_or_suite in used_not_set[suite]: - questions_str = "" - for q in used_not_set[suite][task_or_suite]: - questions_str += q + '\n' - error_msg += (f"In suite {suite}, the {task_or_suite} question configuration " - f"is missing the required question(s):\n{questions_str}\n") - - assert len(used_not_set) == 0, error_msg - - # TODO: Implement a check for set-but-not-used questions - # This will require a fix/adjustment for some suites - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/test/code_tests/slurm_test.py b/src/swell/test/code_tests/slurm_test.py index af5033424..a39665684 100644 --- a/src/swell/test/code_tests/slurm_test.py +++ b/src/swell/test/code_tests/slurm_test.py @@ -11,7 +11,7 @@ from swell.utilities.slurm import prepare_slurm_defaults_and_overrides from swell.utilities.logger import get_logger -from swell.tasks.task_runtimes import TaskRuntimes +from swell.tasks.task_attributes import TaskAttributes from unittest.mock import patch, Mock # -------------------------------------------------------------------------------------------------- @@ -53,25 +53,25 @@ def test_slurm_config(self, platform_mocked: Mock, mock_global_defaults: Mock) - sd_discover_sles15 = prepare_slurm_defaults_and_overrides(logger, 'nccs_discover_sles15', experiment_dict) - run_jedi_var_class = TaskRuntimes.get('RunJediVariationalExecutable') - run_jedi_var_obj = run_jedi_var_class() + run_jedi_var_class = TaskAttributes.get('RunJediVariationalExecutable') + run_jedi_var_obj = run_jedi_var_class('geos_marine', 'nccs_discover_sles15') run_jedi_var_slurm = run_jedi_var_obj.generate_task_slurm_dict( sd_discover_sles15, 'nccs_discover_sles15') self.assertEqual(run_jedi_var_slurm["constraint"], "mil") self.assertEqual(run_jedi_var_slurm["qos"], "dastest") - eva_obs_class = TaskRuntimes.get('EvaObservations') - build_jedi_class = TaskRuntimes.get('BuildJedi') - run_jedi_ufo_class = TaskRuntimes.get('RunJediUfoTestsExecutable') + eva_obs_class = TaskAttributes.get('EvaObservations') + build_jedi_class = TaskAttributes.get('BuildJedi') + run_jedi_ufo_class = TaskAttributes.get('RunJediUfoTestsExecutable') # Platform generic tests for sd in [sd_discover_sles15]: for mc in ["all", "geos_atmosphere", "geos_marine"]: - run_jedi_var_obj = run_jedi_var_class(model=mc) - eva_obs_obj = eva_obs_class(model=mc) - build_jedi_obj = build_jedi_class(model=mc) - run_jedi_ufo_obj = run_jedi_ufo_class(model=mc) + run_jedi_var_obj = run_jedi_var_class(mc, 'nccs_discover_sles15') + eva_obs_obj = eva_obs_class(mc, 'nccs_discover_sles15') + build_jedi_obj = build_jedi_class(mc, 'nccs_discover_sles15') + run_jedi_ufo_obj = run_jedi_ufo_class(mc, 'nccs_discover_sles15') run_jedi_var_dict = run_jedi_var_obj.generate_task_slurm_dict( sd, 'nccs_discover_sles15') diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 6528e9922..8d3109a0a 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -7,12 +7,10 @@ # -------------------------------------------------------------------------------------------------- -from typing import Union, Optional, Tuple +from typing import Tuple from abc import abstractmethod -from swell.utilities.cylc_formatting import CylcSection, indent_lines from swell.utilities.logger import get_logger -from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/scripts/compare_questions.py b/src/swell/utilities/scripts/compare_questions.py deleted file mode 100644 index 246462ffb..000000000 --- a/src/swell/utilities/scripts/compare_questions.py +++ /dev/null @@ -1,292 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - - -import os -from typing import Optional, Tuple -import importlib -import re -from enum import StrEnum, auto - -from swell.swell_path import get_swell_path -from swell.utilities.suite_utils import get_suites -from swell.tasks.task_questions import TaskQuestions as tq -from swell.utilities.swell_questions import QuestionList -from swell.utilities.case_switching import camel_case_to_snake_case -from swell.suites.all_suites import workflows -from swell.utilities.logger import get_logger -from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import ( - PrepareExperimentConfigAndSuite) - - -# -------------------------------------------------------------------------------------------------- - -class CodeDependentQuestions(StrEnum): - """ Questions which are set by swell during experiment creation. """ - EXPERIMENT_ID = auto() - EXPERIMENT_ROOT = auto() - PLATFORM = auto() - - @classmethod - def filter_list(cls, lst: list) -> list: - values = [item.value for item in cls] - return [item for item in lst if item not in values] - -# -------------------------------------------------------------------------------------------------- - - -def get_workflow(suite: str): - """ Parse the suite's flow.cylc file and get all the tasks used by the suite. """ - - logger = get_logger('CodeTests') - - prepare_config = PrepareExperimentConfigAndSuite(logger, - suite, - suite, - 'nccs_discover_sles15', - 'defaults', - None) - - suite_dict = prepare_config.get_experiment_dict() - - workflow_class = workflows.get_workflow(suite) - workflow_obj = workflow_class(suite_dict, {}) - - return workflow_obj - -# -------------------------------------------------------------------------------------------------- - - -def get_scheduling(suite: str): - - workflow_obj = get_workflow(suite) - - scheduling_section = workflow_obj.define_initial_workflow() - - return scheduling_section - -# -------------------------------------------------------------------------------------------------- - - -def get_all_tasks(suite: str) -> list: - """ Parse the suite's flow.cylc file and get all the tasks used by the suite. """ - - workflow_obj = get_workflow(suite) - - tasks = workflow_obj.parse_graph_for_tasks() - - base_tasks = [] - - for task in tasks: - base_task = task.split('-')[0] - if base_task not in ['StageJediCycle', 'sync_point']: - base_tasks.append(base_task) - - return base_tasks - -# -------------------------------------------------------------------------------------------------- - - -def get_question_names(config: QuestionList, model: Optional[str] = None) -> list: - """ Get a list of question names from a QuestionList object. """ - return [q['question_name'] for q in config.expand_question_list(model)] - -# -------------------------------------------------------------------------------------------------- - - -def questions_in_cylc(suite: str) -> list: - """ Parse the suite's flow.cylc file and get a list of external questions. """ - - cylc_questions = [] - - scheduling = get_scheduling(suite) - - lines = scheduling.split('\n') - - for line in lines: - line = line.strip() - - if '.' in line or 'scheduling' in line or 'key' in line: - None - elif re.search(".* = {{.*}}", line): - cylc_questions.append( - line.split('{{')[1].split('}}')[0].strip()) - elif 'models[' in line: - cylc_questions.append( - line.split('["')[-1].split('"]')[0].strip()) - elif re.search(".*{%.* in .* %}", line) and '(' in line: - cylc_questions.append( - line.split('(')[1].split(')')[0].strip()) - elif re.search(".*{%.* in .* %}", line): - cylc_questions.append( - line.split('in ')[1].split(' %}')[0].strip()) - - cylc_questions = sorted(list(set(cylc_questions))) - return cylc_questions - -# -------------------------------------------------------------------------------------------------- - - -def compare_used_and_set_questions() -> Tuple[dict, dict]: - """ - Finds the questions which are set in the suite/task configuration, - and those that are actually used by the suite. - - This method returns two dictionaries, used_not_set and set_not_used, indexed by suite and task. - - used_not_set[suite]['suite'_or_task] is a list of questions which are used in the code, - but are not specified in the suite config or task_questions.py - - set_not_used[suite]['suite'_or_task] consists of questions defined - in the suite or task config, which are not used in the code. - """ - - suites = get_suites() - - # Dictionary for questions used in the suite, but not specified in configuration - used_not_set = {} - # Dictionary for questions set in the configuration, but not actually used - set_not_used = {} - - # GEOS model components - possible_model_components = os.listdir(os.path.join(get_swell_path(), - 'configuration', 'jedi', 'interfaces')) - - for suite in suites: - # Sub-suite dictionary for questions used in the code - used_by = {} - # Sub-suite dictionary for questions set in the configuration - set_for = {} - - # Get the default suite configuration - config_name = ('_' if suite[0].isdigit() else '') + suite - suite_config = getattr(importlib.import_module(f'swell.suites.{suite}.suite_config'), - 'SuiteConfig') - base_config = suite_config[config_name].value - - config_questions = get_question_names(base_config) - - # Get questions which are specified as model-dependent - for model in possible_model_components: - config_questions.extend(get_question_names(base_config, model)) - - config_questions = sorted(list(set(config_questions))) - - # Set suite-defined questions - set_for['suite'] = CodeDependentQuestions.filter_list(config_questions) - # Check for questions used by flow.cylc - used_by['suite'] = CodeDependentQuestions.filter_list(questions_in_cylc(suite)) - - tasks = get_all_tasks(suite) - - for task in tasks: - # Task-specific used and set questions - used_task = [] - set_task = [] - - # Get the set questions for the task - if task in tq.get_all(): - set_task.extend(get_question_names(tq[task].value)) - - for model in possible_model_components: - set_task.extend(get_question_names(tq[task].value, model)) - - # Check the task's code for the questions it uses - task_file = os.path.join(get_swell_path(), 'tasks', - camel_case_to_snake_case(task) + '.py') - - if os.path.exists(task_file): - with open(task_file, 'r') as f: - config_lines = [line for line in f.readlines() if 'self.config.' in line] - for line in config_lines: - if 'get_key_for_model' in line: - field = line.split( - 'self.config.get_key_for_model(')[1].split(')')[0].strip() + ')' - if len(field.split(',')) == 1: - field = field.split(',')[0] + '()' - else: - field = field.split(',')[ - 0].strip() + '(' + field.split(',')[-1].strip() + ')' - - field = field.replace('"', '') - field = field.replace("'", '') - else: - field = line.split('self.config.')[1].split(')')[0].strip() + ')' - - # Include the parentheses, so we can later - # assess whether the key is optional - used_task.append(field) - - set_task = sorted(list(set(set_task))) - used_task = sorted(list(set(used_task))) - - # Filter out the questions set by the code - set_task = CodeDependentQuestions.filter_list(set_task) - used_task = CodeDependentQuestions.filter_list(used_task) - - used_by[task] = used_task - set_for[task] = set_task - - used_not_set[suite] = {} - set_not_used[suite] = {} - - # Set the dictionary for used-but-not-set questions - for suite_task, lst in used_by.items(): - used_not_set[suite][suite_task] = [] - - for question in lst: - question_name = question.split('(')[0].strip() - # Include only non-optional calls from the code - if len(question_name.split(')')[0].strip()) == 0 and ( - question_name not in set_for[suite_task] + set_for['suite']): - used_not_set[suite][suite_task].append(question_name) - - # Clear the suite or task key if there are no discrepancies - if len(used_not_set[suite][suite_task]) == 0: - del used_not_set[suite][suite_task] - - # Get a list of all questions used by tasks across the suite, to compare against the - # set suite questions. - all_used_questions = [] - for key in used_by.keys(): - for question in used_by[key]: - if '(' in question: - all_used_questions.append(question.split('(')[0]) - else: - all_used_questions.append(question) - - # Set the dictionary for set-but-not-used questions - for suite_task, lst in set_for.items(): - set_not_used[suite][suite_task] = [] - - for question in lst: - if suite_task == 'suite': - # Suite questions may be used in tasks throughout the suite - if question not in all_used_questions: - set_not_used[suite]['suite'].append(question) - else: - # Include optional questions - used_by_all = [q if '(' not in q else q.split('(')[0] - for q in used_by[suite_task]] - if question not in used_by_all: - set_not_used[suite][suite_task].append(question) - - # Clear the key if there are no discrepancies - if len(set_not_used[suite][suite_task]) == 0: - del set_not_used[suite][suite_task] - - # Clear the suite if there are no discrepancies - if len(set_not_used[suite]) == 0: - del set_not_used[suite] - - if len(used_not_set[suite]) == 0: - del used_not_set[suite] - - return used_not_set, set_not_used - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index cfe5a84da..462aacd6c 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -9,7 +9,7 @@ import os -from dataclasses import dataclass, asdict, field +from dataclasses import dataclass, asdict from typing import List, Optional, Self, Union, Literal from enum import Enum, StrEnum from isodate import parse_datetime, parse_duration, ISO8601Error From a9022607068a74a891c37b4b5488ba715c250175 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 7 Nov 2025 16:55:59 -0500 Subject: [PATCH 124/299] refactor self.tasks --- src/swell/suites/3dfgat_atmos/workflow.py | 48 +++++++------- src/swell/suites/3dfgat_cycle/workflow.py | 64 +++++++++---------- src/swell/suites/3dvar/workflow.py | 40 ++++++------ src/swell/suites/3dvar_atmos/workflow.py | 44 ++++++------- src/swell/suites/3dvar_cycle/workflow.py | 62 +++++++++--------- src/swell/suites/build_geos/workflow.py | 14 ++-- src/swell/suites/build_jedi/workflow.py | 14 ++-- .../suites/compare_variational/workflow.py | 13 ++-- src/swell/suites/convert_bufr/workflow.py | 26 ++++---- src/swell/suites/convert_ncdiags/workflow.py | 26 ++++---- src/swell/suites/eva_capabilities/workflow.py | 20 +++--- src/swell/suites/forecast_geos/workflow.py | 28 ++++---- src/swell/suites/geosadas/workflow.py | 32 +++++----- src/swell/suites/hofx/workflow.py | 40 ++++++------ src/swell/suites/localensembleda/workflow.py | 46 +++++++------ src/swell/suites/ufo_testing/workflow.py | 36 +++++------ src/swell/utilities/cylc_workflow.py | 11 +++- 17 files changed, 270 insertions(+), 294 deletions(-) diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index de4110c0c..373f8cf1a 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -134,37 +134,35 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneJedi()) - tasks.append(ta.BuildJediByLinking()) - tasks.append(ta.BuildJedi()) + def set_tasks(self) -> None: + + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.BuildJediByLinking()) + self.tasks.append(ta.BuildJedi()) for model in self.experiment_dict['model_components']: - tasks.append(ta.CloneGeosMksi(model=model)) - tasks.append(ta.StageJedi(model=model)) - tasks.append(ta.GetBackground(model=model)) - tasks.append(ta.GetObservations(model=model)) - tasks.append(ta.GenerateBClimatologyByLinking(model=model)) - tasks.append(ta.GenerateBClimatology(model=model)) - tasks.append(ta.GetObsNotInR2d2(model=model)) - tasks.append(ta.GetBackgroundGeosExperiment(model=model)) - tasks.append(ta.GenerateObservingSystemRecords(model=model)) - tasks.append(ta.StageJediCycle(model=model)) - tasks.append(ta.RunJediVariationalExecutable(model=model)) - tasks.append(ta.EvaObservations(model=model)) - tasks.append(ta.EvaJediLog(model=model)) - tasks.append(ta.EvaIncrement(model=model)) - tasks.append(ta.SaveObsDiags(model=model)) - tasks.append(ta.CleanCycle(model=model)) - - return tasks + self.tasks.append(ta.CloneGeosMksi(model=model)) + self.tasks.append(ta.StageJedi(model=model)) + self.tasks.append(ta.GetBackground(model=model)) + self.tasks.append(ta.GetObservations(model=model)) + self.tasks.append(ta.GenerateBClimatologyByLinking(model=model)) + self.tasks.append(ta.GenerateBClimatology(model=model)) + self.tasks.append(ta.GetObsNotInR2d2(model=model)) + self.tasks.append(ta.GetBackgroundGeosExperiment(model=model)) + self.tasks.append(ta.GenerateObservingSystemRecords(model=model)) + self.tasks.append(ta.StageJediCycle(model=model)) + self.tasks.append(ta.RunJediVariationalExecutable(model=model)) + self.tasks.append(ta.EvaObservations(model=model)) + self.tasks.append(ta.EvaJediLog(model=model)) + self.tasks.append(ta.EvaIncrement(model=model)) + self.tasks.append(ta.SaveObsDiags(model=model)) + self.tasks.append(ta.CleanCycle(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index 3da6922f2..d88b04c92 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -151,47 +151,45 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneJedi()) - tasks.append(ta.CloneGeos()) - tasks.append(ta.BuildJediByLinking()) - tasks.append(ta.BuildJedi()) - tasks.append(ta.BuildGeos()) - tasks.append(ta.BuildGeosByLinking()) + def set_tasks(self) -> None: - tasks.append(ta.GetGeosRestart()) - tasks.append(ta.PrepGeosRunDir()) - tasks.append(ta.RunGeosExecutable()) + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.CloneGeos()) + self.tasks.append(ta.BuildJediByLinking()) + self.tasks.append(ta.BuildJedi()) + self.tasks.append(ta.BuildGeos()) + self.tasks.append(ta.BuildGeosByLinking()) + + self.tasks.append(ta.GetGeosRestart()) + self.tasks.append(ta.PrepGeosRunDir()) + self.tasks.append(ta.RunGeosExecutable()) for model in self.experiment_dict['model_components']: - tasks.append(ta.RunJediFgatExecutable(model=model)) - tasks.append(ta.StageJedi(model=model)) - tasks.append(ta.StageJediCycle(model=model)) - tasks.append(ta.MoveDaRestart(model=model)) - tasks.append(ta.LinkGeosOutput(model=model)) - tasks.append(ta.GenerateBClimatology(model=model)) - tasks.append(ta.GenerateBClimatologyByLinking(model=model)) - tasks.append(ta.GetObservations(model=model)) - tasks.append(ta.EvaObservations(model=model)) - tasks.append(ta.EvaJediLog(model=model)) - tasks.append(ta.EvaIncrement(model=model)) - tasks.append(ta.PrepareAnalysis(model=model)) - tasks.append(ta.RunJediConvertStateSoca2ciceExecutable(model=model)) - tasks.append(ta.SaveRestart(model=model)) - tasks.append(ta.CleanCycle(model=model)) - tasks.append(ta.PrepareAnalysis(model=model)) - tasks.append(ta.RemoveForecastDir(model=model)) - tasks.append(ta.SaveObsDiags(model=model)) - - return tasks + self.tasks.append(ta.RunJediFgatExecutable(model=model)) + self.tasks.append(ta.StageJedi(model=model)) + self.tasks.append(ta.StageJediCycle(model=model)) + self.tasks.append(ta.MoveDaRestart(model=model)) + self.tasks.append(ta.LinkGeosOutput(model=model)) + self.tasks.append(ta.GenerateBClimatology(model=model)) + self.tasks.append(ta.GenerateBClimatologyByLinking(model=model)) + self.tasks.append(ta.GetObservations(model=model)) + self.tasks.append(ta.EvaObservations(model=model)) + self.tasks.append(ta.EvaJediLog(model=model)) + self.tasks.append(ta.EvaIncrement(model=model)) + self.tasks.append(ta.PrepareAnalysis(model=model)) + self.tasks.append(ta.RunJediConvertStateSoca2ciceExecutable(model=model)) + self.tasks.append(ta.SaveRestart(model=model)) + self.tasks.append(ta.CleanCycle(model=model)) + self.tasks.append(ta.PrepareAnalysis(model=model)) + self.tasks.append(ta.RemoveForecastDir(model=model)) + self.tasks.append(ta.SaveObsDiags(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 0d3db308d..f5e0d405d 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -120,33 +120,31 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneJedi()) - tasks.append(ta.BuildJediByLinking()) - tasks.append(ta.BuildJedi()) + def set_tasks(self) -> list: + + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.BuildJediByLinking()) + self.tasks.append(ta.BuildJedi()) for model in self.experiment_dict['model_components']: - tasks.append(ta.StageJedi(model=model)) - tasks.append(ta.GetObservations(model=model)) - tasks.append(ta.GenerateBClimatologyByLinking(model=model)) - tasks.append(ta.GenerateBClimatology(model=model)) - tasks.append(ta.StageJediCycle(model=model)) - tasks.append(ta.GetBackground(model=model)) - tasks.append(ta.RunJediVariationalExecutable(model=model)) - tasks.append(ta.EvaObservations(model=model)) - tasks.append(ta.EvaJediLog(model=model)) - tasks.append(ta.EvaIncrement(model=model)) - tasks.append(ta.SaveObsDiags(model=model)) - tasks.append(ta.CleanCycle(model=model)) - - return tasks + self.tasks.append(ta.StageJedi(model=model)) + self.tasks.append(ta.GetObservations(model=model)) + self.tasks.append(ta.GenerateBClimatologyByLinking(model=model)) + self.tasks.append(ta.GenerateBClimatology(model=model)) + self.tasks.append(ta.StageJediCycle(model=model)) + self.tasks.append(ta.GetBackground(model=model)) + self.tasks.append(ta.RunJediVariationalExecutable(model=model)) + self.tasks.append(ta.EvaObservations(model=model)) + self.tasks.append(ta.EvaJediLog(model=model)) + self.tasks.append(ta.EvaIncrement(model=model)) + self.tasks.append(ta.SaveObsDiags(model=model)) + self.tasks.append(ta.CleanCycle(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index 929641635..299700462 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -128,35 +128,33 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneJedi()) - tasks.append(ta.BuildJediByLinking()) - tasks.append(ta.BuildJedi()) + def set_tasks(self) -> list: + + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.BuildJediByLinking()) + self.tasks.append(ta.BuildJedi()) for model in self.experiment_dict['model_components']: - tasks.append(ta.CloneGeosMksi(model=model)) - tasks.append(ta.StageJedi(model=model)) - tasks.append(ta.GetObservations(model=model)) - tasks.append(ta.GenerateBClimatologyByLinking(model=model)) - tasks.append(ta.GenerateBClimatology(model=model)) - tasks.append(ta.GenerateObservingSystemRecords(model=model)) - tasks.append(ta.GetObsNotInR2d2(model=model)) - tasks.append(ta.StageJediCycle(model=model)) - tasks.append(ta.RunJediVariationalExecutable(model=model)) - tasks.append(ta.EvaObservations(model=model)) - tasks.append(ta.EvaJediLog(model=model)) - tasks.append(ta.EvaIncrement(model=model)) - tasks.append(ta.SaveObsDiags(model=model)) - tasks.append(ta.CleanCycle(model=model)) - - return tasks + self.tasks.append(ta.CloneGeosMksi(model=model)) + self.tasks.append(ta.StageJedi(model=model)) + self.tasks.append(ta.GetObservations(model=model)) + self.tasks.append(ta.GenerateBClimatologyByLinking(model=model)) + self.tasks.append(ta.GenerateBClimatology(model=model)) + self.tasks.append(ta.GenerateObservingSystemRecords(model=model)) + self.tasks.append(ta.GetObsNotInR2d2(model=model)) + self.tasks.append(ta.StageJediCycle(model=model)) + self.tasks.append(ta.RunJediVariationalExecutable(model=model)) + self.tasks.append(ta.EvaObservations(model=model)) + self.tasks.append(ta.EvaJediLog(model=model)) + self.tasks.append(ta.EvaIncrement(model=model)) + self.tasks.append(ta.SaveObsDiags(model=model)) + self.tasks.append(ta.CleanCycle(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index 4d35c1e35..0a759844a 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -151,44 +151,42 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneJedi()) - tasks.append(ta.CloneGeos()) - tasks.append(ta.BuildJediByLinking()) - tasks.append(ta.BuildGeosByLinking()) - tasks.append(ta.BuildJedi()) - tasks.append(ta.BuildGeos()) - tasks.append(ta.GetGeosRestart()) - tasks.append(ta.PrepGeosRunDir()) - tasks.append(ta.RunGeosExecutable()) + def set_tasks(self) -> list: + + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.CloneGeos()) + self.tasks.append(ta.BuildJediByLinking()) + self.tasks.append(ta.BuildGeosByLinking()) + self.tasks.append(ta.BuildJedi()) + self.tasks.append(ta.BuildGeos()) + self.tasks.append(ta.GetGeosRestart()) + self.tasks.append(ta.PrepGeosRunDir()) + self.tasks.append(ta.RunGeosExecutable()) for model in self.experiment_dict['model_components']: - tasks.append(ta.StageJedi(model=model)) - tasks.append(ta.StageJediCycle(model=model)) - tasks.append(ta.RunJediVariationalExecutable(model=model)) - tasks.append(ta.MoveDaRestart(model=model)) - tasks.append(ta.LinkGeosOutput(model=model)) - tasks.append(ta.GenerateBClimatology(model=model)) - tasks.append(ta.GenerateBClimatologyByLinking(model=model)) - tasks.append(ta.GetObservations(model=model)) - tasks.append(ta.PrepareAnalysis(model=model)) - tasks.append(ta.RunJediConvertStateSoca2ciceExecutable(model=model)) - tasks.append(ta.SaveRestart(model=model)) - tasks.append(ta.RemoveForecastDir(model=model)) - tasks.append(ta.EvaObservations(model=model)) - tasks.append(ta.EvaJediLog(model=model)) - tasks.append(ta.EvaIncrement(model=model)) - tasks.append(ta.SaveObsDiags(model=model)) - tasks.append(ta.CleanCycle(model=model)) - - return tasks + self.tasks.append(ta.StageJedi(model=model)) + self.tasks.append(ta.StageJediCycle(model=model)) + self.tasks.append(ta.RunJediVariationalExecutable(model=model)) + self.tasks.append(ta.MoveDaRestart(model=model)) + self.tasks.append(ta.LinkGeosOutput(model=model)) + self.tasks.append(ta.GenerateBClimatology(model=model)) + self.tasks.append(ta.GenerateBClimatologyByLinking(model=model)) + self.tasks.append(ta.GetObservations(model=model)) + self.tasks.append(ta.PrepareAnalysis(model=model)) + self.tasks.append(ta.RunJediConvertStateSoca2ciceExecutable(model=model)) + self.tasks.append(ta.SaveRestart(model=model)) + self.tasks.append(ta.RemoveForecastDir(model=model)) + self.tasks.append(ta.EvaObservations(model=model)) + self.tasks.append(ta.EvaJediLog(model=model)) + self.tasks.append(ta.EvaIncrement(model=model)) + self.tasks.append(ta.SaveObsDiags(model=model)) + self.tasks.append(ta.CleanCycle(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py index 9c7b8c168..4ea9d0530 100644 --- a/src/swell/suites/build_geos/workflow.py +++ b/src/swell/suites/build_geos/workflow.py @@ -55,19 +55,17 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneGeos()) - tasks.append(ta.BuildGeos()) - tasks.append(ta.BuildGeosByLinking()) + def set_tasks(self) -> list: - return tasks + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneGeos()) + self.tasks.append(ta.BuildGeos()) + self.tasks.append(ta.BuildGeosByLinking()) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index 33c86e313..0d023a16a 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -55,19 +55,17 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneJedi()) - tasks.append(ta.BuildJedi()) - tasks.append(ta.BuildJediByLinking()) + def set_tasks(self) -> list: - return tasks + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.BuildJedi()) + self.tasks.append(ta.BuildJediByLinking()) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_variational/workflow.py b/src/swell/suites/compare_variational/workflow.py index c23405cd5..b4d3dc791 100644 --- a/src/swell/suites/compare_variational/workflow.py +++ b/src/swell/suites/compare_variational/workflow.py @@ -68,20 +68,17 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] + def set_tasks(self) -> list: for model in self.experiment_dict['model_components']: - tasks.append(ta.EvaComparisonIncrement(model=model)) - tasks.append(ta.EvaComparisonJediLog(model=model)) - tasks.append(ta.JediOopsLogParser(model=model)) - - return tasks + self.tasks.append(ta.EvaComparisonIncrement(model=model)) + self.tasks.append(ta.EvaComparisonJediLog(model=model)) + self.tasks.append(ta.JediOopsLogParser(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_bufr/workflow.py b/src/swell/suites/convert_bufr/workflow.py index 5ffe2bf5b..20637ea40 100644 --- a/src/swell/suites/convert_bufr/workflow.py +++ b/src/swell/suites/convert_bufr/workflow.py @@ -96,26 +96,24 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneJedi()) - tasks.append(ta.BuildJediByLinking()) - tasks.append(ta.BuildJedi()) - tasks.append(ta.CloneGmaoPerllib()) + def set_tasks(self) -> list: - for model in self.experiment_dict['model_components']: - tasks.append(ta.CloneGeosMksi(model=model)) - tasks.append(ta.GetBufr(model=model)) - tasks.append(ta.BufrToIoda(model=model)) - tasks.append(ta.CleanCycle(model=model)) + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.BuildJediByLinking()) + self.tasks.append(ta.BuildJedi()) + self.tasks.append(ta.CloneGmaoPerllib()) - return tasks + for model in self.experiment_dict['model_components']: + self.tasks.append(ta.CloneGeosMksi(model=model)) + self.tasks.append(ta.GetBufr(model=model)) + self.tasks.append(ta.BufrToIoda(model=model)) + self.tasks.append(ta.CleanCycle(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index 4ff88d64f..a959d08d4 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -85,24 +85,22 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneJedi()) - tasks.append(ta.BuildJediByLinking()) - tasks.append(ta.BuildJedi()) - tasks.append(ta.GetGsiBc()) - tasks.append(ta.GsiBcToIoda()) - tasks.append(ta.GetGsiNcdiag()) - tasks.append(ta.GsiNcdiagToIoda()) - tasks.append(ta.CleanCycle()) - - return tasks + def set_tasks(self) -> list: + + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.BuildJediByLinking()) + self.tasks.append(ta.BuildJedi()) + self.tasks.append(ta.GetGsiBc()) + self.tasks.append(ta.GsiBcToIoda()) + self.tasks.append(ta.GetGsiNcdiag()) + self.tasks.append(ta.GsiNcdiagToIoda()) + self.tasks.append(ta.CleanCycle()) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index 1a923ee9d..e4225bc9f 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -81,23 +81,21 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) + def set_tasks(self) -> list: - for model in self.experiment_dict['model_components']: - tasks.append(ta.CloneGeosMksi(model=model)) - tasks.append(ta.GetNcdiags(model=model)) - tasks.append(ta.GenerateObservingSystemRecords(model=model)) - tasks.append(ta.EvaTimeseries(model=model)) - tasks.append(ta.CleanCycle(model=model)) + self.tasks.append(ta.root()) - return tasks + for model in self.experiment_dict['model_components']: + self.tasks.append(ta.CloneGeosMksi(model=model)) + self.tasks.append(ta.GetNcdiags(model=model)) + self.tasks.append(ta.GenerateObservingSystemRecords(model=model)) + self.tasks.append(ta.EvaTimeseries(model=model)) + self.tasks.append(ta.CleanCycle(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index 1ec662ae1..3cff792a7 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -90,25 +90,23 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneGeos()) - tasks.append(ta.BuildGeosByLinking()) - tasks.append(ta.BuildGeos()) - tasks.append(ta.GetGeosRestart()) - tasks.append(ta.PrepGeosRunDir()) - tasks.append(ta.RunGeosExecutable()) - tasks.append(ta.MoveForecastRestart()) - tasks.append(ta.SaveRestart()) - tasks.append(ta.RemoveForecastDir()) - - return tasks + def set_tasks(self) -> list: + + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneGeos()) + self.tasks.append(ta.BuildGeosByLinking()) + self.tasks.append(ta.BuildGeos()) + self.tasks.append(ta.GetGeosRestart()) + self.tasks.append(ta.PrepGeosRunDir()) + self.tasks.append(ta.RunGeosExecutable()) + self.tasks.append(ta.MoveForecastRestart()) + self.tasks.append(ta.SaveRestart()) + self.tasks.append(ta.RemoveForecastDir()) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index cf87e11e3..2d6f5b8c1 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -88,30 +88,28 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneJedi()) - tasks.append(ta.BuildJediByLinking()) - tasks.append(ta.CloneGeosMksi()) + + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.BuildJediByLinking()) + self.tasks.append(ta.CloneGeosMksi()) for model in self.experiment_dict['model_components']: - tasks.append(ta.GenerateObservingSystemRecords(model=model)) - tasks.append(ta.StageJedi(model=model)) - tasks.append(ta.GetGsiBc(model=model)) - tasks.append(ta.GsiBcToIoda(model=model)) - tasks.append(ta.GetGsiNcdiag(model=model)) - tasks.append(ta.GsiNcdiagToIoda(model=model)) - tasks.append(ta.GetGeosAdasBackground(model=model)) - tasks.append(ta.RunJediVariationalExecutable(model=model)) - tasks.append(ta.CleanCycle(model=model)) - - return tasks + self.tasks.append(ta.GenerateObservingSystemRecords(model=model)) + self.tasks.append(ta.StageJedi(model=model)) + self.tasks.append(ta.GetGsiBc(model=model)) + self.tasks.append(ta.GsiBcToIoda(model=model)) + self.tasks.append(ta.GetGsiNcdiag(model=model)) + self.tasks.append(ta.GsiNcdiagToIoda(model=model)) + self.tasks.append(ta.GetGeosAdasBackground(model=model)) + self.tasks.append(ta.RunJediVariationalExecutable(model=model)) + self.tasks.append(ta.CleanCycle(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index 6f58aa766..5fa776579 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -114,33 +114,31 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneJedi()) - tasks.append(ta.BuildJedi()) - tasks.append(ta.BuildJediByLinking()) - tasks.append(ta.CloneGeosMksi()) + def set_tasks(self) -> list: + + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.BuildJedi()) + self.tasks.append(ta.BuildJediByLinking()) + self.tasks.append(ta.CloneGeosMksi()) for model in self.experiment_dict['model_components']: - tasks.append(ta.CloneGeosMksi(model=model)) - tasks.append(ta.GenerateObservingSystemRecords(model=model)) - tasks.append(ta.GetBackgroundGeosExperiment(model=model)) - tasks.append(ta.GetBackground(model=model)) - tasks.append(ta.GetObservations(model=model)) - tasks.append(ta.GetObsNotInR2d2(model=model)) - tasks.append(ta.StageJediCycle(model=model)) - tasks.append(ta.RunJediHofxExecutable(model=model)) - tasks.append(ta.EvaObservations(model=model)) - tasks.append(ta.SaveObsDiags(model=model)) - tasks.append(ta.CleanCycle(model=model)) - - return tasks + self.tasks.append(ta.CloneGeosMksi(model=model)) + self.tasks.append(ta.GenerateObservingSystemRecords(model=model)) + self.tasks.append(ta.GetBackgroundGeosExperiment(model=model)) + self.tasks.append(ta.GetBackground(model=model)) + self.tasks.append(ta.GetObservations(model=model)) + self.tasks.append(ta.GetObsNotInR2d2(model=model)) + self.tasks.append(ta.StageJediCycle(model=model)) + self.tasks.append(ta.RunJediHofxExecutable(model=model)) + self.tasks.append(ta.EvaObservations(model=model)) + self.tasks.append(ta.SaveObsDiags(model=model)) + self.tasks.append(ta.CleanCycle(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 4622c51c4..c22fa2284 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -131,36 +131,34 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneJedi()) - tasks.append(ta.BuildJedi()) - tasks.append(ta.BuildJediByLinking()) + def set_tasks(self) -> list: + + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.BuildJedi()) + self.tasks.append(ta.BuildJediByLinking()) for model in self.experiment_dict['model_components']: - tasks.append(ta.CloneGeosMksi(model=model)) - tasks.append(ta.StageJediCycle(model=model)) - tasks.append(ta.GetObsNotInR2d2(model=model)) - tasks.append(ta.GetObservations(model=model)) - tasks.append(ta.GenerateObservingSystemRecords(model=model)) - tasks.append(ta.GetEnsembleGeosExperiment(model=model)) - tasks.append(ta.sync_point(model=model)) - tasks.append(ta.RunJediObsfiltersExecutable(model=model)) - tasks.append(ta.RunJediLocalEnsembleDaExecutable(model=model)) - tasks.append(ta.RunJediEnsembleMeanVariance(model=model)) - tasks.append(ta.RunJediHofxEnsembleExecutable(model=model)) - tasks.append(ta.EvaIncrement(model=model)) - tasks.append(ta.EvaObservations(model=model)) - tasks.append(ta.SaveObsDiags(model=model)) - tasks.append(ta.CleanCycle(model=model)) - - return tasks + self.tasks.append(ta.CloneGeosMksi(model=model)) + self.tasks.append(ta.StageJediCycle(model=model)) + self.tasks.append(ta.GetObsNotInR2d2(model=model)) + self.tasks.append(ta.GetObservations(model=model)) + self.tasks.append(ta.GenerateObservingSystemRecords(model=model)) + self.tasks.append(ta.GetEnsembleGeosExperiment(model=model)) + self.tasks.append(ta.sync_point(model=model)) + self.tasks.append(ta.RunJediObsfiltersExecutable(model=model)) + self.tasks.append(ta.RunJediLocalEnsembleDaExecutable(model=model)) + self.tasks.append(ta.RunJediEnsembleMeanVariance(model=model)) + self.tasks.append(ta.RunJediHofxEnsembleExecutable(model=model)) + self.tasks.append(ta.EvaIncrement(model=model)) + self.tasks.append(ta.EvaObservations(model=model)) + self.tasks.append(ta.SaveObsDiags(model=model)) + self.tasks.append(ta.CleanCycle(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index b6ae9b0a6..c2acdf719 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -103,31 +103,29 @@ def get_workflow_string(self): dictionary_of_templates=self.experiment_dict, allow_unresolved=True) - for task in self.tasks(): + for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) return workflow_str - def tasks(self) -> list: - tasks = [] - tasks.append(ta.root()) - tasks.append(ta.CloneJedi()) - tasks.append(ta.BuildJedi()) - tasks.append(ta.BuildJediByLinking()) + def set_tasks(self) -> list: + + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.BuildJedi()) + self.tasks.append(ta.BuildJediByLinking()) for model in self.experiment_dict['model_components']: - tasks.append(ta.CloneGeosMksi(model=model)) - tasks.append(ta.GenerateObservingSystemRecords(model=model)) - tasks.append(ta.GetGsiBc(model=model)) - tasks.append(ta.GsiBcToIoda(model=model)) - tasks.append(ta.GetGsiNcdiag(model=model)) - tasks.append(ta.GsiNcdiagToIoda(model=model)) - tasks.append(ta.RunJediUfoTestsExecutable(model=model)) - tasks.append(ta.GetGeovals(model=model)) - tasks.append(ta.EvaObservations(model=model)) - tasks.append(ta.CleanCycle(model=model)) - - return tasks + self.tasks.append(ta.CloneGeosMksi(model=model)) + self.tasks.append(ta.GenerateObservingSystemRecords(model=model)) + self.tasks.append(ta.GetGsiBc(model=model)) + self.tasks.append(ta.GsiBcToIoda(model=model)) + self.tasks.append(ta.GetGsiNcdiag(model=model)) + self.tasks.append(ta.GsiNcdiagToIoda(model=model)) + self.tasks.append(ta.RunJediUfoTestsExecutable(model=model)) + self.tasks.append(ta.GetGeovals(model=model)) + self.tasks.append(ta.EvaObservations(model=model)) + self.tasks.append(ta.CleanCycle(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 8d3109a0a..eeb73fee5 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -41,10 +41,19 @@ def __init__(self, experiment_dict, slurm_external) -> None: self.logger = get_logger(self.__class__.__name__) + self.tasks = [] + self.set_tasks() + # -------------------------------------------------------------------------------------------------- def default_header(self) -> str: return header_str + + # -------------------------------------------------------------------------------------------------- + + @abstractmethod + def set_tasks(self) -> None: + pass # -------------------------------------------------------------------------------------------------- @@ -63,7 +72,7 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: for model in models: model_tasks[model] = [] - for task in self.tasks(): + for task in self.tasks: if task.model is not None: model_tasks[task.model].append(task.base_name) else: From cac1b7ba3cb33be6dd8de41b70e874b71cb70d27 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 7 Nov 2025 16:58:54 -0500 Subject: [PATCH 125/299] code style fix --- src/swell/utilities/cylc_workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index eeb73fee5..576219d37 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -48,9 +48,9 @@ def __init__(self, experiment_dict, slurm_external) -> None: def default_header(self) -> str: return header_str - + # -------------------------------------------------------------------------------------------------- - + @abstractmethod def set_tasks(self) -> None: pass From 90ea975e87d9b3152076332d78413d43c310e16c Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 14 Nov 2025 16:41:38 -0500 Subject: [PATCH 126/299] fixes to flow --- src/swell/suites/3dfgat_cycle/workflow.py | 12 +++++++----- src/swell/suites/3dvar_atmos/workflow.py | 2 ++ src/swell/suites/3dvar_cycle/workflow.py | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index 8d95bb28f..d89d70b82 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -87,8 +87,6 @@ LinkGeosOutput-{{model_component}} => GenerateBClimatology-{{model_component}} # Data assimilation preperation - GetObservations-{{model_component}} - LinkGeosOutput-{{model_component}} => GenerateBClimatology-{{model_component}} StageJediCycle-{{model_component}} => RunJediFgatExecutable-{{model_component}} GenerateBClimatology-{{model_component}} => RunJediFgatExecutable-{{model_component}} GetObservations-{{model_component}} => RunJediFgatExecutable-{{model_component}} @@ -102,14 +100,17 @@ RunJediFgatExecutable-{{model_component}} => EvaIncrement-{{model_component}} {% if 'cice6' in models[model_component]["marine_models"] %} PrepareAnalysis-{{model_component}} => RunJediConvertStateSoca2ciceExecutable-{{model_component}} - RunJediConvertStateSoca2ciceExecutable-{{model_component}} => SaveRestart-{{model_component}} + # RunJediConvertStateSoca2ciceExecutable-{{model_component}} => SaveRestart-{{model_component}} + RunJediConvertStateSoca2ciceExecutable-{{model_component}} => MoveDaRestart-{{model_component}} RunJediConvertStateSoca2ciceExecutable-{{model_component}} => CleanCycle-{{model_component}} {% else %} - PrepareAnalysis-{{model_component}} => SaveRestart-{{model_component}} + # PrepareAnalysis-{{model_component}} => SaveRestart-{{model_component}} + PrepareAnalysis-{{model_component}} => MoveDaRestart-{{model_component}} {% endif %} + # Temporarily disable saving restarts # Move restart to next cycle - SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} + # SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} # Save analysis output # RunJediFgatExecutable-{{model_component}} => SaveAnalysis-{{model_component}} @@ -128,6 +129,7 @@ {% endfor %} """ {% endfor %} + # -------------------------------------------------------------------------------------------------- [runtime] diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index 299700462..bc0608ea7 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -145,6 +145,8 @@ def set_tasks(self) -> list: self.tasks.append(ta.CloneGeosMksi(model=model)) self.tasks.append(ta.StageJedi(model=model)) self.tasks.append(ta.GetObservations(model=model)) + self.tasks.append(ta.GetBackground(model=model)) + self.tasks.append(ta.GetBackgroundGeosExperiment(model=model)) self.tasks.append(ta.GenerateBClimatologyByLinking(model=model)) self.tasks.append(ta.GenerateBClimatology(model=model)) self.tasks.append(ta.GenerateObservingSystemRecords(model=model)) diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index 0734a7ec2..f91a98ae5 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -177,6 +177,7 @@ def set_tasks(self) -> list: self.tasks.append(ta.GetObservations(model=model)) self.tasks.append(ta.PrepareAnalysis(model=model)) self.tasks.append(ta.RunJediConvertStateSoca2ciceExecutable(model=model)) + self.tasks.append(ta.MoveDaRestart(model=model)) self.tasks.append(ta.RemoveForecastDir(model=model)) self.tasks.append(ta.EvaObservations(model=model)) self.tasks.append(ta.EvaJediLog(model=model)) From bdb5fda90b253e8e2c4134edea28b342ad025bb9 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 17 Nov 2025 12:11:48 -0500 Subject: [PATCH 127/299] Add docs for new method --- docs/examples/templating_workflows.md | 95 ++++++++++----------------- 1 file changed, 34 insertions(+), 61 deletions(-) diff --git a/docs/examples/templating_workflows.md b/docs/examples/templating_workflows.md index a59228f9a..498020156 100644 --- a/docs/examples/templating_workflows.md +++ b/docs/examples/templating_workflows.md @@ -3,67 +3,40 @@ The `flow.cylc` file informs the `Cylc` workflow engine on how to run an experim ## Cylc sections -The `flow.cylc` that is generated under this method is not much different from the one generated before, and users shouldn't notice a difference when it comes to creating an experiment, using overrides, etc. When creating an experiment, `swell` consults a file `src/swell/suites//workflow.py` on how to construct the suite. This file should be an extension of the `CylcWorkflow` class (defined in `src/swell/utilities/cylc_workflow.py`). The method `get_workflow_str` is called to return a string which fills the contents of the `flow.cylc` file. Overriding this method can be used to manually specify the contents of the file, but the intended method of using this class is to override methods which comprise the individual sections of the file. Since every suite within Swell contains roughly the same sections, some of which share the same content, it is only necessary to override a few of the methods, most notably the graph section. +The `flow.cylc` that is generated under this method is not much different from the one generated before, and users shouldn't notice a difference when it comes to creating an experiment, using overrides, etc. When creating an experiment, `swell` consults a file `src/swell/suites//workflow.py` on how to construct the suite. This file should be an extension of the `CylcWorkflow` class (defined in `src/swell/utilities/cylc_workflow.py`). The method `get_workflow_str` is called to return a string which fills the contents of the `flow.cylc` file. Overriding this method is used to manually specify the contents of the file. Typically, the graph section is templated in `jinja2`, and the runtime sections for each task are generated using swell's `TaskAttribute` class. However, the entire `flow.cylc` file can be templated in `jinja`, if necessary. -```python -def define_graph_section(self): - # Define the string of the graph section - graph_str = '' - - # Define the string for the R1 (first non-cycling) section - r1 = r1_template - - for model_component in self.experiment_dict['model_components']: - r1 += r1_model.format(model_component=model_component) - - # Format the R1 cycle and add it to the graph - graph_str += self.format_cycle('R1', r1) - - # Format the string for each cycle - for model_component in self.experiment_dict['model_components']: - if 'cycle_times' in self.experiment_dict['models'][model_component]: - for cycle_time in self.experiment_dict['models'][model_component]['cycle_times']: - cycle_str = cycle_template.format(model_component=model_component) - - # Add the cycle string to the graph string - graph_str += self.format_cycle(cycle_time, cycle_str) - - # Create the graph section - graph_section = self.create_new_section('graph', graph_str) - - return graph_section -``` - -The `define_graph_section` task is used to set the graph. There are a few built-in methods used to format the strings into cylc syntax. The `format_cycle` method is used to construct properly indented blocks for cycling intervals. `create_new_section` creates a section object that tracks indentation levels, and can be added to other sections, with properly handled indentation and spacing. Here the `graph` is constructed as one of the sections, since it itself is a sub-member of the `scheduling` block in a cylc graph. Any suite questions in the config can be referenced from `self.experiment_dict`. For example, this is often used to format the model component. ## Tasks and the runtime section -Swell will parse the graph section, which is constructed first, to obtain the tasks which are used by the experiment. It will then build the runtime section by consulting `src/swell/tasks/task_runtimes.py`. Since swell tasks broadly fall into only a few categories (model-dependent or independent, cycling or non-cycling) that do not differ much between suites, they are easily abstracted into a `Task` class. +Swell will parse the graph section, which is constructed first, to obtain the tasks which are used by the experiment. It will then build the runtime section by consulting `src/swell/tasks/task_attributes.py`. Since swell tasks broadly fall into only a few categories (model-dependent or independent, cycling or non-cycling) that do not differ much between suites, they are easily abstracted into a `Task` class. This class will dynamically set attributes such as messaging parameters and slurm settings. ```python -@dataclass class CloneJedi(Task): - pass - -@dataclass -class CloneGeosMksi(Task): - is_model: bool = True + def set_attributes(self): + self.question_list = QuestionList([ + qd.bundles(), + qd.existing_jedi_source_directory(), + qd.existing_jedi_source_directory_pinned(), + qd.jedi_build_method() + ]) -@dataclass -class EvaJediLog(Task): - is_cycling: bool = True - is_model: bool = True -@dataclass class EvaObservations(Task): - time_limit: bool = True - is_cycling: bool = True - is_model: bool = True - slurm: dict = mutable_field({}) - + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {} + self.question_list = QuestionList([ + background_crtm_obs, + qd.marine_models(), + qd.observing_system_records_path(), + qd.window_offset(), + qd.marine_models(), + ]) ``` -Here, the tags `is_cycling` and `is_model` are used to specify what tags the task needs to be appended with in the runtime section. These are set to `False` by default. Tasks with a specified `slurm` dictionary (rather than set to null, as by default) will use their contents to build the `directives` section. +Attributes are set by override the `set_attributes` method in `Task`. This has been combined with the previously-used `task_questions.py` for simplicity. Here, the tags `is_cycling` and `is_model` are used to specify what tags the task needs to be appended with in the runtime section. These are set to `False` by default. Tasks with a specified `slurm` dictionary (rather than set to null, as by default) will use their contents to build the `directives` section. ``` [[EvaObservations-geos_marine]] @@ -80,27 +53,27 @@ Here, the tags `is_cycling` and `is_model` are used to specify what tags the tas --account = ``` -This can be used to set task-specific defaults in `task_runtimes.py`, rather than being set in `slurm.py`: +This can be used to set task-specific defaults in `task_attributes.py`, rather than being set in `slurm.py`: ```python -@dataclass class RunJediConvertStateSoca2ciceExecutable(Task): - is_cycling: bool = True - is_model: bool = True - time_limit: bool = True - slurm: dict = mutable_field({'nodes': 1}) + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {'nodes': 1} ``` This supports setting platform-specific overrides, for example: ```python -@dataclass class RunJediConvertStateSoca2ciceExecutable(Task): - is_cycling: bool = True - is_model: bool = True - time_limit: bool = True - slurm: dict = mutable_field({'nodes': {'all': 1, - 'nccs_discover_cascade': 2}}) + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {'all': 1, + 'nccs_discover_cascade': 2} ``` On the `nccs_discover_cascade` platform, `nodes` will be set as 2, but on any other platform it will be 1. User overrides will still work as they did previously. From 0b6a231afa05e987ad940b94f7151914b36c00bd Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 19 Nov 2025 18:11:43 -0500 Subject: [PATCH 128/299] Add create_task_config --- src/swell/deployment/create_experiment.py | 3 +- src/swell/deployment/create_task_config.py | 193 ++++++++++++++++++ .../prepare_config_and_suite.py | 11 +- src/swell/suites/suite_questions.py | 13 ++ src/swell/swell.py | 30 +++ src/swell/tasks/base/task_base.py | 11 +- src/swell/utilities/cylc_workflow.py | 4 +- src/swell/utilities/question_defaults.py | 11 + 8 files changed, 263 insertions(+), 13 deletions(-) create mode 100644 src/swell/deployment/create_task_config.py diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 6b34bca89..8b889e829 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -153,6 +153,8 @@ def prepare_config( # ---------------------- experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() + workflow.experiment_dict = experiment_dict + # Finalize the workflow by adding the runtime section, and get the contents # ------------------------------------------------------------------------- workflow_string = workflow.get_workflow_string() @@ -361,7 +363,6 @@ def template_modules_file( with open(modules_file, 'w') as modules_file_open: modules_file_open.write(modules_file_str) - # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/create_task_config.py b/src/swell/deployment/create_task_config.py new file mode 100644 index 000000000..374a0e1e1 --- /dev/null +++ b/src/swell/deployment/create_task_config.py @@ -0,0 +1,193 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +from typing import Optional +from importlib import import_module +import yaml + +from swell.tasks.task_attributes import TaskAttributes +from swell.suites.suite_questions import SuiteQuestions +from swell.utilities.logger import get_logger +from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import \ + PrepareExperimentConfigAndSuite +from swell.utilities.slurm import prepare_slurm_defaults_and_overrides +from swell.utilities.dictionary import add_comments_to_dictionary +from swell.deployment.create_experiment import template_modules_file, create_modules_csh +from swell.utilities.jinja2 import template_string_jinja2 +from swell.utilities.shell_commands import create_executable_file + +# -------------------------------------------------------------------------------------------------- + +script_template = ''' +#!{{shell}} +{%- for key, value in task_slurm_dict %} +#SBATCH --{{key}} = {{value}} +{%- endfor %} + +# ------------------- + +source {modules_file} + +# ------------------- + +{{script}} + +# ------------------- +''' + + +# -------------------------------------------------------------------------------------------------- + +def task_config_wrapper(task_name: str, + platform: str, + model: Optional[str], + datetime: Optional[str], + input_method: str, + override: str, + slurm: str) -> None: + + logger = get_logger('SwellTaskConfig') + + task_attr_class = getattr(TaskAttributes, task_name) + + task = task_attr_class(model=model, platform=platform) + + if task.is_model and model is None: + logger.abort('Task requires model (e.g. geos_marine, geos_atmsophere)' + ' but none was specified at the command line.') + + if task.is_cycling and datetime is None: + logger.abort('Task requires datetime (e.g. 20231010T000000Z)' + ' but none was specified at the command line.') + + task_minimum = SuiteQuestions.task_minimum + + if override is None: + override = {} + + if model is not None: + override['model_components'] = [model] + + if 'experiment_root' not in override: + override['experiment_root'] = os.getcwd() + + task_id = f'swell-{task_name}' + if model is not None: + task_id = task_id + f'-{model}' + + if datetime is not None: + task_id = task_id + f'-{datetime}' + + if 'experiment_id' not in override: + override['experiment_id'] = task_id + + if 'use_cycle_dir' not in override: + override['use_cycle_dir'] = False + + prepare_config_and_suite = PrepareExperimentConfigAndSuite(logger=logger, + suite=task_name, + suite_config=task_minimum, + platform=platform, + config_client=input_method, + override=override) + + suite_dict = prepare_config_and_suite.experiment_dict + + model_independent_tasks = [] + model_dependent_tasks = {} + + if model is None: + model_independent_tasks.append(task) + for model_component in suite_dict['model_components']: + if model == model_component: + model_dependent_tasks[model] = [task] + + prepare_config_and_suite.model_independent_tasks = model_independent_tasks + prepare_config_and_suite.model_dependent_tasks = model_dependent_tasks + + experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() + + # Expand all environment vars in the dictionary + # --------------------------------------------- + experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) + experiment_dict_string = os.path.expandvars(experiment_dict_string) + experiment_dict = yaml.safe_load(experiment_dict_string) + + # Add comments to dictionary + # -------------------------- + experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) + + experiment_dict_string_comments = add_comments_to_dictionary(logger, experiment_dict_string, + comment_dict) + + slurm_external_dict = prepare_slurm_defaults_and_overrides(logger, platform, slurm) + + if task.slurm is not None: + task_slurm_dict = task.generate_task_slurm_dict(slurm_external_dict, platform) + else: + task_slurm_dict = None + + experiment_root = experiment_dict['experiment_root'] + experiment_id = experiment_dict['experiment_id'] + + task_path = os.path.join(experiment_root, experiment_id) + + if experiment_dict['use_cycle_dir']: + task_path = os.path.join(task_path, f'{experiment_id}-suite') + + os.makedirs(task_path, exist_ok=True) + + config_file = os.path.join(task_path, 'task_config.yaml') + with open(config_file, 'w') as f: + f.write(experiment_dict_string_comments) + + shell = os.environ.get('SHELL') + if shell is not None and 'bash' in shell: + template_modules_file(logger, experiment_dict, task_path) + modules_file = os.path.join(task_path, 'modules') + shell_type = 'bash' + elif shell is not None and 'csh' in shell: + create_modules_csh(logger, task_path) + modules_file = os.path.join(task_path, 'modules-csh') + shell_type = 'csh' + else: + template_modules_file(logger, experiment_dict, task_path) + create_modules_csh(logger, task_path) + logger.info('Shell type not detected, make sure you have the proper modules' + ' loaded before running experiment') + modules_file = os.path.join(task_path, 'modules') + shell_type = 'bash' + + script = f'swell task {task_name} {config_file}' + if model is not None: + script += f' -m {model}' + + if datetime is not None: + script += f' -d {datetime}' + + script_dict = {} + script_dict['shell'] = shell + script_dict['task_slurm_dict'] = task_slurm_dict + script_dict['modules_file'] = modules_file + script_dict['script'] = script + + script_content = template_string_jinja2(logger, + templated_string=script_template, + dictionary_of_templates=script_dict) + + script_file = os.path.join(task_path, f'{task_id}.{shell_type}') + create_executable_file(logger, script_file, script_content) + + logger.info('Task config generated.') + logger.info('\n\n') + + logger.info(script) + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 2d3578aaf..4129c7276 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -185,11 +185,9 @@ def prepare_task_question_dictionary(self): # Iterate through model independent tasks and update with defaults if not already set for task in self.model_independent_tasks: - task_options.append(task) + task_options.append(task.task_name) + question_list = task.question_list.expand_question_list() - task_class = getattr(task_attributes, task) - - question_list = task_class().question_list.expand_question_list() for question in question_list: question_dict = {question['question_name']: question} @@ -213,10 +211,9 @@ def prepare_task_question_dictionary(self): # Iterate through model dependent tasks and update if not already set for model, task_list in self.model_dependent_tasks.items(): for task in task_list: - task_options.append(task) + task_options.append(task.task_name) - task_class = getattr(task_attributes, task) - question_list = task_class(model=model).question_list.expand_question_list() + question_list = task.question_list.expand_question_list() for question in question_list: question_dict = {question['question_name']: question} diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index ba31f9668..f2e22dbcc 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -65,3 +65,16 @@ class SuiteQuestions(QuestionContainer, Enum): ) # -------------------------------------------------------------------------------------------------- + + task_minimum = QuestionList( + list_name="task_minimum", + questions=[ + qd.experiment_id(), + qd.experiment_root(), + qd.model_components(), + qd.marine_models(), + qd.use_cycle_dir(), + ] + ) + + # -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/swell.py b/src/swell/swell.py index 8854e2e8b..1f2f57062 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -20,6 +20,7 @@ from swell.suites.all_suites import suite_configs from swell.utilities.welcome_message import write_welcome_message from swell.utilities.scripts.utility_driver import get_utilities, utility_wrapper +from swell.deployment.create_task_config import task_config_wrapper # -------------------------------------------------------------------------------------------------- @@ -122,6 +123,35 @@ def create( # Create the experiment directory create_experiment_directory(suite, input_method, platform, override, advanced, slurm) +@swell_driver.command() +@click.argument('task', type=click.Choice(get_tasks())) +@click.option('-p', '--platform', 'platform', default='nccs_discover_sles15', + type=click.Choice(get_platforms()), help=platform_help) +@click.option('-d', '--datetime', 'datetime', default=None, help=datetime_help) +@click.option('-m', '--model', 'model', default=None, help=model_help) +@click.option('-m', '--input_method', 'input_method', default='defaults', + type=click.Choice(['defaults', 'cli']), help=input_method_help) +@click.option('-o', '--override', 'override', default=None, help=override_help) +@click.option('-s', '--slurm', 'slurm', default=None, help=slurm_help) +def create_task_config( + task: str, + platform: str, + datetime: Optional[str], + model: Optional[str], + input_method: str, + override: Optional[str], + slurm: Optional[str], +) -> None: + """ + Create a config for a single task + + This command generates a config to be used to run a single task. + + Arguments:\n + task (str): Name of the task to execute.\n + + """ + task_config_wrapper(task, platform, datetime, model, input_method, override, slurm) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 4ce636df6..cdf93acf3 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -191,9 +191,14 @@ def cycle_dir(self) -> str: self.logger.assert_abort(self.__model__ is not None, 'In get_cycle_dir but this ' + 'should not be called if the task does not receive model.') - # Combine datetime string (directory format) with the model - cycle_dir = os.path.join(self.experiment_path(), 'run', - self.__datetime__.string_directory(), self.__model__) + # Check whether to send to cycle dir + if self.config.use_cycle_dir(True): + + # Combine datetime string (directory format) with the model + cycle_dir = os.path.join(self.experiment_path(), 'run', + self.__datetime__.string_directory(), self.__model__) + else: + return self.experiment_path() # Return return cycle_dir diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/utilities/cylc_workflow.py index 576219d37..af4bb247c 100644 --- a/src/swell/utilities/cylc_workflow.py +++ b/src/swell/utilities/cylc_workflow.py @@ -74,9 +74,9 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: for task in self.tasks: if task.model is not None: - model_tasks[task.model].append(task.base_name) + model_tasks[task.model].append(task) else: - ind_tasks.append(task.base_name) + ind_tasks.append(task) return ind_tasks, model_tasks diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 9e45ba84e..6725b4d5e 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -185,6 +185,17 @@ class start_cycle_point(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class use_cycle_dir(SuiteQuestion): + default_value: bool = True + question_name: str = "use_cycle_dir" + ask_question: bool = False + prompt: str = ("For cycling tasks, send results to the experiment cycle directory? If false, " + "results will be stored in the current working directory.") + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + @dataclass class window_type(SuiteQuestion): default_value: str = "defer_to_model" From 043d34e6fb11cfd6526a6f4a9cb45847ad093666 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 19 Nov 2025 20:13:36 -0500 Subject: [PATCH 129/299] fixes --- src/swell/deployment/create_task_config.py | 12 ++++++------ .../prepare_config_and_suite.py | 18 ++++++------------ src/swell/suites/all_suites.py | 5 ++++- src/swell/suites/suite_questions.py | 6 ++++-- src/swell/swell.py | 3 +++ src/swell/tasks/base/task_base.py | 2 +- 6 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/swell/deployment/create_task_config.py b/src/swell/deployment/create_task_config.py index 374a0e1e1..910cf2324 100644 --- a/src/swell/deployment/create_task_config.py +++ b/src/swell/deployment/create_task_config.py @@ -67,13 +67,13 @@ def task_config_wrapper(task_name: str, logger.abort('Task requires datetime (e.g. 20231010T000000Z)' ' but none was specified at the command line.') - task_minimum = SuiteQuestions.task_minimum - if override is None: override = {} if model is not None: override['model_components'] = [model] + else: + override['model_components'] = [] if 'experiment_root' not in override: override['experiment_root'] = os.getcwd() @@ -92,8 +92,8 @@ def task_config_wrapper(task_name: str, override['use_cycle_dir'] = False prepare_config_and_suite = PrepareExperimentConfigAndSuite(logger=logger, - suite=task_name, - suite_config=task_minimum, + suite='task_minimum', + suite_config='task_minimum', platform=platform, config_client=input_method, override=override) @@ -132,7 +132,7 @@ def task_config_wrapper(task_name: str, if task.slurm is not None: task_slurm_dict = task.generate_task_slurm_dict(slurm_external_dict, platform) else: - task_slurm_dict = None + task_slurm_dict = {} experiment_root = experiment_dict['experiment_root'] experiment_id = experiment_dict['experiment_id'] @@ -190,4 +190,4 @@ def task_config_wrapper(task_name: str, logger.info(script) -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 4129c7276..ae0b9a2cb 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -185,7 +185,7 @@ def prepare_task_question_dictionary(self): # Iterate through model independent tasks and update with defaults if not already set for task in self.model_independent_tasks: - task_options.append(task.task_name) + task_options.append(task.base_name) question_list = task.question_list.expand_question_list() for question in question_list: @@ -211,7 +211,7 @@ def prepare_task_question_dictionary(self): # Iterate through model dependent tasks and update if not already set for model, task_list in self.model_dependent_tasks.items(): for task in task_list: - task_options.append(task.task_name) + task_options.append(task.base_name) question_list = task.question_list.expand_question_list() @@ -225,18 +225,12 @@ def prepare_task_question_dictionary(self): self.question_dictionary_model_dep, {model: question_dict}) # Set options for task email parameters - message_question_dict = {'task_email_parameters': - asdict(qd.task_email_parameters(options=task_options))} - - self.question_dictionary_model_ind = add_dict(self.question_dictionary_model_ind, - message_question_dict) + if 'task_email_parameters' in self.question_dictionary_model_ind: + self.question_dictionary_model_ind['task_email_parameters']['options'] = task_options # Set options for workflow pause - pause_question_dict = {'pause_on_tasks': - asdict(qd.pause_on_tasks(options=task_options))} - - self.question_dictionary_model_ind = add_dict(self.question_dictionary_model_ind, - pause_question_dict) + if 'pause_on_tasks' in self.question_dictionary_model_ind: + self.question_dictionary_model_ind['pause_on_tasks']['options'] = task_options # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/all_suites.py b/src/swell/suites/all_suites.py index 3ac94ae3e..fb1de73e5 100644 --- a/src/swell/suites/all_suites.py +++ b/src/swell/suites/all_suites.py @@ -75,9 +75,12 @@ def __init__(self) -> None: config_dict[format_suite_name(config)] = getattr(suite_container, config) config_map[format_suite_name(config)] = suite else: - config_dict[suite] = SuiteQuestions.all_suites.value + config_dict[suite] = SuiteQuestions.all_suites config_map[suite] = suite + config_dict['task_minimum'] = SuiteQuestions.task_minimum + config_map['task_minimum'] = 'task_minimum' + self.config_dict = config_dict self.__config_map__ = config_map diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index f2e22dbcc..2f9767523 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -26,7 +26,9 @@ class SuiteQuestions(QuestionContainer, Enum): list_name="all_suites", questions=[ qd.experiment_id(), - qd.experiment_root() + qd.experiment_root(), + qd.pause_on_tasks(), + qd.task_email_parameters() ] ) @@ -77,4 +79,4 @@ class SuiteQuestions(QuestionContainer, Enum): ] ) - # -------------------------------------------------------------------------------------------------- \ No newline at end of file + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index 1f2f57062..5aeaa1ecc 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -123,6 +123,9 @@ def create( # Create the experiment directory create_experiment_directory(suite, input_method, platform, override, advanced, slurm) +# -------------------------------------------------------------------------------------------------- + + @swell_driver.command() @click.argument('task', type=click.Choice(get_tasks())) @click.option('-p', '--platform', 'platform', default='nccs_discover_sles15', diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index cdf93acf3..9b9d21f26 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -319,7 +319,7 @@ def get_tasks() -> list: tasks = [] for task_file in task_files: base_name = os.path.basename(task_file) - if '__' not in base_name and base_name not in ['task_questions.py', 'task_runtimes.py']: + if '__' not in base_name and base_name not in ['task_attributes.py']: tasks.append(snake_case_to_camel_case(base_name[0:-3])) # Return list of valid task choices From 5de27b677a7a62eaf490fd4a1ee9614ce0c08293 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 20 Nov 2025 10:38:59 -0500 Subject: [PATCH 130/299] code test fixes --- src/swell/deployment/create_task_config.py | 10 ++++------ .../prepare_config_and_suite.py | 3 --- src/swell/tasks/base/task_base.py | 2 +- src/swell/utilities/question_defaults.py | 4 ++-- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/swell/deployment/create_task_config.py b/src/swell/deployment/create_task_config.py index 910cf2324..201b4c2cf 100644 --- a/src/swell/deployment/create_task_config.py +++ b/src/swell/deployment/create_task_config.py @@ -9,11 +9,9 @@ import os from typing import Optional -from importlib import import_module import yaml from swell.tasks.task_attributes import TaskAttributes -from swell.suites.suite_questions import SuiteQuestions from swell.utilities.logger import get_logger from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import \ PrepareExperimentConfigAndSuite @@ -52,9 +50,9 @@ def task_config_wrapper(task_name: str, input_method: str, override: str, slurm: str) -> None: - + logger = get_logger('SwellTaskConfig') - + task_attr_class = getattr(TaskAttributes, task_name) task = task_attr_class(model=model, platform=platform) @@ -81,7 +79,7 @@ def task_config_wrapper(task_name: str, task_id = f'swell-{task_name}' if model is not None: task_id = task_id + f'-{model}' - + if datetime is not None: task_id = task_id + f'-{datetime}' @@ -181,7 +179,7 @@ def task_config_wrapper(task_name: str, script_content = template_string_jinja2(logger, templated_string=script_template, dictionary_of_templates=script_dict) - + script_file = os.path.join(task_path, f'{task_id}.{shell_type}') create_executable_file(logger, script_file, script_content) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index ae0b9a2cb..7c6c2b58f 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -13,7 +13,6 @@ from collections.abc import Mapping from typing import Union, Tuple, Optional import datetime -from dataclasses import asdict from swell.swell_path import get_swell_path from swell.utilities.suite_utils import get_model_components @@ -22,10 +21,8 @@ from swell.utilities.dictionary import dict_get from swell.utilities.logger import Logger from swell.utilities.dictionary import update_dict, add_dict -from swell.tasks.task_attributes import TaskAttributes as task_attributes from swell.suites.all_suites import suite_configs from swell.utilities.swell_questions import QuestionType -from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 9b9d21f26..742ef4a82 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -196,7 +196,7 @@ def cycle_dir(self) -> str: # Combine datetime string (directory format) with the model cycle_dir = os.path.join(self.experiment_path(), 'run', - self.__datetime__.string_directory(), self.__model__) + self.__datetime__.string_directory(), self.__model__) else: return self.experiment_path() diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 6725b4d5e..88760b2f7 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -190,8 +190,8 @@ class use_cycle_dir(SuiteQuestion): default_value: bool = True question_name: str = "use_cycle_dir" ask_question: bool = False - prompt: str = ("For cycling tasks, send results to the experiment cycle directory? If false, " - "results will be stored in the current working directory.") + prompt: str = ("For cycling tasks, send results to the experiment cycle directory?" + " If false, results will be stored in the current working directory.") widget_type: WType = WType.BOOLEAN # -------------------------------------------------------------------------------------------------- From f7165e9d2a2a252503f46844ef829ef375348ab5 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 20 Nov 2025 11:48:37 -0500 Subject: [PATCH 131/299] Add cwd flag --- docs/examples/templating_workflows.md | 2 +- src/swell/deployment/create_task_config.py | 60 ++++++++++++++++------ src/swell/swell.py | 8 ++- 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/docs/examples/templating_workflows.md b/docs/examples/templating_workflows.md index 498020156..dfd1040b6 100644 --- a/docs/examples/templating_workflows.md +++ b/docs/examples/templating_workflows.md @@ -3,7 +3,7 @@ The `flow.cylc` file informs the `Cylc` workflow engine on how to run an experim ## Cylc sections -The `flow.cylc` that is generated under this method is not much different from the one generated before, and users shouldn't notice a difference when it comes to creating an experiment, using overrides, etc. When creating an experiment, `swell` consults a file `src/swell/suites//workflow.py` on how to construct the suite. This file should be an extension of the `CylcWorkflow` class (defined in `src/swell/utilities/cylc_workflow.py`). The method `get_workflow_str` is called to return a string which fills the contents of the `flow.cylc` file. Overriding this method is used to manually specify the contents of the file. Typically, the graph section is templated in `jinja2`, and the runtime sections for each task are generated using swell's `TaskAttribute` class. However, the entire `flow.cylc` file can be templated in `jinja`, if necessary. +The `flow.cylc` that is generated under this method is not much different from the one generated before, and users shouldn't notice a difference when it comes to creating an experiment, using overrides, etc. When creating an experiment, `swell` consults a file `src/swell/suites//workflow.py` on how to construct the suite. This file should be an extension of the `CylcWorkflow` class (defined in `src/swell/utilities/cylc_workflow.py`). The method `get_workflow_string` is called to return a string which fills the contents of the `flow.cylc` file. Overriding this method is used to manually specify the contents of the file. Typically, the graph section is templated in `jinja2`, and the runtime sections for each task are generated using swell's `TaskAttribute` class. However, the entire `flow.cylc` file can be templated in `jinja`, if necessary. ## Tasks and the runtime section diff --git a/src/swell/deployment/create_task_config.py b/src/swell/deployment/create_task_config.py index 201b4c2cf..3e947ccae 100644 --- a/src/swell/deployment/create_task_config.py +++ b/src/swell/deployment/create_task_config.py @@ -31,7 +31,7 @@ # ------------------- -source {modules_file} +source {{modules_file}} # ------------------- @@ -49,22 +49,29 @@ def task_config_wrapper(task_name: str, datetime: Optional[str], input_method: str, override: str, - slurm: str) -> None: + slurm: str, + cwd: bool) -> None: + # Create logger logger = get_logger('SwellTaskConfig') - task_attr_class = getattr(TaskAttributes, task_name) + logger.info(f'Generating config for task {task_name}') + # Get the task attributes for the class + task_attr_class = getattr(TaskAttributes, task_name) task = task_attr_class(model=model, platform=platform) + # Check that model is specified for the task if task.is_model and model is None: logger.abort('Task requires model (e.g. geos_marine, geos_atmsophere)' ' but none was specified at the command line.') + # Check that datetime is specified for the task if task.is_cycling and datetime is None: logger.abort('Task requires datetime (e.g. 20231010T000000Z)' ' but none was specified at the command line.') + # Construct overrides if override is None: override = {} @@ -73,9 +80,11 @@ def task_config_wrapper(task_name: str, else: override['model_components'] = [] - if 'experiment_root' not in override: + # Build in current working directory + if cwd: override['experiment_root'] = os.getcwd() + # Construct task ID task_id = f'swell-{task_name}' if model is not None: task_id = task_id + f'-{model}' @@ -86,9 +95,11 @@ def task_config_wrapper(task_name: str, if 'experiment_id' not in override: override['experiment_id'] = task_id + # Don't put results in cycle dir if 'use_cycle_dir' not in override: override['use_cycle_dir'] = False + # Build the suite for task minimums prepare_config_and_suite = PrepareExperimentConfigAndSuite(logger=logger, suite='task_minimum', suite_config='task_minimum', @@ -96,56 +107,62 @@ def task_config_wrapper(task_name: str, config_client=input_method, override=override) - suite_dict = prepare_config_and_suite.experiment_dict - + # Set the tasks appropriately model_independent_tasks = [] model_dependent_tasks = {} if model is None: model_independent_tasks.append(task) - for model_component in suite_dict['model_components']: - if model == model_component: - model_dependent_tasks[model] = [task] + else: + model_dependent_tasks[model] = task prepare_config_and_suite.model_independent_tasks = model_independent_tasks prepare_config_and_suite.model_dependent_tasks = model_dependent_tasks + # Configure and ask all questions experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() # Expand all environment vars in the dictionary - # --------------------------------------------- experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) experiment_dict_string = os.path.expandvars(experiment_dict_string) experiment_dict = yaml.safe_load(experiment_dict_string) # Add comments to dictionary - # -------------------------- experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) experiment_dict_string_comments = add_comments_to_dictionary(logger, experiment_dict_string, comment_dict) + # Construct the slurm defaults slurm_external_dict = prepare_slurm_defaults_and_overrides(logger, platform, slurm) + # Construct the slurm dict for the task if task.slurm is not None: task_slurm_dict = task.generate_task_slurm_dict(slurm_external_dict, platform) else: task_slurm_dict = {} + # Determine the path for task results experiment_root = experiment_dict['experiment_root'] experiment_id = experiment_dict['experiment_id'] task_path = os.path.join(experiment_root, experiment_id) + # If use_cycle_dir, construct the experiment directory the same way as a suite if experiment_dict['use_cycle_dir']: task_path = os.path.join(task_path, f'{experiment_id}-suite') + # Create the task directory os.makedirs(task_path, exist_ok=True) + # Write the task config config_file = os.path.join(task_path, 'task_config.yaml') with open(config_file, 'w') as f: f.write(experiment_dict_string_comments) + logger.info(f'Writing task config under {config_file}') + + # Build the modules file depending on the suite shell = os.environ.get('SHELL') if shell is not None and 'bash' in shell: template_modules_file(logger, experiment_dict, task_path) @@ -156,6 +173,7 @@ def task_config_wrapper(task_name: str, modules_file = os.path.join(task_path, 'modules-csh') shell_type = 'csh' else: + # Default to bash template_modules_file(logger, experiment_dict, task_path) create_modules_csh(logger, task_path) logger.info('Shell type not detected, make sure you have the proper modules' @@ -163,6 +181,11 @@ def task_config_wrapper(task_name: str, modules_file = os.path.join(task_path, 'modules') shell_type = 'bash' + file_ext = shell_type + if len(task_slurm_dict) > 0: + file_ext = 'slurm' + + # Build the swell task script script = f'swell task {task_name} {config_file}' if model is not None: script += f' -m {model}' @@ -170,22 +193,29 @@ def task_config_wrapper(task_name: str, if datetime is not None: script += f' -d {datetime}' + # Build the template dict for the shell script file script_dict = {} script_dict['shell'] = shell script_dict['task_slurm_dict'] = task_slurm_dict script_dict['modules_file'] = modules_file script_dict['script'] = script + # Template the shell script script_content = template_string_jinja2(logger, templated_string=script_template, dictionary_of_templates=script_dict) + # Create the shell script file script_file = os.path.join(task_path, f'{task_id}.{shell_type}') create_executable_file(logger, script_file, script_content) - logger.info('Task config generated.') - logger.info('\n\n') - - logger.info(script) + logger.info('\nTask config successfully generated.') + logger.info('To run the task by itself, run: ') + print(f'\n {script}\n') + logger.info('Or, to use auto-generated script, run: ') + if len(task_slurm_dict) > 0: + print(f'\n sbatch {script_file}\n') + else: + print(f' {script_file}\n') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index 5aeaa1ecc..712e7a5d8 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -91,6 +91,10 @@ def swell_driver() -> None: (For diagnostic tasks only) - Define the output directory that diagnostics tests will send their results. Used in comparison suites. """ +cwd_help = """ +For task configs, set flag to create directory at the user's cwd, otherwise directory will be +created in default experiment_root.""" + # -------------------------------------------------------------------------------------------------- @@ -136,6 +140,7 @@ def create( type=click.Choice(['defaults', 'cli']), help=input_method_help) @click.option('-o', '--override', 'override', default=None, help=override_help) @click.option('-s', '--slurm', 'slurm', default=None, help=slurm_help) +@click.option('-c', '--cwd', 'cwd', is_flag=True, help=cwd_help) def create_task_config( task: str, platform: str, @@ -144,6 +149,7 @@ def create_task_config( input_method: str, override: Optional[str], slurm: Optional[str], + cwd: bool, ) -> None: """ Create a config for a single task @@ -154,7 +160,7 @@ def create_task_config( task (str): Name of the task to execute.\n """ - task_config_wrapper(task, platform, datetime, model, input_method, override, slurm) + task_config_wrapper(task, platform, datetime, model, input_method, override, slurm, cwd) # -------------------------------------------------------------------------------------------------- From eac5c0304e8f7755830b4ba8702f8d0678c13bed Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 20 Nov 2025 12:39:31 -0500 Subject: [PATCH 132/299] Fixes --- src/swell/deployment/create_task_config.py | 12 ++++++------ src/swell/swell.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/swell/deployment/create_task_config.py b/src/swell/deployment/create_task_config.py index 3e947ccae..e51ebc06d 100644 --- a/src/swell/deployment/create_task_config.py +++ b/src/swell/deployment/create_task_config.py @@ -25,7 +25,7 @@ script_template = ''' #!{{shell}} -{%- for key, value in task_slurm_dict %} +{%- for key, value in task_slurm_dict.items() %} #SBATCH --{{key}} = {{value}} {%- endfor %} @@ -45,8 +45,8 @@ def task_config_wrapper(task_name: str, platform: str, - model: Optional[str], datetime: Optional[str], + model: Optional[str], input_method: str, override: str, slurm: str, @@ -114,7 +114,7 @@ def task_config_wrapper(task_name: str, if model is None: model_independent_tasks.append(task) else: - model_dependent_tasks[model] = task + model_dependent_tasks[model] = [task] prepare_config_and_suite.model_independent_tasks = model_independent_tasks prepare_config_and_suite.model_dependent_tasks = model_dependent_tasks @@ -206,16 +206,16 @@ def task_config_wrapper(task_name: str, dictionary_of_templates=script_dict) # Create the shell script file - script_file = os.path.join(task_path, f'{task_id}.{shell_type}') + script_file = os.path.join(task_path, f'{task_id}.{file_ext}') create_executable_file(logger, script_file, script_content) - logger.info('\nTask config successfully generated.') + logger.info('Task config successfully generated.') logger.info('To run the task by itself, run: ') print(f'\n {script}\n') logger.info('Or, to use auto-generated script, run: ') if len(task_slurm_dict) > 0: print(f'\n sbatch {script_file}\n') else: - print(f' {script_file}\n') + print(f'\n {script_file}\n') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index 712e7a5d8..120791dbb 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -92,7 +92,7 @@ def swell_driver() -> None: their results. Used in comparison suites. """ cwd_help = """ -For task configs, set flag to create directory at the user's cwd, otherwise directory will be +For task configs, set flag to create directory at the user's cwd, otherwise directory will be created in default experiment_root.""" # -------------------------------------------------------------------------------------------------- @@ -100,7 +100,7 @@ def swell_driver() -> None: @swell_driver.command() @click.argument('suite', type=click.Choice(suite_configs.all_configs())) -@click.option('-m', '--input_method', 'input_method', default='defaults', +@click.option('-i', '--input_method', 'input_method', default='defaults', type=click.Choice(['defaults', 'cli']), help=input_method_help) @click.option('-p', '--platform', 'platform', default='nccs_discover_sles15', type=click.Choice(get_platforms()), help=platform_help) @@ -136,7 +136,7 @@ def create( type=click.Choice(get_platforms()), help=platform_help) @click.option('-d', '--datetime', 'datetime', default=None, help=datetime_help) @click.option('-m', '--model', 'model', default=None, help=model_help) -@click.option('-m', '--input_method', 'input_method', default='defaults', +@click.option('-i', '--input_method', 'input_method', default='defaults', type=click.Choice(['defaults', 'cli']), help=input_method_help) @click.option('-o', '--override', 'override', default=None, help=override_help) @click.option('-s', '--slurm', 'slurm', default=None, help=slurm_help) From 9a4bd06632015396c7b3e3191257e7a98703df85 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 20 Nov 2025 12:46:49 -0500 Subject: [PATCH 133/299] fix input_method --- src/swell/swell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/swell.py b/src/swell/swell.py index 120791dbb..e29819ed7 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -100,7 +100,7 @@ def swell_driver() -> None: @swell_driver.command() @click.argument('suite', type=click.Choice(suite_configs.all_configs())) -@click.option('-i', '--input_method', 'input_method', default='defaults', +@click.option('-m', '--input_method', 'input_method', default='defaults', type=click.Choice(['defaults', 'cli']), help=input_method_help) @click.option('-p', '--platform', 'platform', default='nccs_discover_sles15', type=click.Choice(get_platforms()), help=platform_help) From b0e63e42dcb5ea045a23986f24c278a6f26ce3b3 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 20 Nov 2025 14:01:43 -0500 Subject: [PATCH 134/299] Fixes for questions --- .../prepare_config_and_suite/question_and_answer_cli.py | 3 +++ src/swell/suites/suite_questions.py | 1 + src/swell/utilities/question_defaults.py | 4 ++-- src/swell/utilities/swell_questions.py | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/question_and_answer_cli.py b/src/swell/deployment/prepare_config_and_suite/question_and_answer_cli.py index bc6075020..f253fc61a 100644 --- a/src/swell/deployment/prepare_config_and_suite/question_and_answer_cli.py +++ b/src/swell/deployment/prepare_config_and_suite/question_and_answer_cli.py @@ -27,6 +27,9 @@ def get_answer(self, logger: Logger, key: str, val: dict, model: Optional[str] = widget_type = val['widget_type'] options = val['options'] + if options is None: + options = [] + if model is not None: prompt = f'[{model}] {prompt}' diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index 2f9767523..25df3c48e 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -73,6 +73,7 @@ class SuiteQuestions(QuestionContainer, Enum): questions=[ qd.experiment_id(), qd.experiment_root(), + qd.comparison_experiment_paths(), qd.model_components(), qd.marine_models(), qd.use_cycle_dir(), diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 88760b2f7..73dff7464 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -30,7 +30,7 @@ class comparison_experiment_paths(SuiteQuestion): question_name: str = "comparison_experiment_paths" ask_question: bool = True prompt: str = "Provide paths to two experiments to run comparison tests on." - widget_type: WType = WType.STRING_CHECK_LIST + widget_type: WType = WType.STRING_LIST # -------------------------------------------------------------------------------------------------- @@ -148,7 +148,7 @@ class parser_options(SuiteQuestion): ask_question: bool = True options: list = mutable_field(['fgrep_residual_norm']) prompt: str = "List the test types to run on the JEDI oops log." - widget_type: WType = WType.STRING_DROP_LIST + widget_type: WType = WType.STRING_CHECK_LIST # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index 462aacd6c..35e1e7321 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -31,6 +31,7 @@ class WidgetType(Enum): STRING = "string" STRING_CHECK_LIST = "string-check-list" STRING_DROP_LIST = "string-drop-list" + STRING_LIST = "string-list" BOOLEAN = "boolean" ISO_DURATION = "iso-duration" ISO_DATETIME = "iso-datetime" From 643273793b435e5670863064c145f9217bf76356 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 20 Nov 2025 15:07:41 -0500 Subject: [PATCH 135/299] Fix cycle times --- src/swell/deployment/create_experiment.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 8b889e829..74c3fc64b 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -18,7 +18,7 @@ from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import \ PrepareExperimentConfigAndSuite from swell.swell_path import get_swell_path -from swell.utilities.dictionary import add_comments_to_dictionary, dict_get +from swell.utilities.dictionary import add_comments_to_dictionary, dict_get, update_dict from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.logger import Logger, get_logger from swell.utilities.slurm import prepare_slurm_defaults_and_overrides @@ -153,7 +153,10 @@ def prepare_config( # ---------------------- experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() - workflow.experiment_dict = experiment_dict + # Update dict with cycle times + # ---------------------------- + workflow_dict = update_dict(suite_dict, experiment_dict) + workflow.experiment_dict = workflow_dict # Finalize the workflow by adding the runtime section, and get the contents # ------------------------------------------------------------------------- From f308408ff19ec996cbb8a6b64204cfdb0384c415 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 20 Nov 2025 17:12:52 -0500 Subject: [PATCH 136/299] Fix for comparison suites --- src/swell/deployment/create_experiment.py | 22 +++++++++++++++++++++- src/swell/suites/compare/workflow.py | 12 +++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 74c3fc64b..4ba5d209f 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -23,6 +23,7 @@ from swell.utilities.logger import Logger, get_logger from swell.utilities.slurm import prepare_slurm_defaults_and_overrides from swell.suites.all_suites import suite_configs, workflows +from swell.utilities.check_da_params import check_da_params # -------------------------------------------------------------------------------------------------- @@ -97,6 +98,25 @@ def prepare_config( suite_dict = suite_dict.copy() + # Overrides for comparison suites + if 'start_cycle_point' in suite_dict: + start_cycle_point = suite_dict['start_cycle_point'] + final_cycle_point = suite_dict['final_cycle_point'] + if suite_dict['start_cycle_point'] is None: + config_list = suite_dict['comparison_experiment_paths'] + for model in suite_dict['model_components']: + cycle_times = suite_dict['models'][model]['cycle_times'] + start_cycle_point, final_cycle_point, cycle_times = check_da_params( + config_list, + model, + start_cycle_point, + final_cycle_point, + cycle_times) + + suite_dict['start_cycle_point'] = start_cycle_point + suite_dict['final_cycle_point'] = final_cycle_point + suite_dict['models'][model]['cycle_times'] = cycle_times + # Resolve cycle times for models # ------------------------------ if 'models' in suite_dict and 'start_cycle_point' in suite_dict: @@ -155,7 +175,7 @@ def prepare_config( # Update dict with cycle times # ---------------------------- - workflow_dict = update_dict(suite_dict, experiment_dict) + workflow_dict = update_dict(experiment_dict, suite_dict) workflow.experiment_dict = workflow_dict # Finalize the workflow by adding the runtime section, and get the contents diff --git a/src/swell/suites/compare/workflow.py b/src/swell/suites/compare/workflow.py index 5fdc20339..94e0046ae 100644 --- a/src/swell/suites/compare/workflow.py +++ b/src/swell/suites/compare/workflow.py @@ -7,6 +7,8 @@ # -------------------------------------------------------------------------------------------------- +import yaml + from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow from swell.tasks.task_attributes import TaskAttributes as ta @@ -71,14 +73,22 @@ def get_workflow_string(self): for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) - return workflow_str def set_tasks(self) -> list: + for path in self.experiment_dict['comparison_experiment_paths']: + with open(path, 'r') as f: + config_dict = yaml.safe_load(f) + for model in self.experiment_dict['model_components']: + num_of_iterations = config_dict['models'][model]['number_of_iterations'] + + self.experiment_dict['models'][model]['number_of_iterations'] = num_of_iterations + for model in self.experiment_dict['model_components']: self.tasks.append(ta.EvaComparisonIncrement(model=model)) self.tasks.append(ta.EvaComparisonJediLog(model=model)) self.tasks.append(ta.JediOopsLogParser(model=model)) + self.tasks.append(ta.JediLogComparison(model=model)) # -------------------------------------------------------------------------------------------------- From 1ea1be31dd3b518a0668a186e6e1be505cacf73a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 24 Nov 2025 16:43:23 -0500 Subject: [PATCH 137/299] add time limit to slurm directives --- src/swell/deployment/create_task_config.py | 7 +++++++ src/swell/utilities/task_specification.py | 22 +++++++++++++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/swell/deployment/create_task_config.py b/src/swell/deployment/create_task_config.py index e51ebc06d..3fb207d0a 100644 --- a/src/swell/deployment/create_task_config.py +++ b/src/swell/deployment/create_task_config.py @@ -10,6 +10,7 @@ import os from typing import Optional import yaml +import isodate from swell.tasks.task_attributes import TaskAttributes from swell.utilities.logger import get_logger @@ -139,6 +140,12 @@ def task_config_wrapper(task_name: str, # Construct the slurm dict for the task if task.slurm is not None: task_slurm_dict = task.generate_task_slurm_dict(slurm_external_dict, platform) + + time_limit = task.get_time_limit() + if time_limit is not None: + time_limit_dto = isodate.parse_duration(time_limit) + task_slurm_dict['time'] = isodate.strftime(time_limit_dto, '%H:%M:%S') + else: task_slurm_dict = {} diff --git a/src/swell/utilities/task_specification.py b/src/swell/utilities/task_specification.py index 1f2f41686..532b9ede7 100644 --- a/src/swell/utilities/task_specification.py +++ b/src/swell/utilities/task_specification.py @@ -161,6 +161,20 @@ def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Ma # -------------------------------------------------------------------------------------------------- + def get_time_limit(self) -> str: + + # Set the time limit, default is 1 hour + if self.time_limit is True: + time_limit = 'PT1H' + elif self.time_limit: + time_limit = self.match_platform(self.time_limit, platform) + else: + time_limit = None + + return time_limit + + # -------------------------------------------------------------------------------------------------- + def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): ''' Return the runtime section for the given task. ''' @@ -186,11 +200,9 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): if self.slurm is not None: runtime_dict['platform'] = platform - # Set the time limit, default is 1 hour - if self.time_limit is True: - runtime_dict['execution time limit'] = 'PT1H' - elif self.time_limit: - time_limit = self.match_platform(self.time_limit, platform) + time_limit = self.get_time_limit() + + if time_limit is not None: runtime_dict['execution time limit'] = time_limit # Set the retry if this task needs it From 33f911f77e63c832a482ed0e2df0df910633905b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 16 Dec 2025 16:47:31 -0500 Subject: [PATCH 138/299] address comments --- docs/examples/templating_workflows.md | 4 +- src/swell/deployment/create_task_config.py | 13 +- src/swell/tasks/base/task_base.py | 8 +- src/swell/utilities/cylc_formatting.py | 45 ++-- src/swell/utilities/cylc_runtime.py | 243 --------------------- src/swell/utilities/question_defaults.py | 4 +- 6 files changed, 34 insertions(+), 283 deletions(-) delete mode 100644 src/swell/utilities/cylc_runtime.py diff --git a/docs/examples/templating_workflows.md b/docs/examples/templating_workflows.md index dfd1040b6..9c3d021d8 100644 --- a/docs/examples/templating_workflows.md +++ b/docs/examples/templating_workflows.md @@ -36,7 +36,7 @@ class EvaObservations(Task): ]) ``` -Attributes are set by override the `set_attributes` method in `Task`. This has been combined with the previously-used `task_questions.py` for simplicity. Here, the tags `is_cycling` and `is_model` are used to specify what tags the task needs to be appended with in the runtime section. These are set to `False` by default. Tasks with a specified `slurm` dictionary (rather than set to null, as by default) will use their contents to build the `directives` section. +Attributes are set by override the `set_attributes` method in `Task`. This has been combined with the previously-used `task_questions.py` for simplicity. Here, the tags `is_cycling` and `is_model` are used to specify what tags the task needs to be appended with in the runtime section. These are set to `False` by default. Tasks with a specified `slurm` dictionary (rather than set to null, as by default) will use their contents to build the `directives` section. For the task specification above for `EvaObservations`, the runtime section will be renderend as the following: ``` [[EvaObservations-geos_marine]] @@ -53,7 +53,7 @@ Attributes are set by override the `set_attributes` method in `Task`. This has b --account = ``` -This can be used to set task-specific defaults in `task_attributes.py`, rather than being set in `slurm.py`: +This can be used to set task-specific defaults in `task_attributes.py`, rather than being set in `slurm.py`. For example, the task below defaults to slurm setting `--nodes=1`. ```python class RunJediConvertStateSoca2ciceExecutable(Task): diff --git a/src/swell/deployment/create_task_config.py b/src/swell/deployment/create_task_config.py index 3fb207d0a..0b78c7584 100644 --- a/src/swell/deployment/create_task_config.py +++ b/src/swell/deployment/create_task_config.py @@ -169,8 +169,11 @@ def task_config_wrapper(task_name: str, logger.info(f'Writing task config under {config_file}') - # Build the modules file depending on the suite + # Build the modules file depending on the shell type shell = os.environ.get('SHELL') + if shell is None: + logger.abort('Could not ascertaine $SHELL') + if shell is not None and 'bash' in shell: template_modules_file(logger, experiment_dict, task_path) modules_file = os.path.join(task_path, 'modules') @@ -180,13 +183,7 @@ def task_config_wrapper(task_name: str, modules_file = os.path.join(task_path, 'modules-csh') shell_type = 'csh' else: - # Default to bash - template_modules_file(logger, experiment_dict, task_path) - create_modules_csh(logger, task_path) - logger.info('Shell type not detected, make sure you have the proper modules' - ' loaded before running experiment') - modules_file = os.path.join(task_path, 'modules') - shell_type = 'bash' + logger.abort(f'Failed to deduce the target shell. $SHELL is currently set to {shell}') file_ext = shell_type if len(task_slurm_dict) > 0: diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 742ef4a82..7f4ae2f8e 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -41,7 +41,7 @@ def __init__( datetime_input: Optional[str], model: str, ensemblePacket: Optional[str], - task_name: str, + task_name: str ) -> None: # Create message logger @@ -192,7 +192,7 @@ def cycle_dir(self) -> str: 'should not be called if the task does not receive model.') # Check whether to send to cycle dir - if self.config.use_cycle_dir(True): + if self.config.use_cycle_dir(default_value=True): # Combine datetime string (directory format) with the model cycle_dir = os.path.join(self.experiment_path(), 'run', @@ -280,7 +280,7 @@ def create_task( config: str, datetime: Union[str, dt, None], model: str, - ensemblePacket: Optional[str], + ensemblePacket: Optional[str] ) -> taskBase: # Load R2D2 credentials before importing any task modules @@ -319,7 +319,7 @@ def get_tasks() -> list: tasks = [] for task_file in task_files: base_name = os.path.basename(task_file) - if '__' not in base_name and base_name not in ['task_attributes.py']: + if '__' not in base_name and base_name != 'task_attributes.py': tasks.append(snake_case_to_camel_case(base_name[0:-3])) # Return list of valid task choices diff --git a/src/swell/utilities/cylc_formatting.py b/src/swell/utilities/cylc_formatting.py index c86e3c1be..7ca854706 100644 --- a/src/swell/utilities/cylc_formatting.py +++ b/src/swell/utilities/cylc_formatting.py @@ -42,29 +42,6 @@ def indent_lines(string: str, level: int = 0, reset: bool = False): # -------------------------------------------------------------------------------------------------- -def format_section(section: Self, level: int = 0) -> str: - # Format a string to match cylc's section syntax - # format the header with the appropriate amount of enclosing brackets and indents - - section_str = '' - - name = section.name - if name is not None: - section_str += textwrap.indent(f'{(level+1)*"["}{name}{"]"*(level+1)}\n', INDENT*level) - else: - level -= 1 - - content = section.content - if isinstance(content, Mapping): - content = format_dict(content) - - section_str += indent_lines(content, level+1) - - return section_str - -# -------------------------------------------------------------------------------------------------- - - class CylcSection(): ''' Holds the information contained in a section, including the name and contents, which can be a @@ -78,11 +55,31 @@ def __init__(self, name: Optional[str] = None, content: Union[str, dict] = '') - self.subsections = [] + def format_section(self, section: Self, level: int = 0) -> str: + # Format a string to match cylc's section syntax + # format the header with the appropriate amount of enclosing brackets and indents + + section_str = '' + + name = section.name + if name is not None: + section_str += textwrap.indent(f'{(level+1)*"["}{name}{"]"*(level+1)}\n', INDENT*level) + else: + level -= 1 + + content = section.content + if isinstance(content, Mapping): + content = format_dict(content) + + section_str += indent_lines(content, level+1) + + return section_str + def add_subsection(self, subsection: Self) -> None: self.subsections.append(subsection) def get_section_str(self, level: int = 0) -> str: - section_str = format_section(self, level) + section_str = self.format_section(self, level) for subsection in self.subsections: section_str += subsection.get_section_str(level+1) diff --git a/src/swell/utilities/cylc_runtime.py b/src/swell/utilities/cylc_runtime.py deleted file mode 100644 index 7bb3ada50..000000000 --- a/src/swell/utilities/cylc_runtime.py +++ /dev/null @@ -1,243 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - -import os -import yaml -from typing import Union, Optional -from collections.abc import Mapping -from dataclasses import dataclass - -from swell.utilities.cylc_formatting import CylcSection, indent_lines -from swell.utilities.suite_utils import get_model_components -from swell.utilities.dictionary import update_dict -from swell.utilities.dataclass_utils import mutable_field - -# -------------------------------------------------------------------------------------------------- - - -@dataclass -class Task: - - ''' - Contains the basic properties and information needed to format the cylc [runtime] section. - ''' - - base_name: Optional[str] = None - scheduling_name: Optional[str] = None - - model: Optional[str] = None - - pre_script: Union[str, bool, None] = False - script: Union[str, bool, None] = None - - retry: Union[str, Mapping, None] = None - time_limit: Union[str, Mapping, None] = None - environment: Optional[Mapping] = None - slurm: Optional[Mapping] = None - - is_cycling: bool = False - is_model: bool = False - - additional_sections: list = mutable_field([]) - mail_events: list = mutable_field(['failed', 'submit-failed']) - - # -------------------------------------------------------------------------------------------------- - - def __post_init__(self): - - if self.base_name is None: - self.base_name = self.__class__.__name__ - - if self.scheduling_name is None: - self.scheduling_name = self.base_name - - if self.is_model and self.model is not None: - self.scheduling_name += f'-{self.model}' - - if self.script is None: - self.script = f'swell task {self.base_name} $config' - - if self.is_cycling: - self.script += ' -d $datetime' - - if self.is_model and self.model is not None: - self.script += ' -m {model}' - - if self.is_model and self.model is not None: - self.script = self.script.format(model=self.model) - self.scheduling_name = self.scheduling_name.format(model=self.model) - - # -------------------------------------------------------------------------------------------------- - - def format_string_block(self, string: str) -> str: - out_string = '"""\n' - out_string += indent_lines(string, 1) - out_string += '"""' - - return out_string - - # -------------------------------------------------------------------------------------------------- - - def match_platform(self, content: Union[str, dict], platform: str): - # Resolve platform-specific entries in the task object - - if isinstance(content, Mapping): - if platform in content.keys(): - content = content[platform] - elif 'all' in content.keys(): - content = content['all'] - - return content - - # -------------------------------------------------------------------------------------------------- - - def create_new_section(self, - name: Optional[str] = None, - content: Union[str, dict] = '' - ) -> CylcSection: - return CylcSection(name, content) - - # -------------------------------------------------------------------------------------------------- - - def resolve_model(self, slurm_dict: Mapping) -> dict: - ''' Resolve "all" and "model" entries in slurm dictionary ''' - if 'all' in slurm_dict.keys() and isinstance(slurm_dict['all'], Mapping): - slurm_dict = update_dict(slurm_dict, slurm_dict['all']) - del slurm_dict['all'] - if self.model in slurm_dict.keys() and isinstance(slurm_dict[self.model], Mapping): - slurm_dict = update_dict(slurm_dict, slurm_dict[self.model]) - - for model in get_model_components(): - if model in slurm_dict.keys(): - del slurm_dict[model] - - return slurm_dict - - # -------------------------------------------------------------------------------------------------- - - def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Mapping: - # Take the external slurm dictionary and merge it with the task's parameters - # to get the dict that will be output in the flow.cylc - - slurm_dict = {} - if self.slurm is not None: - for key, value in self.slurm.items(): - slurm_dict[key] = self.match_platform(value, platform) - - slurm_globals = slurm_external['slurm_directives_global'] - slurm_task = {} - - if 'slurm_directives_tasks' in slurm_external.keys(): - task_directives = slurm_external['slurm_directives_tasks'] - - if self.base_name in task_directives: - slurm_task = task_directives[self.base_name] - if self.scheduling_name in task_directives: - slurm_task = task_directives[self.scheduling_name] - - slurm_dict = {'job-name': self.scheduling_name, - **self.resolve_model(slurm_globals), - **self.resolve_model(slurm_dict), - **self.resolve_model(slurm_task)} - - return slurm_dict - - # -------------------------------------------------------------------------------------------------- - - def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): - ''' Return the runtime section for the given task. ''' - - platform = experiment_dict['platform'] - runtime_dict = {} - - # Set the pre_script only if it is specified - if self.pre_script: - runtime_dict['pre-script'] = self.format_string_block(self.pre_script) - - # Set the script - if self.script: - script_str = self.script - - if 'pause_on_tasks' in experiment_dict.keys(): - if len(set([self.base_name, self.scheduling_name]) - & set(experiment_dict['pause_on_tasks'])) > 0: - script_str += '\ncylc pause $CYLC_WORKFLOW_ID' - - runtime_dict['script'] = self.format_string_block(script_str) - - # Specify the platform if this is a slurm task - if self.slurm is not None: - runtime_dict['platform'] = platform - - # Set the time limit, default is 1 hour - if self.time_limit is True: - runtime_dict['execution time limit'] = 'PT1H' - elif self.time_limit: - time_limit = self.match_platform(self.time_limit, platform) - runtime_dict['execution time limit'] = time_limit - - # Set the retry if this task needs it - if self.retry: - if self.retry is True: - retry = '2*PT1M' - else: - retry = self.match_platform(self.retry, platform) - - runtime_dict['execution retry delays'] = retry - - runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) - - # Set the environment dictionary - if self.environment is not None: - environment_section = self.create_new_section('environment', self.environment) - runtime_section.add_subsection(environment_section) - - # Specify the slurm dictionary with defaults from user and global settings - if self.slurm is not None: - - slurm_dict = self.generate_task_slurm_dict(slurm_external, platform) - - slurm_section_dict = {} - for key, value in slurm_dict.items(): - slurm_section_dict[f'--{key}'] = value - - directive_section = self.create_new_section('directives', slurm_section_dict) - - runtime_section.add_subsection(directive_section) - - # Append additional sections to runtime - for section in self.additional_sections: - runtime_section.add_subsection(section) - - # Check slurm messaging parameters - events = [] - events = self.mail_events - - # Add messaging section - settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) - if os.path.exists(settings_file) and len(events) > 0: - with open(settings_file, 'r') as f: - settings_dict = yaml.safe_load(f) - if 'email_address' in settings_dict.keys(): - email_address = settings_dict['email_address'] - address_section = self.create_new_section('mail', f'to = {email_address}') - runtime_section.add_subsection(address_section) - - event_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" - event_str += "mail events = " + ', '.join(events) - event_str += "\n{% endif %}\n" - - event_section = self.create_new_section('events', event_str) - runtime_section.add_subsection(event_section) - - runtime_string = runtime_section.get_section_str(1) - - return runtime_string - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 7ad940340..abed13fe9 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -9,7 +9,7 @@ from dataclasses import dataclass -from typing import List, Dict, Union +from typing import List, Dict, Union, Literal from swell.utilities.swell_questions import SuiteQuestion, TaskQuestion from swell.utilities.swell_questions import WidgetType as WType @@ -1286,7 +1286,7 @@ class swell_static_files_user(TaskQuestion): @dataclass class task_email_parameters(TaskQuestion): - default_value: Union[str, dict] = "auto" + default_value: Union[Literal["auto"], dict] = "auto" question_name: str = "task_email_parameters" prompt: str = ("Provide a dictionary mapping tasks to cylc event statuses, or 'auto' to " "automatically configure these based on the graph.") From ca92d9fd846448d3c6a84eba6041b3e3b5b5cdf2 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 17 Dec 2025 10:01:33 -0500 Subject: [PATCH 139/299] pycodestyle fix --- src/swell/utilities/question_defaults.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index abed13fe9..b1429d484 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -990,7 +990,7 @@ class ncdiag_experiments(TaskQuestion): # -------------------------------------------------------------------------------------------------- @dataclass - class npx(TaskQuestion): + class npx(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "npx" ask_question: bool = True @@ -1016,7 +1016,7 @@ class npx_proc(TaskQuestion): # -------------------------------------------------------------------------------------------------- @dataclass - class npy(TaskQuestion): + class npy(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "npy" ask_question: bool = True From 038ddb96ea9e24c2db1112a471e50da54524d129 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 17 Dec 2025 16:06:18 -0500 Subject: [PATCH 140/299] Fix default_value --- src/swell/tasks/base/task_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 7f4ae2f8e..5ca570fff 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -192,7 +192,7 @@ def cycle_dir(self) -> str: 'should not be called if the task does not receive model.') # Check whether to send to cycle dir - if self.config.use_cycle_dir(default_value=True): + if self.config.use_cycle_dir(True): # Combine datetime string (directory format) with the model cycle_dir = os.path.join(self.experiment_path(), 'run', From 7ddae1c93b41703ff268a7c384000e30fa9bb731 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 18 Dec 2025 14:53:01 -0500 Subject: [PATCH 141/299] Remove task questions --- src/swell/tasks/task_questions.py | 735 ------------------------------ 1 file changed, 735 deletions(-) delete mode 100644 src/swell/tasks/task_questions.py diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py deleted file mode 100644 index af4406437..000000000 --- a/src/swell/tasks/task_questions.py +++ /dev/null @@ -1,735 +0,0 @@ -# -------------------------------------------------------------------------------------------------- -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -from enum import Enum - -from swell.utilities.swell_questions import QuestionList, QuestionContainer -from swell.utilities.question_defaults import QuestionDefaults as qd - - -# -------------------------------------------------------------------------------------------------- - -class TaskQuestions(QuestionContainer, Enum): - - # -------------------------------------------------------------------------------------------------- - # Helper question lists used by multiple tasks (in order of use) - # -------------------------------------------------------------------------------------------------- - - background_crtm_obs = QuestionList( - list_name="background_crtm_obs", - questions=[ - qd.background_time_offset(), - qd.crtm_coeff_dir(), - qd.observations(), - qd.observing_system_records_path() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - np_proc_resolution = QuestionList( - list_name="np_resolution", - questions=[ - qd.npx_proc(), - qd.npy_proc(), - qd.npx(), - qd.npy(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - window_questions = QuestionList( - list_name="window_questions", - questions=[ - qd.window_length(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - run_jedi_executable = QuestionList( - list_name="run_jedi_executable", - questions=[ - background_crtm_obs, - np_proc_resolution, - window_questions, - qd.analysis_variables(), - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.gradient_norm_reduction(), - qd.gsibec_configuration(), - qd.jedi_forecast_model(), - qd.minimizer(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.number_of_iterations(), - qd.total_processors(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - swell_static_file_questions = QuestionList( - list_name="swell_static_file_questions", - questions=[ - qd.swell_static_files(), - qd.swell_static_files_user() - ] - ) - - # -------------------------------------------------------------------------------------------------- - # Task-specific question lists (in alphabetical order) - # -------------------------------------------------------------------------------------------------- - - BuildGeos = QuestionList( - list_name="BuildGeos", - questions=[ - qd.geos_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - BuildGeosByLinking = QuestionList( - list_name="BuildGeosByLinking", - questions=[ - qd.existing_geos_gcm_build_path(), - qd.geos_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - BuildJedi = QuestionList( - list_name="BuildJedi", - questions=[ - qd.bundles(), - qd.jedi_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - BuildJediByLinking = QuestionList( - list_name="BuildJediByLinking", - questions=[ - qd.existing_jedi_build_directory(), - qd.existing_jedi_build_directory_pinned(), - qd.jedi_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CleanCycle = QuestionList( - list_name="CleanCycle", - questions=[ - qd.clean_patterns() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CloneGeos = QuestionList( - list_name="CloneGeos", - questions=[ - qd.existing_geos_gcm_source_path(), - qd.geos_build_method(), - qd.geos_gcm_tag() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CloneGeosMksi = QuestionList( - list_name="CloneGeosMksi", - questions=[ - qd.observing_system_records_mksi_path(), - qd.observing_system_records_mksi_path_tag() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CloneGmaoPerllib = QuestionList( - list_name="CloneGmaoPerllib", - questions=[ - qd.existing_perllib_path(), - qd.gmao_perllib_tag() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CloneJedi = QuestionList( - list_name="CloneJedi", - questions=[ - qd.bundles(), - qd.existing_jedi_source_directory(), - qd.existing_jedi_source_directory_pinned(), - qd.jedi_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaComparisonJediLog = QuestionList( - list_name="EvaJediLog", - questions=[ - qd.comparison_log_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaIncrement = QuestionList( - list_name="EvaIncrement", - questions=[ - qd.marine_models(), - qd.window_length(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaObservations = QuestionList( - list_name="EvaObservations", - questions=[ - background_crtm_obs, - qd.marine_models(), - qd.observing_system_records_path(), - qd.window_length(), - qd.marine_models(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaTimeseries = QuestionList( - list_name="EvaTimeseries", - questions=[ - background_crtm_obs, - qd.window_length(), - qd.ncdiag_experiments(), - qd.marine_models(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GenerateBClimatology = QuestionList( - list_name="GenerateBClimatology", - questions=[ - np_proc_resolution, - swell_static_file_questions, - qd.analysis_variables(), - qd.background_error_model(), - qd.generate_yaml_and_exit(), - qd.gradient_norm_reduction(), - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.jedi_forecast_model(), - qd.marine_models(), - qd.minimizer(), - qd.number_of_iterations(), - qd.observing_system_records_path(), - qd.total_processors(), - qd.window_length(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GenerateBClimatologyByLinking = QuestionList( - list_name="GenerateBClimatologyByLinking", - questions=[ - swell_static_file_questions, - qd.background_error_model(), - qd.horizontal_resolution(), - qd.vertical_resolution(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GenerateObservingSystemRecords = QuestionList( - list_name="GenerateObservingSystemRecords", - questions=[ - qd.observations(), - qd.observing_system_records_mksi_path(), - qd.observing_system_records_path() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetBackground = QuestionList( - list_name="GetBackground", - questions=[ - window_questions, - qd.background_experiment(), - qd.background_frequency(), - qd.horizontal_resolution(), - qd.marine_models(), - qd.r2d2_local_path(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetBackgroundGeosExperiment = QuestionList( - list_name="GetBackgroundGeosExperiment", - questions=[ - qd.horizontal_resolution(), - qd.background_experiment(), - qd.background_time_offset(), - qd.geos_x_background_directory() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetBufr = QuestionList( - list_name="GetBufr", - questions=[ - qd.bufr_obs_classes() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetEnsemble = QuestionList( - list_name="GetEnsemble", - questions=[ - qd.path_to_ensemble() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetEnsembleGeosExperiment = QuestionList( - list_name="GetEnsembleGeosExperiment", - questions=[ - qd.background_experiment(), - qd.background_time_offset(), - qd.geos_x_ensemble_directory() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGeovals = QuestionList( - list_name="GetGeovals", - questions=[ - background_crtm_obs, - qd.geovals_experiment(), - qd.geovals_provider(), - qd.r2d2_local_path(), - qd.window_length(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGeosAdasBackground = QuestionList( - list_name="GetGeosAdasBackground", - questions=[ - qd.path_to_geos_adas_background() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGeosRestart = QuestionList( - list_name="GetGeosRestart", - questions=[ - swell_static_file_questions, - qd.geos_restarts_directory() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGsiBc = QuestionList( - list_name="GetGsiBc", - questions=[ - qd.path_to_gsi_bc_coefficients(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGsiNcdiag = QuestionList( - list_name="GetGsiNcdiag", - questions=[ - qd.path_to_gsi_nc_diags() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetNcdiags = QuestionList( - list_name="GetNcdiags", - questions=[ - background_crtm_obs, - qd.ncdiag_experiments(), - qd.marine_models(), - qd.r2d2_local_path(), - qd.window_length(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetObservations = QuestionList( - list_name="GetObservations", - questions=[ - background_crtm_obs, - qd.cycling_varbc(), - qd.obs_experiment(), - qd.observing_system_records_path(), - qd.r2d2_local_path(), - qd.window_length(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetObsNotInR2d2 = QuestionList( - list_name="GetExistingObservations", - questions=[ - qd.ioda_locations_not_in_r2d2(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GsiBcToIoda = QuestionList( - list_name="GsiBcToIoda", - questions=[ - background_crtm_obs, - qd.observing_system_records_path(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GsiNcdiagToIoda = QuestionList( - list_name="GsiNcdiagToIoda", - questions=[ - qd.observations(), - qd.produce_geovals(), - qd.single_observations(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - JediLogComparison = QuestionList( - list_name="JediComparisonLog", - questions=[ - qd.comparison_log_type(), - qd.number_of_iterations() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - JediOopsLogParser = QuestionList( - list_name="JediOopsLogParser", - questions=[ - qd.comparison_log_type(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - LinkGeosOutput = QuestionList( - list_name="LinkGeosOutput", - questions=[ - window_questions, - qd.background_frequency(), - qd.marine_models() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - MoveDaRestart = QuestionList( - list_name="MoveDaRestart", - questions=[ - qd.mom6_iau(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - MoveForecastRestart = QuestionList( - list_name="MoveForecastRestart", - questions=[ - qd.forecast_duration() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - PrepareAnalysis = QuestionList( - list_name="PrepareAnalysis", - questions=[ - qd.analysis_variables(), - qd.mom6_iau(), - qd.total_processors(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - PrepGeosRunDir = QuestionList( - list_name="PrepGeosRunDir", - questions=[ - swell_static_file_questions, - qd.existing_geos_gcm_build_path(), - qd.forecast_duration(), - qd.geos_experiment_directory(), - qd.mom6_iau_nhours() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediConvertStateSoca2ciceExecutable = QuestionList( - list_name="RunJediConvertStateSoca2ciceExecutable", - questions=[ - qd.analysis_variables(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.marine_models(), - qd.observations(), - qd.total_processors(), - qd.window_length(), - qd.window_type(), - qd.comparison_log_type('convert_state_soca2cice'), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediEnsembleMeanVariance = QuestionList( - list_name="RunJediEnsembleMeanVariance", - questions=[ - np_proc_resolution, - window_questions, - qd.analysis_variables(), - qd.ensemble_num_members(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.observations(), - qd.observing_system_records_path(), - qd.comparison_log_type('ensmeanvariance'), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediFgatExecutable = QuestionList( - list_name="RunJediFgatExecutable", - questions=[ - run_jedi_executable, - qd.marine_models(), - qd.comparison_log_type('fgat') - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediHofxEnsembleExecutable = QuestionList( - list_name="RunJediHofxEnsembleExecutable", - questions=[ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.ensemble_num_members(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.total_processors(), - qd.comparison_log_type('hofx') - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediHofxExecutable = QuestionList( - list_name="RunJediHofxExecutable", - questions=[ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.save_geovals(), - qd.total_processors(), - qd.comparison_log_type('ensemblehofx'), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediLocalEnsembleDaExecutable = QuestionList( - list_name="RunJediLocalEnsembleDaExecutable", - questions=[ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.ensemble_num_members(), - qd.ensmean_only(), - qd.ensmeanvariance_only(), - qd.generate_yaml_and_exit(), - qd.horizontal_localization_lengthscale(), - qd.horizontal_localization_max_nobs(), - qd.horizontal_localization_method(), - qd.jedi_forecast_model(), - qd.local_ensemble_inflation_mult(), - qd.local_ensemble_inflation_rtpp(), - qd.local_ensemble_inflation_rtps(), - qd.local_ensemble_save_posterior_ensemble(), - qd.local_ensemble_save_posterior_ensemble_increments(), - qd.local_ensemble_save_posterior_mean(), - qd.local_ensemble_save_posterior_mean_increment(), - qd.local_ensemble_solver(), - qd.local_ensemble_use_linear_observer(), - qd.skip_ensemble_hofx(), - qd.total_processors(), - qd.vertical_localization_apply_log_transform(), - qd.vertical_localization_function(), - qd.vertical_localization_ioda_vertical_coord(), - qd.vertical_localization_ioda_vertical_coord_group(), - qd.vertical_localization_lengthscale(), - qd.vertical_localization_method(), - qd.perhost(), - qd.comparison_log_type('localensembleda'), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediObsfiltersExecutable = QuestionList( - list_name="RunJediObsfiltersExecutable", - questions=[ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.observing_system_records_path(), - qd.total_processors(), - qd.obs_thinning_rej_fraction(), - qd.comparison_log_type('obsfilters') - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediUfoTestsExecutable = QuestionList( - list_name="RunJediUfoTestsExecutable", - questions=[ - background_crtm_obs, - qd.generate_yaml_and_exit(), - qd.single_observations(), - qd.window_length(), - qd.comparison_log_type('ufo_tests'), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediVariationalExecutable = QuestionList( - list_name="RunJediVariationalExecutable", - questions=[ - run_jedi_executable, - qd.perhost(), - qd.comparison_log_type('variational'), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - SaveObsDiags = QuestionList( - list_name="SaveObsDiags", - questions=[ - background_crtm_obs, - qd.window_length(), - qd.r2d2_local_path(), - qd.marine_models() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - SaveRestart = QuestionList( - list_name="SaveRestart", - questions=[ - window_questions, - qd.background_time_offset(), - qd.forecast_duration(), - qd.horizontal_resolution(), - qd.marine_models(), - qd.r2d2_local_path() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - StageJedi = QuestionList( - list_name="StageJedi", - questions=[ - swell_static_file_questions, - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - StoreBackground = QuestionList( - list_name="StoreBackground", - questions=[ - window_questions, - qd.background_experiment(), - qd.background_frequency(), - qd.horizontal_resolution(), - qd.r2d2_local_path(), - ] - ) - - # -------------------------------------------------------------------------------------------------- From bf1eaf24624f768c759ae4001f0ed57de0776d62 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 18 Dec 2025 14:53:30 -0500 Subject: [PATCH 142/299] Rework settings files --- src/swell/utilities/settings.py | 36 +++++++++++++++++++++++ src/swell/utilities/task_specification.py | 30 +++++++++---------- 2 files changed, 50 insertions(+), 16 deletions(-) create mode 100644 src/swell/utilities/settings.py diff --git a/src/swell/utilities/settings.py b/src/swell/utilities/settings.py new file mode 100644 index 000000000..3ffb85a55 --- /dev/null +++ b/src/swell/utilities/settings.py @@ -0,0 +1,36 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +import yaml + +# -------------------------------------------------------------------------------------------------- + +def read_settings() -> dict: + ''' + Reads user settings from yaml file under ~/.swell/swell-settings.yaml + + Args: + None + + Returns: + Dictionary of settings specified in file. + ''' + settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) + + if os.path.exists(settings_file): + with open(settings_file, 'r') as f: + settings_dict = yaml.safe_load(f) + + else: + settings_dict = {} + + return settings_dict + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/utilities/task_specification.py b/src/swell/utilities/task_specification.py index 532b9ede7..ed2cad0cb 100644 --- a/src/swell/utilities/task_specification.py +++ b/src/swell/utilities/task_specification.py @@ -16,6 +16,7 @@ from swell.utilities.suite_utils import get_model_components from swell.utilities.dictionary import update_dict from swell.utilities.swell_questions import QuestionList +from swell.utilities.settings import read_settings # -------------------------------------------------------------------------------------------------- @@ -234,25 +235,22 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): runtime_section.add_subsection(section) # Check slurm messaging parameters - events = [] events = self.mail_events + settings_dict = read_settings() + # Add messaging section - settings_file = os.path.expanduser(os.path.join('~', '.swell', 'swell-settings.yaml')) - if os.path.exists(settings_file) and len(events) > 0: - with open(settings_file, 'r') as f: - settings_dict = yaml.safe_load(f) - if 'email_address' in settings_dict.keys(): - email_address = settings_dict['email_address'] - address_section = self.create_new_section('mail', f'to = {email_address}') - runtime_section.add_subsection(address_section) - - event_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" - event_str += "mail events = " + ', '.join(events) - event_str += "\n{% endif %}\n" - - event_section = self.create_new_section('events', event_str) - runtime_section.add_subsection(event_section) + if len(events) > 0 and 'email_address' in settings_dict.keys(): + email_address = settings_dict['email_address'] + address_section = self.create_new_section('mail', f'to = {email_address}') + runtime_section.add_subsection(address_section) + + event_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" + event_str += "mail events = " + ', '.join(events) + event_str += "\n{% endif %}\n" + + event_section = self.create_new_section('events', event_str) + runtime_section.add_subsection(event_section) runtime_string = runtime_section.get_section_str(1) From 2b8f8ff5f597891ed14ee5e43d45613d4f215eff Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 23 Dec 2025 10:23:42 -0500 Subject: [PATCH 143/299] Refactor location of task attributes --- src/swell/deployment/create_task_config.py | 2 +- src/swell/suites/3dfgat_atmos/workflow.py | 2 +- src/swell/suites/3dfgat_cycle/workflow.py | 2 +- src/swell/suites/3dvar/workflow.py | 2 +- src/swell/suites/3dvar_atmos/workflow.py | 2 +- src/swell/suites/3dvar_cycle/workflow.py | 2 +- src/swell/suites/build_geos/workflow.py | 2 +- src/swell/suites/build_jedi/workflow.py | 2 +- src/swell/suites/compare/workflow.py | 2 +- src/swell/suites/convert_bufr/workflow.py | 2 +- src/swell/suites/convert_ncdiags/workflow.py | 2 +- src/swell/suites/eva_capabilities/workflow.py | 2 +- src/swell/suites/forecast_geos/workflow.py | 2 +- src/swell/suites/geosadas/workflow.py | 2 +- src/swell/suites/hofx/workflow.py | 2 +- src/swell/suites/localensembleda/workflow.py | 2 +- src/swell/suites/ufo_testing/workflow.py | 2 +- src/swell/tasks/task_attributes.py | 859 ------------------ src/swell/test/code_tests/slurm_test.py | 2 +- src/swell/utilities/config.py | 2 +- src/swell/utilities/task_specification.py | 261 ------ 21 files changed, 19 insertions(+), 1139 deletions(-) delete mode 100644 src/swell/tasks/task_attributes.py delete mode 100644 src/swell/utilities/task_specification.py diff --git a/src/swell/deployment/create_task_config.py b/src/swell/deployment/create_task_config.py index 0b78c7584..cadad9cf3 100644 --- a/src/swell/deployment/create_task_config.py +++ b/src/swell/deployment/create_task_config.py @@ -12,7 +12,7 @@ import yaml import isodate -from swell.tasks.task_attributes import TaskAttributes +from swell.tasks.base.task_attributes import TaskAttributes from swell.utilities.logger import get_logger from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import \ PrepareExperimentConfigAndSuite diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 373f8cf1a..97e22831d 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index d89d70b82..043ec3eb6 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 294c726a5..6ecb10b16 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index bc0608ea7..02f3da14c 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index f91a98ae5..aaa81aa59 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py index 4ea9d0530..3cd046576 100644 --- a/src/swell/suites/build_geos/workflow.py +++ b/src/swell/suites/build_geos/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index 0d023a16a..9673f5eaf 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare/workflow.py b/src/swell/suites/compare/workflow.py index 94e0046ae..ce47e7f84 100644 --- a/src/swell/suites/compare/workflow.py +++ b/src/swell/suites/compare/workflow.py @@ -11,7 +11,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_bufr/workflow.py b/src/swell/suites/convert_bufr/workflow.py index 20637ea40..dc9525494 100644 --- a/src/swell/suites/convert_bufr/workflow.py +++ b/src/swell/suites/convert_bufr/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index a959d08d4..9d67a36ae 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index e4225bc9f..8efc9c062 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index 3cff792a7..e5288b812 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index a2d2c1017..31ca3c2c9 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index 5fa776579..596f317e1 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index c22fa2284..12ef77d82 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index c2acdf719..339fbe08c 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.cylc_workflow import CylcWorkflow -from swell.tasks.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_attributes.py b/src/swell/tasks/task_attributes.py deleted file mode 100644 index 16769129a..000000000 --- a/src/swell/tasks/task_attributes.py +++ /dev/null @@ -1,859 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - -from swell.utilities.task_specification import Task -from swell.utilities.swell_questions import QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd - -# -------------------------------------------------------------------------------------------------- - -background_crtm_obs = QuestionList( - list_name="background_crtm_obs", - questions=[ - qd.background_time_offset(), - qd.crtm_coeff_dir(), - qd.observations(), - qd.observing_system_records_path() - ] -) - -# -------------------------------------------------------------------------------------------------- - -np_proc_resolution = QuestionList( - list_name="np_resolution", - questions=[ - qd.npx_proc(), - qd.npy_proc(), - qd.npx(), - qd.npy(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ] -) - -# -------------------------------------------------------------------------------------------------- - -window_questions = QuestionList( - list_name="window_questions", - questions=[ - qd.window_length(), - qd.window_type() - ] -) - -# -------------------------------------------------------------------------------------------------- - -run_jedi_executable = QuestionList( - list_name="run_jedi_executable", - questions=[ - background_crtm_obs, - np_proc_resolution, - window_questions, - qd.analysis_variables(), - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.gradient_norm_reduction(), - qd.gsibec_configuration(), - qd.jedi_forecast_model(), - qd.minimizer(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.number_of_iterations(), - qd.total_processors(), - ] -) - -# -------------------------------------------------------------------------------------------------- - -swell_static_file_questions = QuestionList( - list_name="swell_static_file_questions", - questions=[ - qd.swell_static_files(), - qd.swell_static_files_user() - ] -) - -# -------------------------------------------------------------------------------------------------- - - -class TaskAttributes(): - - # -------------------------------------------------------------------------------------------------- - - class root(Task): - - def set_attributes(self): - self.script = False - self.pre_script = "source $CYLC_SUITE_DEF_PATH/modules" - self.additional_sections = [self.create_new_section('environment', - {'datetime': '$CYLC_TASK_CYCLE_POINT', - 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'})] # noqa - - # -------------------------------------------------------------------------------------------------- - - class BuildGeos(Task): - def set_attributes(self): - self.question_list = QuestionList([ - qd.geos_build_method() - ]) - - # -------------------------------------------------------------------------------------------------- - - class BuildGeosByLinking(Task): - def set_attributes(self): - self.mail_events = ['submit-failed'] - self.question_list = QuestionList([ - qd.existing_geos_gcm_build_path(), - qd.geos_build_method() - ]) - - # -------------------------------------------------------------------------------------------------- - - class BuildJediByLinking(Task): - def set_attributes(self): - self.mail_events = ['submit-failed'] - self.question_list = QuestionList([ - qd.existing_jedi_build_directory(), - qd.existing_jedi_build_directory_pinned(), - qd.jedi_build_method() - ]) - - # -------------------------------------------------------------------------------------------------- - - class BuildJedi(Task): - def set_attributes(self): - self.time_limit = True - self.slurm = {} - self.question_list = QuestionList([ - qd.bundles(), - qd.jedi_build_method() - ]) - - # -------------------------------------------------------------------------------------------------- - - class CleanCycle(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.clean_patterns() - ]) - - # -------------------------------------------------------------------------------------------------- - - class CloneGeos(Task): - def set_attributes(self): - self.question_list = QuestionList([ - qd.existing_geos_gcm_source_path(), - qd.geos_build_method(), - qd.geos_gcm_tag() - ]) - - # -------------------------------------------------------------------------------------------------- - - class CloneJedi(Task): - def set_attributes(self): - self.question_list = QuestionList([ - qd.bundles(), - qd.existing_jedi_source_directory(), - qd.existing_jedi_source_directory_pinned(), - qd.jedi_build_method() - ]) - - # -------------------------------------------------------------------------------------------------- - - class CloneGeosMksi(Task): - def set_attributes(self): - self.is_model = True - self.question_list = QuestionList([ - qd.observing_system_records_mksi_path(), - qd.observing_system_records_mksi_path_tag() - ]) - - # -------------------------------------------------------------------------------------------------- - - class CloneGmaoPerllib(Task): - def set_attributes(self): - self.question_list = QuestionList([ - qd.existing_perllib_path(), - qd.gmao_perllib_tag() - ]) - - # -------------------------------------------------------------------------------------------------- - - class EvaJediLog(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - - # -------------------------------------------------------------------------------------------------- - - class EvaComparisonIncrement(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.marine_models(), - qd.window_length(), - qd.window_type() - ]) - - # -------------------------------------------------------------------------------------------------- - - class EvaComparisonJediLog(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - - # -------------------------------------------------------------------------------------------------- - - class EvaIncrement(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.marine_models(), - qd.window_length(), - qd.window_type() - ]) - - # -------------------------------------------------------------------------------------------------- - - class EvaObservations(Task): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.slurm = {} - self.question_list = QuestionList([ - background_crtm_obs, - qd.marine_models(), - qd.observing_system_records_path(), - qd.window_length(), - qd.marine_models(), - ]) - - # -------------------------------------------------------------------------------------------------- - - class EvaTimeseries(Task): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.slurm = {} - self.question_list = QuestionList([ - background_crtm_obs, - qd.window_length(), - qd.ncdiag_experiments(), - qd.marine_models(), - ]) - - # -------------------------------------------------------------------------------------------------- - - class JediOopsLogParser(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.parser_options(), - ]) - - # -------------------------------------------------------------------------------------------------- - - class GetBackground(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - window_questions, - qd.window_length(), - qd.background_experiment(), - qd.background_frequency(), - qd.horizontal_resolution(), - qd.marine_models(), - qd.r2d2_local_path(), - ]) - - # -------------------------------------------------------------------------------------------------- - - class GetBackgroundGeosExperiment(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.mail_events = ['submit-failed'] - self.question_list = QuestionList([ - qd.horizontal_resolution(), - qd.background_experiment(), - qd.background_time_offset(), - qd.geos_x_background_directory() - ]) - - # -------------------------------------------------------------------------------------------------- - - class GetBufr(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.bufr_obs_classes() - ]) - - # -------------------------------------------------------------------------------------------------- - - class BufrToIoda(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - - # -------------------------------------------------------------------------------------------------- - - class GetEnsembleGeosExperiment(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.background_experiment(), - qd.background_time_offset(), - qd.geos_x_ensemble_directory() - ]) - - # -------------------------------------------------------------------------------------------------- - - class GetGeosRestart(Task): - def set_attributes(self): - self.is_cycling = True - self.question_list = QuestionList([ - swell_static_file_questions, - qd.geos_restarts_directory() - ]) - - # -------------------------------------------------------------------------------------------------- - - class GetGeovals(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - background_crtm_obs, - qd.geovals_experiment(), - qd.geovals_provider(), - qd.r2d2_local_path(), - qd.window_length(), - ]) - - # -------------------------------------------------------------------------------------------------- - - class GetGsiBc(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.path_to_gsi_bc_coefficients(), - qd.window_length() - ]) - - # -------------------------------------------------------------------------------------------------- - - class GsiBcToIoda(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - background_crtm_obs, - qd.observing_system_records_path(), - qd.window_length() - ]) - - # -------------------------------------------------------------------------------------------------- - - class GetGsiNcdiag(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.path_to_gsi_nc_diags() - ]) - - # -------------------------------------------------------------------------------------------------- - - class GsiNcdiagToIoda(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.observations(), - qd.produce_geovals(), - qd.single_observations(), - qd.window_length() - ]) - - # -------------------------------------------------------------------------------------------------- - - class GetNcdiags(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - background_crtm_obs, - qd.ncdiag_experiments(), - qd.marine_models(), - qd.r2d2_local_path(), - qd.window_length(), - ]) - - # -------------------------------------------------------------------------------------------------- - - class GetGeosAdasBackground(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.path_to_geos_adas_background() - ]) - - # -------------------------------------------------------------------------------------------------- - - class GetObservations(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - background_crtm_obs, - qd.cycling_varbc(), - qd.obs_experiment(), - qd.observing_system_records_path(), - qd.r2d2_local_path(), - qd.window_length(), - ]) - - # -------------------------------------------------------------------------------------------------- - - class GetObsNotInR2d2(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.mail_events = ['submit-failed'] - self.question_list = QuestionList([ - qd.ioda_locations_not_in_r2d2(), - ]) - - # -------------------------------------------------------------------------------------------------- - - class GenerateBClimatology(Task): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.retry = '2*PT1M' - self.slurm = {} - self.question_list = QuestionList([ - np_proc_resolution, - swell_static_file_questions, - qd.analysis_variables(), - qd.background_error_model(), - qd.generate_yaml_and_exit(), - qd.gradient_norm_reduction(), - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.jedi_forecast_model(), - qd.marine_models(), - qd.minimizer(), - qd.number_of_iterations(), - qd.observing_system_records_path(), - qd.total_processors(), - qd.window_length(), - qd.window_type() - ]) - - # -------------------------------------------------------------------------------------------------- - - class GenerateBClimatologyByLinking(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - swell_static_file_questions, - qd.background_error_model(), - qd.horizontal_resolution(), - qd.vertical_resolution(), - qd.window_length(), - qd.window_type() - ]) - - # -------------------------------------------------------------------------------------------------- - - class GenerateObservingSystemRecords(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.observations(), - qd.observing_system_records_mksi_path(), - qd.observing_system_records_path() - ]) - - # -------------------------------------------------------------------------------------------------- - - class LinkGeosOutput(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - window_questions, - qd.background_frequency(), - qd.marine_models() - ]) - - # -------------------------------------------------------------------------------------------------- - - class MoveDaRestart(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.mom6_iau(), - qd.window_length() - ]) - - # -------------------------------------------------------------------------------------------------- - - class MoveForecastRestart(Task): - def set_attributes(self): - self.is_cycling = True - self.question_list = QuestionList([ - qd.forecast_duration() - ]) - - # -------------------------------------------------------------------------------------------------- - - class PrepGeosRunDir(Task): - def set_attributes(self): - self.is_cycling = True - self.question_list = QuestionList([ - swell_static_file_questions, - qd.existing_geos_gcm_build_path(), - qd.forecast_duration(), - qd.geos_experiment_directory(), - qd.mom6_iau_nhours() - ]) - - # -------------------------------------------------------------------------------------------------- - - class PrepareAnalysis(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - qd.analysis_variables(), - qd.mom6_iau(), - qd.total_processors(), - qd.window_length(), - ]) - - # -------------------------------------------------------------------------------------------------- - - class RunJediFgatExecutable(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.question_list = QuestionList([ - run_jedi_executable, - qd.marine_models(), - qd.comparison_log_type('fgat'), - ]) - - # -------------------------------------------------------------------------------------------------- - - class RunJediEnsembleMeanVariance(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.question_list = QuestionList([ - np_proc_resolution, - window_questions, - qd.analysis_variables(), - qd.ensemble_num_members(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.observations(), - qd.observing_system_records_path(), - qd.comparison_log_type('ensmeanvariance'), - ]) - - # -------------------------------------------------------------------------------------------------- - - class RunJediHofxEnsembleExecutable(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.question_list = QuestionList([ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.ensemble_num_members(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.total_processors(), - qd.comparison_log_type('hofx'), - ]) - - # -------------------------------------------------------------------------------------------------- - - class RunJediHofxExecutable(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.question_list = QuestionList([ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.save_geovals(), - qd.total_processors(), - qd.comparison_log_type('ensemblehofx'), - ]) - - # -------------------------------------------------------------------------------------------------- - - class RunJediLocalEnsembleDaExecutable(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.question_list = QuestionList([ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.ensemble_num_members(), - qd.ensmean_only(), - qd.ensmeanvariance_only(), - qd.generate_yaml_and_exit(), - qd.horizontal_localization_lengthscale(), - qd.horizontal_localization_max_nobs(), - qd.horizontal_localization_method(), - qd.jedi_forecast_model(), - qd.local_ensemble_inflation_mult(), - qd.local_ensemble_inflation_rtpp(), - qd.local_ensemble_inflation_rtps(), - qd.local_ensemble_save_posterior_ensemble(), - qd.local_ensemble_save_posterior_ensemble_increments(), - qd.local_ensemble_save_posterior_mean(), - qd.local_ensemble_save_posterior_mean_increment(), - qd.local_ensemble_solver(), - qd.local_ensemble_use_linear_observer(), - qd.skip_ensemble_hofx(), - qd.total_processors(), - qd.vertical_localization_apply_log_transform(), - qd.vertical_localization_function(), - qd.vertical_localization_ioda_vertical_coord(), - qd.vertical_localization_ioda_vertical_coord_group(), - qd.vertical_localization_lengthscale(), - qd.vertical_localization_method(), - qd.perhost(), - qd.comparison_log_type('localensembleda'), - ]) - - # -------------------------------------------------------------------------------------------------- - - class RunJediVariationalExecutable(Task): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.slurm = {'nodes': 3} - self.question_list = QuestionList([ - run_jedi_executable, - qd.perhost(), - qd.comparison_log_type('variational'), - ]) - - # -------------------------------------------------------------------------------------------------- - - class RemoveForecastDir(Task): - def set_attributes(self): - self.is_cycling = True - - # -------------------------------------------------------------------------------------------------- - - class RunGeosExecutable(Task): - def set_attributes(self): - self.is_cycling = True - - # -------------------------------------------------------------------------------------------------- - - class RunJediUfoTestsExecutable(Task): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.slurm = {'ntasks-per-node': 1} - self.question_list = QuestionList([ - background_crtm_obs, - qd.generate_yaml_and_exit(), - qd.single_observations(), - qd.window_length(), - qd.comparison_log_type('ufo_tests'), - ]) - - # -------------------------------------------------------------------------------------------------- - - class RunJediConvertStateSoca2ciceExecutable(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {'nodes': 1} - self.question_list = QuestionList([ - qd.analysis_variables(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.marine_models(), - qd.observations(), - qd.total_processors(), - qd.window_length(), - qd.window_type(), - qd.comparison_log_type('convert_state_soca2cice'), - ]) - - # -------------------------------------------------------------------------------------------------- - - class RunJediFgatExecutable(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.question_list = QuestionList([ - run_jedi_executable, - qd.marine_models(), - qd.comparison_log_type('fgat'), - ]) - - # -------------------------------------------------------------------------------------------------- - - class SaveObsDiags(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - background_crtm_obs, - qd.r2d2_local_path(), - qd.window_length(), - qd.marine_models() - ]) - - # -------------------------------------------------------------------------------------------------- - - class SaveRestart(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.question_list = QuestionList([ - window_questions, - qd.background_time_offset(), - qd.forecast_duration(), - qd.horizontal_resolution(), - qd.marine_models(), - qd.r2d2_local_path() - ]) - - # -------------------------------------------------------------------------------------------------- - - class StageJedi(Task): - def set_attributes(self): - self.is_model = True - self.question_list = QuestionList([ - swell_static_file_questions, - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ]) - - # -------------------------------------------------------------------------------------------------- - - class StageJediCycle(Task): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.base_name = "StageJedi" - self.scheduling_name = "StageJediCycle-{model}" - self.question_list = QuestionList([ - swell_static_file_questions, - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ]) - - # -------------------------------------------------------------------------------------------------- - - class sync_point(Task): - def set_attributes(self): - self.script = "true" - - # -------------------------------------------------------------------------------------------------- - - class JediLogComparison(Task): - def set_attributes(self): - self.is_model = True - self.question_list = QuestionList([ - qd.number_of_iterations(), - qd.comparison_log_type(), - ]) - - # -------------------------------------------------------------------------------------------------- - - class RunJediObsfiltersExecutable(Task): - def set_attributes(self): - self.script = ("swell task RunJediObsfiltersExecutable $config" - " -d $datetime -m geos_atmosphere") - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.question_list = QuestionList([ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.observing_system_records_path(), - qd.total_processors(), - qd.obs_thinning_rej_fraction(), - qd.comparison_log_type('obsfilters') - ]) - - # -------------------------------------------------------------------------------------------------- - - @classmethod - def get(cls, name: str) -> Task: - return getattr(cls, name) - -# -------------------------------------------------------------------------------------------- diff --git a/src/swell/test/code_tests/slurm_test.py b/src/swell/test/code_tests/slurm_test.py index a39665684..bcad1bd58 100644 --- a/src/swell/test/code_tests/slurm_test.py +++ b/src/swell/test/code_tests/slurm_test.py @@ -11,7 +11,7 @@ from swell.utilities.slurm import prepare_slurm_defaults_and_overrides from swell.utilities.logger import get_logger -from swell.tasks.task_attributes import TaskAttributes +from swell.tasks.base.task_attributes import TaskAttributes from unittest.mock import patch, Mock # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 4a0d10fa2..84227722b 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -10,7 +10,7 @@ import yaml from typing import Callable -from swell.tasks.task_attributes import TaskAttributes as task_attributes +from swell.tasks.base.task_attributes import TaskAttributes as task_attributes from swell.utilities.logger import Logger from swell.suites.all_suites import suite_configs diff --git a/src/swell/utilities/task_specification.py b/src/swell/utilities/task_specification.py deleted file mode 100644 index ed2cad0cb..000000000 --- a/src/swell/utilities/task_specification.py +++ /dev/null @@ -1,261 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - -import os -import yaml -from typing import Union, Optional -from collections.abc import Mapping - -from swell.utilities.cylc_formatting import CylcSection, indent_lines -from swell.utilities.suite_utils import get_model_components -from swell.utilities.dictionary import update_dict -from swell.utilities.swell_questions import QuestionList -from swell.utilities.settings import read_settings - -# -------------------------------------------------------------------------------------------------- - - -class Task: - - ''' - Contains the basic properties and information needed to format the cylc [runtime] section. - ''' - - def __init__(self, model: Optional[str] = None, platform: Optional[str] = None) -> None: - - self.model = model - self.platform = platform - - self.base_name = None - self.scheduling_name = None - - self.is_cycling = False - self.is_model = False - - self.pre_script = False - self.script = None - - self.retry = None - self.time_limit = None - self.slurm = None - - self.subsections = [] - self.mail_events = ['failed', 'submit-failed'] - - self.question_list = QuestionList([]) - self.additional_sections = [] - - self.set_attributes() - self.post_init() - - # -------------------------------------------------------------------------------------------------- - - def set_attributes(self) -> None: - pass - - # -------------------------------------------------------------------------------------------------- - - def post_init(self): - - if self.base_name is None: - self.base_name = self.__class__.__name__ - - if self.scheduling_name is None: - self.scheduling_name = self.base_name - - if self.is_model and self.model is not None: - self.scheduling_name += f'-{self.model}' - - if self.script is None: - self.script = f'swell task {self.base_name} $config' - - if self.is_cycling: - self.script += ' -d $datetime' - - if self.is_model and self.model is not None: - self.script += ' -m {model}' - - if self.is_model and self.model is not None: - self.script = self.script.format(model=self.model) - self.scheduling_name = self.scheduling_name.format(model=self.model) - - # -------------------------------------------------------------------------------------------------- - - def format_string_block(self, string: str) -> str: - out_string = '"""\n' - out_string += indent_lines(string, 1) - out_string += '"""' - - return out_string - - # -------------------------------------------------------------------------------------------------- - - def match_platform(self, content: Union[str, dict], platform: str): - # Resolve platform-specific entries in the task object - - if isinstance(content, Mapping): - if platform in content.keys(): - content = content[platform] - elif 'all' in content.keys(): - content = content['all'] - - return content - - # -------------------------------------------------------------------------------------------------- - - def create_new_section(self, - name: Optional[str] = None, - content: Union[str, dict] = '' - ) -> CylcSection: - return CylcSection(name, content) - - # -------------------------------------------------------------------------------------------------- - - def resolve_model(self, slurm_dict: Mapping) -> dict: - ''' Resolve "all" and "model" entries in slurm dictionary ''' - if 'all' in slurm_dict.keys() and isinstance(slurm_dict['all'], Mapping): - slurm_dict = update_dict(slurm_dict, slurm_dict['all']) - del slurm_dict['all'] - if self.model in slurm_dict.keys() and isinstance(slurm_dict[self.model], Mapping): - slurm_dict = update_dict(slurm_dict, slurm_dict[self.model]) - - for model in get_model_components(): - if model in slurm_dict.keys(): - del slurm_dict[model] - - return slurm_dict - - # -------------------------------------------------------------------------------------------------- - - def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Mapping: - # Take the external slurm dictionary and merge it with the task's parameters - # to get the dict that will be output in the flow.cylc - - slurm_dict = {} - if self.slurm is not None: - for key, value in self.slurm.items(): - slurm_dict[key] = self.match_platform(value, platform) - - slurm_globals = slurm_external['slurm_directives_global'] - slurm_task = {} - - if 'slurm_directives_tasks' in slurm_external.keys(): - task_directives = slurm_external['slurm_directives_tasks'] - - if self.base_name in task_directives: - slurm_task = task_directives[self.base_name] - if self.scheduling_name in task_directives: - slurm_task = task_directives[self.scheduling_name] - - slurm_dict = {'job-name': self.scheduling_name, - **self.resolve_model(slurm_globals), - **self.resolve_model(slurm_dict), - **self.resolve_model(slurm_task)} - - return slurm_dict - - # -------------------------------------------------------------------------------------------------- - - def get_time_limit(self) -> str: - - # Set the time limit, default is 1 hour - if self.time_limit is True: - time_limit = 'PT1H' - elif self.time_limit: - time_limit = self.match_platform(self.time_limit, platform) - else: - time_limit = None - - return time_limit - - # -------------------------------------------------------------------------------------------------- - - def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): - ''' Return the runtime section for the given task. ''' - - platform = experiment_dict['platform'] - runtime_dict = {} - - # Set the pre_script only if it is specified - if self.pre_script: - runtime_dict['pre-script'] = self.format_string_block(self.pre_script) - - # Set the script - if self.script: - script_str = self.script - - if 'pause_on_tasks' in experiment_dict.keys(): - if len(set([self.base_name, self.scheduling_name]) - & set(experiment_dict['pause_on_tasks'])) > 0: - script_str += '\ncylc pause $CYLC_WORKFLOW_ID' - - runtime_dict['script'] = self.format_string_block(script_str) - - # Specify the platform if this is a slurm task - if self.slurm is not None: - runtime_dict['platform'] = platform - - time_limit = self.get_time_limit() - - if time_limit is not None: - runtime_dict['execution time limit'] = time_limit - - # Set the retry if this task needs it - if self.retry: - if self.retry is True: - retry = '2*PT1M' - else: - retry = self.match_platform(self.retry, platform) - - runtime_dict['execution retry delays'] = retry - - runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) - - # Specify the slurm dictionary with defaults from user and global settings - if self.slurm is not None: - - slurm_dict = self.generate_task_slurm_dict(slurm_external, platform) - - slurm_section_dict = {} - for key, value in slurm_dict.items(): - slurm_section_dict[f'--{key}'] = value - - directive_section = self.create_new_section('directives', slurm_section_dict) - - runtime_section.add_subsection(directive_section) - - # Append additional sections to runtime - for section in self.additional_sections: - runtime_section.add_subsection(section) - - # Check slurm messaging parameters - events = self.mail_events - - settings_dict = read_settings() - - # Add messaging section - if len(events) > 0 and 'email_address' in settings_dict.keys(): - email_address = settings_dict['email_address'] - address_section = self.create_new_section('mail', f'to = {email_address}') - runtime_section.add_subsection(address_section) - - event_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" - event_str += "mail events = " + ', '.join(events) - event_str += "\n{% endif %}\n" - - event_section = self.create_new_section('events', event_str) - runtime_section.add_subsection(event_section) - - runtime_string = runtime_section.get_section_str(1) - - runtime_string += ' # ' + '-' * 96 + '\n\n' - - return runtime_string - -# -------------------------------------------------------------------------------------------------- From 7332d637e3c9debb38f205f4517db6d2ce1e3a1a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 23 Dec 2025 10:32:53 -0500 Subject: [PATCH 144/299] Rename task_setup --- src/swell/tasks/base/task_attributes.py | 859 ++++++++++++++++++++++++ src/swell/tasks/base/task_setup.py | 261 +++++++ 2 files changed, 1120 insertions(+) create mode 100644 src/swell/tasks/base/task_attributes.py create mode 100644 src/swell/tasks/base/task_setup.py diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py new file mode 100644 index 000000000..446226d3a --- /dev/null +++ b/src/swell/tasks/base/task_attributes.py @@ -0,0 +1,859 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.swell_questions import QuestionList +from swell.utilities.question_defaults import QuestionDefaults as qd + +# -------------------------------------------------------------------------------------------------- + +background_crtm_obs = QuestionList( + list_name="background_crtm_obs", + questions=[ + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path() + ] +) + +# -------------------------------------------------------------------------------------------------- + +np_proc_resolution = QuestionList( + list_name="np_resolution", + questions=[ + qd.npx_proc(), + qd.npy_proc(), + qd.npx(), + qd.npy(), + qd.horizontal_resolution(), + qd.vertical_resolution() + ] +) + +# -------------------------------------------------------------------------------------------------- + +window_questions = QuestionList( + list_name="window_questions", + questions=[ + qd.window_length(), + qd.window_type() + ] +) + +# -------------------------------------------------------------------------------------------------- + +run_jedi_executable = QuestionList( + list_name="run_jedi_executable", + questions=[ + background_crtm_obs, + np_proc_resolution, + window_questions, + qd.analysis_variables(), + qd.background_frequency(), + qd.generate_yaml_and_exit(), + qd.gradient_norm_reduction(), + qd.gsibec_configuration(), + qd.jedi_forecast_model(), + qd.minimizer(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.number_of_iterations(), + qd.total_processors(), + ] +) + +# -------------------------------------------------------------------------------------------------- + +swell_static_file_questions = QuestionList( + list_name="swell_static_file_questions", + questions=[ + qd.swell_static_files(), + qd.swell_static_files_user() + ] +) + +# -------------------------------------------------------------------------------------------------- + + +class TaskAttributes(): + + # -------------------------------------------------------------------------------------------------- + + class root(TaskSetup): + + def set_attributes(self): + self.script = False + self.pre_script = "source $CYLC_SUITE_DEF_PATH/modules" + self.additional_sections = [self.create_new_section('environment', + {'datetime': '$CYLC_TASK_CYCLE_POINT', + 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'})] # noqa + + # -------------------------------------------------------------------------------------------------- + + class BuildGeos(TaskSetup): + def set_attributes(self): + self.question_list = QuestionList([ + qd.geos_build_method() + ]) + + # -------------------------------------------------------------------------------------------------- + + class BuildGeosByLinking(TaskSetup): + def set_attributes(self): + self.mail_events = ['submit-failed'] + self.question_list = QuestionList([ + qd.existing_geos_gcm_build_path(), + qd.geos_build_method() + ]) + + # -------------------------------------------------------------------------------------------------- + + class BuildJediByLinking(TaskSetup): + def set_attributes(self): + self.mail_events = ['submit-failed'] + self.question_list = QuestionList([ + qd.existing_jedi_build_directory(), + qd.existing_jedi_build_directory_pinned(), + qd.jedi_build_method() + ]) + + # -------------------------------------------------------------------------------------------------- + + class BuildJedi(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + qd.bundles(), + qd.jedi_build_method() + ]) + + # -------------------------------------------------------------------------------------------------- + + class CleanCycle(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.clean_patterns() + ]) + + # -------------------------------------------------------------------------------------------------- + + class CloneGeos(TaskSetup): + def set_attributes(self): + self.question_list = QuestionList([ + qd.existing_geos_gcm_source_path(), + qd.geos_build_method(), + qd.geos_gcm_tag() + ]) + + # -------------------------------------------------------------------------------------------------- + + class CloneJedi(TaskSetup): + def set_attributes(self): + self.question_list = QuestionList([ + qd.bundles(), + qd.existing_jedi_source_directory(), + qd.existing_jedi_source_directory_pinned(), + qd.jedi_build_method() + ]) + + # -------------------------------------------------------------------------------------------------- + + class CloneGeosMksi(TaskSetup): + def set_attributes(self): + self.is_model = True + self.question_list = QuestionList([ + qd.observing_system_records_mksi_path(), + qd.observing_system_records_mksi_path_tag() + ]) + + # -------------------------------------------------------------------------------------------------- + + class CloneGmaoPerllib(TaskSetup): + def set_attributes(self): + self.question_list = QuestionList([ + qd.existing_perllib_path(), + qd.gmao_perllib_tag() + ]) + + # -------------------------------------------------------------------------------------------------- + + class EvaJediLog(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + + # -------------------------------------------------------------------------------------------------- + + class EvaComparisonIncrement(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.marine_models(), + qd.window_length(), + qd.window_type() + ]) + + # -------------------------------------------------------------------------------------------------- + + class EvaComparisonJediLog(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + + # -------------------------------------------------------------------------------------------------- + + class EvaIncrement(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.marine_models(), + qd.window_length(), + qd.window_type() + ]) + + # -------------------------------------------------------------------------------------------------- + + class EvaObservations(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {} + self.question_list = QuestionList([ + background_crtm_obs, + qd.marine_models(), + qd.observing_system_records_path(), + qd.window_length(), + qd.marine_models(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class EvaTimeseries(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {} + self.question_list = QuestionList([ + background_crtm_obs, + qd.window_length(), + qd.ncdiag_experiments(), + qd.marine_models(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class JediOopsLogParser(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.parser_options(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetBackground(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + window_questions, + qd.window_length(), + qd.background_experiment(), + qd.background_frequency(), + qd.horizontal_resolution(), + qd.marine_models(), + qd.r2d2_local_path(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetBackgroundGeosExperiment(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.mail_events = ['submit-failed'] + self.question_list = QuestionList([ + qd.horizontal_resolution(), + qd.background_experiment(), + qd.background_time_offset(), + qd.geos_x_background_directory() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetBufr(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.bufr_obs_classes() + ]) + + # -------------------------------------------------------------------------------------------------- + + class BufrToIoda(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + + # -------------------------------------------------------------------------------------------------- + + class GetEnsembleGeosExperiment(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.background_experiment(), + qd.background_time_offset(), + qd.geos_x_ensemble_directory() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetGeosRestart(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.question_list = QuestionList([ + swell_static_file_questions, + qd.geos_restarts_directory() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetGeovals(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + background_crtm_obs, + qd.geovals_experiment(), + qd.geovals_provider(), + qd.r2d2_local_path(), + qd.window_length(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetGsiBc(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.path_to_gsi_bc_coefficients(), + qd.window_length() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GsiBcToIoda(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + background_crtm_obs, + qd.observing_system_records_path(), + qd.window_length() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetGsiNcdiag(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.path_to_gsi_nc_diags() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GsiNcdiagToIoda(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.observations(), + qd.produce_geovals(), + qd.single_observations(), + qd.window_length() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetNcdiags(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + background_crtm_obs, + qd.ncdiag_experiments(), + qd.marine_models(), + qd.r2d2_local_path(), + qd.window_length(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetGeosAdasBackground(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.path_to_geos_adas_background() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetObservations(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + background_crtm_obs, + qd.cycling_varbc(), + qd.obs_experiment(), + qd.observing_system_records_path(), + qd.r2d2_local_path(), + qd.window_length(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class GetObsNotInR2d2(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.mail_events = ['submit-failed'] + self.question_list = QuestionList([ + qd.ioda_locations_not_in_r2d2(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class GenerateBClimatology(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.retry = '2*PT1M' + self.slurm = {} + self.question_list = QuestionList([ + np_proc_resolution, + swell_static_file_questions, + qd.analysis_variables(), + qd.background_error_model(), + qd.generate_yaml_and_exit(), + qd.gradient_norm_reduction(), + qd.gsibec_configuration(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.jedi_forecast_model(), + qd.marine_models(), + qd.minimizer(), + qd.number_of_iterations(), + qd.observing_system_records_path(), + qd.total_processors(), + qd.window_length(), + qd.window_type() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GenerateBClimatologyByLinking(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + swell_static_file_questions, + qd.background_error_model(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.window_length(), + qd.window_type() + ]) + + # -------------------------------------------------------------------------------------------------- + + class GenerateObservingSystemRecords(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.observations(), + qd.observing_system_records_mksi_path(), + qd.observing_system_records_path() + ]) + + # -------------------------------------------------------------------------------------------------- + + class LinkGeosOutput(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + window_questions, + qd.background_frequency(), + qd.marine_models() + ]) + + # -------------------------------------------------------------------------------------------------- + + class MoveDaRestart(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.mom6_iau(), + qd.window_length() + ]) + + # -------------------------------------------------------------------------------------------------- + + class MoveForecastRestart(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.question_list = QuestionList([ + qd.forecast_duration() + ]) + + # -------------------------------------------------------------------------------------------------- + + class PrepGeosRunDir(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.question_list = QuestionList([ + swell_static_file_questions, + qd.existing_geos_gcm_build_path(), + qd.forecast_duration(), + qd.geos_experiment_directory(), + qd.mom6_iau_nhours() + ]) + + # -------------------------------------------------------------------------------------------------- + + class PrepareAnalysis(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + qd.analysis_variables(), + qd.mom6_iau(), + qd.total_processors(), + qd.window_length(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediFgatExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + run_jedi_executable, + qd.marine_models(), + qd.comparison_log_type('fgat'), + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediEnsembleMeanVariance(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + np_proc_resolution, + window_questions, + qd.analysis_variables(), + qd.ensemble_num_members(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.observations(), + qd.observing_system_records_path(), + qd.comparison_log_type('ensmeanvariance'), + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediHofxEnsembleExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.background_frequency(), + qd.ensemble_hofx_packets(), + qd.ensemble_hofx_strategy(), + qd.ensemble_num_members(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.total_processors(), + qd.comparison_log_type('hofx'), + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediHofxExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.background_frequency(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.save_geovals(), + qd.total_processors(), + qd.comparison_log_type('ensemblehofx'), + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediLocalEnsembleDaExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.ensemble_hofx_packets(), + qd.ensemble_hofx_strategy(), + qd.ensemble_num_members(), + qd.ensmean_only(), + qd.ensmeanvariance_only(), + qd.generate_yaml_and_exit(), + qd.horizontal_localization_lengthscale(), + qd.horizontal_localization_max_nobs(), + qd.horizontal_localization_method(), + qd.jedi_forecast_model(), + qd.local_ensemble_inflation_mult(), + qd.local_ensemble_inflation_rtpp(), + qd.local_ensemble_inflation_rtps(), + qd.local_ensemble_save_posterior_ensemble(), + qd.local_ensemble_save_posterior_ensemble_increments(), + qd.local_ensemble_save_posterior_mean(), + qd.local_ensemble_save_posterior_mean_increment(), + qd.local_ensemble_solver(), + qd.local_ensemble_use_linear_observer(), + qd.skip_ensemble_hofx(), + qd.total_processors(), + qd.vertical_localization_apply_log_transform(), + qd.vertical_localization_function(), + qd.vertical_localization_ioda_vertical_coord(), + qd.vertical_localization_ioda_vertical_coord_group(), + qd.vertical_localization_lengthscale(), + qd.vertical_localization_method(), + qd.perhost(), + qd.comparison_log_type('localensembleda'), + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediVariationalExecutable(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {'nodes': 3} + self.question_list = QuestionList([ + run_jedi_executable, + qd.perhost(), + qd.comparison_log_type('variational'), + ]) + + # -------------------------------------------------------------------------------------------------- + + class RemoveForecastDir(TaskSetup): + def set_attributes(self): + self.is_cycling = True + + # -------------------------------------------------------------------------------------------------- + + class RunGeosExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + + # -------------------------------------------------------------------------------------------------- + + class RunJediUfoTestsExecutable(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {'ntasks-per-node': 1} + self.question_list = QuestionList([ + background_crtm_obs, + qd.generate_yaml_and_exit(), + qd.single_observations(), + qd.window_length(), + qd.comparison_log_type('ufo_tests'), + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediConvertStateSoca2ciceExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {'nodes': 1} + self.question_list = QuestionList([ + qd.analysis_variables(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.marine_models(), + qd.observations(), + qd.total_processors(), + qd.window_length(), + qd.window_type(), + qd.comparison_log_type('convert_state_soca2cice'), + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediFgatExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + run_jedi_executable, + qd.marine_models(), + qd.comparison_log_type('fgat'), + ]) + + # -------------------------------------------------------------------------------------------------- + + class SaveObsDiags(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + background_crtm_obs, + qd.r2d2_local_path(), + qd.window_length(), + qd.marine_models() + ]) + + # -------------------------------------------------------------------------------------------------- + + class SaveRestart(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.question_list = QuestionList([ + window_questions, + qd.background_time_offset(), + qd.forecast_duration(), + qd.horizontal_resolution(), + qd.marine_models(), + qd.r2d2_local_path() + ]) + + # -------------------------------------------------------------------------------------------------- + + class StageJedi(TaskSetup): + def set_attributes(self): + self.is_model = True + self.question_list = QuestionList([ + swell_static_file_questions, + qd.gsibec_configuration(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.horizontal_resolution(), + qd.vertical_resolution() + ]) + + # -------------------------------------------------------------------------------------------------- + + class StageJediCycle(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.base_name = "StageJedi" + self.scheduling_name = "StageJediCycle-{model}" + self.question_list = QuestionList([ + swell_static_file_questions, + qd.gsibec_configuration(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.horizontal_resolution(), + qd.vertical_resolution() + ]) + + # -------------------------------------------------------------------------------------------------- + + class sync_point(TaskSetup): + def set_attributes(self): + self.script = "true" + + # -------------------------------------------------------------------------------------------------- + + class JediLogComparison(TaskSetup): + def set_attributes(self): + self.is_model = True + self.question_list = QuestionList([ + qd.number_of_iterations(), + qd.comparison_log_type(), + ]) + + # -------------------------------------------------------------------------------------------------- + + class RunJediObsfiltersExecutable(TaskSetup): + def set_attributes(self): + self.script = ("swell task RunJediObsfiltersExecutable $config" + " -d $datetime -m geos_atmosphere") + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.question_list = QuestionList([ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.background_frequency(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.observing_system_records_path(), + qd.total_processors(), + qd.obs_thinning_rej_fraction(), + qd.comparison_log_type('obsfilters') + ]) + + # -------------------------------------------------------------------------------------------------- + + @classmethod + def get(cls, name: str) -> Task: + return getattr(cls, name) + +# -------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py new file mode 100644 index 000000000..4233ffb78 --- /dev/null +++ b/src/swell/tasks/base/task_setup.py @@ -0,0 +1,261 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +import yaml +from typing import Union, Optional +from collections.abc import Mapping + +from swell.utilities.cylc_formatting import CylcSection, indent_lines +from swell.utilities.suite_utils import get_model_components +from swell.utilities.dictionary import update_dict +from swell.utilities.swell_questions import QuestionList +from swell.utilities.settings import read_settings + +# -------------------------------------------------------------------------------------------------- + + +class TaskSetup: + + ''' + Contains the basic properties and information needed to format the cylc [runtime] section. + ''' + + def __init__(self, model: Optional[str] = None, platform: Optional[str] = None) -> None: + + self.model = model + self.platform = platform + + self.base_name = None + self.scheduling_name = None + + self.is_cycling = False + self.is_model = False + + self.pre_script = False + self.script = None + + self.retry = None + self.time_limit = None + self.slurm = None + + self.subsections = [] + self.mail_events = ['failed', 'submit-failed'] + + self.question_list = QuestionList([]) + self.additional_sections = [] + + self.set_attributes() + self.post_init() + + # -------------------------------------------------------------------------------------------------- + + def set_attributes(self) -> None: + pass + + # -------------------------------------------------------------------------------------------------- + + def post_init(self): + + if self.base_name is None: + self.base_name = self.__class__.__name__ + + if self.scheduling_name is None: + self.scheduling_name = self.base_name + + if self.is_model and self.model is not None: + self.scheduling_name += f'-{self.model}' + + if self.script is None: + self.script = f'swell task {self.base_name} $config' + + if self.is_cycling: + self.script += ' -d $datetime' + + if self.is_model and self.model is not None: + self.script += ' -m {model}' + + if self.is_model and self.model is not None: + self.script = self.script.format(model=self.model) + self.scheduling_name = self.scheduling_name.format(model=self.model) + + # -------------------------------------------------------------------------------------------------- + + def format_string_block(self, string: str) -> str: + out_string = '"""\n' + out_string += indent_lines(string, 1) + out_string += '"""' + + return out_string + + # -------------------------------------------------------------------------------------------------- + + def match_platform(self, content: Union[str, dict], platform: str): + # Resolve platform-specific entries in the task object + + if isinstance(content, Mapping): + if platform in content.keys(): + content = content[platform] + elif 'all' in content.keys(): + content = content['all'] + + return content + + # -------------------------------------------------------------------------------------------------- + + def create_new_section(self, + name: Optional[str] = None, + content: Union[str, dict] = '' + ) -> CylcSection: + return CylcSection(name, content) + + # -------------------------------------------------------------------------------------------------- + + def resolve_model(self, slurm_dict: Mapping) -> dict: + ''' Resolve "all" and "model" entries in slurm dictionary ''' + if 'all' in slurm_dict.keys() and isinstance(slurm_dict['all'], Mapping): + slurm_dict = update_dict(slurm_dict, slurm_dict['all']) + del slurm_dict['all'] + if self.model in slurm_dict.keys() and isinstance(slurm_dict[self.model], Mapping): + slurm_dict = update_dict(slurm_dict, slurm_dict[self.model]) + + for model in get_model_components(): + if model in slurm_dict.keys(): + del slurm_dict[model] + + return slurm_dict + + # -------------------------------------------------------------------------------------------------- + + def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Mapping: + # Take the external slurm dictionary and merge it with the task's parameters + # to get the dict that will be output in the flow.cylc + + slurm_dict = {} + if self.slurm is not None: + for key, value in self.slurm.items(): + slurm_dict[key] = self.match_platform(value, platform) + + slurm_globals = slurm_external['slurm_directives_global'] + slurm_task = {} + + if 'slurm_directives_tasks' in slurm_external.keys(): + task_directives = slurm_external['slurm_directives_tasks'] + + if self.base_name in task_directives: + slurm_task = task_directives[self.base_name] + if self.scheduling_name in task_directives: + slurm_task = task_directives[self.scheduling_name] + + slurm_dict = {'job-name': self.scheduling_name, + **self.resolve_model(slurm_globals), + **self.resolve_model(slurm_dict), + **self.resolve_model(slurm_task)} + + return slurm_dict + + # -------------------------------------------------------------------------------------------------- + + def get_time_limit(self) -> str: + + # Set the time limit, default is 1 hour + if self.time_limit is True: + time_limit = 'PT1H' + elif self.time_limit: + time_limit = self.match_platform(self.time_limit, platform) + else: + time_limit = None + + return time_limit + + # -------------------------------------------------------------------------------------------------- + + def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): + ''' Return the runtime section for the given task. ''' + + platform = experiment_dict['platform'] + runtime_dict = {} + + # Set the pre_script only if it is specified + if self.pre_script: + runtime_dict['pre-script'] = self.format_string_block(self.pre_script) + + # Set the script + if self.script: + script_str = self.script + + if 'pause_on_tasks' in experiment_dict.keys(): + if len(set([self.base_name, self.scheduling_name]) + & set(experiment_dict['pause_on_tasks'])) > 0: + script_str += '\ncylc pause $CYLC_WORKFLOW_ID' + + runtime_dict['script'] = self.format_string_block(script_str) + + # Specify the platform if this is a slurm task + if self.slurm is not None: + runtime_dict['platform'] = platform + + time_limit = self.get_time_limit() + + if time_limit is not None: + runtime_dict['execution time limit'] = time_limit + + # Set the retry if this task needs it + if self.retry: + if self.retry is True: + retry = '2*PT1M' + else: + retry = self.match_platform(self.retry, platform) + + runtime_dict['execution retry delays'] = retry + + runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) + + # Specify the slurm dictionary with defaults from user and global settings + if self.slurm is not None: + + slurm_dict = self.generate_task_slurm_dict(slurm_external, platform) + + slurm_section_dict = {} + for key, value in slurm_dict.items(): + slurm_section_dict[f'--{key}'] = value + + directive_section = self.create_new_section('directives', slurm_section_dict) + + runtime_section.add_subsection(directive_section) + + # Append additional sections to runtime + for section in self.additional_sections: + runtime_section.add_subsection(section) + + # Check slurm messaging parameters + events = self.mail_events + + settings_dict = read_settings() + + # Add messaging section + if len(events) > 0 and 'email_address' in settings_dict.keys(): + email_address = settings_dict['email_address'] + address_section = self.create_new_section('mail', f'to = {email_address}') + runtime_section.add_subsection(address_section) + + event_str = "{% if environ['SWELL_SEND_MESSAGES'] %}\n" + event_str += "mail events = " + ', '.join(events) + event_str += "\n{% endif %}\n" + + event_section = self.create_new_section('events', event_str) + runtime_section.add_subsection(event_section) + + runtime_string = runtime_section.get_section_str(1) + + runtime_string += ' # ' + '-' * 96 + '\n\n' + + return runtime_string + +# -------------------------------------------------------------------------------------------------- From ec177742058f671b4bcb469a046a68301fcbe29e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 23 Dec 2025 10:36:26 -0500 Subject: [PATCH 145/299] Fix --- src/swell/tasks/base/task_attributes.py | 2 +- src/swell/tasks/base/task_setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index 446226d3a..7cf8cd187 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -853,7 +853,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- @classmethod - def get(cls, name: str) -> Task: + def get(cls, name: str) -> TaskSetup: return getattr(cls, name) # -------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index 4233ffb78..b1808a3f4 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -162,7 +162,7 @@ def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Ma # -------------------------------------------------------------------------------------------------- - def get_time_limit(self) -> str: + def get_time_limit(self, platform) -> str: # Set the time limit, default is 1 hour if self.time_limit is True: @@ -201,7 +201,7 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): if self.slurm is not None: runtime_dict['platform'] = platform - time_limit = self.get_time_limit() + time_limit = self.get_time_limit(platform) if time_limit is not None: runtime_dict['execution time limit'] = time_limit From 9c8dc2be0307efcdb7d1cf78134664a6dc92307c Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 23 Dec 2025 16:47:07 -0500 Subject: [PATCH 146/299] Address comments --- src/swell/deployment/create_experiment.py | 2 +- .../prepare_config_and_suite.py | 4 - src/swell/tasks/base/task_setup.py | 82 +++++++++++-------- 3 files changed, 47 insertions(+), 41 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 4ba5d209f..6054ca578 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -94,7 +94,7 @@ def prepare_config( # Retrieved the answered suite questions # -------------------------------------- - suite_dict = prepare_config_and_suite.get_experiment_dict() + suite_dict = prepare_config_and_suite.experiment_dict suite_dict = suite_dict.copy() diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 7c6c2b58f..c4ced5475 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -119,10 +119,6 @@ def configure_and_ask_task_questions(self) -> None: return self.experiment_dict, self.comment_dict - # ---------------------------------------------------------------------------------------------- - - def get_experiment_dict(self) -> Mapping: - return self.experiment_dict # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index b1808a3f4..4f3fd45c4 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -9,8 +9,8 @@ import os import yaml -from typing import Union, Optional from collections.abc import Mapping +from abc import abstractmethod, ABC from swell.utilities.cylc_formatting import CylcSection, indent_lines from swell.utilities.suite_utils import get_model_components @@ -21,13 +21,30 @@ # -------------------------------------------------------------------------------------------------- -class TaskSetup: +class TaskSetup(ABC): ''' Contains the basic properties and information needed to format the cylc [runtime] section. + + Attributes: + model: model the task is being run under at runtime + platform: platform the task is being run on + + base_name: basic name of the task within Swell + scheduling_name: name for the task within cylc, typically model type is appended to the base name + is_cycling: boolean for whether the task is run on cycles + is_model: boolean for whether the task is run on a certain model + pre_script: cylc setting for scripts run before the main script + script: string of shell code to be run by cylc for the task + retry: times * time interval cylc should retry the task, e.g. 2*PT10s + time_limit: execution time limit for slurm + slurm: dictionary of slurm parameters + mail events: list of events for email messaging through cylc + question_list: list of questions keys used by the task + additional_sections: list of additional CylcSection objects to append to the runtime section of the task ''' - def __init__(self, model: Optional[str] = None, platform: Optional[str] = None) -> None: + def __init__(self, model: str | None = None, platform: str | None = None) -> None: self.model = model self.platform = platform @@ -45,7 +62,6 @@ def __init__(self, model: Optional[str] = None, platform: Optional[str] = None) self.time_limit = None self.slurm = None - self.subsections = [] self.mail_events = ['failed', 'submit-failed'] self.question_list = QuestionList([]) @@ -56,7 +72,10 @@ def __init__(self, model: Optional[str] = None, platform: Optional[str] = None) # -------------------------------------------------------------------------------------------------- + @abstractmethod def set_attributes(self) -> None: + '''Abstract method to be overridden by each task in order to set attributes. + ''' pass # -------------------------------------------------------------------------------------------------- @@ -85,6 +104,18 @@ def post_init(self): self.script = self.script.format(model=self.model) self.scheduling_name = self.scheduling_name.format(model=self.model) + # Set retry defaults + if self.retry is True: + self.retry = '2*PT1M' + else: + self.retry = self.match_platform(self.retry) + + # Set time limit defaults + if self.time_limit is True: + self.time_limit = 'PT1H' + elif self.time_limit: + self.time_limit = self.match_platform(self.time_limit) + # -------------------------------------------------------------------------------------------------- def format_string_block(self, string: str) -> str: @@ -96,12 +127,12 @@ def format_string_block(self, string: str) -> str: # -------------------------------------------------------------------------------------------------- - def match_platform(self, content: Union[str, dict], platform: str): + def match_platform(self, content: str | dict): # Resolve platform-specific entries in the task object if isinstance(content, Mapping): - if platform in content.keys(): - content = content[platform] + if self.platform in content.keys(): + content = content[self.platform] elif 'all' in content.keys(): content = content['all'] @@ -110,8 +141,8 @@ def match_platform(self, content: Union[str, dict], platform: str): # -------------------------------------------------------------------------------------------------- def create_new_section(self, - name: Optional[str] = None, - content: Union[str, dict] = '' + name: str | None = None, + content: str | dict = '' ) -> CylcSection: return CylcSection(name, content) @@ -140,7 +171,7 @@ def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Ma slurm_dict = {} if self.slurm is not None: for key, value in self.slurm.items(): - slurm_dict[key] = self.match_platform(value, platform) + slurm_dict[key] = self.match_platform(value) slurm_globals = slurm_external['slurm_directives_global'] slurm_task = {} @@ -162,21 +193,7 @@ def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Ma # -------------------------------------------------------------------------------------------------- - def get_time_limit(self, platform) -> str: - - # Set the time limit, default is 1 hour - if self.time_limit is True: - time_limit = 'PT1H' - elif self.time_limit: - time_limit = self.match_platform(self.time_limit, platform) - else: - time_limit = None - - return time_limit - - # -------------------------------------------------------------------------------------------------- - - def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): + def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping) -> str: ''' Return the runtime section for the given task. ''' platform = experiment_dict['platform'] @@ -201,19 +218,12 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): if self.slurm is not None: runtime_dict['platform'] = platform - time_limit = self.get_time_limit(platform) - - if time_limit is not None: - runtime_dict['execution time limit'] = time_limit + if self.time_limit is not None: + runtime_dict['execution time limit'] = self.time_limit # Set the retry if this task needs it if self.retry: - if self.retry is True: - retry = '2*PT1M' - else: - retry = self.match_platform(self.retry, platform) - - runtime_dict['execution retry delays'] = retry + runtime_dict['execution retry delays'] = self.retry runtime_section = self.create_new_section(self.scheduling_name, runtime_dict) @@ -252,7 +262,7 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping): event_section = self.create_new_section('events', event_str) runtime_section.add_subsection(event_section) - runtime_string = runtime_section.get_section_str(1) + runtime_string = runtime_section.get_section_str(level=1) runtime_string += ' # ' + '-' * 96 + '\n\n' From 1f897cfaf9f2061038f01c9138ef255992cbd126 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 23 Dec 2025 16:57:39 -0500 Subject: [PATCH 147/299] code style fixes --- .../prepare_config_and_suite/prepare_config_and_suite.py | 1 - src/swell/tasks/base/task_setup.py | 8 +++----- src/swell/utilities/settings.py | 5 +++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index c4ced5475..66c9c7ece 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -119,7 +119,6 @@ def configure_and_ask_task_questions(self) -> None: return self.experiment_dict, self.comment_dict - # ---------------------------------------------------------------------------------------------- def prepare_suite_question_dictionary(self) -> None: diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index 4f3fd45c4..2cd83f12a 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -7,8 +7,6 @@ # -------------------------------------------------------------------------------------------------- -import os -import yaml from collections.abc import Mapping from abc import abstractmethod, ABC @@ -28,10 +26,10 @@ class TaskSetup(ABC): Attributes: model: model the task is being run under at runtime - platform: platform the task is being run on + platform: platform the task is being run on base_name: basic name of the task within Swell - scheduling_name: name for the task within cylc, typically model type is appended to the base name + scheduling_name: name for the task within cylc is_cycling: boolean for whether the task is run on cycles is_model: boolean for whether the task is run on a certain model pre_script: cylc setting for scripts run before the main script @@ -41,7 +39,7 @@ class TaskSetup(ABC): slurm: dictionary of slurm parameters mail events: list of events for email messaging through cylc question_list: list of questions keys used by the task - additional_sections: list of additional CylcSection objects to append to the runtime section of the task + additional_sections: list of additional CylcSection objects to append to the runtime section ''' def __init__(self, model: str | None = None, platform: str | None = None) -> None: diff --git a/src/swell/utilities/settings.py b/src/swell/utilities/settings.py index 3ffb85a55..9f595e197 100644 --- a/src/swell/utilities/settings.py +++ b/src/swell/utilities/settings.py @@ -12,6 +12,7 @@ # -------------------------------------------------------------------------------------------------- + def read_settings() -> dict: ''' Reads user settings from yaml file under ~/.swell/swell-settings.yaml @@ -27,10 +28,10 @@ def read_settings() -> dict: if os.path.exists(settings_file): with open(settings_file, 'r') as f: settings_dict = yaml.safe_load(f) - + else: settings_dict = {} return settings_dict -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- From a136729420b63b885622e4b1fd35c21794bf2cb6 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 29 Dec 2025 10:18:42 -0500 Subject: [PATCH 148/299] Add comments and refactoring changes --- src/swell/deployment/create_experiment.py | 2 +- .../prepare_config_and_suite.py | 2 +- src/swell/suites/3dfgat_atmos/suite_config.py | 2 +- src/swell/suites/3dfgat_atmos/workflow.py | 2 +- src/swell/suites/3dfgat_cycle/suite_config.py | 2 +- src/swell/suites/3dfgat_cycle/workflow.py | 2 +- src/swell/suites/3dvar/suite_config.py | 2 +- src/swell/suites/3dvar/workflow.py | 2 +- src/swell/suites/3dvar_atmos/suite_config.py | 2 +- src/swell/suites/3dvar_atmos/workflow.py | 2 +- src/swell/suites/3dvar_cycle/suite_config.py | 2 +- src/swell/suites/3dvar_cycle/workflow.py | 2 +- src/swell/suites/{ => base}/all_suites.py | 4 +- .../base}/cylc_workflow.py | 0 .../suites/{ => base}/suite_questions.py | 0 src/swell/suites/build_geos/suite_config.py | 2 +- src/swell/suites/build_geos/workflow.py | 2 +- src/swell/suites/build_jedi/suite_config.py | 2 +- src/swell/suites/build_jedi/workflow.py | 2 +- src/swell/suites/compare/suite_config.py | 2 +- src/swell/suites/compare/workflow.py | 2 +- src/swell/suites/convert_bufr/suite_config.py | 2 +- src/swell/suites/convert_bufr/workflow.py | 2 +- .../suites/convert_ncdiags/suite_config.py | 2 +- src/swell/suites/convert_ncdiags/workflow.py | 2 +- .../suites/eva_capabilities/suite_config.py | 2 +- src/swell/suites/eva_capabilities/workflow.py | 2 +- .../suites/forecast_geos/suite_config.py | 2 +- src/swell/suites/forecast_geos/workflow.py | 2 +- src/swell/suites/geosadas/suite_config.py | 2 +- src/swell/suites/geosadas/workflow.py | 2 +- src/swell/suites/hofx/suite_config.py | 2 +- src/swell/suites/hofx/workflow.py | 2 +- .../suites/localensembleda/suite_config.py | 2 +- src/swell/suites/localensembleda/workflow.py | 2 +- src/swell/suites/ufo_testing/suite_config.py | 2 +- src/swell/suites/ufo_testing/workflow.py | 2 +- src/swell/swell.py | 2 +- src/swell/tasks/base/task_setup.py | 72 +++++++++++++++++-- src/swell/utilities/config.py | 2 +- 40 files changed, 104 insertions(+), 44 deletions(-) rename src/swell/suites/{ => base}/all_suites.py (97%) rename src/swell/{utilities => suites/base}/cylc_workflow.py (100%) rename src/swell/suites/{ => base}/suite_questions.py (100%) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 6054ca578..8965b0156 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -22,7 +22,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.logger import Logger, get_logger from swell.utilities.slurm import prepare_slurm_defaults_and_overrides -from swell.suites.all_suites import suite_configs, workflows +from swell.suites.base.all_suites import suite_configs, workflows from swell.utilities.check_da_params import check_da_params diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 66c9c7ece..9ca24477b 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -21,7 +21,7 @@ from swell.utilities.dictionary import dict_get from swell.utilities.logger import Logger from swell.utilities.dictionary import update_dict, add_dict -from swell.suites.all_suites import suite_configs +from swell.suites.base.all_suites import suite_configs from swell.utilities.swell_questions import QuestionType diff --git a/src/swell/suites/3dfgat_atmos/suite_config.py b/src/swell/suites/3dfgat_atmos/suite_config.py index b61ee9c9e..74d195eed 100644 --- a/src/swell/suites/3dfgat_atmos/suite_config.py +++ b/src/swell/suites/3dfgat_atmos/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 97e22831d..4f539e32b 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/suite_config.py b/src/swell/suites/3dfgat_cycle/suite_config.py index 97bc057ee..28c3d9844 100644 --- a/src/swell/suites/3dfgat_cycle/suite_config.py +++ b/src/swell/suites/3dfgat_cycle/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index 043ec3eb6..3960a43f4 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/suite_config.py b/src/swell/suites/3dvar/suite_config.py index 916107429..df33a9ec5 100644 --- a/src/swell/suites/3dvar/suite_config.py +++ b/src/swell/suites/3dvar/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 6ecb10b16..3929f7ad7 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/suite_config.py b/src/swell/suites/3dvar_atmos/suite_config.py index ec98f11c3..711f225b8 100644 --- a/src/swell/suites/3dvar_atmos/suite_config.py +++ b/src/swell/suites/3dvar_atmos/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index 02f3da14c..a05b1e6cb 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/suite_config.py b/src/swell/suites/3dvar_cycle/suite_config.py index 38377d1dd..49ef97202 100644 --- a/src/swell/suites/3dvar_cycle/suite_config.py +++ b/src/swell/suites/3dvar_cycle/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index aaa81aa59..1b5f3016b 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/all_suites.py b/src/swell/suites/base/all_suites.py similarity index 97% rename from src/swell/suites/all_suites.py rename to src/swell/suites/base/all_suites.py index fb1de73e5..a0ead6f54 100644 --- a/src/swell/suites/all_suites.py +++ b/src/swell/suites/base/all_suites.py @@ -13,8 +13,8 @@ from swell.swell_path import get_swell_path from swell.utilities.suite_utils import get_suites -from swell.suites.suite_questions import SuiteQuestions -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.suite_questions import SuiteQuestions +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.utilities.swell_questions import QuestionList # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/cylc_workflow.py b/src/swell/suites/base/cylc_workflow.py similarity index 100% rename from src/swell/utilities/cylc_workflow.py rename to src/swell/suites/base/cylc_workflow.py diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/base/suite_questions.py similarity index 100% rename from src/swell/suites/suite_questions.py rename to src/swell/suites/base/suite_questions.py diff --git a/src/swell/suites/build_geos/suite_config.py b/src/swell/suites/build_geos/suite_config.py index a61136668..4327d1eb0 100644 --- a/src/swell/suites/build_geos/suite_config.py +++ b/src/swell/suites/build_geos/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py index 3cd046576..f795e2b1d 100644 --- a/src/swell/suites/build_geos/workflow.py +++ b/src/swell/suites/build_geos/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/suite_config.py b/src/swell/suites/build_jedi/suite_config.py index 4da92ab6b..65ae82a9e 100644 --- a/src/swell/suites/build_jedi/suite_config.py +++ b/src/swell/suites/build_jedi/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index 9673f5eaf..cf212f783 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare/suite_config.py b/src/swell/suites/compare/suite_config.py index 9f8c4807e..a41df7f40 100644 --- a/src/swell/suites/compare/suite_config.py +++ b/src/swell/suites/compare/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList, WidgetType from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/compare/workflow.py b/src/swell/suites/compare/workflow.py index ce47e7f84..f403420b8 100644 --- a/src/swell/suites/compare/workflow.py +++ b/src/swell/suites/compare/workflow.py @@ -10,7 +10,7 @@ import yaml from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_bufr/suite_config.py b/src/swell/suites/convert_bufr/suite_config.py index ce1e0f4cc..c74e0b701 100644 --- a/src/swell/suites/convert_bufr/suite_config.py +++ b/src/swell/suites/convert_bufr/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/convert_bufr/workflow.py b/src/swell/suites/convert_bufr/workflow.py index dc9525494..db6715944 100644 --- a/src/swell/suites/convert_bufr/workflow.py +++ b/src/swell/suites/convert_bufr/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/suite_config.py b/src/swell/suites/convert_ncdiags/suite_config.py index c8669546c..a758b4faf 100644 --- a/src/swell/suites/convert_ncdiags/suite_config.py +++ b/src/swell/suites/convert_ncdiags/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index 9d67a36ae..b975437ce 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eva_capabilities/suite_config.py b/src/swell/suites/eva_capabilities/suite_config.py index a5dc9af50..edd7b7369 100644 --- a/src/swell/suites/eva_capabilities/suite_config.py +++ b/src/swell/suites/eva_capabilities/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index 8efc9c062..4d1e7c5ee 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/suite_config.py b/src/swell/suites/forecast_geos/suite_config.py index 4fb2db306..246550c07 100644 --- a/src/swell/suites/forecast_geos/suite_config.py +++ b/src/swell/suites/forecast_geos/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index e5288b812..9894a363c 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/suite_config.py b/src/swell/suites/geosadas/suite_config.py index ebe2710b6..daf3a6bc8 100644 --- a/src/swell/suites/geosadas/suite_config.py +++ b/src/swell/suites/geosadas/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index 31ca3c2c9..a4c220e67 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/suite_config.py b/src/swell/suites/hofx/suite_config.py index 2448ab528..46d486482 100644 --- a/src/swell/suites/hofx/suite_config.py +++ b/src/swell/suites/hofx/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index 596f317e1..1969453cd 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index e395d740c..fc881d1d9 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 12ef77d82..a3bf23184 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/suite_config.py b/src/swell/suites/ufo_testing/suite_config.py index ab10cc015..fd04b66a9 100644 --- a/src/swell/suites/ufo_testing/suite_config.py +++ b/src/swell/suites/ufo_testing/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index 339fbe08c..0de652bfc 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.cylc_workflow import CylcWorkflow +from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import TaskAttributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index e29819ed7..376dd43c8 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import task_wrapper, get_tasks from swell.test.test_driver import test_wrapper, valid_tests from swell.test.suite_tests.suite_tests import run_suite, TestSuite -from swell.suites.all_suites import suite_configs +from swell.suites.base.all_suites import suite_configs from swell.utilities.welcome_message import write_welcome_message from swell.utilities.scripts.utility_driver import get_utilities, utility_wrapper from swell.deployment.create_task_config import task_config_wrapper diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index 2cd83f12a..6aba68b1e 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -79,6 +79,8 @@ def set_attributes(self) -> None: # -------------------------------------------------------------------------------------------------- def post_init(self): + '''Sets and resolves defaults for tasks after assignment + ''' if self.base_name is None: self.base_name = self.__class__.__name__ @@ -117,6 +119,14 @@ def post_init(self): # -------------------------------------------------------------------------------------------------- def format_string_block(self, string: str) -> str: + """Format a string block with indentation for use in cylc. + + Arguments: + string: string to be placed in quotes and indented + + Returns: + Indented and quoted string. + """ out_string = '"""\n' out_string += indent_lines(string, 1) out_string += '"""' @@ -126,7 +136,22 @@ def format_string_block(self, string: str) -> str: # -------------------------------------------------------------------------------------------------- def match_platform(self, content: str | dict): - # Resolve platform-specific entries in the task object + '''Resolve platform-specific entries in mapping. + + Arguments: + content: string or mapping containing platform-designated entries + + Returns: + content filtered by the current platform, if specified + + Examples: + >>> self.match_platform('a') + 'a' + + self.platform = 'nccs_discover_sles15' + >>> self.match_platform({'nccs_discover_sles15': 'a', 'nccs_discover_cascade': 'b'}) + 'a' + ''' if isinstance(content, Mapping): if self.platform in content.keys(): @@ -142,12 +167,31 @@ def create_new_section(self, name: str | None = None, content: str | dict = '' ) -> CylcSection: + '''Create and retrun a new CylcSection object for use in formatting. + + Arguments: + name: Name of cylc section to be created + content: string or dictionary of contents for the section + ''' return CylcSection(name, content) # -------------------------------------------------------------------------------------------------- def resolve_model(self, slurm_dict: Mapping) -> dict: - ''' Resolve "all" and "model" entries in slurm dictionary ''' + '''Resolve model-specific entries in slurm dictionary specification, if they exist. + + Arguments: + slurm_dict: dictionary of slurm settings + + Returns: + dictionary of slurm settings with any model-specific defaults resolved + + Examples: + >>> self.model = 'geos_marine' + >>> self.resolve_model({'time': '01:00:00', 'nodes': {'geos_atmosphere': 1, 'geos_marine': 3}}) + + {'time': '01:00:00', 'nodes': 3} + ''' if 'all' in slurm_dict.keys() and isinstance(slurm_dict['all'], Mapping): slurm_dict = update_dict(slurm_dict, slurm_dict['all']) del slurm_dict['all'] @@ -162,9 +206,17 @@ def resolve_model(self, slurm_dict: Mapping) -> dict: # -------------------------------------------------------------------------------------------------- - def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Mapping: - # Take the external slurm dictionary and merge it with the task's parameters - # to get the dict that will be output in the flow.cylc + def generate_task_slurm_dict(self, slurm_external: Mapping) -> Mapping: + '''Take the external slurm dictionary and merge it with the task's parameters + to get the dict that will be output in the runtime section + + Arguments: + slurm_external: dictionary from `utilities/slurm.py` with defaults from the + platform and user. + + Returns: + Finalized dictionary of slurm defaults for the task. + ''' slurm_dict = {} if self.slurm is not None: @@ -192,7 +244,15 @@ def generate_task_slurm_dict(self, slurm_external: Mapping, platform: str) -> Ma # -------------------------------------------------------------------------------------------------- def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping) -> str: - ''' Return the runtime section for the given task. ''' + '''Return the runtime section for the given task. + + Arguments: + experiment_dict: experiment dictionary from `create_experiment` + slurm_external: external slurm defaults from globals and user defaults + + Returns: + String to place in flow.cylc. + ''' platform = experiment_dict['platform'] runtime_dict = {} diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 84227722b..432e1e927 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -12,7 +12,7 @@ from swell.tasks.base.task_attributes import TaskAttributes as task_attributes from swell.utilities.logger import Logger -from swell.suites.all_suites import suite_configs +from swell.suites.base.all_suites import suite_configs # -------------------------------------------------------------------------------------------------- From db60a4b3ec10ec644e794fa2814ec8f328c4c2c0 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 29 Dec 2025 10:36:07 -0500 Subject: [PATCH 149/299] Fixes for refactoring --- src/swell/tasks/base/task_setup.py | 16 ++++++++-------- src/swell/test/code_tests/slurm_test.py | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index 6aba68b1e..ff3e88fc6 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -120,7 +120,7 @@ def post_init(self): def format_string_block(self, string: str) -> str: """Format a string block with indentation for use in cylc. - + Arguments: string: string to be placed in quotes and indented @@ -137,7 +137,7 @@ def format_string_block(self, string: str) -> str: def match_platform(self, content: str | dict): '''Resolve platform-specific entries in mapping. - + Arguments: content: string or mapping containing platform-designated entries @@ -168,7 +168,7 @@ def create_new_section(self, content: str | dict = '' ) -> CylcSection: '''Create and retrun a new CylcSection object for use in formatting. - + Arguments: name: Name of cylc section to be created content: string or dictionary of contents for the section @@ -191,7 +191,7 @@ def resolve_model(self, slurm_dict: Mapping) -> dict: >>> self.resolve_model({'time': '01:00:00', 'nodes': {'geos_atmosphere': 1, 'geos_marine': 3}}) {'time': '01:00:00', 'nodes': 3} - ''' + ''' # noqa if 'all' in slurm_dict.keys() and isinstance(slurm_dict['all'], Mapping): slurm_dict = update_dict(slurm_dict, slurm_dict['all']) del slurm_dict['all'] @@ -211,7 +211,7 @@ def generate_task_slurm_dict(self, slurm_external: Mapping) -> Mapping: to get the dict that will be output in the runtime section Arguments: - slurm_external: dictionary from `utilities/slurm.py` with defaults from the + slurm_external: dictionary from `utilities/slurm.py` with defaults from the platform and user. Returns: @@ -244,8 +244,8 @@ def generate_task_slurm_dict(self, slurm_external: Mapping) -> Mapping: # -------------------------------------------------------------------------------------------------- def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping) -> str: - '''Return the runtime section for the given task. - + '''Return the runtime section for the given task. + Arguments: experiment_dict: experiment dictionary from `create_experiment` slurm_external: external slurm defaults from globals and user defaults @@ -288,7 +288,7 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping) -> s # Specify the slurm dictionary with defaults from user and global settings if self.slurm is not None: - slurm_dict = self.generate_task_slurm_dict(slurm_external, platform) + slurm_dict = self.generate_task_slurm_dict(slurm_external) slurm_section_dict = {} for key, value in slurm_dict.items(): diff --git a/src/swell/test/code_tests/slurm_test.py b/src/swell/test/code_tests/slurm_test.py index bcad1bd58..506ce69fc 100644 --- a/src/swell/test/code_tests/slurm_test.py +++ b/src/swell/test/code_tests/slurm_test.py @@ -56,7 +56,7 @@ def test_slurm_config(self, platform_mocked: Mock, mock_global_defaults: Mock) - run_jedi_var_class = TaskAttributes.get('RunJediVariationalExecutable') run_jedi_var_obj = run_jedi_var_class('geos_marine', 'nccs_discover_sles15') run_jedi_var_slurm = run_jedi_var_obj.generate_task_slurm_dict( - sd_discover_sles15, 'nccs_discover_sles15') + sd_discover_sles15) self.assertEqual(run_jedi_var_slurm["constraint"], "mil") self.assertEqual(run_jedi_var_slurm["qos"], "dastest") @@ -74,13 +74,13 @@ def test_slurm_config(self, platform_mocked: Mock, mock_global_defaults: Mock) - run_jedi_ufo_obj = run_jedi_ufo_class(mc, 'nccs_discover_sles15') run_jedi_var_dict = run_jedi_var_obj.generate_task_slurm_dict( - sd, 'nccs_discover_sles15') + sd) eva_obs_dict = eva_obs_obj.generate_task_slurm_dict( - sd, 'nccs_discover_sles15') + sd) build_jedi_dict = build_jedi_obj.generate_task_slurm_dict( - sd, 'nccs_discover_sles15') + sd) run_jedi_ufo_dict = run_jedi_ufo_obj.generate_task_slurm_dict( - sd, 'nccs_discover_sles15') + sd) # Hard-coded task-specific defaults self.assertEqual(run_jedi_var_dict["nodes"], 3) From 6399bcac29db75e47ba8b9a36d0b17a24f4821cf Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 29 Dec 2025 11:14:54 -0500 Subject: [PATCH 150/299] Add docstrings --- src/swell/deployment/create_experiment.py | 1 + src/swell/utilities/cylc_formatting.py | 52 ++++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 8965b0156..e36849ad5 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -36,6 +36,7 @@ def clone_config( platform: str, advanced: bool ) -> str: + # Create a logger logger = get_logger('SwellCloneExperiment') diff --git a/src/swell/utilities/cylc_formatting.py b/src/swell/utilities/cylc_formatting.py index 7ca854706..19f9f488b 100644 --- a/src/swell/utilities/cylc_formatting.py +++ b/src/swell/utilities/cylc_formatting.py @@ -17,7 +17,31 @@ def format_dict(dictionary: Mapping): - # Convert a dictionary into a string + """Convert a dictionary into a string. + + Args: + dictionary (Mapping): The dictionary to format. + + Returns: + str: A string representation of the dictionary, with each key-value pair on a new line in the format 'key = value'. + + Examples: + >>> format_dict({'a': 1, 'b': "test"}) + 'a = 1\nb = test\n' + + # NOTE: Strings are not quoted + >>> print(format_dict({'a': "1", 'b': "test"})) + a = 1 + b = test + + # NOTE: Nested dictionaries are printed in native dict/JSON format + >>> print(format_dict({'a': "this", 'b': {"b1": 1, "b2": 2}})) + a = this + b = {'b1': 1, 'b2': 2} + + >>> format_dict({}) + '' + """ dict_str = '' @@ -30,7 +54,13 @@ def format_dict(dictionary: Mapping): def indent_lines(string: str, level: int = 0, reset: bool = False): - # Reset line indentation for string, and indent lines by level + """Indent and/or reset string lines by multiple of level + + Arguments: + string: String to indent + level: multiple of indentation + reset: boolean of whether or not to reset string indentation + """ if reset: string = textwrap.dedent(string) @@ -47,6 +77,11 @@ class CylcSection(): Holds the information contained in a section, including the name and contents, which can be a string or dictionary. Also tracks child subsections, automatically handling indentation and syntax at the time when the string is retrieved. + + Attributes: + name: Header name of section + content: String or mapping of cylc section content + subsections: tracking of additional subsections to append to the section content ''' def __init__(self, name: Optional[str] = None, content: Union[str, dict] = '') -> None: @@ -76,9 +111,22 @@ def format_section(self, section: Self, level: int = 0) -> str: return section_str def add_subsection(self, subsection: Self) -> None: + """Add subsection to section tracking. + + Arguments: + subsection: CylcSection object to append + """ self.subsections.append(subsection) def get_section_str(self, level: int = 0) -> str: + """Get string of section contents for flow.cylc + + Arguments: + level: int of indent level multiple + + Returns: + String of section content + """ section_str = self.format_section(self, level) for subsection in self.subsections: From a304a1df37a77add43292cce4572cc4085582114 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 29 Dec 2025 11:21:48 -0500 Subject: [PATCH 151/299] pycodestyle fix --- src/swell/utilities/cylc_formatting.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/swell/utilities/cylc_formatting.py b/src/swell/utilities/cylc_formatting.py index 19f9f488b..4a1b9f043 100644 --- a/src/swell/utilities/cylc_formatting.py +++ b/src/swell/utilities/cylc_formatting.py @@ -23,7 +23,8 @@ def format_dict(dictionary: Mapping): dictionary (Mapping): The dictionary to format. Returns: - str: A string representation of the dictionary, with each key-value pair on a new line in the format 'key = value'. + str: A string representation of the dictionary, with each key-value pair on a + new line in the format 'key = value'. Examples: >>> format_dict({'a': 1, 'b': "test"}) @@ -112,7 +113,7 @@ def format_section(self, section: Self, level: int = 0) -> str: def add_subsection(self, subsection: Self) -> None: """Add subsection to section tracking. - + Arguments: subsection: CylcSection object to append """ From 256fd3300cf7a4ad0b1df3f56efd1bbd88e89260 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 29 Dec 2025 17:01:55 -0500 Subject: [PATCH 152/299] Add docstrings --- src/swell/suites/base/cylc_workflow.py | 30 +++++++++++++++++--------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/swell/suites/base/cylc_workflow.py b/src/swell/suites/base/cylc_workflow.py index af4bb247c..21cc54c84 100644 --- a/src/swell/suites/base/cylc_workflow.py +++ b/src/swell/suites/base/cylc_workflow.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from typing import Tuple -from abc import abstractmethod +from abc import abstractmethod, ABC from swell.utilities.logger import get_logger @@ -25,15 +25,17 @@ ''' -class CylcWorkflow(): +class CylcWorkflow(ABC): - ''' - Handles generating the flow.cylc file contents using the CylcSection syntax for each - necessary section in the cylc file. Since Swell workflows share a lot of common language, - this method has the convenience of automatically setting a lot of the contents. This means - that the graph section is the only part that will need to be adjusted in many cases, - and tasks may need to be altered in src/swell/tasks/task_runtimes.py. - ''' + """Abstract class setting tasks to be run by the workflow, + as well as specifying the contents of flow.cylc. + + Attributes: + experiment_dict: Mapping of suite config to use in configuring the graph + slurm_external: Mapping of user and global slurm settings + tasks: list of TaskSetup objects which specify questions used by the + suite and cylc runtime attributes + """ def __init__(self, experiment_dict, slurm_external) -> None: self.experiment_dict = experiment_dict @@ -47,18 +49,25 @@ def __init__(self, experiment_dict, slurm_external) -> None: # -------------------------------------------------------------------------------------------------- def default_header(self) -> str: + """Set the default header, contains copyright information for Swell.""" return header_str # -------------------------------------------------------------------------------------------------- @abstractmethod def set_tasks(self) -> None: + """Abstract method to be overridden by child workflows, sets a list of TaskSetup objects.""" pass # -------------------------------------------------------------------------------------------------- def get_independent_and_model_tasks(self) -> Tuple[list, dict]: - # Separate the tasks into model independent and dependent + """Iterates through tasks and separate questions into model-independent and dependent. + + Returns: + List of model-independent questions. + Mapping of model to list of questions associated with that model. + """ ind_tasks = [] model_tasks = {} @@ -84,6 +93,7 @@ def get_independent_and_model_tasks(self) -> Tuple[list, dict]: @abstractmethod def get_workflow_string(self) -> str: + """Abstract method containing instructions for constructing flow.cylc contents.""" return '' # -------------------------------------------------------------------------------------------------- From 4872c61b16d5fa157f05eacb8343c91a831d22ee Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 29 Dec 2025 17:05:05 -0500 Subject: [PATCH 153/299] pycodestyle fix --- src/swell/suites/base/cylc_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/suites/base/cylc_workflow.py b/src/swell/suites/base/cylc_workflow.py index 21cc54c84..eeb9273f6 100644 --- a/src/swell/suites/base/cylc_workflow.py +++ b/src/swell/suites/base/cylc_workflow.py @@ -63,7 +63,7 @@ def set_tasks(self) -> None: def get_independent_and_model_tasks(self) -> Tuple[list, dict]: """Iterates through tasks and separate questions into model-independent and dependent. - + Returns: List of model-independent questions. Mapping of model to list of questions associated with that model. From 8dd603cbed67736fab43bf6f786868b004684a0b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 30 Dec 2025 14:44:26 -0500 Subject: [PATCH 154/299] comparison test fixes --- src/swell/deployment/create_experiment.py | 11 +++++++---- src/swell/suites/compare/workflow.py | 14 ++++++++++++-- src/swell/tasks/base/task_attributes.py | 4 ++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index e36849ad5..4219f891f 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -95,15 +95,14 @@ def prepare_config( # Retrieved the answered suite questions # -------------------------------------- - suite_dict = prepare_config_and_suite.experiment_dict - - suite_dict = suite_dict.copy() + suite_dict = prepare_config_and_suite.experiment_dict.copy() # Overrides for comparison suites if 'start_cycle_point' in suite_dict: start_cycle_point = suite_dict['start_cycle_point'] final_cycle_point = suite_dict['final_cycle_point'] - if suite_dict['start_cycle_point'] is None: + if 'comparison_experiment_paths' in suite_dict and \ + suite_dict['start_cycle_point'] is None: config_list = suite_dict['comparison_experiment_paths'] for model in suite_dict['model_components']: cycle_times = suite_dict['models'][model]['cycle_times'] @@ -174,6 +173,10 @@ def prepare_config( # ---------------------- experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() + if 'start_cycle_point' in suite_dict: + experiment_dict['start_cycle_point'] = suite_dict['start_cycle_point'] + experiment_dict['final_cycle_point'] = suite_dict['final_cycle_point'] + # Update dict with cycle times # ---------------------------- workflow_dict = update_dict(experiment_dict, suite_dict) diff --git a/src/swell/suites/compare/workflow.py b/src/swell/suites/compare/workflow.py index f403420b8..77c09568e 100644 --- a/src/swell/suites/compare/workflow.py +++ b/src/swell/suites/compare/workflow.py @@ -77,7 +77,9 @@ def get_workflow_string(self): def set_tasks(self) -> list: - for path in self.experiment_dict['comparison_experiment_paths']: + paths = self.experiment_dict['comparison_experiment_paths'] + + for path in paths: with open(path, 'r') as f: config_dict = yaml.safe_load(f) for model in self.experiment_dict['model_components']: @@ -85,10 +87,18 @@ def set_tasks(self) -> list: self.experiment_dict['models'][model]['number_of_iterations'] = num_of_iterations + self.tasks.append(ta.root()) + for model in self.experiment_dict['model_components']: self.tasks.append(ta.EvaComparisonIncrement(model=model)) self.tasks.append(ta.EvaComparisonJediLog(model=model)) - self.tasks.append(ta.JediOopsLogParser(model=model)) self.tasks.append(ta.JediLogComparison(model=model)) + for i, path in enumerate(paths): + log_parser = ta.JediOopsLogParser(model=model) + log_parser.scheduling_name = f'JediOopsLogParser-{model}-{i}' + log_parser.script = (f'swell task JediOopsLogParser {paths[i]}' + f' -d $datetime -m {model}') + self.tasks.append(log_parser) + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index 7cf8cd187..4e9b097ec 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -210,6 +210,9 @@ class EvaComparisonJediLog(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True + self.question_list = QuestionList([ + qd.comparison_log_type() + ]) # -------------------------------------------------------------------------------------------------- @@ -262,6 +265,7 @@ def set_attributes(self): self.is_model = True self.question_list = QuestionList([ qd.parser_options(), + qd.comparison_log_type() ]) # -------------------------------------------------------------------------------------------------- From 862b1b71258240415b90f7e00b67d0e3bc0b428c Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 30 Dec 2025 15:02:48 -0500 Subject: [PATCH 155/299] refactor questions --- src/swell/tasks/base/task_attributes.py | 208 ++++++++++++------------ src/swell/tasks/base/task_setup.py | 5 +- 2 files changed, 108 insertions(+), 105 deletions(-) diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index 4e9b097ec..9ebef1fc1 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -99,30 +99,30 @@ def set_attributes(self): class BuildGeos(TaskSetup): def set_attributes(self): - self.question_list = QuestionList([ + self.questions = [ qd.geos_build_method() - ]) + ] # -------------------------------------------------------------------------------------------------- class BuildGeosByLinking(TaskSetup): def set_attributes(self): self.mail_events = ['submit-failed'] - self.question_list = QuestionList([ + self.questions = [ qd.existing_geos_gcm_build_path(), qd.geos_build_method() - ]) + ] # -------------------------------------------------------------------------------------------------- class BuildJediByLinking(TaskSetup): def set_attributes(self): self.mail_events = ['submit-failed'] - self.question_list = QuestionList([ + self.questions = [ qd.existing_jedi_build_directory(), qd.existing_jedi_build_directory_pinned(), qd.jedi_build_method() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -130,10 +130,10 @@ class BuildJedi(TaskSetup): def set_attributes(self): self.time_limit = True self.slurm = {} - self.question_list = QuestionList([ + self.questions = [ qd.bundles(), qd.jedi_build_method() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -141,49 +141,49 @@ class CleanCycle(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.clean_patterns() - ]) + ] # -------------------------------------------------------------------------------------------------- class CloneGeos(TaskSetup): def set_attributes(self): - self.question_list = QuestionList([ + self.questions = [ qd.existing_geos_gcm_source_path(), qd.geos_build_method(), qd.geos_gcm_tag() - ]) + ] # -------------------------------------------------------------------------------------------------- class CloneJedi(TaskSetup): def set_attributes(self): - self.question_list = QuestionList([ + self.questions = [ qd.bundles(), qd.existing_jedi_source_directory(), qd.existing_jedi_source_directory_pinned(), qd.jedi_build_method() - ]) + ] # -------------------------------------------------------------------------------------------------- class CloneGeosMksi(TaskSetup): def set_attributes(self): self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.observing_system_records_mksi_path(), qd.observing_system_records_mksi_path_tag() - ]) + ] # -------------------------------------------------------------------------------------------------- class CloneGmaoPerllib(TaskSetup): def set_attributes(self): - self.question_list = QuestionList([ + self.questions = [ qd.existing_perllib_path(), qd.gmao_perllib_tag() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -198,11 +198,11 @@ class EvaComparisonIncrement(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.marine_models(), qd.window_length(), qd.window_type() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -210,9 +210,9 @@ class EvaComparisonJediLog(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.comparison_log_type() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -220,11 +220,11 @@ class EvaIncrement(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.marine_models(), qd.window_length(), qd.window_type() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -234,13 +234,13 @@ def set_attributes(self): self.is_cycling = True self.is_model = True self.slurm = {} - self.question_list = QuestionList([ + self.questions = [ background_crtm_obs, qd.marine_models(), qd.observing_system_records_path(), qd.window_length(), qd.marine_models(), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -250,12 +250,12 @@ def set_attributes(self): self.is_cycling = True self.is_model = True self.slurm = {} - self.question_list = QuestionList([ + self.questions = [ background_crtm_obs, qd.window_length(), qd.ncdiag_experiments(), qd.marine_models(), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -263,10 +263,10 @@ class JediOopsLogParser(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.parser_options(), qd.comparison_log_type() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -274,7 +274,7 @@ class GetBackground(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ window_questions, qd.window_length(), qd.background_experiment(), @@ -282,7 +282,7 @@ def set_attributes(self): qd.horizontal_resolution(), qd.marine_models(), qd.r2d2_local_path(), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -291,12 +291,12 @@ def set_attributes(self): self.is_cycling = True self.is_model = True self.mail_events = ['submit-failed'] - self.question_list = QuestionList([ + self.questions = [ qd.horizontal_resolution(), qd.background_experiment(), qd.background_time_offset(), qd.geos_x_background_directory() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -304,9 +304,9 @@ class GetBufr(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.bufr_obs_classes() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -321,21 +321,21 @@ class GetEnsembleGeosExperiment(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.background_experiment(), qd.background_time_offset(), qd.geos_x_ensemble_directory() - ]) + ] # -------------------------------------------------------------------------------------------------- class GetGeosRestart(TaskSetup): def set_attributes(self): self.is_cycling = True - self.question_list = QuestionList([ + self.questions = [ swell_static_file_questions, qd.geos_restarts_directory() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -343,13 +343,13 @@ class GetGeovals(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ background_crtm_obs, qd.geovals_experiment(), qd.geovals_provider(), qd.r2d2_local_path(), qd.window_length(), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -357,10 +357,10 @@ class GetGsiBc(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.path_to_gsi_bc_coefficients(), qd.window_length() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -368,11 +368,11 @@ class GsiBcToIoda(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ background_crtm_obs, qd.observing_system_records_path(), qd.window_length() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -380,9 +380,9 @@ class GetGsiNcdiag(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.path_to_gsi_nc_diags() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -390,12 +390,12 @@ class GsiNcdiagToIoda(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.observations(), qd.produce_geovals(), qd.single_observations(), qd.window_length() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -403,13 +403,13 @@ class GetNcdiags(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ background_crtm_obs, qd.ncdiag_experiments(), qd.marine_models(), qd.r2d2_local_path(), qd.window_length(), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -417,9 +417,9 @@ class GetGeosAdasBackground(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.path_to_geos_adas_background() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -427,14 +427,14 @@ class GetObservations(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ background_crtm_obs, qd.cycling_varbc(), qd.obs_experiment(), qd.observing_system_records_path(), qd.r2d2_local_path(), qd.window_length(), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -443,9 +443,9 @@ def set_attributes(self): self.is_cycling = True self.is_model = True self.mail_events = ['submit-failed'] - self.question_list = QuestionList([ + self.questions = [ qd.ioda_locations_not_in_r2d2(), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -456,7 +456,7 @@ def set_attributes(self): self.is_model = True self.retry = '2*PT1M' self.slurm = {} - self.question_list = QuestionList([ + self.questions = [ np_proc_resolution, swell_static_file_questions, qd.analysis_variables(), @@ -474,7 +474,7 @@ def set_attributes(self): qd.total_processors(), qd.window_length(), qd.window_type() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -482,14 +482,14 @@ class GenerateBClimatologyByLinking(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ swell_static_file_questions, qd.background_error_model(), qd.horizontal_resolution(), qd.vertical_resolution(), qd.window_length(), qd.window_type() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -497,11 +497,11 @@ class GenerateObservingSystemRecords(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.observations(), qd.observing_system_records_mksi_path(), qd.observing_system_records_path() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -509,11 +509,11 @@ class LinkGeosOutput(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ window_questions, qd.background_frequency(), qd.marine_models() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -521,32 +521,32 @@ class MoveDaRestart(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.mom6_iau(), qd.window_length() - ]) + ] # -------------------------------------------------------------------------------------------------- class MoveForecastRestart(TaskSetup): def set_attributes(self): self.is_cycling = True - self.question_list = QuestionList([ + self.questions = [ qd.forecast_duration() - ]) + ] # -------------------------------------------------------------------------------------------------- class PrepGeosRunDir(TaskSetup): def set_attributes(self): self.is_cycling = True - self.question_list = QuestionList([ + self.questions = [ swell_static_file_questions, qd.existing_geos_gcm_build_path(), qd.forecast_duration(), qd.geos_experiment_directory(), qd.mom6_iau_nhours() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -554,12 +554,12 @@ class PrepareAnalysis(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.analysis_variables(), qd.mom6_iau(), qd.total_processors(), qd.window_length(), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -569,11 +569,11 @@ def set_attributes(self): self.is_model = True self.time_limit = True self.slurm = {} - self.question_list = QuestionList([ + self.questions = [ run_jedi_executable, qd.marine_models(), qd.comparison_log_type('fgat'), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -583,7 +583,7 @@ def set_attributes(self): self.is_model = True self.time_limit = True self.slurm = {} - self.question_list = QuestionList([ + self.questions = [ np_proc_resolution, window_questions, qd.analysis_variables(), @@ -593,7 +593,7 @@ def set_attributes(self): qd.observations(), qd.observing_system_records_path(), qd.comparison_log_type('ensmeanvariance'), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -603,7 +603,7 @@ def set_attributes(self): self.is_model = True self.time_limit = True self.slurm = {} - self.question_list = QuestionList([ + self.questions = [ np_proc_resolution, window_questions, background_crtm_obs, @@ -615,7 +615,7 @@ def set_attributes(self): qd.jedi_forecast_model(), qd.total_processors(), qd.comparison_log_type('hofx'), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -625,7 +625,7 @@ def set_attributes(self): self.is_model = True self.time_limit = True self.slurm = {} - self.question_list = QuestionList([ + self.questions = [ np_proc_resolution, window_questions, background_crtm_obs, @@ -635,7 +635,7 @@ def set_attributes(self): qd.save_geovals(), qd.total_processors(), qd.comparison_log_type('ensemblehofx'), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -645,7 +645,7 @@ def set_attributes(self): self.is_model = True self.time_limit = True self.slurm = {} - self.question_list = QuestionList([ + self.questions = [ np_proc_resolution, window_questions, background_crtm_obs, @@ -678,7 +678,7 @@ def set_attributes(self): qd.vertical_localization_method(), qd.perhost(), qd.comparison_log_type('localensembleda'), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -688,11 +688,11 @@ def set_attributes(self): self.is_cycling = True self.is_model = True self.slurm = {'nodes': 3} - self.question_list = QuestionList([ + self.questions = [ run_jedi_executable, qd.perhost(), qd.comparison_log_type('variational'), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -714,13 +714,13 @@ def set_attributes(self): self.is_cycling = True self.is_model = True self.slurm = {'ntasks-per-node': 1} - self.question_list = QuestionList([ + self.questions = [ background_crtm_obs, qd.generate_yaml_and_exit(), qd.single_observations(), qd.window_length(), qd.comparison_log_type('ufo_tests'), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -730,7 +730,7 @@ def set_attributes(self): self.is_model = True self.time_limit = True self.slurm = {'nodes': 1} - self.question_list = QuestionList([ + self.questions = [ qd.analysis_variables(), qd.generate_yaml_and_exit(), qd.jedi_forecast_model(), @@ -740,7 +740,7 @@ def set_attributes(self): qd.window_length(), qd.window_type(), qd.comparison_log_type('convert_state_soca2cice'), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -750,11 +750,11 @@ def set_attributes(self): self.is_model = True self.time_limit = True self.slurm = {} - self.question_list = QuestionList([ + self.questions = [ run_jedi_executable, qd.marine_models(), qd.comparison_log_type('fgat'), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -762,12 +762,12 @@ class SaveObsDiags(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ background_crtm_obs, qd.r2d2_local_path(), qd.window_length(), qd.marine_models() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -775,28 +775,28 @@ class SaveRestart(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.question_list = QuestionList([ + self.questions = [ window_questions, qd.background_time_offset(), qd.forecast_duration(), qd.horizontal_resolution(), qd.marine_models(), qd.r2d2_local_path() - ]) + ] # -------------------------------------------------------------------------------------------------- class StageJedi(TaskSetup): def set_attributes(self): self.is_model = True - self.question_list = QuestionList([ + self.questions = [ swell_static_file_questions, qd.gsibec_configuration(), qd.gsibec_nlats(), qd.gsibec_nlons(), qd.horizontal_resolution(), qd.vertical_resolution() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -806,14 +806,14 @@ def set_attributes(self): self.is_model = True self.base_name = "StageJedi" self.scheduling_name = "StageJediCycle-{model}" - self.question_list = QuestionList([ + self.questions = [ swell_static_file_questions, qd.gsibec_configuration(), qd.gsibec_nlats(), qd.gsibec_nlons(), qd.horizontal_resolution(), qd.vertical_resolution() - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -826,10 +826,10 @@ def set_attributes(self): class JediLogComparison(TaskSetup): def set_attributes(self): self.is_model = True - self.question_list = QuestionList([ + self.questions = [ qd.number_of_iterations(), qd.comparison_log_type(), - ]) + ] # -------------------------------------------------------------------------------------------------- @@ -841,7 +841,7 @@ def set_attributes(self): self.is_model = True self.time_limit = True self.slurm = {} - self.question_list = QuestionList([ + self.questions = [ np_proc_resolution, window_questions, background_crtm_obs, @@ -852,7 +852,7 @@ def set_attributes(self): qd.total_processors(), qd.obs_thinning_rej_fraction(), qd.comparison_log_type('obsfilters') - ]) + ] # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index ff3e88fc6..5e2835ebc 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -62,7 +62,7 @@ def __init__(self, model: str | None = None, platform: str | None = None) -> Non self.mail_events = ['failed', 'submit-failed'] - self.question_list = QuestionList([]) + self.questions = [] self.additional_sections = [] self.set_attributes() @@ -116,6 +116,9 @@ def post_init(self): elif self.time_limit: self.time_limit = self.match_platform(self.time_limit) + # Convert questions list into object + self.question_list = QuestionList(self.questions) + # -------------------------------------------------------------------------------------------------- def format_string_block(self, string: str) -> str: From e9304bb80b623545b5401e5b50efb3cc0429558e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 30 Dec 2025 16:35:14 -0500 Subject: [PATCH 156/299] Update docs --- docs/examples/templating_workflows.md | 20 ++++++++++---------- src/swell/tasks/base/task_setup.py | 13 +++++++++++++ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/docs/examples/templating_workflows.md b/docs/examples/templating_workflows.md index 9c3d021d8..1cc5acf5b 100644 --- a/docs/examples/templating_workflows.md +++ b/docs/examples/templating_workflows.md @@ -8,35 +8,35 @@ The `flow.cylc` that is generated under this method is not much different from t ## Tasks and the runtime section -Swell will parse the graph section, which is constructed first, to obtain the tasks which are used by the experiment. It will then build the runtime section by consulting `src/swell/tasks/task_attributes.py`. Since swell tasks broadly fall into only a few categories (model-dependent or independent, cycling or non-cycling) that do not differ much between suites, they are easily abstracted into a `Task` class. This class will dynamically set attributes such as messaging parameters and slurm settings. +Swell will parse the graph section, which is constructed first, to obtain the tasks which are used by the experiment. It will then build the runtime section by consulting `src/swell/tasks/base/task_attributes.py`. Since swell tasks broadly fall into only a few categories (model-dependent or independent, cycling or non-cycling) that do not differ much between suites, they are easily abstracted into a `TaskSetup` class. This class will dynamically set attributes such as messaging parameters and slurm settings. ```python -class CloneJedi(Task): +class CloneJedi(TaskSetup): def set_attributes(self): - self.question_list = QuestionList([ + self.questions = [ qd.bundles(), qd.existing_jedi_source_directory(), qd.existing_jedi_source_directory_pinned(), qd.jedi_build_method() - ]) + ] -class EvaObservations(Task): +class EvaObservations(TaskSetup): def set_attributes(self): self.time_limit = True self.is_cycling = True self.is_model = True self.slurm = {} - self.question_list = QuestionList([ + self.questions = [ background_crtm_obs, qd.marine_models(), qd.observing_system_records_path(), qd.window_offset(), qd.marine_models(), - ]) + ] ``` -Attributes are set by override the `set_attributes` method in `Task`. This has been combined with the previously-used `task_questions.py` for simplicity. Here, the tags `is_cycling` and `is_model` are used to specify what tags the task needs to be appended with in the runtime section. These are set to `False` by default. Tasks with a specified `slurm` dictionary (rather than set to null, as by default) will use their contents to build the `directives` section. For the task specification above for `EvaObservations`, the runtime section will be renderend as the following: +Attributes are set by override the `set_attributes` method in `TaskSetup`. This has been combined with the previously-used `task_questions.py` for simplicity. Here, the tags `is_cycling` and `is_model` are used to specify what tags the task needs to be appended with in the runtime section. These are set to `False` by default. Tasks with a specified `slurm` dictionary (rather than set to null, as by default) will use their contents to build the `directives` section. For the task specification above for `EvaObservations`, the runtime section will be renderend as the following: ``` [[EvaObservations-geos_marine]] @@ -56,7 +56,7 @@ Attributes are set by override the `set_attributes` method in `Task`. This has b This can be used to set task-specific defaults in `task_attributes.py`, rather than being set in `slurm.py`. For example, the task below defaults to slurm setting `--nodes=1`. ```python -class RunJediConvertStateSoca2ciceExecutable(Task): +class RunJediConvertStateSoca2ciceExecutable(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True @@ -67,7 +67,7 @@ class RunJediConvertStateSoca2ciceExecutable(Task): This supports setting platform-specific overrides, for example: ```python -class RunJediConvertStateSoca2ciceExecutable(Task): +class RunJediConvertStateSoca2ciceExecutable(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index 5e2835ebc..861cff69f 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -249,6 +249,19 @@ def generate_task_slurm_dict(self, slurm_external: Mapping) -> Mapping: def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping) -> str: '''Return the runtime section for the given task. + Constructs a CylcSection object by filling in a dictionary with the following components + from the task: + + 1) pre-script + 2) script + 3) platform + 4) execution time limit + 5) execution retry delays + 6) slurm subsection + 7) any additional subsections + + The CylcSection's contents is then converted into a string + Arguments: experiment_dict: experiment dictionary from `create_experiment` slurm_external: external slurm defaults from globals and user defaults From 57730a2f2a1a2f9c0e9bc9ba7bce249f0bb08f80 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 30 Dec 2025 16:58:43 -0500 Subject: [PATCH 157/299] Refactor email address --- .../prepare_config_and_suite.py | 18 ++++++++++++++++++ src/swell/suites/base/suite_questions.py | 3 ++- src/swell/tasks/base/task_setup.py | 7 ++----- src/swell/utilities/question_defaults.py | 9 +++++++++ 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 9ca24477b..8ff7d1dfa 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -247,6 +247,24 @@ def override_with_defaults(self, suite_task: QuestionType) -> None: question[platform_key] == 'defer_to_platform': question[platform_key] = platform_val + # Construct the dictionary for user defaults + # ------------------------------------------ + user_defaults = {} + settings_file = os.path.expanduser('~/.swell/swell-settings.yaml') + if os.path.exists(settings_file): + with open(settings_file, 'r') as f: + user_defaults = yaml.safe_load(f) + + # See if any questions have user defaults + # --------------------------------------- + for question_name, question in self.question_dictionary_model_ind.items(): + if question['question_type'] == suite_task: + if question_name in user_defaults.keys(): + for user_key, user_val in user_defaults[question_name].items(): + if platform_key not in question.keys() or \ + question[platform_key] == 'defer_to_user': + question[user_key] = user_val + # Perform a model override on the model_dep dictionary # ---------------------------------------------------- if self.suite_needs_model_components: diff --git a/src/swell/suites/base/suite_questions.py b/src/swell/suites/base/suite_questions.py index 25df3c48e..5cd1aa02d 100644 --- a/src/swell/suites/base/suite_questions.py +++ b/src/swell/suites/base/suite_questions.py @@ -28,7 +28,8 @@ class SuiteQuestions(QuestionContainer, Enum): qd.experiment_id(), qd.experiment_root(), qd.pause_on_tasks(), - qd.task_email_parameters() + qd.task_email_parameters(), + qd.email_address() ] ) diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index 861cff69f..342375ce9 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -14,7 +14,6 @@ from swell.utilities.suite_utils import get_model_components from swell.utilities.dictionary import update_dict from swell.utilities.swell_questions import QuestionList -from swell.utilities.settings import read_settings # -------------------------------------------------------------------------------------------------- @@ -321,11 +320,9 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping) -> s # Check slurm messaging parameters events = self.mail_events - settings_dict = read_settings() - # Add messaging section - if len(events) > 0 and 'email_address' in settings_dict.keys(): - email_address = settings_dict['email_address'] + if len(events) > 0 and 'email_address' in experiment_dict: + email_address = experiment_dict['email_address'] address_section = self.create_new_section('mail', f'to = {email_address}') runtime_section.add_subsection(address_section) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index b1429d484..dce1eacd6 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -61,6 +61,15 @@ class cycling_varbc(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class email_address(SuiteQuestion): + default_value: str = "defer_to_user" + question_name: str = "email_address" + prompt: str = "What email address should cylc messages be sent to?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_hofx_packets(SuiteQuestion): default_value: str = "defer_to_model" From f4f677cae48785ea81b11e723c360c4d0c329154 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 30 Dec 2025 17:09:42 -0500 Subject: [PATCH 158/299] Don't use unset email address --- src/swell/tasks/base/task_setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index 342375ce9..2eb9d890d 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -321,7 +321,8 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping) -> s events = self.mail_events # Add messaging section - if len(events) > 0 and 'email_address' in experiment_dict: + if len(events) > 0 and "email_address" in experiment_dict and \ + experiment_dict["email_address"] != "defer_to_user": email_address = experiment_dict['email_address'] address_section = self.create_new_section('mail', f'to = {email_address}') runtime_section.add_subsection(address_section) From dc92d3b1cd5e33c095f63365d2f28409711bfffd Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 7 Jan 2026 11:53:20 -0500 Subject: [PATCH 159/299] Update docs/examples/templating_workflows.md Co-authored-by: Alexey Shiklomanov --- docs/examples/templating_workflows.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/examples/templating_workflows.md b/docs/examples/templating_workflows.md index 1cc5acf5b..6929216ff 100644 --- a/docs/examples/templating_workflows.md +++ b/docs/examples/templating_workflows.md @@ -36,7 +36,20 @@ class EvaObservations(TaskSetup): ] ``` -Attributes are set by override the `set_attributes` method in `TaskSetup`. This has been combined with the previously-used `task_questions.py` for simplicity. Here, the tags `is_cycling` and `is_model` are used to specify what tags the task needs to be appended with in the runtime section. These are set to `False` by default. Tasks with a specified `slurm` dictionary (rather than set to null, as by default) will use their contents to build the `directives` section. For the task specification above for `EvaObservations`, the runtime section will be renderend as the following: +Attributes are set by override the `set_attributes` method in `TaskSetup`. This has been combined with the previously-used `task_questions.py` for simplicity. + +The tags `is_cycling` and `is_model` (both `False` by default) modify the script command (`swell task $config`): + +- `is_cycling = True` adds `-d $datetime` for cycling tasks +- `is_model = True` adds `-m {model}` to indicate model-specific tasks. + +The `slurm` attribute determines where or not the task requires Slurm and provides a way to set task-specific overrides: + +- `slurm = None` means the task is not a Slurm task, so no `[[[directives]]]` section will be written. +- `slurm = {}` means the task *is* a Slurm task, so the `[[[directives]]]` will be populated according to the platform's default slurm settings (in `src/swell/deployment/platforms`) along with user-specific overrides. +- `slurm = {}` will optionally override the platform defaults with task-specific ones (but note that *user-configured overrides always have the highest priority*). + +For the task specification above for `EvaObservations`, the runtime section will be renderend as the following: ``` [[EvaObservations-geos_marine]] From 635122447f849a3930ef45724d80c5d9411580a9 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 7 Jan 2026 14:32:10 -0500 Subject: [PATCH 160/299] Update slurm settings --- src/swell/deployment/create_task_config.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/swell/deployment/create_task_config.py b/src/swell/deployment/create_task_config.py index cadad9cf3..828887d19 100644 --- a/src/swell/deployment/create_task_config.py +++ b/src/swell/deployment/create_task_config.py @@ -24,11 +24,12 @@ # -------------------------------------------------------------------------------------------------- -script_template = ''' -#!{{shell}} +script_template = '''#!{{shell}} +{% if task_slurm_dict != None %} {%- for key, value in task_slurm_dict.items() %} #SBATCH --{{key}} = {{value}} {%- endfor %} +{% endif %} # ------------------- @@ -134,21 +135,18 @@ def task_config_wrapper(task_name: str, experiment_dict_string_comments = add_comments_to_dictionary(logger, experiment_dict_string, comment_dict) - # Construct the slurm defaults - slurm_external_dict = prepare_slurm_defaults_and_overrides(logger, platform, slurm) - # Construct the slurm dict for the task + task_slurm_dict = None if task.slurm is not None: - task_slurm_dict = task.generate_task_slurm_dict(slurm_external_dict, platform) + # Construct the slurm defaults + slurm_external_dict = prepare_slurm_defaults_and_overrides(logger, platform, slurm) + task_slurm_dict = task.generate_task_slurm_dict(slurm_external_dict) - time_limit = task.get_time_limit() + time_limit = task.time_limit if time_limit is not None: time_limit_dto = isodate.parse_duration(time_limit) task_slurm_dict['time'] = isodate.strftime(time_limit_dto, '%H:%M:%S') - else: - task_slurm_dict = {} - # Determine the path for task results experiment_root = experiment_dict['experiment_root'] experiment_id = experiment_dict['experiment_id'] @@ -186,7 +184,7 @@ def task_config_wrapper(task_name: str, logger.abort(f'Failed to deduce the target shell. $SHELL is currently set to {shell}') file_ext = shell_type - if len(task_slurm_dict) > 0: + if task_slurm_dict is not None: file_ext = 'slurm' # Build the swell task script @@ -217,7 +215,7 @@ def task_config_wrapper(task_name: str, logger.info('To run the task by itself, run: ') print(f'\n {script}\n') logger.info('Or, to use auto-generated script, run: ') - if len(task_slurm_dict) > 0: + if task_slurm_dict is not None: print(f'\n sbatch {script_file}\n') else: print(f'\n {script_file}\n') From 10b55bc2b7ae002c59ee5fb0942542c2e501cae5 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 8 Jan 2026 14:39:43 -0500 Subject: [PATCH 161/299] pycodestyle fix --- src/swell/tasks/base/task_attributes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index 7d4591531..566f04200 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -567,7 +567,7 @@ class RenderJediObservations(TaskSetup): def set_attributes(self): self.is_cycling = True self.is_model = True - self.questions=[ + self.questions = [ qd.check_for_obs(), qd.crtm_coeff_dir(), qd.background_time_offset(), @@ -576,7 +576,7 @@ def set_attributes(self): qd.set_obs_as_local(), qd.window_length() ] - + # -------------------------------------------------------------------------------------------------- class RunJediFgatExecutable(TaskSetup): From 46143e364b6a6bf27b24f3f8a30c73cbded42584 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 12 Jan 2026 10:57:17 -0500 Subject: [PATCH 162/299] working --- src/swell/tasks/base/task_attributes.py | 1582 +++++++++-------- src/swell/tasks/bufr_to_ioda.py | 7 + src/swell/tasks/build_geos.py | 15 +- src/swell/tasks/build_geos_by_linking.py | 11 + src/swell/tasks/build_jedi.py | 12 + src/swell/tasks/build_jedi_by_linking.py | 11 + src/swell/tasks/clean_cycle.py | 11 + src/swell/tasks/clone_geos.py | 11 + src/swell/tasks/clone_geos_mksi.py | 11 + src/swell/tasks/clone_gmao_perllib.py | 10 + src/swell/tasks/clone_jedi.py | 20 +- src/swell/tasks/eva_comparison_increment.py | 17 +- src/swell/tasks/eva_comparison_jedi_log.py | 15 +- src/swell/tasks/eva_increment.py | 17 +- src/swell/tasks/eva_jedi_log.py | 10 + src/swell/tasks/eva_observations.py | 24 +- src/swell/tasks/eva_timeseries.py | 23 +- src/swell/tasks/generate_b_climatology.py | 36 + .../generate_b_climatology_by_linking.py | 19 +- .../generate_observing_system_records.py | 13 + src/swell/tasks/get_background.py | 24 +- .../tasks/get_background_geos_experiment.py | 15 + src/swell/tasks/get_bufr.py | 11 + src/swell/tasks/get_ensemble.py | 4 +- .../tasks/get_ensemble_geos_experiment.py | 13 + src/swell/tasks/get_geos_adas_background.py | 11 + src/swell/tasks/get_geos_restart.py | 12 + src/swell/tasks/get_geovals.py | 22 +- src/swell/tasks/get_gsi_bc.py | 12 + src/swell/tasks/get_gsi_ncdiag.py | 11 + src/swell/tasks/get_ncdiags.py | 25 +- src/swell/tasks/get_obs_not_in_r2d2.py | 12 + src/swell/tasks/get_observations.py | 26 +- src/swell/tasks/gsi_bc_to_ioda.py | 12 + src/swell/tasks/gsi_ncdiag_to_ioda.py | 22 +- src/swell/tasks/jedi_c_test.py | 52 - src/swell/tasks/jedi_log_comparison.py | 11 + src/swell/tasks/jedi_oops_log_parser.py | 12 + src/swell/tasks/link_geos_output.py | 17 +- src/swell/tasks/move_da_restart.py | 12 + src/swell/tasks/move_forecast_restart.py | 10 + src/swell/tasks/prep_geos_run_dir.py | 15 + src/swell/tasks/prepare_analysis.py | 14 + src/swell/tasks/remove_forecast_dir.py | 8 +- src/swell/tasks/render_jedi_observations.py | 17 + src/swell/tasks/run_geos_executable.py | 7 + ...jedi_convert_state_soca2cice_executable.py | 21 + .../tasks/run_jedi_ensemble_mean_variance.py | 27 + src/swell/tasks/run_jedi_fgat_executable.py | 37 + .../run_jedi_hofx_ensemble_executable.py | 32 + src/swell/tasks/run_jedi_hofx_executable.py | 30 + .../run_jedi_local_ensemble_da_executable.py | 55 + .../tasks/run_jedi_obsfilters_executable.py | 33 + .../tasks/run_jedi_ufo_tests_executable.py | 20 + .../tasks/run_jedi_variational_executable.py | 37 + src/swell/tasks/save_obs_diags.py | 21 +- src/swell/tasks/save_restart.py | 20 +- src/swell/tasks/stage_jedi.py | 16 + src/swell/tasks/store_background.py | 21 +- 59 files changed, 1778 insertions(+), 874 deletions(-) delete mode 100644 src/swell/tasks/jedi_c_test.py diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index 566f04200..d8d69e102 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -11,6 +11,19 @@ from swell.utilities.swell_questions import QuestionList from swell.utilities.question_defaults import QuestionDefaults as qd + qd.npx_proc(), + qd.npy_proc(), + qd.npx(), + qd.npy(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.window_length(), + qd.window_type(), + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + # -------------------------------------------------------------------------------------------------- background_crtm_obs = QuestionList( @@ -19,7 +32,7 @@ qd.background_time_offset(), qd.crtm_coeff_dir(), qd.observations(), - qd.observing_system_records_path() + qd.observing_system_records_path(), ] ) @@ -84,793 +97,786 @@ class TaskAttributes(): - # -------------------------------------------------------------------------------------------------- - - class root(TaskSetup): - - def set_attributes(self): - self.script = False - self.pre_script = "source $CYLC_SUITE_DEF_PATH/modules" - self.additional_sections = [self.create_new_section('environment', - {'datetime': '$CYLC_TASK_CYCLE_POINT', - 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'})] # noqa - - # -------------------------------------------------------------------------------------------------- - - class BuildGeos(TaskSetup): - def set_attributes(self): - self.questions = [ - qd.geos_build_method() - ] - - # -------------------------------------------------------------------------------------------------- - - class BuildGeosByLinking(TaskSetup): - def set_attributes(self): - self.mail_events = ['submit-failed'] - self.questions = [ - qd.existing_geos_gcm_build_path(), - qd.geos_build_method() - ] - - # -------------------------------------------------------------------------------------------------- - - class BuildJediByLinking(TaskSetup): - def set_attributes(self): - self.mail_events = ['submit-failed'] - self.questions = [ - qd.existing_jedi_build_directory(), - qd.existing_jedi_build_directory_pinned(), - qd.jedi_build_method() - ] - - # -------------------------------------------------------------------------------------------------- - - class BuildJedi(TaskSetup): - def set_attributes(self): - self.time_limit = True - self.slurm = {} - self.questions = [ - qd.bundles(), - qd.jedi_build_method() - ] - - # -------------------------------------------------------------------------------------------------- - - class CleanCycle(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.clean_patterns() - ] - - # -------------------------------------------------------------------------------------------------- - - class CloneGeos(TaskSetup): - def set_attributes(self): - self.questions = [ - qd.existing_geos_gcm_source_path(), - qd.geos_build_method(), - qd.geos_gcm_tag() - ] - - # -------------------------------------------------------------------------------------------------- - - class CloneJedi(TaskSetup): - def set_attributes(self): - self.questions = [ - qd.bundles(), - qd.existing_jedi_source_directory(), - qd.existing_jedi_source_directory_pinned(), - qd.jedi_build_method() - ] - - # -------------------------------------------------------------------------------------------------- - - class CloneGeosMksi(TaskSetup): - def set_attributes(self): - self.is_model = True - self.questions = [ - qd.observing_system_records_mksi_path(), - qd.observing_system_records_mksi_path_tag() - ] - - # -------------------------------------------------------------------------------------------------- - - class CloneGmaoPerllib(TaskSetup): - def set_attributes(self): - self.questions = [ - qd.existing_perllib_path(), - qd.gmao_perllib_tag() - ] - - # -------------------------------------------------------------------------------------------------- - - class EvaJediLog(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - - # -------------------------------------------------------------------------------------------------- - - class EvaComparisonIncrement(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.marine_models(), - qd.window_length(), - qd.window_type() - ] - - # -------------------------------------------------------------------------------------------------- - - class EvaComparisonJediLog(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.comparison_log_type() - ] - - # -------------------------------------------------------------------------------------------------- - - class EvaIncrement(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.marine_models(), - qd.window_length(), - qd.window_type() - ] - - # -------------------------------------------------------------------------------------------------- - - class EvaObservations(TaskSetup): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.slurm = {} - self.questions = [ - background_crtm_obs, - qd.marine_models(), - qd.observing_system_records_path(), - qd.window_length(), - qd.marine_models(), - ] - - # -------------------------------------------------------------------------------------------------- - - class EvaTimeseries(TaskSetup): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.slurm = {} - self.questions = [ - background_crtm_obs, - qd.window_length(), - qd.ncdiag_experiments(), - qd.marine_models(), - ] - - # -------------------------------------------------------------------------------------------------- - - class JediOopsLogParser(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.parser_options(), - qd.comparison_log_type() - ] - - # -------------------------------------------------------------------------------------------------- - - class GetBackground(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - window_questions, - qd.window_length(), - qd.background_experiment(), - qd.background_frequency(), - qd.horizontal_resolution(), - qd.marine_models(), - qd.r2d2_local_path(), - ] - - # -------------------------------------------------------------------------------------------------- - - class GetBackgroundGeosExperiment(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.mail_events = ['submit-failed'] - self.questions = [ - qd.horizontal_resolution(), - qd.background_experiment(), - qd.background_time_offset(), - qd.geos_x_background_directory() - ] - - # -------------------------------------------------------------------------------------------------- - - class GetBufr(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.bufr_obs_classes() - ] - - # -------------------------------------------------------------------------------------------------- - - class BufrToIoda(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - - # -------------------------------------------------------------------------------------------------- - - class GetEnsembleGeosExperiment(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.background_experiment(), - qd.background_time_offset(), - qd.geos_x_ensemble_directory() - ] - - # -------------------------------------------------------------------------------------------------- - - class GetGeosRestart(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.questions = [ - swell_static_file_questions, - qd.geos_restarts_directory() - ] - - # -------------------------------------------------------------------------------------------------- - - class GetGeovals(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - background_crtm_obs, - qd.geovals_experiment(), - qd.geovals_provider(), - qd.r2d2_local_path(), - qd.window_length(), - ] - - # -------------------------------------------------------------------------------------------------- - - class GetGsiBc(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.path_to_gsi_bc_coefficients(), - qd.window_length() - ] - - # -------------------------------------------------------------------------------------------------- - - class GsiBcToIoda(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - background_crtm_obs, - qd.observing_system_records_path(), - qd.window_length() - ] - - # -------------------------------------------------------------------------------------------------- - - class GetGsiNcdiag(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.path_to_gsi_nc_diags() - ] - - # -------------------------------------------------------------------------------------------------- - - class GsiNcdiagToIoda(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.observations(), - qd.produce_geovals(), - qd.single_observations(), - qd.window_length() - ] - - # -------------------------------------------------------------------------------------------------- - - class GetNcdiags(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - background_crtm_obs, - qd.ncdiag_experiments(), - qd.marine_models(), - qd.r2d2_local_path(), - qd.window_length(), - ] - - # -------------------------------------------------------------------------------------------------- - - class GetGeosAdasBackground(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.path_to_geos_adas_background() - ] - - # -------------------------------------------------------------------------------------------------- - - class GetObservations(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - background_crtm_obs, - qd.cycling_varbc(), - qd.obs_experiment(), - qd.observing_system_records_path(), - qd.r2d2_local_path(), - qd.window_length(), - ] - - # -------------------------------------------------------------------------------------------------- - - class GetObsNotInR2d2(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.mail_events = ['submit-failed'] - self.questions = [ - qd.ioda_locations_not_in_r2d2(), - ] - - # -------------------------------------------------------------------------------------------------- - - class GenerateBClimatology(TaskSetup): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.retry = '2*PT1M' - self.slurm = {} - self.questions = [ - np_proc_resolution, - swell_static_file_questions, - qd.analysis_variables(), - qd.background_error_model(), - qd.generate_yaml_and_exit(), - qd.gradient_norm_reduction(), - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.jedi_forecast_model(), - qd.marine_models(), - qd.minimizer(), - qd.number_of_iterations(), - qd.observing_system_records_path(), - qd.total_processors(), - qd.window_length(), - qd.window_type() - ] - - # -------------------------------------------------------------------------------------------------- - - class GenerateBClimatologyByLinking(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - swell_static_file_questions, - qd.background_error_model(), - qd.horizontal_resolution(), - qd.vertical_resolution(), - qd.window_length(), - qd.window_type() - ] - - # -------------------------------------------------------------------------------------------------- - - class GenerateObservingSystemRecords(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.observations(), - qd.observing_system_records_mksi_path(), - qd.observing_system_records_path() - ] - - # -------------------------------------------------------------------------------------------------- - - class LinkGeosOutput(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - window_questions, - qd.background_frequency(), - qd.marine_models() - ] - - # -------------------------------------------------------------------------------------------------- - - class MoveDaRestart(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.mom6_iau(), - qd.window_length() - ] - - # -------------------------------------------------------------------------------------------------- - - class MoveForecastRestart(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.questions = [ - qd.forecast_duration() - ] - - # -------------------------------------------------------------------------------------------------- - - class PrepGeosRunDir(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.questions = [ - swell_static_file_questions, - qd.existing_geos_gcm_build_path(), - qd.forecast_duration(), - qd.geos_experiment_directory(), - qd.mom6_iau_nhours() - ] - - # -------------------------------------------------------------------------------------------------- - - class PrepareAnalysis(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.analysis_variables(), - qd.mom6_iau(), - qd.total_processors(), - qd.window_length(), - ] - - # -------------------------------------------------------------------------------------------------- - - class RenderJediObservations(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.check_for_obs(), - qd.crtm_coeff_dir(), - qd.background_time_offset(), - qd.observing_system_records_path(), - qd.observations(), - qd.set_obs_as_local(), - qd.window_length() - ] - - # -------------------------------------------------------------------------------------------------- - - class RunJediFgatExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - run_jedi_executable, - qd.marine_models(), - qd.comparison_log_type('fgat'), - ] - - # -------------------------------------------------------------------------------------------------- - - class RunJediEnsembleMeanVariance(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - np_proc_resolution, - window_questions, - qd.analysis_variables(), - qd.ensemble_num_members(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.observations(), - qd.observing_system_records_path(), - qd.comparison_log_type('ensmeanvariance'), - ] - - # -------------------------------------------------------------------------------------------------- - - class RunJediHofxEnsembleExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.ensemble_num_members(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.total_processors(), - qd.comparison_log_type('hofx'), - ] - - # -------------------------------------------------------------------------------------------------- - - class RunJediHofxExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.save_geovals(), - qd.total_processors(), - qd.comparison_log_type('ensemblehofx'), - ] - - # -------------------------------------------------------------------------------------------------- - - class RunJediLocalEnsembleDaExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.ensemble_num_members(), - qd.ensmean_only(), - qd.ensmeanvariance_only(), - qd.generate_yaml_and_exit(), - qd.horizontal_localization_lengthscale(), - qd.horizontal_localization_max_nobs(), - qd.horizontal_localization_method(), - qd.jedi_forecast_model(), - qd.local_ensemble_inflation_mult(), - qd.local_ensemble_inflation_rtpp(), - qd.local_ensemble_inflation_rtps(), - qd.local_ensemble_save_posterior_ensemble(), - qd.local_ensemble_save_posterior_ensemble_increments(), - qd.local_ensemble_save_posterior_mean(), - qd.local_ensemble_save_posterior_mean_increment(), - qd.local_ensemble_solver(), - qd.local_ensemble_use_linear_observer(), - qd.skip_ensemble_hofx(), - qd.total_processors(), - qd.vertical_localization_apply_log_transform(), - qd.vertical_localization_function(), - qd.vertical_localization_ioda_vertical_coord(), - qd.vertical_localization_ioda_vertical_coord_group(), - qd.vertical_localization_lengthscale(), - qd.vertical_localization_method(), - qd.perhost(), - qd.comparison_log_type('localensembleda'), - ] - - # -------------------------------------------------------------------------------------------------- - - class RunJediVariationalExecutable(TaskSetup): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.slurm = {'nodes': 3} - self.questions = [ - run_jedi_executable, - qd.perhost(), - qd.comparison_log_type('variational'), - ] - - # -------------------------------------------------------------------------------------------------- - - class RemoveForecastDir(TaskSetup): - def set_attributes(self): - self.is_cycling = True - - # -------------------------------------------------------------------------------------------------- - - class RunGeosExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - - # -------------------------------------------------------------------------------------------------- - - class RunJediUfoTestsExecutable(TaskSetup): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.slurm = {'ntasks-per-node': 1} - self.questions = [ - background_crtm_obs, - qd.generate_yaml_and_exit(), - qd.single_observations(), - qd.window_length(), - qd.comparison_log_type('ufo_tests'), - ] - - # -------------------------------------------------------------------------------------------------- - - class RunJediConvertStateSoca2ciceExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {'nodes': 1} - self.questions = [ - qd.analysis_variables(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.marine_models(), - qd.observations(), - qd.total_processors(), - qd.window_length(), - qd.window_type(), - qd.comparison_log_type('convert_state_soca2cice'), - ] - - # -------------------------------------------------------------------------------------------------- - - class RunJediFgatExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - run_jedi_executable, - qd.marine_models(), - qd.comparison_log_type('fgat'), - ] - - # -------------------------------------------------------------------------------------------------- - - class SaveObsDiags(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - background_crtm_obs, - qd.r2d2_local_path(), - qd.window_length(), - qd.marine_models() - ] - - # -------------------------------------------------------------------------------------------------- - - class SaveRestart(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - window_questions, - qd.background_time_offset(), - qd.forecast_duration(), - qd.horizontal_resolution(), - qd.marine_models(), - qd.r2d2_local_path() - ] - - # -------------------------------------------------------------------------------------------------- - - class StageJedi(TaskSetup): - def set_attributes(self): - self.is_model = True - self.questions = [ - swell_static_file_questions, - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ] - - # -------------------------------------------------------------------------------------------------- - - class StageJediCycle(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.base_name = "StageJedi" - self.scheduling_name = "StageJediCycle-{model}" - self.questions = [ - swell_static_file_questions, - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ] - - # -------------------------------------------------------------------------------------------------- - - class sync_point(TaskSetup): - def set_attributes(self): - self.script = "true" - - # -------------------------------------------------------------------------------------------------- - - class JediLogComparison(TaskSetup): - def set_attributes(self): - self.is_model = True - self.questions = [ - qd.number_of_iterations(), - qd.comparison_log_type(), - ] - - # -------------------------------------------------------------------------------------------------- - - class RunJediObsfiltersExecutable(TaskSetup): - def set_attributes(self): - self.script = ("swell task RunJediObsfiltersExecutable $config" - " -d $datetime -m geos_atmosphere") - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.observing_system_records_path(), - qd.total_processors(), - qd.obs_thinning_rej_fraction(), - qd.comparison_log_type('obsfilters') - ] - - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- + +class root(TaskSetup): + + def set_attributes(self): + self.script = False + self.pre_script = "source $CYLC_SUITE_DEF_PATH/modules" + self.additional_sections = [self.create_new_section('environment', + {'datetime': '$CYLC_TASK_CYCLE_POINT', + 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'})] # noqa + +# -------------------------------------------------------------------------------------------------- + +class BuildGeos(TaskSetup): + def set_attributes(self): + self.questions = [ + qd.geos_build_method() + ] + +# -------------------------------------------------------------------------------------------------- + +class BuildGeosByLinking(TaskSetup): + def set_attributes(self): + self.mail_events = ['submit-failed'] + self.questions = [ + qd.existing_geos_gcm_build_path(), + qd.geos_build_method() + ] + +# -------------------------------------------------------------------------------------------------- + +class BuildJediByLinking(TaskSetup): + def set_attributes(self): + self.mail_events = ['submit-failed'] + self.questions = [ + qd.existing_jedi_build_directory(), + qd.existing_jedi_build_directory_pinned(), + qd.jedi_build_method() + ] + +# -------------------------------------------------------------------------------------------------- + +class BuildJedi(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.slurm = {} + self.questions = [ + qd.bundles(), + qd.jedi_build_method() + ] + +# -------------------------------------------------------------------------------------------------- + +class CleanCycle(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.clean_patterns() + ] + +# -------------------------------------------------------------------------------------------------- + +class CloneGeos(TaskSetup): + def set_attributes(self): + self.questions = [ + qd.existing_geos_gcm_source_path(), + qd.geos_build_method(), + qd.geos_gcm_tag() + ] + +# -------------------------------------------------------------------------------------------------- + +class CloneJedi(TaskSetup): + def set_attributes(self): + self.questions = [ + qd.bundles(), + qd.existing_jedi_source_directory(), + qd.existing_jedi_source_directory_pinned(), + qd.jedi_build_method() + ] + +# -------------------------------------------------------------------------------------------------- + +class CloneGeosMksi(TaskSetup): + def set_attributes(self): + self.is_model = True + self.questions = [ + qd.observing_system_records_mksi_path(), + qd.observing_system_records_mksi_path_tag() + ] + +# -------------------------------------------------------------------------------------------------- + +class CloneGmaoPerllib(TaskSetup): + def set_attributes(self): + self.questions = [ + qd.existing_perllib_path(), + qd.gmao_perllib_tag() + ] + +# -------------------------------------------------------------------------------------------------- + +class EvaJediLog(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + +# -------------------------------------------------------------------------------------------------- + +class EvaComparisonIncrement(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.marine_models(), + qd.window_length(), + qd.window_type() + ] + +# -------------------------------------------------------------------------------------------------- + +class EvaComparisonJediLog(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.comparison_log_type() + ] + +# -------------------------------------------------------------------------------------------------- + +class EvaIncrement(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.marine_models(), + qd.window_length(), + qd.window_type() + ] + +# -------------------------------------------------------------------------------------------------- + +class EvaObservations(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {} + self.questions = [ + background_crtm_obs, + qd.marine_models(), + qd.observing_system_records_path(), + qd.window_length(), + qd.marine_models(), + ] + +# -------------------------------------------------------------------------------------------------- + +class EvaTimeseries(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {} + self.questions = [ + background_crtm_obs, + qd.window_length(), + qd.ncdiag_experiments(), + qd.marine_models(), + ] + +# -------------------------------------------------------------------------------------------------- + +class JediOopsLogParser(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.parser_options(), + qd.comparison_log_type() + ] + +# -------------------------------------------------------------------------------------------------- + +class GetBackground(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + window_questions, + qd.window_length(), + qd.background_experiment(), + qd.background_frequency(), + qd.horizontal_resolution(), + qd.marine_models(), + qd.r2d2_local_path(), + ] + +# -------------------------------------------------------------------------------------------------- + +class GetBackgroundGeosExperiment(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.mail_events = ['submit-failed'] + self.questions = [ + qd.horizontal_resolution(), + qd.background_experiment(), + qd.background_time_offset(), + qd.geos_x_background_directory() + ] + +# -------------------------------------------------------------------------------------------------- + +class GetBufr(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.bufr_obs_classes() + ] + +# -------------------------------------------------------------------------------------------------- + +class GetEnsembleGeosExperiment(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.background_experiment(), + qd.background_time_offset(), + qd.geos_x_ensemble_directory() + ] + +# -------------------------------------------------------------------------------------------------- + +class GetGeosRestart(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.questions = [ + swell_static_file_questions, + qd.geos_restarts_directory() + ] + +# -------------------------------------------------------------------------------------------------- + +class GetGeovals(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + background_crtm_obs, + qd.geovals_experiment(), + qd.geovals_provider(), + qd.r2d2_local_path(), + qd.window_length(), + ] + +# -------------------------------------------------------------------------------------------------- + +class GetGsiBc(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.path_to_gsi_bc_coefficients(), + qd.window_length() + ] + +# -------------------------------------------------------------------------------------------------- + +class GsiBcToIoda(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + background_crtm_obs, + qd.observing_system_records_path(), + qd.window_length() + ] + +# -------------------------------------------------------------------------------------------------- + +class GetGsiNcdiag(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.path_to_gsi_nc_diags() + ] + +# -------------------------------------------------------------------------------------------------- + +class GsiNcdiagToIoda(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.observations(), + qd.produce_geovals(), + qd.single_observations(), + qd.window_length() + ] + +# -------------------------------------------------------------------------------------------------- + +class GetNcdiags(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + background_crtm_obs, + qd.ncdiag_experiments(), + qd.marine_models(), + qd.r2d2_local_path(), + qd.window_length(), + ] + +# -------------------------------------------------------------------------------------------------- + +class GetGeosAdasBackground(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.path_to_geos_adas_background() + ] + +# -------------------------------------------------------------------------------------------------- + +class GetObservations(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + background_crtm_obs, + qd.cycling_varbc(), + qd.obs_experiment(), + qd.observing_system_records_path(), + qd.r2d2_local_path(), + qd.window_length(), + ] + +# -------------------------------------------------------------------------------------------------- + +class GetObsNotInR2d2(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.mail_events = ['submit-failed'] + self.questions = [ + qd.ioda_locations_not_in_r2d2(), + ] + +# -------------------------------------------------------------------------------------------------- + +class GenerateBClimatology(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.retry = '2*PT1M' + self.slurm = {} + self.questions = [ + np_proc_resolution, + swell_static_file_questions, + qd.analysis_variables(), + qd.background_error_model(), + qd.generate_yaml_and_exit(), + qd.gradient_norm_reduction(), + qd.gsibec_configuration(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.jedi_forecast_model(), + qd.marine_models(), + qd.minimizer(), + qd.number_of_iterations(), + qd.observing_system_records_path(), + qd.total_processors(), + qd.window_length(), + qd.window_type() + ] + +# -------------------------------------------------------------------------------------------------- + +class GenerateBClimatologyByLinking(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + swell_static_file_questions, + qd.background_error_model(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.window_length(), + qd.window_type() + ] + +# -------------------------------------------------------------------------------------------------- + +class GenerateObservingSystemRecords(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.observations(), + qd.observing_system_records_mksi_path(), + qd.observing_system_records_path() + ] + +# -------------------------------------------------------------------------------------------------- + +class LinkGeosOutput(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + window_questions, + qd.background_frequency(), + qd.marine_models() + ] + +# -------------------------------------------------------------------------------------------------- + +class MoveDaRestart(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.mom6_iau(), + qd.window_length() + ] + +# -------------------------------------------------------------------------------------------------- + +class MoveForecastRestart(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.questions = [ + qd.forecast_duration() + ] + +# -------------------------------------------------------------------------------------------------- + +class PrepGeosRunDir(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.questions = [ + swell_static_file_questions, + qd.existing_geos_gcm_build_path(), + qd.forecast_duration(), + qd.geos_experiment_directory(), + qd.mom6_iau_nhours() + ] + +# -------------------------------------------------------------------------------------------------- + +class PrepareAnalysis(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.analysis_variables(), + qd.mom6_iau(), + qd.total_processors(), + qd.window_length(), + ] + +# -------------------------------------------------------------------------------------------------- + +class RenderJediObservations(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.check_for_obs(), + qd.crtm_coeff_dir(), + qd.background_time_offset(), + qd.observing_system_records_path(), + qd.observations(), + qd.set_obs_as_local(), + qd.window_length() + ] + +# -------------------------------------------------------------------------------------------------- + +class RunJediFgatExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + run_jedi_executable, + qd.marine_models(), + qd.comparison_log_type('fgat'), + ] + +# -------------------------------------------------------------------------------------------------- + +class RunJediEnsembleMeanVariance(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + np_proc_resolution, + window_questions, + qd.analysis_variables(), + qd.ensemble_num_members(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.observations(), + qd.observing_system_records_path(), + qd.comparison_log_type('ensmeanvariance'), + ] + +# -------------------------------------------------------------------------------------------------- + +class RunJediHofxEnsembleExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.background_frequency(), + qd.ensemble_hofx_packets(), + qd.ensemble_hofx_strategy(), + qd.ensemble_num_members(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.total_processors(), + qd.comparison_log_type('hofx'), + ] + +# -------------------------------------------------------------------------------------------------- + +class RunJediHofxExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.background_frequency(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.save_geovals(), + qd.total_processors(), + qd.comparison_log_type('ensemblehofx'), + ] + +# -------------------------------------------------------------------------------------------------- + +class RunJediLocalEnsembleDaExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.ensemble_hofx_packets(), + qd.ensemble_hofx_strategy(), + qd.ensemble_num_members(), + qd.ensmean_only(), + qd.ensmeanvariance_only(), + qd.generate_yaml_and_exit(), + qd.horizontal_localization_lengthscale(), + qd.horizontal_localization_max_nobs(), + qd.horizontal_localization_method(), + qd.jedi_forecast_model(), + qd.local_ensemble_inflation_mult(), + qd.local_ensemble_inflation_rtpp(), + qd.local_ensemble_inflation_rtps(), + qd.local_ensemble_save_posterior_ensemble(), + qd.local_ensemble_save_posterior_ensemble_increments(), + qd.local_ensemble_save_posterior_mean(), + qd.local_ensemble_save_posterior_mean_increment(), + qd.local_ensemble_solver(), + qd.local_ensemble_use_linear_observer(), + qd.skip_ensemble_hofx(), + qd.total_processors(), + qd.vertical_localization_apply_log_transform(), + qd.vertical_localization_function(), + qd.vertical_localization_ioda_vertical_coord(), + qd.vertical_localization_ioda_vertical_coord_group(), + qd.vertical_localization_lengthscale(), + qd.vertical_localization_method(), + qd.perhost(), + qd.comparison_log_type('localensembleda'), + ] + +# -------------------------------------------------------------------------------------------------- + +class RunJediVariationalExecutable(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {'nodes': 3} + self.questions = [ + run_jedi_executable, + qd.perhost(), + qd.comparison_log_type('variational'), + ] + +# -------------------------------------------------------------------------------------------------- + +class RemoveForecastDir(TaskSetup): + def set_attributes(self): + self.is_cycling = True + +# -------------------------------------------------------------------------------------------------- + +class RunGeosExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + +# -------------------------------------------------------------------------------------------------- + +class RunJediUfoTestsExecutable(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {'ntasks-per-node': 1} + self.questions = [ + background_crtm_obs, + qd.generate_yaml_and_exit(), + qd.single_observations(), + qd.window_length(), + qd.comparison_log_type('ufo_tests'), + ] + +# -------------------------------------------------------------------------------------------------- + +class RunJediConvertStateSoca2ciceExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {'nodes': 1} + self.questions = [ + qd.analysis_variables(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.marine_models(), + qd.observations(), + qd.total_processors(), + qd.window_length(), + qd.window_type(), + qd.comparison_log_type('convert_state_soca2cice'), + ] + +# -------------------------------------------------------------------------------------------------- + +class RunJediFgatExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + run_jedi_executable, + qd.marine_models(), + qd.comparison_log_type('fgat'), + ] + +# -------------------------------------------------------------------------------------------------- + +class SaveObsDiags(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + background_crtm_obs, + qd.r2d2_local_path(), + qd.window_length(), + qd.marine_models() + ] + +# -------------------------------------------------------------------------------------------------- + +class SaveRestart(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + window_questions, + qd.background_time_offset(), + qd.forecast_duration(), + qd.horizontal_resolution(), + qd.marine_models(), + qd.r2d2_local_path() + ] + +# -------------------------------------------------------------------------------------------------- + +class StageJedi(TaskSetup): + def set_attributes(self): + self.is_model = True + self.questions = [ + swell_static_file_questions, + qd.gsibec_configuration(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.horizontal_resolution(), + qd.vertical_resolution() + ] + +# -------------------------------------------------------------------------------------------------- + +class StageJediCycle(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.base_name = "StageJedi" + self.scheduling_name = "StageJediCycle-{model}" + self.questions = [ + swell_static_file_questions, + qd.gsibec_configuration(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.horizontal_resolution(), + qd.vertical_resolution() + ] + +# -------------------------------------------------------------------------------------------------- + +class sync_point(TaskSetup): + def set_attributes(self): + self.script = "true" + +# -------------------------------------------------------------------------------------------------- + +class JediLogComparison(TaskSetup): + def set_attributes(self): + self.is_model = True + self.questions = [ + qd.number_of_iterations(), + qd.comparison_log_type(), + ] + +# -------------------------------------------------------------------------------------------------- + +class RunJediObsfiltersExecutable(TaskSetup): + def set_attributes(self): + self.script = ("swell task RunJediObsfiltersExecutable $config" + " -d $datetime -m geos_atmosphere") + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + np_proc_resolution, + window_questions, + background_crtm_obs, + qd.background_frequency(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.observing_system_records_path(), + qd.total_processors(), + qd.obs_thinning_rej_fraction(), + qd.comparison_log_type('obsfilters') + ] + +# -------------------------------------------------------------------------------------------------- @classmethod def get(cls, name: str) -> TaskSetup: diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 7e8cdd7db..8e4b9ce34 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -14,6 +14,7 @@ import yaml from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- @@ -34,6 +35,12 @@ # -------------------------------------------------------------------------------------------------- +class BufrToIodaSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + +# -------------------------------------------------------------------------------------------------- class BufrToIoda(taskBase): diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index 614db7023..0163085c0 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -11,12 +11,21 @@ import os from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import build_and_source_dirs from swell.utilities.shell_commands import run_subprocess, create_executable_file # -------------------------------------------------------------------------------------------------- +class BuildGeosSetup(TaskSetup): + def set_attributes(self): + self.questions = [ + qd.geos_build_method() + ] + +# -------------------------------------------------------------------------------------------------- class BuildGeos(taskBase): @@ -32,10 +41,12 @@ def execute(self) -> None: geos_gcm_build_path, geos_gcm_source_path = build_and_source_dirs(geos_gcm_path) os.makedirs(geos_gcm_build_path, exist_ok=True) + geos_build_method = self.config.geos_build_method() + # Check that the choice is to create build # ---------------------------------------- - if not self.config.geos_build_method() == 'create': - self.logger.abort(f'Found \'{jedi_build_method}\' for jedi_build_method in the ' + if not geos_build_method == 'create': + self.logger.abort(f'Found \'{geos_build_method}\' for jedi_build_method in the ' f'experiment dictionary. Must be \'create\'.') # Create script that encapsulates the steps of building GEOS diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index 16fc91ee6..44c3450e4 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -11,11 +11,22 @@ import os from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import build_and_source_dirs, link_path # -------------------------------------------------------------------------------------------------- +class BuildGeosByLinkingSetup(TaskSetup): + def set_attributes(self): + self.mail_events = ['submit-failed'] + self.questions = [ + qd.existing_geos_gcm_build_path(), + qd.geos_build_method() + ] + +# -------------------------------------------------------------------------------------------------- class BuildGeosByLinking(taskBase): diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index d64bd0b62..07b0576d1 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -13,10 +13,22 @@ from jedi_bundle.bin.jedi_bundle import execute_tasks, get_bundles from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs # -------------------------------------------------------------------------------------------------- +class BuildJediSetup(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.slurm = {} + self.questions = [ + qd.bundles(), + qd.jedi_build_method() + ] + +# -------------------------------------------------------------------------------------------------- class BuildJedi(taskBase): diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index f746a70f8..1e3daa687 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -11,10 +11,21 @@ import os from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import build_and_source_dirs, link_path # -------------------------------------------------------------------------------------------------- +class BuildGeosByLinkingSetup(TaskSetup): + def set_attributes(self): + self.mail_events = ['submit-failed'] + self.questions = [ + qd.existing_geos_gcm_build_path(), + qd.geos_build_method() + ] + +# -------------------------------------------------------------------------------------------------- class BuildJediByLinking(taskBase): diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index 30221edf8..5eaeae39a 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -9,12 +9,23 @@ import os from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats from datetime import datetime as dt import glob # -------------------------------------------------------------------------------------------------- +class CleanCycleSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.clean_patterns() + ] + +# -------------------------------------------------------------------------------------------------- class CleanCycle(taskBase): diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 6190a81a0..48759ab88 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -11,12 +11,23 @@ import os from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import build_and_source_dirs, link_path from swell.utilities.git_utils import git_clone from swell.utilities.shell_commands import run_subprocess # -------------------------------------------------------------------------------------------------- +class CloneGeosSetup(TaskSetup): + def set_attributes(self): + self.questions = [ + qd.existing_geos_gcm_source_path(), + qd.geos_build_method(), + qd.geos_gcm_tag() + ] + +# -------------------------------------------------------------------------------------------------- class CloneGeos(taskBase): diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index 555b6638a..c2af75d45 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -10,10 +10,21 @@ import os from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import link_path # -------------------------------------------------------------------------------------------------- +class CloneGeosMksiSetup(TaskSetup): + def set_attributes(self): + self.is_model = True + self.questions = [ + qd.observing_system_records_mksi_path(), + qd.observing_system_records_mksi_path_tag() + ] + +# -------------------------------------------------------------------------------------------------- class CloneGeosMksi(taskBase): diff --git a/src/swell/tasks/clone_gmao_perllib.py b/src/swell/tasks/clone_gmao_perllib.py index 573876822..8a6211f07 100644 --- a/src/swell/tasks/clone_gmao_perllib.py +++ b/src/swell/tasks/clone_gmao_perllib.py @@ -12,9 +12,19 @@ import subprocess from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- +class CloneGmaoPerllibSetup(TaskSetup): + def set_attributes(self): + self.questions = [ + qd.existing_perllib_path(), + qd.gmao_perllib_tag() + ] + +# -------------------------------------------------------------------------------------------------- class CloneGmaoPerllib(taskBase): diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index 043975437..e48533cdf 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -10,21 +10,35 @@ import os -from jedi_bundle.bin.jedi_bundle import execute_tasks, get_bundles - from swell.utilities.build import link_path from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.pinned_versions.check_hashes import check_hashes from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs # -------------------------------------------------------------------------------------------------- +class CloneJedi(TaskSetup): + def set_attributes(self): + self.questions = [ + qd.bundles(), + qd.existing_jedi_source_directory(), + qd.existing_jedi_source_directory_pinned(), + qd.jedi_build_method() + ] + +# -------------------------------------------------------------------------------------------------- -class CloneJedi(taskBase): +class CloneJediSetup(taskBase): def execute(self) -> None: + # Import JEDI modules + # ------------------- + from jedi_bundle.bin.jedi_bundle import execute_tasks, get_bundles + # Get the experiment/jedi_bundle directory # ---------------------------------------- swell_exp_path = self.experiment_path() diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index b6855011e..b45c4e6d4 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -12,14 +12,25 @@ import yaml import glob -from eva.eva_driver import eva - from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.data_assimilation_window_params import DataAssimilationWindowParams # -------------------------------------------------------------------------------------------------- +class EvaComparisonIncrementSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.marine_models(), + qd.window_length(), + qd.window_type() + ] + +# -------------------------------------------------------------------------------------------------- class EvaComparisonIncrement(taskBase): @@ -37,6 +48,8 @@ def window_info_from_config(self, path: str): def execute(self) -> None: + from eva.eva_driver import eva + model = self.get_model() # Open the eva configuration file diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 51bb18757..d79674ea1 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -11,19 +11,30 @@ import os import yaml -from eva.eva_driver import eva - from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- +class EvaComparisonJediLogSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.comparison_log_type() + ] + +# -------------------------------------------------------------------------------------------------- class EvaComparisonJediLog(taskBase): def execute(self) -> None: + from eva.eva_driver import eva + # Get the model # ------------- model = self.get_model() diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index ea95837cc..e79a37034 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -11,18 +11,31 @@ import os import yaml -from eva.eva_driver import eva - from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- +class EvaIncrementSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.marine_models(), + qd.window_length(), + qd.window_type() + ] + +# -------------------------------------------------------------------------------------------------- class EvaIncrement(taskBase): def execute(self) -> None: + from eva.eva_driver import eva + # Get the model and window type # ----------------------------- model = self.get_model() diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index 5a73e2558..ece6713e5 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -14,16 +14,26 @@ from eva.eva_driver import eva from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- +class EvaJediLogSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + +# -------------------------------------------------------------------------------------------------- class EvaJediLog(taskBase): def execute(self) -> None: + from eva.eva_driver import eva + # Get the model # ------------- model = self.get_model() diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 93353d96b..6d937119f 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -12,10 +12,10 @@ import os import yaml -from eva.eva_driver import eva - from swell.deployment.platforms.platforms import login_or_compute from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.observations import ioda_name_to_long_name @@ -31,11 +31,31 @@ def run_eva(eva_dict: dict) -> eva: # -------------------------------------------------------------------------------------------------- +class EvaObservationsSetup(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {} + self.questions = [ + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.marine_models(), + qd.observing_system_records_path(), + qd.window_length(), + qd.marine_models(), + ] + +# -------------------------------------------------------------------------------------------------- class EvaObservations(taskBase): def execute(self) -> None: + from eva.eva_driver import eva + window_length = self.config.window_length() # Compute window beginning time diff --git a/src/swell/tasks/eva_timeseries.py b/src/swell/tasks/eva_timeseries.py index 20695acb6..cc16f8c6c 100644 --- a/src/swell/tasks/eva_timeseries.py +++ b/src/swell/tasks/eva_timeseries.py @@ -14,10 +14,10 @@ import os import yaml -from eva.eva_driver import eva - from swell.deployment.platforms.platforms import login_or_compute from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 @@ -33,11 +33,30 @@ def run_eva(eva_dict: dict) -> eva: # -------------------------------------------------------------------------------------------------- +class EvaTimeseriesSetup(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {} + self.questions = [ + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.window_length(), + qd.ncdiag_experiments(), + qd.marine_models(), + ] + +# -------------------------------------------------------------------------------------------------- class EvaTimeseries(taskBase): def execute(self) -> None: + from eva.eva_driver import eva + window_length = self.config.window_length() # Compute window beginning time diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index ac31fb57b..ad3145425 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -9,11 +9,47 @@ import yaml from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.shell_commands import run_track_log_subprocess from swell.utilities.file_system_operations import check_if_files_exist_in_path # -------------------------------------------------------------------------------------------------- +class GenerateBClimatologySetup(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.retry = '2*PT1M' + self.slurm = {} + self.questions = [ + qd.npx_proc(), + qd.npy_proc(), + qd.npx(), + qd.npy(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.swell_static_files(), + qd.swell_static_files_user(), + qd.analysis_variables(), + qd.background_error_model(), + qd.generate_yaml_and_exit(), + qd.gradient_norm_reduction(), + qd.gsibec_configuration(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.jedi_forecast_model(), + qd.marine_models(), + qd.minimizer(), + qd.number_of_iterations(), + qd.observing_system_records_path(), + qd.total_processors(), + qd.window_length(), + qd.window_type() + ] + +# -------------------------------------------------------------------------------------------------- class GenerateBClimatology(taskBase): diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 11ffe21c1..bb52802f4 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -8,13 +8,30 @@ import os from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import link_all_files_from_first_in_hierarchy_of_sources # -------------------------------------------------------------------------------------------------- +class GenerateBClimatologyByLinking(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.swell_static_files(), + qd.swell_static_files_user(), + qd.background_error_model(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.window_length(), + qd.window_type() + ] -class GenerateBClimatologyByLinking(taskBase): +# -------------------------------------------------------------------------------------------------- + +class GenerateBClimatologyByLinkingSetup(taskBase): def execute(self) -> None: """Acquires B Matrix files for background error model(s): diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index bf8d71fdc..0a7ad251c 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -11,10 +11,23 @@ import os from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.observing_system_records import ObservingSystemRecords # -------------------------------------------------------------------------------------------------- +class GenerateObservingSystemRecordsSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.observations(), + qd.observing_system_records_mksi_path(), + qd.observing_system_records_path() + ] + +# -------------------------------------------------------------------------------------------------- class GenerateObservingSystemRecords(taskBase): diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 3090a21c1..efd33e39f 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -9,7 +9,8 @@ from swell.tasks.base.task_base import taskBase -from swell.utilities.r2d2 import create_r2d2_config +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd import isodate import os @@ -25,7 +26,24 @@ # -------------------------------------------------------------------------------------------------- -class GetBackground(taskBase): +class GetBackground(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.window_length(), + qd.window_type(), + qd.window_length(), + qd.background_experiment(), + qd.background_frequency(), + qd.horizontal_resolution(), + qd.marine_models(), + qd.r2d2_local_path(), + ] + +# -------------------------------------------------------------------------------------------------- + +class GetBackgroundSetup(taskBase): def execute(self) -> None: """Acquires background files for a given experiment and cycle @@ -36,6 +54,8 @@ def execute(self) -> None: See the taskBase constructor for more information. """ + from swell.utilities.r2d2 import create_r2d2_config + # Get duration into forecast for first background file # ---------------------------------------------------- bkg_steps = [] diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index 778a0406c..4c5cffbf3 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -14,10 +14,25 @@ import tarfile from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats # -------------------------------------------------------------------------------------------------- +class GetBackgroundGeosExperimentSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.mail_events = ['submit-failed'] + self.questions = [ + qd.horizontal_resolution(), + qd.background_experiment(), + qd.background_time_offset(), + qd.geos_x_background_directory() + ] + +# -------------------------------------------------------------------------------------------------- class GetBackgroundGeosExperiment(taskBase): diff --git a/src/swell/tasks/get_bufr.py b/src/swell/tasks/get_bufr.py index bef9fbedd..534f4c1a5 100644 --- a/src/swell/tasks/get_bufr.py +++ b/src/swell/tasks/get_bufr.py @@ -14,9 +14,20 @@ from datetime import datetime as dt from swell.utilities.datetime_util import datetime_formats from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- +class GetBufrSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.bufr_obs_classes() + ] + +# -------------------------------------------------------------------------------------------------- class GetBufr(taskBase): ''' diff --git a/src/swell/tasks/get_ensemble.py b/src/swell/tasks/get_ensemble.py index 08b77b97b..446c58b69 100644 --- a/src/swell/tasks/get_ensemble.py +++ b/src/swell/tasks/get_ensemble.py @@ -12,12 +12,14 @@ import os from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- -class GetEnsemble(taskBase): +class GetEnsembleSetup(taskBase): def execute(self) -> None: """Acquires ensemble member files for a given experiment and cycle diff --git a/src/swell/tasks/get_ensemble_geos_experiment.py b/src/swell/tasks/get_ensemble_geos_experiment.py index 11d69feab..a005e69d2 100644 --- a/src/swell/tasks/get_ensemble_geos_experiment.py +++ b/src/swell/tasks/get_ensemble_geos_experiment.py @@ -13,10 +13,23 @@ import tarfile from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats # -------------------------------------------------------------------------------------------------- +class GetEnsembleGeosExperimentSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.background_experiment(), + qd.background_time_offset(), + qd.geos_x_ensemble_directory() + ] + +# -------------------------------------------------------------------------------------------------- class GetEnsembleGeosExperiment(taskBase): diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py index 354d53515..48f33f4d8 100644 --- a/src/swell/tasks/get_geos_adas_background.py +++ b/src/swell/tasks/get_geos_adas_background.py @@ -14,10 +14,21 @@ import re from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- +class GetGeosAdasBackgroundSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.path_to_geos_adas_background() + ] + +# -------------------------------------------------------------------------------------------------- class GetGeosAdasBackground(taskBase): diff --git a/src/swell/tasks/get_geos_restart.py b/src/swell/tasks/get_geos_restart.py index db9175636..2a62903dd 100644 --- a/src/swell/tasks/get_geos_restart.py +++ b/src/swell/tasks/get_geos_restart.py @@ -11,10 +11,22 @@ import glob from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import copy_to_dst_dir, check_if_files_exist_in_path # -------------------------------------------------------------------------------------------------- +class GetGeosRestartSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.questions = [ + qd.swell_static_files(), + qd.swell_static_files_user(), + qd.geos_restarts_directory() + ] + +# -------------------------------------------------------------------------------------------------- class GetGeosRestart(taskBase): diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index a11b46221..7e11cdb09 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -11,16 +11,36 @@ import os from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.r2d2 import create_r2d2_config -from r2d2 import fetch +# -------------------------------------------------------------------------------------------------- + +class GetGeovalsSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.geovals_experiment(), + qd.geovals_provider(), + qd.r2d2_local_path(), + qd.window_length(), + ] + # -------------------------------------------------------------------------------------------------- class GetGeovals(taskBase): def execute(self) -> None: + from r2d2 import fetch + # Parse config # ------------ geovals_experiment = self.config.geovals_experiment() diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index 9ca84ae20..47d144ebd 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -15,10 +15,22 @@ import tarfile from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- +class GetGsiBcSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.path_to_gsi_bc_coefficients(), + qd.window_length() + ] + +# -------------------------------------------------------------------------------------------------- class GetGsiBc(taskBase): diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index 3567e9581..c8463ce0c 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -12,10 +12,21 @@ import os from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- +class GetGsiNcdiagSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.path_to_gsi_nc_diags() + ] + +# -------------------------------------------------------------------------------------------------- class GetGsiNcdiag(taskBase): diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index 7178518a1..abf64edec 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -9,11 +9,27 @@ import os from swell.tasks.base.task_base import taskBase -from r2d2 import fetch -from swell.utilities.r2d2 import create_r2d2_config +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- +class GetNcdiagsSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.ncdiag_experiments(), + qd.marine_models(), + qd.r2d2_local_path(), + qd.window_length(), + ] + +# -------------------------------------------------------------------------------------------------- class GetNcdiags(taskBase): @@ -23,6 +39,11 @@ class GetNcdiags(taskBase): def execute(self) -> None: + # Import modules + # -------------- + from r2d2 import fetch + from swell.utilities.r2d2 import create_r2d2_config + # Parse config # ------------ ncdiag_experiments = self.config.ncdiag_experiments() diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py index 6ad2e21e2..339be0e60 100644 --- a/src/swell/tasks/get_obs_not_in_r2d2.py +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -13,10 +13,22 @@ import subprocess from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- +class GetObsNotInR2d2Setup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.mail_events = ['submit-failed'] + self.questions = [ + qd.ioda_locations_not_in_r2d2(), + ] + +# -------------------------------------------------------------------------------------------------- class GetObsNotInR2d2(taskBase): diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index e7a95d05c..93fde6626 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -10,12 +10,12 @@ import isodate import numpy as np import os -import r2d2 -import netCDF4 as nc from typing import Union from datetime import timedelta, datetime as dt from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.r2d2 import create_r2d2_config from swell.utilities.datetime_util import datetime_formats from swell.utilities.observations import get_ioda_names_list, get_provider_for_observation @@ -30,6 +30,23 @@ # -------------------------------------------------------------------------------------------------- +class GetObservationsSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.cycling_varbc(), + qd.obs_experiment(), + qd.observing_system_records_path(), + qd.r2d2_local_path(), + qd.window_length(), + ] + +# -------------------------------------------------------------------------------------------------- class GetObservations(taskBase): @@ -97,6 +114,11 @@ def execute(self) -> None: "tlapse" files need to be fetched. """ + # Import modules + # -------------- + import r2d2 + import netCDF4 as nc + # Parse config # ------------ obs_experiment = self.config.obs_experiment() diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index e86e61921..09d8e2037 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -13,12 +13,24 @@ from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.dictionary import write_dict_to_yaml from swell.utilities.shell_commands import run_track_log_subprocess # -------------------------------------------------------------------------------------------------- +class GetGsiBcSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.path_to_gsi_bc_coefficients(), + qd.window_length() + ] + +# -------------------------------------------------------------------------------------------------- class GsiBcToIoda(taskBase): diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 18d296cda..eb4439b75 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -14,22 +14,36 @@ import os import re -# Ioda converters -import pyiodaconv.gsi_ncdiag as gsid -from pyiodaconv.combine_obsspace import combine_obsspace - from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.shell_commands import run_subprocess, create_executable_file # -------------------------------------------------------------------------------------------------- +class GsiNcdiagToIodaSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.observations(), + qd.produce_geovals(), + qd.single_observations(), + qd.window_length() + ] + +# -------------------------------------------------------------------------------------------------- class GsiNcdiagToIoda(taskBase): def execute(self) -> None: + # Ioda converters + import pyiodaconv.gsi_ncdiag as gsid + from pyiodaconv.combine_obsspace import combine_obsspace + # Parse configuration # ------------------- observations = self.config.observations() diff --git a/src/swell/tasks/jedi_c_test.py b/src/swell/tasks/jedi_c_test.py deleted file mode 100644 index 8c115c519..000000000 --- a/src/swell/tasks/jedi_c_test.py +++ /dev/null @@ -1,52 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import os -import subprocess - -from swell.tasks.base.task_base import taskBase - -# -------------------------------------------------------------------------------------------------- - - -class JediCTest(taskBase): - - def execute(self) -> None: - - # Locate the soca directory - build_dir = self.config.existing_jedi_build_directory() - soca_dir = os.path.join(build_dir, 'soca') - - # Run the ctests - cwd = os.getcwd() - os.chdir(soca_dir) - command = ['ctest', '-V', '-I'] - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - # Record the output - results, error = process.communicate() - os.chdir(cwd) - - # Get the output file name - out_name = f'jedi_ctest_results' - if self.test_iteration is not None: - out_name += f'-{self.test_iteration}' - out_name += '.txt' - - # Make the output directory - out_path = os.path.join(self.test_output, 'comparison_tests') - os.makedirs(out_path, exist_ok=True) - - # Write the results - with open(os.path.join(out_path, out_name), 'w') as f: - f.write(f'JEDI CTest results for build located at: {soca_dir}\n') - f.write(results.decode('utf-8')) - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py index ca8e8ea72..d6b6cefdf 100644 --- a/src/swell/tasks/jedi_log_comparison.py +++ b/src/swell/tasks/jedi_log_comparison.py @@ -14,6 +14,8 @@ import numpy as np from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- @@ -21,6 +23,15 @@ # -------------------------------------------------------------------------------------------------- +class JediLogComparisonSetup(TaskSetup): + def set_attributes(self): + self.is_model = True + self.questions = [ + qd.number_of_iterations(), + qd.comparison_log_type(), + ] + +# -------------------------------------------------------------------------------------------------- class JediLogComparison(taskBase): diff --git a/src/swell/tasks/jedi_oops_log_parser.py b/src/swell/tasks/jedi_oops_log_parser.py index 543f77dda..c19fbbbe5 100644 --- a/src/swell/tasks/jedi_oops_log_parser.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -12,9 +12,21 @@ import subprocess from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- +class JediOopsLogParserSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.parser_options(), + qd.comparison_log_type() + ] + +# -------------------------------------------------------------------------------------------------- class JediOopsLogParser(taskBase): diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index f2853728a..21f68ad2c 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -12,14 +12,27 @@ import os from netCDF4 import Dataset import numpy as np -import xarray as xr from typing import Tuple from swell.utilities.datetime_util import datetime_formats from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- +class LinkGeosOutputSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.window_length(), + qd.window_type(), + qd.background_frequency(), + qd.marine_models() + ] + +# -------------------------------------------------------------------------------------------------- class LinkGeosOutput(taskBase): @@ -33,6 +46,8 @@ def execute(self) -> None: type (history vs. restart), DA method, and window length. """ + import xarray as xr + # Parse configuration # ------------------- self.marine_models = self.config.marine_models(None) or [] diff --git a/src/swell/tasks/move_da_restart.py b/src/swell/tasks/move_da_restart.py index 31851a6ef..ba446aca2 100644 --- a/src/swell/tasks/move_da_restart.py +++ b/src/swell/tasks/move_da_restart.py @@ -13,10 +13,22 @@ from typing import Union from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import move_files # -------------------------------------------------------------------------------------------------- +class MoveDaRestartSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.mom6_iau(), + qd.window_length() + ] + +# -------------------------------------------------------------------------------------------------- class MoveDaRestart(taskBase): diff --git a/src/swell/tasks/move_forecast_restart.py b/src/swell/tasks/move_forecast_restart.py index d4e297539..bc489c8f4 100644 --- a/src/swell/tasks/move_forecast_restart.py +++ b/src/swell/tasks/move_forecast_restart.py @@ -12,10 +12,20 @@ from typing import Union from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import move_files # -------------------------------------------------------------------------------------------------- +class MoveForecastRestartSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.questions = [ + qd.forecast_duration() + ] + +# -------------------------------------------------------------------------------------------------- class MoveForecastRestart(taskBase): diff --git a/src/swell/tasks/prep_geos_run_dir.py b/src/swell/tasks/prep_geos_run_dir.py index 9183f20ce..033538a29 100644 --- a/src/swell/tasks/prep_geos_run_dir.py +++ b/src/swell/tasks/prep_geos_run_dir.py @@ -15,10 +15,25 @@ from datetime import datetime as dt from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import copy_to_dst_dir, check_if_files_exist_in_path # -------------------------------------------------------------------------------------------------- +class PrepGeosRunDirSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.questions = [ + qd.swell_static_files(), + qd.swell_static_files_user(), + qd.existing_geos_gcm_build_path(), + qd.forecast_duration(), + qd.geos_experiment_directory(), + qd.mom6_iau_nhours() + ] + +# -------------------------------------------------------------------------------------------------- class PrepGeosRunDir(taskBase): diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index 83c56861b..0a2095cc9 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -15,9 +15,23 @@ from swell.utilities.shell_commands import run_subprocess from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- +class PrepareAnalysisSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.analysis_variables(), + qd.mom6_iau(), + qd.total_processors(), + qd.window_length(), + ] + +# -------------------------------------------------------------------------------------------------- class PrepareAnalysis(taskBase): diff --git a/src/swell/tasks/remove_forecast_dir.py b/src/swell/tasks/remove_forecast_dir.py index a527e680d..b96d1579e 100644 --- a/src/swell/tasks/remove_forecast_dir.py +++ b/src/swell/tasks/remove_forecast_dir.py @@ -10,10 +10,16 @@ import shutil from swell.tasks.base.task_base import taskBase - +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- +class RemoveForecastDirSetup(TaskSetup): + def set_attributes(self): + self.is_cycling = True + +# -------------------------------------------------------------------------------------------------- class RemoveForecastDir(taskBase): diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index f5b8dffe3..8a7deadb9 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -12,10 +12,27 @@ from ruamel.yaml import YAML from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import check_obs # -------------------------------------------------------------------------------------------------- +class RenderJediObservations(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.check_for_obs(), + qd.crtm_coeff_dir(), + qd.background_time_offset(), + qd.observing_system_records_path(), + qd.observations(), + qd.set_obs_as_local(), + qd.window_length() + ] + +# -------------------------------------------------------------------------------------------------- class RenderJediObservations(taskBase): diff --git a/src/swell/tasks/run_geos_executable.py b/src/swell/tasks/run_geos_executable.py index 88489986f..d799b462d 100644 --- a/src/swell/tasks/run_geos_executable.py +++ b/src/swell/tasks/run_geos_executable.py @@ -11,10 +11,17 @@ from typing import Optional from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.shell_commands import run_track_log_subprocess # -------------------------------------------------------------------------------------------------- +class RunGeosExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + +# -------------------------------------------------------------------------------------------------- class RunGeosExecutable(taskBase): diff --git a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py index 2a4c1234e..40d1b6e16 100644 --- a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py +++ b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py @@ -12,11 +12,32 @@ from ruamel.yaml import YAML from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- +class RunJediConvertStateSoca2ciceExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {'nodes': 1} + self.questions = [ + qd.analysis_variables(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.marine_models(), + qd.observations(), + qd.total_processors(), + qd.window_length(), + qd.window_type(), + qd.comparison_log_type('convert_state_soca2cice'), + ] + +# -------------------------------------------------------------------------------------------------- class RunJediConvertStateSoca2ciceExecutable(taskBase): diff --git a/src/swell/tasks/run_jedi_ensemble_mean_variance.py b/src/swell/tasks/run_jedi_ensemble_mean_variance.py index 1f8be3705..ca179abef 100644 --- a/src/swell/tasks/run_jedi_ensemble_mean_variance.py +++ b/src/swell/tasks/run_jedi_ensemble_mean_variance.py @@ -12,11 +12,38 @@ from ruamel.yaml import YAML from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- +class RunJediEnsembleMeanVariance(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + qd.npx_proc(), + qd.npy_proc(), + qd.npx(), + qd.npy(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.window_length(), + qd.window_type(), + qd.analysis_variables(), + qd.ensemble_num_members(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.observations(), + qd.observing_system_records_path(), + qd.comparison_log_type('ensmeanvariance'), + ] + +# -------------------------------------------------------------------------------------------------- class RunJediEnsembleMeanVariance(taskBase): diff --git a/src/swell/tasks/run_jedi_fgat_executable.py b/src/swell/tasks/run_jedi_fgat_executable.py index e15222b65..8dd815ea7 100644 --- a/src/swell/tasks/run_jedi_fgat_executable.py +++ b/src/swell/tasks/run_jedi_fgat_executable.py @@ -11,11 +11,48 @@ from ruamel.yaml import YAML from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- +class RunJediFgatExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.npx_proc(), + qd.npy_proc(), + qd.npx(), + qd.npy(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.window_length(), + qd.window_type(), + qd.analysis_variables(), + qd.background_frequency(), + qd.generate_yaml_and_exit(), + qd.gradient_norm_reduction(), + qd.gsibec_configuration(), + qd.jedi_forecast_model(), + qd.minimizer(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.number_of_iterations(), + qd.total_processors(), + qd.marine_models(), + qd.comparison_log_type('fgat'), + ] + +# -------------------------------------------------------------------------------------------------- class RunJediFgatExecutable(taskBase): diff --git a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py index fac823e17..fb9d871fc 100644 --- a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py +++ b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py @@ -12,11 +12,43 @@ from ruamel.yaml import YAML from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable from swell.tasks.run_jedi_hofx_executable import RunJediHofxExecutable # -------------------------------------------------------------------------------------------------- +class RunJediHofxEnsembleExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + qd.npx_proc(), + qd.npy_proc(), + qd.npx(), + qd.npy(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.window_length(), + qd.window_type(), + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.background_frequency(), + qd.ensemble_hofx_packets(), + qd.ensemble_hofx_strategy(), + qd.ensemble_num_members(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.total_processors(), + qd.comparison_log_type('hofx'), + ] + +# -------------------------------------------------------------------------------------------------- class RunJediHofxEnsembleExecutable(RunJediHofxExecutable, taskBase): diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index d0e7808d6..7dbd74408 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -14,12 +14,42 @@ from typing import Optional from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.netcdf_files import combine_files_without_groups from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- +class RunJediHofxExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + qd.npx_proc(), + qd.npy_proc(), + qd.npx(), + qd.npy(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.window_length(), + qd.window_type(), + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.background_frequency(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.save_geovals(), + qd.total_processors(), + qd.comparison_log_type('ensemblehofx'), + ] + +# -------------------------------------------------------------------------------------------------- class RunJediHofxExecutable(taskBase): diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index fb71a5d1a..08644bab7 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -13,6 +13,8 @@ from swell.swell_path import get_swell_path from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- @@ -33,6 +35,59 @@ def replace_key(obj, old_key, new_key): else: return obj +# -------------------------------------------------------------------------------------------------- + +class RunJediLocalEnsembleDaExecutable(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + qd.npx_proc(), + qd.npy_proc(), + qd.npx(), + qd.npy(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.window_length(), + qd.window_type(), + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.ensemble_hofx_packets(), + qd.ensemble_hofx_strategy(), + qd.ensemble_num_members(), + qd.ensmean_only(), + qd.ensmeanvariance_only(), + qd.generate_yaml_and_exit(), + qd.horizontal_localization_lengthscale(), + qd.horizontal_localization_max_nobs(), + qd.horizontal_localization_method(), + qd.jedi_forecast_model(), + qd.local_ensemble_inflation_mult(), + qd.local_ensemble_inflation_rtpp(), + qd.local_ensemble_inflation_rtps(), + qd.local_ensemble_save_posterior_ensemble(), + qd.local_ensemble_save_posterior_ensemble_increments(), + qd.local_ensemble_save_posterior_mean(), + qd.local_ensemble_save_posterior_mean_increment(), + qd.local_ensemble_solver(), + qd.local_ensemble_use_linear_observer(), + qd.skip_ensemble_hofx(), + qd.total_processors(), + qd.vertical_localization_apply_log_transform(), + qd.vertical_localization_function(), + qd.vertical_localization_ioda_vertical_coord(), + qd.vertical_localization_ioda_vertical_coord_group(), + qd.vertical_localization_lengthscale(), + qd.vertical_localization_method(), + qd.perhost(), + qd.comparison_log_type('localensembleda'), + ] + +# -------------------------------------------------------------------------------------------------- class RunJediLocalEnsembleDaExecutable(taskBase): diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index a76b6ba8f..025312708 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -13,10 +13,43 @@ from typing import Optional import random from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- +class RunJediObsfiltersExecutable(TaskSetup): + def set_attributes(self): + self.script = ("swell task RunJediObsfiltersExecutable $config" + " -d $datetime -m geos_atmosphere") + self.is_cycling = True + self.is_model = True + self.time_limit = True + self.slurm = {} + self.questions = [ + qd.npx_proc(), + qd.npy_proc(), + qd.npx(), + qd.npy(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.window_length(), + qd.window_type(), + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.background_frequency(), + qd.generate_yaml_and_exit(), + qd.jedi_forecast_model(), + qd.observing_system_records_path(), + qd.total_processors(), + qd.obs_thinning_rej_fraction(), + qd.comparison_log_type('obsfilters') + ] + +# -------------------------------------------------------------------------------------------------- class RunJediObsfiltersExecutable(taskBase): diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index a330e72fe..962def625 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -13,12 +13,32 @@ from ruamel.yaml import YAML from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.dictionary import update_dict from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- +class RunJediUfoTestsExecutable(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {'ntasks-per-node': 1} + self.questions = [ + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.generate_yaml_and_exit(), + qd.single_observations(), + qd.window_length(), + qd.comparison_log_type('ufo_tests'), + ] + +# -------------------------------------------------------------------------------------------------- class RunJediUfoTestsExecutable(taskBase): diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index e9d5d70df..ee1830dbc 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -11,11 +11,48 @@ from ruamel.yaml import YAML from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- +class RunJediVariationalExecutable(TaskSetup): + def set_attributes(self): + self.time_limit = True + self.is_cycling = True + self.is_model = True + self.slurm = {'nodes': 3} + self.questions = [ + qd.npx_proc(), + qd.npy_proc(), + qd.npx(), + qd.npy(), + qd.horizontal_resolution(), + qd.vertical_resolution(), + qd.window_length(), + qd.window_type(), + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.analysis_variables(), + qd.background_frequency(), + qd.generate_yaml_and_exit(), + qd.gradient_norm_reduction(), + qd.gsibec_configuration(), + qd.jedi_forecast_model(), + qd.minimizer(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.number_of_iterations(), + qd.total_processors(), + qd.perhost(), + qd.comparison_log_type('variational'), + ] + +# -------------------------------------------------------------------------------------------------- class RunJediVariationalExecutable(taskBase): diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 0a4ee4f29..fb4b1b5b8 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -8,13 +8,30 @@ # -------------------------------------------------------------------------------------------------- import os -import r2d2 + from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.r2d2 import create_r2d2_config from swell.utilities.run_jedi_executables import check_obs # -------------------------------------------------------------------------------------------------- +class SaveObsDiags(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), + qd.r2d2_local_path(), + qd.window_length(), + qd.marine_models() + ] + +# -------------------------------------------------------------------------------------------------- class SaveObsDiags(taskBase): @@ -24,6 +41,8 @@ class SaveObsDiags(taskBase): def execute(self) -> None: + import r2d2 + # Parse config # ------------ background_time_offset = self.config.background_time_offset() diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index c251cebe1..69296d752 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -13,12 +13,28 @@ from r2d2 import store from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.file_system_operations import copy_to_dst_dir -from swell.utilities.r2d2 import create_r2d2_config # -------------------------------------------------------------------------------------------------- +class SaveRestart(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.window_length(), + qd.window_type(), + qd.background_time_offset(), + qd.forecast_duration(), + qd.horizontal_resolution(), + qd.marine_models(), + qd.r2d2_local_path() + ] + +# -------------------------------------------------------------------------------------------------- class SaveRestart(taskBase): @@ -30,6 +46,8 @@ def execute(self): Does not handle 4d backgrounds properly """ + from swell.utilities.r2d2 import create_r2d2_config + # Parse config window_type = self.config.window_type() window_length = self.config.window_length() diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index a9baa19ab..2281e1ca3 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -12,6 +12,8 @@ from swell.swell_path import get_swell_path from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.filehandler import get_file_handler from swell.utilities.exceptions import SwellError from swell.utilities.file_system_operations import check_if_files_exist_in_path @@ -19,6 +21,20 @@ # -------------------------------------------------------------------------------------------------- +class StageJedi(TaskSetup): + def set_attributes(self): + self.is_model = True + self.questions = [ + qd.swell_static_files(), + qd.swell_static_files_user(), + qd.gsibec_configuration(), + qd.gsibec_nlats(), + qd.gsibec_nlons(), + qd.horizontal_resolution(), + qd.vertical_resolution() + ] + +# -------------------------------------------------------------------------------------------------- class StageJedi(taskBase): diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index 43d9095e4..7004c019d 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -11,16 +11,30 @@ from datetime import datetime as dt import isodate import os -from r2d2 import store from swell.tasks.base.task_base import taskBase +from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats -from swell.utilities.r2d2 import create_r2d2_config # -------------------------------------------------------------------------------------------------- +class StoreBackground(TaskSetup): + def set_attributes(self): + self.is_cycling = True + self.is_model = True + self.questions = [ + qd.window_length(), + qd.window_type(), + qd.background_experiment(), + qd.background_frequency(), + qd.horizontal_resolution(), + qd.r2d2_local_path(), + ] + +# -------------------------------------------------------------------------------------------------- class StoreBackground(taskBase): @@ -34,6 +48,9 @@ def execute(self) -> None: See the taskBase constructor for more information. """ + from r2d2 import store + from swell.utilities.r2d2 import create_r2d2_config + # Current cycle time object # ------------------------- current_cycle_dto = dt.strptime(self.cycle_time(), datetime_formats['iso_format']) From a6cacfa7557e829b8c86eed730df019329c92284 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 16 Jan 2026 12:03:30 -0500 Subject: [PATCH 163/299] Refactor --- src/swell/tasks/base/task_attributes.py | 884 +----------------- src/swell/tasks/bufr_to_ioda.py | 4 +- src/swell/tasks/build_geos.py | 4 +- src/swell/tasks/build_geos_by_linking.py | 4 +- src/swell/tasks/build_jedi.py | 4 +- src/swell/tasks/build_jedi_by_linking.py | 4 +- src/swell/tasks/clean_cycle.py | 2 + src/swell/tasks/clone_geos.py | 4 +- src/swell/tasks/clone_geos_mksi.py | 4 +- src/swell/tasks/clone_gmao_perllib.py | 4 +- src/swell/tasks/clone_jedi.py | 6 +- src/swell/tasks/eva_comparison_increment.py | 4 +- src/swell/tasks/eva_comparison_jedi_log.py | 4 +- src/swell/tasks/eva_increment.py | 4 +- src/swell/tasks/eva_jedi_log.py | 4 +- src/swell/tasks/eva_observations.py | 4 +- src/swell/tasks/eva_timeseries.py | 4 +- src/swell/tasks/generate_b_climatology.py | 4 +- .../generate_b_climatology_by_linking.py | 6 +- .../generate_observing_system_records.py | 4 +- src/swell/tasks/get_background.py | 6 +- .../tasks/get_background_geos_experiment.py | 4 +- src/swell/tasks/get_bufr.py | 4 +- src/swell/tasks/get_ensemble.py | 2 +- .../tasks/get_ensemble_geos_experiment.py | 4 +- src/swell/tasks/get_geos_adas_background.py | 4 +- src/swell/tasks/get_geos_restart.py | 4 +- src/swell/tasks/get_geovals.py | 4 +- src/swell/tasks/get_gsi_bc.py | 4 +- src/swell/tasks/get_gsi_ncdiag.py | 4 +- src/swell/tasks/get_ncdiags.py | 4 +- src/swell/tasks/get_obs_not_in_r2d2.py | 4 +- src/swell/tasks/get_observations.py | 4 +- src/swell/tasks/gsi_bc_to_ioda.py | 4 +- src/swell/tasks/gsi_ncdiag_to_ioda.py | 4 +- src/swell/tasks/jedi_log_comparison.py | 4 +- src/swell/tasks/jedi_oops_log_parser.py | 4 +- src/swell/tasks/link_geos_output.py | 4 +- src/swell/tasks/move_da_restart.py | 4 +- src/swell/tasks/move_forecast_restart.py | 4 +- src/swell/tasks/prep_geos_run_dir.py | 4 +- src/swell/tasks/prepare_analysis.py | 4 +- src/swell/tasks/remove_forecast_dir.py | 4 +- src/swell/tasks/render_jedi_observations.py | 4 +- src/swell/tasks/run_geos_executable.py | 4 +- ...jedi_convert_state_soca2cice_executable.py | 5 +- .../tasks/run_jedi_ensemble_mean_variance.py | 4 +- src/swell/tasks/run_jedi_fgat_executable.py | 4 +- .../run_jedi_hofx_ensemble_executable.py | 4 +- src/swell/tasks/run_jedi_hofx_executable.py | 4 +- .../run_jedi_local_ensemble_da_executable.py | 4 +- .../tasks/run_jedi_obsfilters_executable.py | 4 +- .../tasks/run_jedi_ufo_tests_executable.py | 4 +- .../tasks/run_jedi_variational_executable.py | 4 +- src/swell/tasks/save_obs_diags.py | 4 +- src/swell/tasks/save_restart.py | 4 +- src/swell/tasks/stage_jedi.py | 4 +- src/swell/tasks/store_background.py | 4 +- 58 files changed, 192 insertions(+), 923 deletions(-) diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index d8d69e102..194ac4260 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -7,879 +7,37 @@ # -------------------------------------------------------------------------------------------------- -from swell.tasks.base.task_setup import TaskSetup -from swell.utilities.swell_questions import QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import os +import glob +import importlib - qd.npx_proc(), - qd.npy_proc(), - qd.npx(), - qd.npy(), - qd.horizontal_resolution(), - qd.vertical_resolution(), - qd.window_length(), - qd.window_type(), - qd.background_time_offset(), - qd.crtm_coeff_dir(), - qd.observations(), - qd.observing_system_records_path(), +from swell.swell_path import get_swell_path +from swell.utilities.case_switching import snake_case_to_camel_case # -------------------------------------------------------------------------------------------------- -background_crtm_obs = QuestionList( - list_name="background_crtm_obs", - questions=[ - qd.background_time_offset(), - qd.crtm_coeff_dir(), - qd.observations(), - qd.observing_system_records_path(), - ] -) - -# -------------------------------------------------------------------------------------------------- - -np_proc_resolution = QuestionList( - list_name="np_resolution", - questions=[ - qd.npx_proc(), - qd.npy_proc(), - qd.npx(), - qd.npy(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ] -) - -# -------------------------------------------------------------------------------------------------- - -window_questions = QuestionList( - list_name="window_questions", - questions=[ - qd.window_length(), - qd.window_type() - ] -) - -# -------------------------------------------------------------------------------------------------- - -run_jedi_executable = QuestionList( - list_name="run_jedi_executable", - questions=[ - background_crtm_obs, - np_proc_resolution, - window_questions, - qd.analysis_variables(), - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.gradient_norm_reduction(), - qd.gsibec_configuration(), - qd.jedi_forecast_model(), - qd.minimizer(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.number_of_iterations(), - qd.total_processors(), - ] -) - -# -------------------------------------------------------------------------------------------------- - -swell_static_file_questions = QuestionList( - list_name="swell_static_file_questions", - questions=[ - qd.swell_static_files(), - qd.swell_static_files_user() - ] -) - -# -------------------------------------------------------------------------------------------------- - - class TaskAttributes(): + def __init__(self) -> None: + task_path = os.path.join(get_swell_path(), 'tasks', '*.py') -# -------------------------------------------------------------------------------------------------- - -class root(TaskSetup): - - def set_attributes(self): - self.script = False - self.pre_script = "source $CYLC_SUITE_DEF_PATH/modules" - self.additional_sections = [self.create_new_section('environment', - {'datetime': '$CYLC_TASK_CYCLE_POINT', - 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'})] # noqa - -# -------------------------------------------------------------------------------------------------- - -class BuildGeos(TaskSetup): - def set_attributes(self): - self.questions = [ - qd.geos_build_method() - ] - -# -------------------------------------------------------------------------------------------------- - -class BuildGeosByLinking(TaskSetup): - def set_attributes(self): - self.mail_events = ['submit-failed'] - self.questions = [ - qd.existing_geos_gcm_build_path(), - qd.geos_build_method() - ] - -# -------------------------------------------------------------------------------------------------- - -class BuildJediByLinking(TaskSetup): - def set_attributes(self): - self.mail_events = ['submit-failed'] - self.questions = [ - qd.existing_jedi_build_directory(), - qd.existing_jedi_build_directory_pinned(), - qd.jedi_build_method() - ] - -# -------------------------------------------------------------------------------------------------- - -class BuildJedi(TaskSetup): - def set_attributes(self): - self.time_limit = True - self.slurm = {} - self.questions = [ - qd.bundles(), - qd.jedi_build_method() - ] - -# -------------------------------------------------------------------------------------------------- - -class CleanCycle(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.clean_patterns() - ] - -# -------------------------------------------------------------------------------------------------- - -class CloneGeos(TaskSetup): - def set_attributes(self): - self.questions = [ - qd.existing_geos_gcm_source_path(), - qd.geos_build_method(), - qd.geos_gcm_tag() - ] - -# -------------------------------------------------------------------------------------------------- - -class CloneJedi(TaskSetup): - def set_attributes(self): - self.questions = [ - qd.bundles(), - qd.existing_jedi_source_directory(), - qd.existing_jedi_source_directory_pinned(), - qd.jedi_build_method() - ] - -# -------------------------------------------------------------------------------------------------- - -class CloneGeosMksi(TaskSetup): - def set_attributes(self): - self.is_model = True - self.questions = [ - qd.observing_system_records_mksi_path(), - qd.observing_system_records_mksi_path_tag() - ] - -# -------------------------------------------------------------------------------------------------- - -class CloneGmaoPerllib(TaskSetup): - def set_attributes(self): - self.questions = [ - qd.existing_perllib_path(), - qd.gmao_perllib_tag() - ] - -# -------------------------------------------------------------------------------------------------- - -class EvaJediLog(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - -# -------------------------------------------------------------------------------------------------- - -class EvaComparisonIncrement(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.marine_models(), - qd.window_length(), - qd.window_type() - ] - -# -------------------------------------------------------------------------------------------------- - -class EvaComparisonJediLog(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.comparison_log_type() - ] - -# -------------------------------------------------------------------------------------------------- - -class EvaIncrement(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.marine_models(), - qd.window_length(), - qd.window_type() - ] - -# -------------------------------------------------------------------------------------------------- - -class EvaObservations(TaskSetup): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.slurm = {} - self.questions = [ - background_crtm_obs, - qd.marine_models(), - qd.observing_system_records_path(), - qd.window_length(), - qd.marine_models(), - ] - -# -------------------------------------------------------------------------------------------------- - -class EvaTimeseries(TaskSetup): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.slurm = {} - self.questions = [ - background_crtm_obs, - qd.window_length(), - qd.ncdiag_experiments(), - qd.marine_models(), - ] - -# -------------------------------------------------------------------------------------------------- - -class JediOopsLogParser(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.parser_options(), - qd.comparison_log_type() - ] - -# -------------------------------------------------------------------------------------------------- + self.task_map = {} -class GetBackground(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - window_questions, - qd.window_length(), - qd.background_experiment(), - qd.background_frequency(), - qd.horizontal_resolution(), - qd.marine_models(), - qd.r2d2_local_path(), - ] + for task_file in glob.glob(task_path): + module_name = os.path.basename(task_file).split('.py')[0] + module_path = f'swell.tasks.{module_name}' -# -------------------------------------------------------------------------------------------------- + try: + module = importlib.import_module(module_path) + setup = getattr(module, 'Setup') -class GetBackgroundGeosExperiment(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.mail_events = ['submit-failed'] - self.questions = [ - qd.horizontal_resolution(), - qd.background_experiment(), - qd.background_time_offset(), - qd.geos_x_background_directory() - ] + task_name = snake_case_to_camel_case(module_name) -# -------------------------------------------------------------------------------------------------- + self.task_map[task_name] = setup -class GetBufr(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.bufr_obs_classes() - ] - -# -------------------------------------------------------------------------------------------------- - -class GetEnsembleGeosExperiment(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.background_experiment(), - qd.background_time_offset(), - qd.geos_x_ensemble_directory() - ] - -# -------------------------------------------------------------------------------------------------- - -class GetGeosRestart(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.questions = [ - swell_static_file_questions, - qd.geos_restarts_directory() - ] - -# -------------------------------------------------------------------------------------------------- - -class GetGeovals(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - background_crtm_obs, - qd.geovals_experiment(), - qd.geovals_provider(), - qd.r2d2_local_path(), - qd.window_length(), - ] - -# -------------------------------------------------------------------------------------------------- - -class GetGsiBc(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.path_to_gsi_bc_coefficients(), - qd.window_length() - ] - -# -------------------------------------------------------------------------------------------------- - -class GsiBcToIoda(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - background_crtm_obs, - qd.observing_system_records_path(), - qd.window_length() - ] - -# -------------------------------------------------------------------------------------------------- - -class GetGsiNcdiag(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.path_to_gsi_nc_diags() - ] - -# -------------------------------------------------------------------------------------------------- - -class GsiNcdiagToIoda(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.observations(), - qd.produce_geovals(), - qd.single_observations(), - qd.window_length() - ] - -# -------------------------------------------------------------------------------------------------- - -class GetNcdiags(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - background_crtm_obs, - qd.ncdiag_experiments(), - qd.marine_models(), - qd.r2d2_local_path(), - qd.window_length(), - ] - -# -------------------------------------------------------------------------------------------------- - -class GetGeosAdasBackground(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.path_to_geos_adas_background() - ] - -# -------------------------------------------------------------------------------------------------- - -class GetObservations(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - background_crtm_obs, - qd.cycling_varbc(), - qd.obs_experiment(), - qd.observing_system_records_path(), - qd.r2d2_local_path(), - qd.window_length(), - ] - -# -------------------------------------------------------------------------------------------------- - -class GetObsNotInR2d2(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.mail_events = ['submit-failed'] - self.questions = [ - qd.ioda_locations_not_in_r2d2(), - ] - -# -------------------------------------------------------------------------------------------------- - -class GenerateBClimatology(TaskSetup): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.retry = '2*PT1M' - self.slurm = {} - self.questions = [ - np_proc_resolution, - swell_static_file_questions, - qd.analysis_variables(), - qd.background_error_model(), - qd.generate_yaml_and_exit(), - qd.gradient_norm_reduction(), - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.jedi_forecast_model(), - qd.marine_models(), - qd.minimizer(), - qd.number_of_iterations(), - qd.observing_system_records_path(), - qd.total_processors(), - qd.window_length(), - qd.window_type() - ] - -# -------------------------------------------------------------------------------------------------- - -class GenerateBClimatologyByLinking(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - swell_static_file_questions, - qd.background_error_model(), - qd.horizontal_resolution(), - qd.vertical_resolution(), - qd.window_length(), - qd.window_type() - ] - -# -------------------------------------------------------------------------------------------------- - -class GenerateObservingSystemRecords(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.observations(), - qd.observing_system_records_mksi_path(), - qd.observing_system_records_path() - ] - -# -------------------------------------------------------------------------------------------------- - -class LinkGeosOutput(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - window_questions, - qd.background_frequency(), - qd.marine_models() - ] - -# -------------------------------------------------------------------------------------------------- - -class MoveDaRestart(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.mom6_iau(), - qd.window_length() - ] - -# -------------------------------------------------------------------------------------------------- - -class MoveForecastRestart(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.questions = [ - qd.forecast_duration() - ] - -# -------------------------------------------------------------------------------------------------- - -class PrepGeosRunDir(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.questions = [ - swell_static_file_questions, - qd.existing_geos_gcm_build_path(), - qd.forecast_duration(), - qd.geos_experiment_directory(), - qd.mom6_iau_nhours() - ] - -# -------------------------------------------------------------------------------------------------- - -class PrepareAnalysis(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.analysis_variables(), - qd.mom6_iau(), - qd.total_processors(), - qd.window_length(), - ] - -# -------------------------------------------------------------------------------------------------- - -class RenderJediObservations(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - qd.check_for_obs(), - qd.crtm_coeff_dir(), - qd.background_time_offset(), - qd.observing_system_records_path(), - qd.observations(), - qd.set_obs_as_local(), - qd.window_length() - ] - -# -------------------------------------------------------------------------------------------------- - -class RunJediFgatExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - run_jedi_executable, - qd.marine_models(), - qd.comparison_log_type('fgat'), - ] - -# -------------------------------------------------------------------------------------------------- - -class RunJediEnsembleMeanVariance(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - np_proc_resolution, - window_questions, - qd.analysis_variables(), - qd.ensemble_num_members(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.observations(), - qd.observing_system_records_path(), - qd.comparison_log_type('ensmeanvariance'), - ] - -# -------------------------------------------------------------------------------------------------- - -class RunJediHofxEnsembleExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.ensemble_num_members(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.total_processors(), - qd.comparison_log_type('hofx'), - ] - -# -------------------------------------------------------------------------------------------------- - -class RunJediHofxExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.save_geovals(), - qd.total_processors(), - qd.comparison_log_type('ensemblehofx'), - ] - -# -------------------------------------------------------------------------------------------------- - -class RunJediLocalEnsembleDaExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.ensemble_num_members(), - qd.ensmean_only(), - qd.ensmeanvariance_only(), - qd.generate_yaml_and_exit(), - qd.horizontal_localization_lengthscale(), - qd.horizontal_localization_max_nobs(), - qd.horizontal_localization_method(), - qd.jedi_forecast_model(), - qd.local_ensemble_inflation_mult(), - qd.local_ensemble_inflation_rtpp(), - qd.local_ensemble_inflation_rtps(), - qd.local_ensemble_save_posterior_ensemble(), - qd.local_ensemble_save_posterior_ensemble_increments(), - qd.local_ensemble_save_posterior_mean(), - qd.local_ensemble_save_posterior_mean_increment(), - qd.local_ensemble_solver(), - qd.local_ensemble_use_linear_observer(), - qd.skip_ensemble_hofx(), - qd.total_processors(), - qd.vertical_localization_apply_log_transform(), - qd.vertical_localization_function(), - qd.vertical_localization_ioda_vertical_coord(), - qd.vertical_localization_ioda_vertical_coord_group(), - qd.vertical_localization_lengthscale(), - qd.vertical_localization_method(), - qd.perhost(), - qd.comparison_log_type('localensembleda'), - ] - -# -------------------------------------------------------------------------------------------------- - -class RunJediVariationalExecutable(TaskSetup): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.slurm = {'nodes': 3} - self.questions = [ - run_jedi_executable, - qd.perhost(), - qd.comparison_log_type('variational'), - ] - -# -------------------------------------------------------------------------------------------------- - -class RemoveForecastDir(TaskSetup): - def set_attributes(self): - self.is_cycling = True - -# -------------------------------------------------------------------------------------------------- - -class RunGeosExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - -# -------------------------------------------------------------------------------------------------- - -class RunJediUfoTestsExecutable(TaskSetup): - def set_attributes(self): - self.time_limit = True - self.is_cycling = True - self.is_model = True - self.slurm = {'ntasks-per-node': 1} - self.questions = [ - background_crtm_obs, - qd.generate_yaml_and_exit(), - qd.single_observations(), - qd.window_length(), - qd.comparison_log_type('ufo_tests'), - ] - -# -------------------------------------------------------------------------------------------------- - -class RunJediConvertStateSoca2ciceExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {'nodes': 1} - self.questions = [ - qd.analysis_variables(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.marine_models(), - qd.observations(), - qd.total_processors(), - qd.window_length(), - qd.window_type(), - qd.comparison_log_type('convert_state_soca2cice'), - ] - -# -------------------------------------------------------------------------------------------------- - -class RunJediFgatExecutable(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - run_jedi_executable, - qd.marine_models(), - qd.comparison_log_type('fgat'), - ] - -# -------------------------------------------------------------------------------------------------- - -class SaveObsDiags(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - background_crtm_obs, - qd.r2d2_local_path(), - qd.window_length(), - qd.marine_models() - ] - -# -------------------------------------------------------------------------------------------------- - -class SaveRestart(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.questions = [ - window_questions, - qd.background_time_offset(), - qd.forecast_duration(), - qd.horizontal_resolution(), - qd.marine_models(), - qd.r2d2_local_path() - ] - -# -------------------------------------------------------------------------------------------------- - -class StageJedi(TaskSetup): - def set_attributes(self): - self.is_model = True - self.questions = [ - swell_static_file_questions, - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ] - -# -------------------------------------------------------------------------------------------------- - -class StageJediCycle(TaskSetup): - def set_attributes(self): - self.is_cycling = True - self.is_model = True - self.base_name = "StageJedi" - self.scheduling_name = "StageJediCycle-{model}" - self.questions = [ - swell_static_file_questions, - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ] - -# -------------------------------------------------------------------------------------------------- - -class sync_point(TaskSetup): - def set_attributes(self): - self.script = "true" - -# -------------------------------------------------------------------------------------------------- - -class JediLogComparison(TaskSetup): - def set_attributes(self): - self.is_model = True - self.questions = [ - qd.number_of_iterations(), - qd.comparison_log_type(), - ] - -# -------------------------------------------------------------------------------------------------- - -class RunJediObsfiltersExecutable(TaskSetup): - def set_attributes(self): - self.script = ("swell task RunJediObsfiltersExecutable $config" - " -d $datetime -m geos_atmosphere") - self.is_cycling = True - self.is_model = True - self.time_limit = True - self.slurm = {} - self.questions = [ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.observing_system_records_path(), - qd.total_processors(), - qd.obs_thinning_rej_fraction(), - qd.comparison_log_type('obsfilters') - ] - -# -------------------------------------------------------------------------------------------------- + except (ImportError, AttributeError): + pass - @classmethod - def get(cls, name: str) -> TaskSetup: - return getattr(cls, name) + def get(self, task_name: str): + return self.task_map[task_name] -# -------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 8e4b9ce34..4e3f9ddea 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -35,8 +35,10 @@ # -------------------------------------------------------------------------------------------------- -class BufrToIodaSetup(TaskSetup): +task_name = 'BufrToIoda' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index 0163085c0..ab01e21a9 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -19,8 +19,10 @@ # -------------------------------------------------------------------------------------------------- -class BuildGeosSetup(TaskSetup): +task_name = 'BuildGeos' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.questions = [ qd.geos_build_method() ] diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index 44c3450e4..983d6a601 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -18,8 +18,10 @@ # -------------------------------------------------------------------------------------------------- -class BuildGeosByLinkingSetup(TaskSetup): +task_name = 'BuildGeosByLinking' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.mail_events = ['submit-failed'] self.questions = [ qd.existing_geos_gcm_build_path(), diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index 07b0576d1..7282b20b9 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -19,8 +19,10 @@ # -------------------------------------------------------------------------------------------------- -class BuildJediSetup(TaskSetup): +task_name = 'BuildJedi' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.time_limit = True self.slurm = {} self.questions = [ diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index 1e3daa687..5b16713c3 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -17,8 +17,10 @@ # -------------------------------------------------------------------------------------------------- -class BuildGeosByLinkingSetup(TaskSetup): +task_name = 'BuildJediByLinking' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.mail_events = ['submit-failed'] self.questions = [ qd.existing_geos_gcm_build_path(), diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index 5eaeae39a..693e15e1f 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -17,8 +17,10 @@ # -------------------------------------------------------------------------------------------------- +task_name = 'CleanCycle' class CleanCycleSetup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 48759ab88..3a75a2c98 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -19,8 +19,10 @@ # -------------------------------------------------------------------------------------------------- -class CloneGeosSetup(TaskSetup): +task_name = 'CloneGeos' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.questions = [ qd.existing_geos_gcm_source_path(), qd.geos_build_method(), diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index c2af75d45..266137432 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -16,8 +16,10 @@ # -------------------------------------------------------------------------------------------------- -class CloneGeosMksiSetup(TaskSetup): +task_name = 'CloneGeosMksi' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_model = True self.questions = [ qd.observing_system_records_mksi_path(), diff --git a/src/swell/tasks/clone_gmao_perllib.py b/src/swell/tasks/clone_gmao_perllib.py index 8a6211f07..2255d4283 100644 --- a/src/swell/tasks/clone_gmao_perllib.py +++ b/src/swell/tasks/clone_gmao_perllib.py @@ -17,8 +17,10 @@ # -------------------------------------------------------------------------------------------------- -class CloneGmaoPerllibSetup(TaskSetup): +task_name = 'CloneGmaoPerllib' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.questions = [ qd.existing_perllib_path(), qd.gmao_perllib_tag() diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index e48533cdf..76d262604 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -20,8 +20,10 @@ # -------------------------------------------------------------------------------------------------- -class CloneJedi(TaskSetup): +task_name = 'CloneJedi' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.questions = [ qd.bundles(), qd.existing_jedi_source_directory(), @@ -31,7 +33,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- -class CloneJediSetup(taskBase): +class CloneJedi(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index b45c4e6d4..e4bb07ac9 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -20,8 +20,10 @@ # -------------------------------------------------------------------------------------------------- -class EvaComparisonIncrementSetup(TaskSetup): +task_name = 'EvaComparisonIncrement' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index d79674ea1..905571dc0 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -19,8 +19,10 @@ # -------------------------------------------------------------------------------------------------- -class EvaComparisonJediLogSetup(TaskSetup): +task_name = 'EvaComparisonJediLog' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index e79a37034..4f239a774 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -18,8 +18,10 @@ # -------------------------------------------------------------------------------------------------- -class EvaIncrementSetup(TaskSetup): +task_name = 'EvaIncrement' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index ece6713e5..c54a821f2 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -21,8 +21,10 @@ # -------------------------------------------------------------------------------------------------- -class EvaJediLogSetup(TaskSetup): +task_name = 'EvaJediLog' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 6d937119f..d8f7fb3cc 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -31,8 +31,10 @@ def run_eva(eva_dict: dict) -> eva: # -------------------------------------------------------------------------------------------------- -class EvaObservationsSetup(TaskSetup): +task_name = 'EvaObservations' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.time_limit = True self.is_cycling = True self.is_model = True diff --git a/src/swell/tasks/eva_timeseries.py b/src/swell/tasks/eva_timeseries.py index cc16f8c6c..4f70dfbff 100644 --- a/src/swell/tasks/eva_timeseries.py +++ b/src/swell/tasks/eva_timeseries.py @@ -33,8 +33,10 @@ def run_eva(eva_dict: dict) -> eva: # -------------------------------------------------------------------------------------------------- -class EvaTimeseriesSetup(TaskSetup): +task_name = 'EvaTimeseries' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.time_limit = True self.is_cycling = True self.is_model = True diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index ad3145425..3126eb199 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -16,8 +16,10 @@ # -------------------------------------------------------------------------------------------------- -class GenerateBClimatologySetup(TaskSetup): +task_name = 'GenerateBClimatology' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.time_limit = True self.is_cycling = True self.is_model = True diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index bb52802f4..18cb35e22 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -15,8 +15,10 @@ # -------------------------------------------------------------------------------------------------- -class GenerateBClimatologyByLinking(TaskSetup): +task_name = 'GenerateBClimatologyByLinking' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ @@ -31,7 +33,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- -class GenerateBClimatologyByLinkingSetup(taskBase): +class GenerateBClimatologyByLinking(taskBase): def execute(self) -> None: """Acquires B Matrix files for background error model(s): diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index 0a7ad251c..d62ed5788 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -17,8 +17,10 @@ # -------------------------------------------------------------------------------------------------- -class GenerateObservingSystemRecordsSetup(TaskSetup): +task_name = 'GenerateObservingSystemRecords' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index efd33e39f..55ff47dff 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -26,8 +26,10 @@ # -------------------------------------------------------------------------------------------------- -class GetBackground(TaskSetup): +task_name = 'GetBackground' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ @@ -43,7 +45,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- -class GetBackgroundSetup(taskBase): +class GetBackground(taskBase): def execute(self) -> None: """Acquires background files for a given experiment and cycle diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index 4c5cffbf3..b6b21fbf9 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -20,8 +20,10 @@ # -------------------------------------------------------------------------------------------------- -class GetBackgroundGeosExperimentSetup(TaskSetup): +task_name = 'GetBackgroundGeosExperiment' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.mail_events = ['submit-failed'] diff --git a/src/swell/tasks/get_bufr.py b/src/swell/tasks/get_bufr.py index 534f4c1a5..589d0cbe2 100644 --- a/src/swell/tasks/get_bufr.py +++ b/src/swell/tasks/get_bufr.py @@ -19,8 +19,10 @@ # -------------------------------------------------------------------------------------------------- -class GetBufrSetup(TaskSetup): +task_name = 'GetBufr' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/get_ensemble.py b/src/swell/tasks/get_ensemble.py index 446c58b69..d78cea89b 100644 --- a/src/swell/tasks/get_ensemble.py +++ b/src/swell/tasks/get_ensemble.py @@ -19,7 +19,7 @@ # -------------------------------------------------------------------------------------------------- -class GetEnsembleSetup(taskBase): +class GetEnsemble(taskBase): def execute(self) -> None: """Acquires ensemble member files for a given experiment and cycle diff --git a/src/swell/tasks/get_ensemble_geos_experiment.py b/src/swell/tasks/get_ensemble_geos_experiment.py index a005e69d2..8e7d178d4 100644 --- a/src/swell/tasks/get_ensemble_geos_experiment.py +++ b/src/swell/tasks/get_ensemble_geos_experiment.py @@ -19,8 +19,10 @@ # -------------------------------------------------------------------------------------------------- -class GetEnsembleGeosExperimentSetup(TaskSetup): +task_name = 'GetEnsembleGeosExperiment' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py index 48f33f4d8..e988ace64 100644 --- a/src/swell/tasks/get_geos_adas_background.py +++ b/src/swell/tasks/get_geos_adas_background.py @@ -20,8 +20,10 @@ # -------------------------------------------------------------------------------------------------- -class GetGeosAdasBackgroundSetup(TaskSetup): +task_name = 'GetGeosAdasBackground' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/get_geos_restart.py b/src/swell/tasks/get_geos_restart.py index 2a62903dd..629d0a5d6 100644 --- a/src/swell/tasks/get_geos_restart.py +++ b/src/swell/tasks/get_geos_restart.py @@ -17,8 +17,10 @@ # -------------------------------------------------------------------------------------------------- -class GetGeosRestartSetup(TaskSetup): +task_name = 'GetGeosRestart' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.questions = [ qd.swell_static_files(), diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index 7e11cdb09..62419e2a2 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -18,8 +18,10 @@ # -------------------------------------------------------------------------------------------------- -class GetGeovalsSetup(TaskSetup): +task_name = 'GetGeovals' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index 47d144ebd..c08694dd9 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -21,8 +21,10 @@ # -------------------------------------------------------------------------------------------------- -class GetGsiBcSetup(TaskSetup): +task_name = 'GetGsiBc' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index c8463ce0c..507a9c90c 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -18,8 +18,10 @@ # -------------------------------------------------------------------------------------------------- -class GetGsiNcdiagSetup(TaskSetup): +task_name = 'GetGsiNcdiag' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index abf64edec..5d833bf14 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -14,8 +14,10 @@ # -------------------------------------------------------------------------------------------------- -class GetNcdiagsSetup(TaskSetup): +task_name = 'GetNcdiags' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py index 339be0e60..b873f41e1 100644 --- a/src/swell/tasks/get_obs_not_in_r2d2.py +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -19,8 +19,10 @@ # -------------------------------------------------------------------------------------------------- -class GetObsNotInR2d2Setup(TaskSetup): +task_name = 'GetObsNotInR2d2' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.mail_events = ['submit-failed'] diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 93fde6626..a7dcb19ad 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -30,8 +30,10 @@ # -------------------------------------------------------------------------------------------------- -class GetObservationsSetup(TaskSetup): +task_name = 'GetObservations' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 09d8e2037..c4fdfd2cb 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -21,8 +21,10 @@ # -------------------------------------------------------------------------------------------------- -class GetGsiBcSetup(TaskSetup): +task_name = 'GsiBcToIoda' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index eb4439b75..91c9275d7 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -23,8 +23,10 @@ # -------------------------------------------------------------------------------------------------- -class GsiNcdiagToIodaSetup(TaskSetup): +task_name = 'GsiNcdiagToIoda' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py index d6b6cefdf..29ee0a927 100644 --- a/src/swell/tasks/jedi_log_comparison.py +++ b/src/swell/tasks/jedi_log_comparison.py @@ -23,8 +23,10 @@ # -------------------------------------------------------------------------------------------------- -class JediLogComparisonSetup(TaskSetup): +task_name = 'JediLogComparison' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_model = True self.questions = [ qd.number_of_iterations(), diff --git a/src/swell/tasks/jedi_oops_log_parser.py b/src/swell/tasks/jedi_oops_log_parser.py index c19fbbbe5..5e893d6f3 100644 --- a/src/swell/tasks/jedi_oops_log_parser.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -17,8 +17,10 @@ # -------------------------------------------------------------------------------------------------- -class JediOopsLogParserSetup(TaskSetup): +task_name = 'JediOopsLogParser' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index 21f68ad2c..5dfde9515 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -21,8 +21,10 @@ # -------------------------------------------------------------------------------------------------- -class LinkGeosOutputSetup(TaskSetup): +task_name = 'LinkGeosOutput' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/move_da_restart.py b/src/swell/tasks/move_da_restart.py index ba446aca2..fa86ab408 100644 --- a/src/swell/tasks/move_da_restart.py +++ b/src/swell/tasks/move_da_restart.py @@ -19,8 +19,10 @@ # -------------------------------------------------------------------------------------------------- -class MoveDaRestartSetup(TaskSetup): +task_name = 'MoveDaRestart' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/move_forecast_restart.py b/src/swell/tasks/move_forecast_restart.py index bc489c8f4..9bffd3d40 100644 --- a/src/swell/tasks/move_forecast_restart.py +++ b/src/swell/tasks/move_forecast_restart.py @@ -18,8 +18,10 @@ # -------------------------------------------------------------------------------------------------- -class MoveForecastRestartSetup(TaskSetup): +task_name = 'MoveForecastRestart' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.questions = [ qd.forecast_duration() diff --git a/src/swell/tasks/prep_geos_run_dir.py b/src/swell/tasks/prep_geos_run_dir.py index 033538a29..a8ab8942f 100644 --- a/src/swell/tasks/prep_geos_run_dir.py +++ b/src/swell/tasks/prep_geos_run_dir.py @@ -21,8 +21,10 @@ # -------------------------------------------------------------------------------------------------- -class PrepGeosRunDirSetup(TaskSetup): +task_name = 'PrepGeosRunDir' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.questions = [ qd.swell_static_files(), diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index 0a2095cc9..93cb705ae 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -20,8 +20,10 @@ # -------------------------------------------------------------------------------------------------- -class PrepareAnalysisSetup(TaskSetup): +task_name = 'PrepareAnalysis' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/remove_forecast_dir.py b/src/swell/tasks/remove_forecast_dir.py index b96d1579e..26ea86b4d 100644 --- a/src/swell/tasks/remove_forecast_dir.py +++ b/src/swell/tasks/remove_forecast_dir.py @@ -15,8 +15,10 @@ # -------------------------------------------------------------------------------------------------- -class RemoveForecastDirSetup(TaskSetup): +task_name = 'RemoveForecastDir' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index 8a7deadb9..9ccac7825 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -18,8 +18,10 @@ # -------------------------------------------------------------------------------------------------- -class RenderJediObservations(TaskSetup): +task_name = 'RenderJediObservations' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/run_geos_executable.py b/src/swell/tasks/run_geos_executable.py index d799b462d..1fd9fe3a9 100644 --- a/src/swell/tasks/run_geos_executable.py +++ b/src/swell/tasks/run_geos_executable.py @@ -17,8 +17,10 @@ # -------------------------------------------------------------------------------------------------- -class RunGeosExecutable(TaskSetup): +task_name = 'RunGeosExecutable' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py index 40d1b6e16..28a634604 100644 --- a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py +++ b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py @@ -19,8 +19,10 @@ # -------------------------------------------------------------------------------------------------- -class RunJediConvertStateSoca2ciceExecutable(TaskSetup): +task_name = 'RunJediConvertStateSoca2ciceExecutable' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.time_limit = True @@ -41,7 +43,6 @@ def set_attributes(self): class RunJediConvertStateSoca2ciceExecutable(taskBase): - # ---------------------------------------------------------------------------------------------- def execute(self) -> None: diff --git a/src/swell/tasks/run_jedi_ensemble_mean_variance.py b/src/swell/tasks/run_jedi_ensemble_mean_variance.py index ca179abef..57411ed96 100644 --- a/src/swell/tasks/run_jedi_ensemble_mean_variance.py +++ b/src/swell/tasks/run_jedi_ensemble_mean_variance.py @@ -19,8 +19,10 @@ # -------------------------------------------------------------------------------------------------- -class RunJediEnsembleMeanVariance(TaskSetup): +task_name = 'RunJediEnsembleMeanVariance' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.time_limit = True diff --git a/src/swell/tasks/run_jedi_fgat_executable.py b/src/swell/tasks/run_jedi_fgat_executable.py index 8dd815ea7..e345623a6 100644 --- a/src/swell/tasks/run_jedi_fgat_executable.py +++ b/src/swell/tasks/run_jedi_fgat_executable.py @@ -18,8 +18,10 @@ # -------------------------------------------------------------------------------------------------- -class RunJediFgatExecutable(TaskSetup): +task_name = 'RunJediFgatExecutable' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.time_limit = True diff --git a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py index fb9d871fc..4644bdaf9 100644 --- a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py +++ b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py @@ -19,8 +19,10 @@ # -------------------------------------------------------------------------------------------------- -class RunJediHofxEnsembleExecutable(TaskSetup): +task_name = 'RunJediHofxEnsembleExecutable' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.time_limit = True diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 7dbd74408..522f7a72b 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -22,8 +22,10 @@ # -------------------------------------------------------------------------------------------------- -class RunJediHofxExecutable(TaskSetup): +task_name = 'RunJediHofxExecutable' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.time_limit = True diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 08644bab7..fbb62bcd2 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -37,8 +37,10 @@ def replace_key(obj, old_key, new_key): # -------------------------------------------------------------------------------------------------- -class RunJediLocalEnsembleDaExecutable(TaskSetup): +task_name = 'RunJediLocalEnsembleDaExecutable' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.time_limit = True diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index 025312708..393badf10 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -19,8 +19,10 @@ # -------------------------------------------------------------------------------------------------- -class RunJediObsfiltersExecutable(TaskSetup): +task_name = 'RunJediObsfiltersExecutable' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.script = ("swell task RunJediObsfiltersExecutable $config" " -d $datetime -m geos_atmosphere") self.is_cycling = True diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 962def625..f2e897b91 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -21,8 +21,10 @@ # -------------------------------------------------------------------------------------------------- -class RunJediUfoTestsExecutable(TaskSetup): +task_name = 'RunJediUfoTestsExecutable' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.time_limit = True self.is_cycling = True self.is_model = True diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index ee1830dbc..ddafc8eff 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -18,8 +18,10 @@ # -------------------------------------------------------------------------------------------------- -class RunJediVariationalExecutable(TaskSetup): +task_name = 'RunJediVariationalExecutable' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.time_limit = True self.is_cycling = True self.is_model = True diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index fb4b1b5b8..96afbf966 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -17,8 +17,10 @@ # -------------------------------------------------------------------------------------------------- -class SaveObsDiags(TaskSetup): +task_name = 'SaveObsDiags' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index 69296d752..540c48b85 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -20,8 +20,10 @@ # -------------------------------------------------------------------------------------------------- -class SaveRestart(TaskSetup): +task_name = 'SaveRestart' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 2281e1ca3..93d6d6bd6 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -21,8 +21,10 @@ # -------------------------------------------------------------------------------------------------- -class StageJedi(TaskSetup): +task_name = 'StageJedi' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_model = True self.questions = [ qd.swell_static_files(), diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index 7004c019d..a1f51a449 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -21,8 +21,10 @@ # -------------------------------------------------------------------------------------------------- -class StoreBackground(TaskSetup): +task_name = 'StoreBackground' +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.is_cycling = True self.is_model = True self.questions = [ From 5fa08bf3545e9eaba43e3fc51d8c00d31c538ac4 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 16 Jan 2026 16:40:49 -0500 Subject: [PATCH 164/299] Refactor --- src/swell/suites/3dfgat_atmos/workflow.py | 2 +- src/swell/suites/3dfgat_cycle/workflow.py | 2 +- src/swell/suites/3dvar/workflow.py | 2 +- src/swell/suites/3dvar_atmos/workflow.py | 2 +- src/swell/suites/3dvar_cycle/workflow.py | 2 +- src/swell/suites/build_geos/workflow.py | 2 +- src/swell/suites/build_jedi/workflow.py | 2 +- src/swell/suites/compare/workflow.py | 2 +- src/swell/suites/convert_bufr/workflow.py | 2 +- src/swell/suites/convert_ncdiags/workflow.py | 2 +- src/swell/suites/eva_capabilities/workflow.py | 2 +- src/swell/suites/forecast_geos/workflow.py | 2 +- src/swell/suites/geosadas/workflow.py | 2 +- src/swell/suites/hofx/workflow.py | 2 +- src/swell/suites/localensembleda/workflow.py | 2 +- src/swell/suites/ufo_testing/workflow.py | 2 +- src/swell/tasks/base/task_attributes.py | 47 ++++++++++++++++--- src/swell/tasks/base/task_base.py | 3 +- src/swell/tasks/bufr_to_ioda.py | 3 ++ src/swell/tasks/build_geos.py | 3 ++ src/swell/tasks/build_geos_by_linking.py | 3 ++ src/swell/tasks/build_jedi.py | 3 ++ src/swell/tasks/build_jedi_by_linking.py | 8 +++- src/swell/tasks/clean_cycle.py | 5 +- src/swell/tasks/clone_geos.py | 3 ++ src/swell/tasks/clone_geos_mksi.py | 3 ++ src/swell/tasks/clone_gmao_perllib.py | 3 ++ src/swell/tasks/clone_jedi.py | 3 ++ src/swell/tasks/eva_comparison_increment.py | 3 ++ src/swell/tasks/eva_comparison_jedi_log.py | 3 ++ src/swell/tasks/eva_increment.py | 3 ++ src/swell/tasks/eva_jedi_log.py | 6 +-- src/swell/tasks/eva_observations.py | 8 ++-- src/swell/tasks/eva_timeseries.py | 9 ++-- src/swell/tasks/generate_b_climatology.py | 3 ++ .../generate_b_climatology_by_linking.py | 3 ++ .../generate_observing_system_records.py | 3 ++ src/swell/tasks/get_background.py | 5 +- .../tasks/get_background_geos_experiment.py | 3 ++ src/swell/tasks/get_bufr.py | 3 ++ src/swell/tasks/get_ensemble.py | 12 +++++ .../tasks/get_ensemble_geos_experiment.py | 3 ++ src/swell/tasks/get_geos_adas_background.py | 3 ++ src/swell/tasks/get_geos_restart.py | 3 ++ src/swell/tasks/get_geovals.py | 3 ++ src/swell/tasks/get_gsi_bc.py | 3 ++ src/swell/tasks/get_gsi_ncdiag.py | 3 ++ src/swell/tasks/get_ncdiags.py | 3 ++ src/swell/tasks/get_obs_not_in_r2d2.py | 3 ++ src/swell/tasks/get_observations.py | 7 ++- src/swell/tasks/gsi_bc_to_ioda.py | 3 ++ src/swell/tasks/gsi_ncdiag_to_ioda.py | 3 ++ src/swell/tasks/jedi_log_comparison.py | 3 ++ src/swell/tasks/jedi_oops_log_parser.py | 3 ++ src/swell/tasks/link_geos_output.py | 9 +++- src/swell/tasks/move_da_restart.py | 3 ++ src/swell/tasks/move_forecast_restart.py | 3 ++ src/swell/tasks/prep_geos_run_dir.py | 3 ++ src/swell/tasks/prepare_analysis.py | 3 ++ src/swell/tasks/remove_forecast_dir.py | 4 +- src/swell/tasks/render_jedi_observations.py | 3 ++ src/swell/tasks/run_geos_executable.py | 4 +- ...jedi_convert_state_soca2cice_executable.py | 4 +- .../tasks/run_jedi_ensemble_mean_variance.py | 3 ++ src/swell/tasks/run_jedi_fgat_executable.py | 3 ++ .../run_jedi_hofx_ensemble_executable.py | 3 ++ src/swell/tasks/run_jedi_hofx_executable.py | 3 ++ .../run_jedi_local_ensemble_da_executable.py | 4 ++ .../tasks/run_jedi_obsfilters_executable.py | 5 +- .../tasks/run_jedi_ufo_tests_executable.py | 3 ++ .../tasks/run_jedi_variational_executable.py | 3 ++ src/swell/tasks/save_obs_diags.py | 3 ++ src/swell/tasks/save_restart.py | 5 +- src/swell/tasks/stage_jedi.py | 3 ++ src/swell/tasks/store_background.py | 3 ++ src/swell/test/code_tests/slurm_test.py | 10 ++-- src/swell/utilities/config.py | 2 +- 77 files changed, 265 insertions(+), 50 deletions(-) diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 7d9a4e979..82f36a088 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index a854fa4a0..38201c2e4 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 4bf42c154..f28bf5089 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index aa4fd9b25..674b379c6 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index 265333c93..634128548 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py index f795e2b1d..e770703ad 100644 --- a/src/swell/suites/build_geos/workflow.py +++ b/src/swell/suites/build_geos/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index cf212f783..186971a49 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare/workflow.py b/src/swell/suites/compare/workflow.py index 77c09568e..35c6daa07 100644 --- a/src/swell/suites/compare/workflow.py +++ b/src/swell/suites/compare/workflow.py @@ -11,7 +11,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_bufr/workflow.py b/src/swell/suites/convert_bufr/workflow.py index db6715944..1823efc4c 100644 --- a/src/swell/suites/convert_bufr/workflow.py +++ b/src/swell/suites/convert_bufr/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index b975437ce..a25c5c736 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index 4d1e7c5ee..a1e1d5e11 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index 9894a363c..5c4caffb5 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index 868da09a9..375892dc4 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index e841c2870..e0751a713 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 82824d742..6709335ac 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index 65208144c..341567f0b 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -9,7 +9,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import TaskAttributes as ta +from swell.tasks.base.task_attributes import task_attributes as ta # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index 194ac4260..c0c6c7dd9 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -13,6 +13,34 @@ from swell.swell_path import get_swell_path from swell.utilities.case_switching import snake_case_to_camel_case +from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.stage_jedi import Setup as StageJedi + +# -------------------------------------------------------------------------------------------------- + + +class root(TaskSetup): + def set_attributes(self): + self.script = False + self.pre_script = "source $CYLC_SUITE_DEF_PATH/modules" + self.additional_sections = [self.create_new_section('environment', + {'datetime': '$CYLC_TASK_CYCLE_POINT', + 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'})] # noqa + + +class StageJediCycle(StageJedi): + def set_attributes(self): + super().set_attributes() + self.base_name = "StageJedi" + self.scheduling_name = "StageJediCycle-{model}" + self.is_cycling = True + self.is_model = True + + +class sync_point(TaskSetup): + def set_attributes(self): + self.script = "true" + # -------------------------------------------------------------------------------------------------- @@ -20,7 +48,9 @@ class TaskAttributes(): def __init__(self) -> None: task_path = os.path.join(get_swell_path(), 'tasks', '*.py') - self.task_map = {} + setattr(self, 'root', root) + setattr(self, 'StageJediCycle', StageJediCycle) + setattr(self, 'sync_point', sync_point) for task_file in glob.glob(task_path): module_name = os.path.basename(task_file).split('.py')[0] @@ -32,12 +62,17 @@ def __init__(self) -> None: task_name = snake_case_to_camel_case(module_name) - self.task_map[task_name] = setup + setattr(self, task_name, setup) - except (ImportError, AttributeError): + except AttributeError: pass - def get(self, task_name: str): - return self.task_map[task_name] + def get(self, task_name): + return getattr(self, task_name) -# -------------------------------------------------------------------------------------------------- \ No newline at end of file + +# -------------------------------------------------------------------------------------------------- + +task_attributes = TaskAttributes() + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 5ca570fff..8ab7bd882 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -21,7 +21,6 @@ # swell imports from swell.swell_path import get_swell_path from swell.utilities.case_switching import camel_case_to_snake_case, snake_case_to_camel_case -from swell.utilities.config import Config from swell.utilities.data_assimilation_window_params import DataAssimilationWindowParams from swell.utilities.datetime_util import Datetime from swell.utilities.logger import get_logger @@ -48,6 +47,8 @@ def __init__( # --------------------- self.logger = get_logger(task_name) + from swell.utilities.config import Config + # Write out the initialization info # --------------------------------- self.logger.info(' Initializing task with the following parameters:') diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 4e3f9ddea..38899f5db 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -36,6 +36,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'BufrToIoda' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -44,6 +46,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class BufrToIoda(taskBase): # python split filename by delimiter period and then search for string in the resulting list diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index ab01e21a9..9c0ee9547 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -20,6 +20,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'BuildGeos' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -29,6 +31,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class BuildGeos(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index 983d6a601..f1265d553 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -19,6 +19,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'BuildGeosByLinking' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -30,6 +32,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class BuildGeosByLinking(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index 7282b20b9..9ee68b9c4 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -20,6 +20,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'BuildJedi' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -32,6 +34,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class BuildJedi(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index 5b16713c3..133c417fc 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -18,17 +18,21 @@ # -------------------------------------------------------------------------------------------------- task_name = 'BuildJediByLinking' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.mail_events = ['submit-failed'] self.questions = [ - qd.existing_geos_gcm_build_path(), - qd.geos_build_method() + qd.existing_jedi_build_directory(), + qd.existing_jedi_build_directory_pinned(), + qd.jedi_build_method() ] # -------------------------------------------------------------------------------------------------- + class BuildJediByLinking(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index 693e15e1f..488900569 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -18,7 +18,9 @@ # -------------------------------------------------------------------------------------------------- task_name = 'CleanCycle' -class CleanCycleSetup(TaskSetup): + + +class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True @@ -29,6 +31,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class CleanCycle(taskBase): """Cleans current cycle based on list defined in the configuration file diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 3a75a2c98..1f791064a 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -20,6 +20,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'CloneGeos' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -31,6 +33,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class CloneGeos(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index 266137432..8dcffcfd1 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -17,6 +17,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'CloneGeosMksi' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -28,6 +30,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class CloneGeosMksi(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/clone_gmao_perllib.py b/src/swell/tasks/clone_gmao_perllib.py index 2255d4283..b36be7941 100644 --- a/src/swell/tasks/clone_gmao_perllib.py +++ b/src/swell/tasks/clone_gmao_perllib.py @@ -18,6 +18,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'CloneGmaoPerllib' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -28,6 +30,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class CloneGmaoPerllib(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index 76d262604..d7eace30f 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -21,6 +21,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'CloneJedi' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -33,6 +35,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class CloneJedi(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index e4bb07ac9..b14c9c6e2 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -21,6 +21,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'EvaComparisonIncrement' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -34,6 +36,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class EvaComparisonIncrement(taskBase): def window_info_from_config(self, path: str): diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 905571dc0..c996abc99 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -20,6 +20,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'EvaComparisonJediLog' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -31,6 +33,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class EvaComparisonJediLog(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index 4f239a774..d7fc5b866 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -19,6 +19,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'EvaIncrement' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -32,6 +34,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class EvaIncrement(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index c54a821f2..530570a02 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -11,17 +11,16 @@ import os import yaml -from eva.eva_driver import eva - from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup -from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- task_name = 'EvaJediLog' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -30,6 +29,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class EvaJediLog(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index d8f7fb3cc..a4911d02b 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -25,13 +25,16 @@ # Pass through to avoid confusion with optional logger argument inside eva -def run_eva(eva_dict: dict) -> eva: +def run_eva(eva_dict: dict): + from eva.eva_driver import eva eva(eva_dict) # -------------------------------------------------------------------------------------------------- task_name = 'EvaObservations' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -52,12 +55,11 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class EvaObservations(taskBase): def execute(self) -> None: - from eva.eva_driver import eva - window_length = self.config.window_length() # Compute window beginning time diff --git a/src/swell/tasks/eva_timeseries.py b/src/swell/tasks/eva_timeseries.py index 4f70dfbff..77690a96b 100644 --- a/src/swell/tasks/eva_timeseries.py +++ b/src/swell/tasks/eva_timeseries.py @@ -27,13 +27,17 @@ # Pass through to avoid confusion with optional logger argument inside eva -def run_eva(eva_dict: dict) -> eva: +def run_eva(eva_dict: dict): + + from eva.eva_driver import eva eva(eva_dict) # -------------------------------------------------------------------------------------------------- task_name = 'EvaTimeseries' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -53,12 +57,11 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class EvaTimeseries(taskBase): def execute(self) -> None: - from eva.eva_driver import eva - window_length = self.config.window_length() # Compute window beginning time diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index 3126eb199..cfd217fcd 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -17,6 +17,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GenerateBClimatology' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -53,6 +55,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GenerateBClimatology(taskBase): def jedi_dictionary_iterator(self, jedi_config_dict: dict) -> None: diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 18cb35e22..9aefba725 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -16,6 +16,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GenerateBClimatologyByLinking' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -33,6 +35,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GenerateBClimatologyByLinking(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index d62ed5788..5164ae6d8 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -18,6 +18,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GenerateObservingSystemRecords' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -31,6 +33,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GenerateObservingSystemRecords(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 55ff47dff..8044bd067 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -14,7 +14,6 @@ import isodate import os -import r2d2 # -------------------------------------------------------------------------------------------------- @@ -27,6 +26,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GetBackground' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -45,6 +46,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GetBackground(taskBase): def execute(self) -> None: @@ -57,6 +59,7 @@ def execute(self) -> None: """ from swell.utilities.r2d2 import create_r2d2_config + import r2d2 # Get duration into forecast for first background file # ---------------------------------------------------- diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index b6b21fbf9..cd81ecbb8 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -21,6 +21,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GetBackgroundGeosExperiment' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -36,6 +38,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GetBackgroundGeosExperiment(taskBase): def execute(self): diff --git a/src/swell/tasks/get_bufr.py b/src/swell/tasks/get_bufr.py index 589d0cbe2..86b5894fb 100644 --- a/src/swell/tasks/get_bufr.py +++ b/src/swell/tasks/get_bufr.py @@ -20,6 +20,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GetBufr' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -31,6 +33,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GetBufr(taskBase): ''' diff --git a/src/swell/tasks/get_ensemble.py b/src/swell/tasks/get_ensemble.py index d78cea89b..3a66eeb34 100644 --- a/src/swell/tasks/get_ensemble.py +++ b/src/swell/tasks/get_ensemble.py @@ -18,6 +18,18 @@ # -------------------------------------------------------------------------------------------------- +task_name = 'GetEnsemble' + + +class Setup(TaskSetup): + def set_attributes(self): + self.base_name = task_name + self.questions = [ + qd.path_to_ensemble() + ] + +# -------------------------------------------------------------------------------------------------- + class GetEnsemble(taskBase): diff --git a/src/swell/tasks/get_ensemble_geos_experiment.py b/src/swell/tasks/get_ensemble_geos_experiment.py index 8e7d178d4..a023e5368 100644 --- a/src/swell/tasks/get_ensemble_geos_experiment.py +++ b/src/swell/tasks/get_ensemble_geos_experiment.py @@ -20,6 +20,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GetEnsembleGeosExperiment' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -33,6 +35,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GetEnsembleGeosExperiment(taskBase): def execute(self): diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py index e988ace64..d273bc3df 100644 --- a/src/swell/tasks/get_geos_adas_background.py +++ b/src/swell/tasks/get_geos_adas_background.py @@ -21,6 +21,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GetGeosAdasBackground' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -32,6 +34,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GetGeosAdasBackground(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/get_geos_restart.py b/src/swell/tasks/get_geos_restart.py index 629d0a5d6..39e0363b7 100644 --- a/src/swell/tasks/get_geos_restart.py +++ b/src/swell/tasks/get_geos_restart.py @@ -18,6 +18,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GetGeosRestart' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -30,6 +32,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GetGeosRestart(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index 62419e2a2..cd4cf8eb4 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -19,6 +19,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GetGeovals' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -37,6 +39,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GetGeovals(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index c08694dd9..a4ec7d2d8 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -22,6 +22,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GetGsiBc' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -34,6 +36,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GetGsiBc(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index 507a9c90c..ee698d669 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -19,6 +19,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GetGsiNcdiag' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -30,6 +32,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GetGsiNcdiag(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index 5d833bf14..a69ecb3b3 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -15,6 +15,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GetNcdiags' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -33,6 +35,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GetNcdiags(taskBase): """ diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py index b873f41e1..c19289d45 100644 --- a/src/swell/tasks/get_obs_not_in_r2d2.py +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -20,6 +20,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GetObsNotInR2d2' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -32,6 +34,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GetObsNotInR2d2(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index a7dcb19ad..aa6682b20 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -31,6 +31,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GetObservations' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -50,6 +52,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GetObservations(taskBase): def execute(self) -> None: @@ -119,7 +122,6 @@ def execute(self) -> None: # Import modules # -------------- import r2d2 - import netCDF4 as nc # Parse config # ------------ @@ -435,6 +437,7 @@ def create_obs_time_list( # Get the target data from the netcdf file # ---------------------------------------- def get_data(self, input_file: str, group: str, var_name: str) -> object: + import netCDF4 as nc with nc.Dataset(input_file, 'r') as ds: return ds[group][var_name][:] @@ -460,6 +463,8 @@ def read_and_combine(self, input_filenames: list, output_filename: str) -> None: if os.path.exists(output_filename): os.remove(output_filename) + import netCDF4 as nc + # Reduce the list of input files to only those that exist # ------------------------------------------------------------- existing_files = [f for f in input_filenames if os.path.exists(f)] diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index c4fdfd2cb..ac418e9d4 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -22,6 +22,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GsiBcToIoda' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -34,6 +36,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GsiBcToIoda(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 91c9275d7..108ba05ba 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -24,6 +24,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'GsiNcdiagToIoda' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -38,6 +40,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class GsiNcdiagToIoda(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py index 29ee0a927..5589d941c 100644 --- a/src/swell/tasks/jedi_log_comparison.py +++ b/src/swell/tasks/jedi_log_comparison.py @@ -24,6 +24,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'JediLogComparison' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -35,6 +37,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class JediLogComparison(taskBase): def execute(self): diff --git a/src/swell/tasks/jedi_oops_log_parser.py b/src/swell/tasks/jedi_oops_log_parser.py index 5e893d6f3..640049966 100644 --- a/src/swell/tasks/jedi_oops_log_parser.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -18,6 +18,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'JediOopsLogParser' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -30,6 +32,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class JediOopsLogParser(taskBase): def fgrep_residual_norm(self, output_file): diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index 5dfde9515..3b5b4b21d 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -22,6 +22,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'LinkGeosOutput' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -36,6 +38,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class LinkGeosOutput(taskBase): # ---------------------------------------------------------------------------------------------- @@ -48,8 +51,6 @@ def execute(self) -> None: type (history vs. restart), DA method, and window length. """ - import xarray as xr - # Parse configuration # ------------------- self.marine_models = self.config.marine_models(None) or [] @@ -205,6 +206,8 @@ def prepare_cice6_history(self, dst_history: str, ) -> None: + import xarray as xr + # Since history already has the aggregated variables, we just need to rename # the dimensions and variables to match SOCA requirements ds = xr.open_dataset(src_history) @@ -227,6 +230,8 @@ def prepare_cice6_restart(self) -> Tuple[str, str]: 'hi_h': 'vicen', 'hs_h': 'vsnon'} + import xarray as xr + # read CICE6 restart # ----------------- ds = xr.open_dataset(self.forecast_dir(['RESTART', 'iced.nc'])) diff --git a/src/swell/tasks/move_da_restart.py b/src/swell/tasks/move_da_restart.py index fa86ab408..5cc6b2411 100644 --- a/src/swell/tasks/move_da_restart.py +++ b/src/swell/tasks/move_da_restart.py @@ -20,6 +20,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'MoveDaRestart' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -32,6 +34,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class MoveDaRestart(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/move_forecast_restart.py b/src/swell/tasks/move_forecast_restart.py index 9bffd3d40..64b8748b7 100644 --- a/src/swell/tasks/move_forecast_restart.py +++ b/src/swell/tasks/move_forecast_restart.py @@ -19,6 +19,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'MoveForecastRestart' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -29,6 +31,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class MoveForecastRestart(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/prep_geos_run_dir.py b/src/swell/tasks/prep_geos_run_dir.py index a8ab8942f..896685656 100644 --- a/src/swell/tasks/prep_geos_run_dir.py +++ b/src/swell/tasks/prep_geos_run_dir.py @@ -22,6 +22,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'PrepGeosRunDir' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -37,6 +39,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class PrepGeosRunDir(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index 93cb705ae..7340c4a5a 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -21,6 +21,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'PrepareAnalysis' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -35,6 +37,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class PrepareAnalysis(taskBase): # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/remove_forecast_dir.py b/src/swell/tasks/remove_forecast_dir.py index 26ea86b4d..66271fd0a 100644 --- a/src/swell/tasks/remove_forecast_dir.py +++ b/src/swell/tasks/remove_forecast_dir.py @@ -11,11 +11,12 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup -from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- task_name = 'RemoveForecastDir' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -23,6 +24,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class RemoveForecastDir(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index 9ccac7825..ade6113d1 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -19,6 +19,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'RenderJediObservations' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -36,6 +38,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class RenderJediObservations(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/run_geos_executable.py b/src/swell/tasks/run_geos_executable.py index 1fd9fe3a9..128feccf0 100644 --- a/src/swell/tasks/run_geos_executable.py +++ b/src/swell/tasks/run_geos_executable.py @@ -12,12 +12,13 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup -from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.shell_commands import run_track_log_subprocess # -------------------------------------------------------------------------------------------------- task_name = 'RunGeosExecutable' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -25,6 +26,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class RunGeosExecutable(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py index 28a634604..7c032ab65 100644 --- a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py +++ b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py @@ -20,6 +20,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'RunJediConvertStateSoca2ciceExecutable' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -41,8 +43,8 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- -class RunJediConvertStateSoca2ciceExecutable(taskBase): +class RunJediConvertStateSoca2ciceExecutable(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/run_jedi_ensemble_mean_variance.py b/src/swell/tasks/run_jedi_ensemble_mean_variance.py index 57411ed96..0fc525c7f 100644 --- a/src/swell/tasks/run_jedi_ensemble_mean_variance.py +++ b/src/swell/tasks/run_jedi_ensemble_mean_variance.py @@ -20,6 +20,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'RunJediEnsembleMeanVariance' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -47,6 +49,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class RunJediEnsembleMeanVariance(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_fgat_executable.py b/src/swell/tasks/run_jedi_fgat_executable.py index e345623a6..428c2c0e6 100644 --- a/src/swell/tasks/run_jedi_fgat_executable.py +++ b/src/swell/tasks/run_jedi_fgat_executable.py @@ -19,6 +19,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'RunJediFgatExecutable' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -56,6 +58,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class RunJediFgatExecutable(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py index 4644bdaf9..ec5dfa91f 100644 --- a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py +++ b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py @@ -20,6 +20,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'RunJediHofxEnsembleExecutable' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -52,6 +54,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class RunJediHofxEnsembleExecutable(RunJediHofxExecutable, taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 522f7a72b..deb166bf1 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -23,6 +23,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'RunJediHofxExecutable' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -53,6 +55,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class RunJediHofxExecutable(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index fbb62bcd2..ea9746275 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -37,7 +37,10 @@ def replace_key(obj, old_key, new_key): # -------------------------------------------------------------------------------------------------- + task_name = 'RunJediLocalEnsembleDaExecutable' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -91,6 +94,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class RunJediLocalEnsembleDaExecutable(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index 393badf10..a07e3d447 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -20,11 +20,13 @@ # -------------------------------------------------------------------------------------------------- task_name = 'RunJediObsfiltersExecutable' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.script = ("swell task RunJediObsfiltersExecutable $config" - " -d $datetime -m geos_atmosphere") + " -d $datetime -m geos_atmosphere") self.is_cycling = True self.is_model = True self.time_limit = True @@ -53,6 +55,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class RunJediObsfiltersExecutable(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index f2e897b91..cc2a08803 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -22,6 +22,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'RunJediUfoTestsExecutable' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -42,6 +44,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class RunJediUfoTestsExecutable(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index ddafc8eff..6da894c41 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -19,6 +19,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'RunJediVariationalExecutable' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -56,6 +58,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class RunJediVariationalExecutable(taskBase): # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 96afbf966..8d8ed0675 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -18,6 +18,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'SaveObsDiags' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -35,6 +37,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class SaveObsDiags(taskBase): """ diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index 540c48b85..c51d4d741 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -10,7 +10,6 @@ from datetime import datetime as dt import isodate import os -from r2d2 import store from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup @@ -21,6 +20,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'SaveRestart' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -38,6 +39,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class SaveRestart(taskBase): def execute(self): @@ -49,6 +51,7 @@ def execute(self): """ from swell.utilities.r2d2 import create_r2d2_config + from r2d2 import store # Parse config window_type = self.config.window_type() diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 93d6d6bd6..d820d9eaf 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -22,6 +22,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'StageJedi' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -38,6 +40,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class StageJedi(taskBase): def execute(self) -> None: diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index a1f51a449..1530c646e 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -22,6 +22,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'StoreBackground' + + class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name @@ -38,6 +40,7 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- + class StoreBackground(taskBase): def execute(self) -> None: diff --git a/src/swell/test/code_tests/slurm_test.py b/src/swell/test/code_tests/slurm_test.py index 506ce69fc..7295a6860 100644 --- a/src/swell/test/code_tests/slurm_test.py +++ b/src/swell/test/code_tests/slurm_test.py @@ -11,7 +11,7 @@ from swell.utilities.slurm import prepare_slurm_defaults_and_overrides from swell.utilities.logger import get_logger -from swell.tasks.base.task_attributes import TaskAttributes +from swell.tasks.base.task_attributes import task_attributes from unittest.mock import patch, Mock # -------------------------------------------------------------------------------------------------- @@ -53,7 +53,7 @@ def test_slurm_config(self, platform_mocked: Mock, mock_global_defaults: Mock) - sd_discover_sles15 = prepare_slurm_defaults_and_overrides(logger, 'nccs_discover_sles15', experiment_dict) - run_jedi_var_class = TaskAttributes.get('RunJediVariationalExecutable') + run_jedi_var_class = task_attributes.get('RunJediVariationalExecutable') run_jedi_var_obj = run_jedi_var_class('geos_marine', 'nccs_discover_sles15') run_jedi_var_slurm = run_jedi_var_obj.generate_task_slurm_dict( sd_discover_sles15) @@ -61,9 +61,9 @@ def test_slurm_config(self, platform_mocked: Mock, mock_global_defaults: Mock) - self.assertEqual(run_jedi_var_slurm["constraint"], "mil") self.assertEqual(run_jedi_var_slurm["qos"], "dastest") - eva_obs_class = TaskAttributes.get('EvaObservations') - build_jedi_class = TaskAttributes.get('BuildJedi') - run_jedi_ufo_class = TaskAttributes.get('RunJediUfoTestsExecutable') + eva_obs_class = task_attributes.get('EvaObservations') + build_jedi_class = task_attributes.get('BuildJedi') + run_jedi_ufo_class = task_attributes.get('RunJediUfoTestsExecutable') # Platform generic tests for sd in [sd_discover_sles15]: diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 432e1e927..1c62f8fac 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -10,7 +10,7 @@ import yaml from typing import Callable -from swell.tasks.base.task_attributes import TaskAttributes as task_attributes +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.logger import Logger from swell.suites.base.all_suites import suite_configs From 7ec1c51f0dbd098cc5da8d83f298075efb85fc13 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 16 Jan 2026 16:54:21 -0500 Subject: [PATCH 165/299] syntax fix --- src/swell/tasks/get_gsi_ncdiag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index ee698d669..d169bcb14 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -57,7 +57,7 @@ def execute(self) -> None: os.makedirs(gsi_diag_dir, 0o755, exist_ok=True) # Assert that some files were found - self.logger.assert_abort(len(gsi_diag_path_files) != 0 is not None, f'No ncdiag ' + + self.logger.assert_abort(len(gsi_diag_path_files) != 0, f'No ncdiag ' + f'files found in the source directory ' + f'\'{gsi_diag_path}\'') From 688cfd9b9c93eff379bc1af3a56f8be79400b517 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 16 Jan 2026 17:11:21 -0500 Subject: [PATCH 166/299] fixes for flow.cylc --- src/swell/suites/3dvar_cycle/workflow.py | 1 + src/swell/suites/hofx/workflow.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index 634128548..c0842d5b6 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -136,6 +136,7 @@ # Task defaults # ------------- + ''' # noqa # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index e0751a713..412cb0c3a 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -100,6 +100,7 @@ # Task defaults # ------------- + ''' # noqa # -------------------------------------------------------------------------------------------------- From 020838d4dc305cf53afb5c834699ec85654cf99c Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 10:30:34 -0500 Subject: [PATCH 167/299] discover --- src/swell/tasks/base/task_attributes.py | 28 ++++++++++++------------- src/swell/tasks/base/task_setup.py | 9 ++++++++ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index c0c6c7dd9..77588ba3c 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -10,11 +10,13 @@ import os import glob import importlib +import pkgutil from swell.swell_path import get_swell_path from swell.utilities.case_switching import snake_case_to_camel_case from swell.tasks.base.task_setup import TaskSetup from swell.tasks.stage_jedi import Setup as StageJedi +import swell.tasks # -------------------------------------------------------------------------------------------------- @@ -44,28 +46,26 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- +def discover_plugins(package): + + for loader, module_name, is_pkg in pkgutil.walk_packages(package.__path__): + full_module_name = f"{package.__name__}.{module_name}" + + importlib.import_module(full_module_name) + +# -------------------------------------------------------------------------------------------------- + class TaskAttributes(): def __init__(self) -> None: - task_path = os.path.join(get_swell_path(), 'tasks', '*.py') setattr(self, 'root', root) setattr(self, 'StageJediCycle', StageJediCycle) setattr(self, 'sync_point', sync_point) - for task_file in glob.glob(task_path): - module_name = os.path.basename(task_file).split('.py')[0] - module_path = f'swell.tasks.{module_name}' - - try: - module = importlib.import_module(module_path) - setup = getattr(module, 'Setup') - - task_name = snake_case_to_camel_case(module_name) - - setattr(self, task_name, setup) + discover_plugins(swell.tasks) - except AttributeError: - pass + for module_name, module in TaskSetup._registry.items(): + setattr(self, module_name, module) def get(self, task_name): return getattr(self, task_name) diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index 2eb9d890d..91b5e1f5c 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -41,6 +41,8 @@ class TaskSetup(ABC): additional_sections: list of additional CylcSection objects to append to the runtime section ''' + _registry = {} + def __init__(self, model: str | None = None, platform: str | None = None) -> None: self.model = model @@ -69,6 +71,13 @@ def __init__(self, model: str | None = None, platform: str | None = None) -> Non # -------------------------------------------------------------------------------------------------- + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + + cls._registry[cls.__name__.lower()] = cls + + # -------------------------------------------------------------------------------------------------- + @abstractmethod def set_attributes(self) -> None: '''Abstract method to be overridden by each task in order to set attributes. From c48a32ecfce4ccd3910de748d44e48541abc6b66 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 13:21:40 -0500 Subject: [PATCH 168/299] try registry --- src/swell/tasks/base/task_attributes.py | 10 +++++----- src/swell/tasks/bufr_to_ioda.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index 77588ba3c..6b003252c 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -57,15 +57,15 @@ def discover_plugins(package): class TaskAttributes(): def __init__(self) -> None: - setattr(self, 'root', root) setattr(self, 'StageJediCycle', StageJediCycle) setattr(self, 'sync_point', sync_point) - discover_plugins(swell.tasks) - - for module_name, module in TaskSetup._registry.items(): - setattr(self, module_name, module) + def register(self, name): + def wrapper(cls): + setattr(self, name, cls) + return cls + return wrapper def get(self, task_name): return getattr(self, task_name) diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 38899f5db..6cfd52afe 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -15,6 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- @@ -37,7 +38,7 @@ task_name = 'BufrToIoda' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name From 77d5b1f01e3a9c57df22c5ae09856a09758b77c4 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 13:25:29 -0500 Subject: [PATCH 169/299] fixes --- src/swell/tasks/base/task_attributes.py | 2 ++ src/swell/tasks/build_jedi_by_linking.py | 3 ++- src/swell/tasks/clone_jedi.py | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index 6b003252c..99afb9254 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -75,4 +75,6 @@ def get(self, task_name): task_attributes = TaskAttributes() +discover_plugins(swell.tasks) + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index 133c417fc..42739efa7 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -14,12 +14,13 @@ from swell.tasks.base.task_setup import TaskSetup from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import build_and_source_dirs, link_path +from swell.tasks.base.task_attributes import task_attributes # -------------------------------------------------------------------------------------------------- task_name = 'BuildJediByLinking' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index d7eace30f..25795d74b 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -16,13 +16,14 @@ from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.pinned_versions.check_hashes import check_hashes from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs +from swell.tasks.base.task_attributes import task_attributes # -------------------------------------------------------------------------------------------------- task_name = 'CloneJedi' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name From c7477c9bc27553616e5d5cedd1414a0d8ac6b607 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 13:37:52 -0500 Subject: [PATCH 170/299] register all tasks --- src/swell/tasks/build_geos.py | 3 ++- src/swell/tasks/build_geos_by_linking.py | 3 ++- src/swell/tasks/build_jedi.py | 3 ++- src/swell/tasks/build_jedi_by_linking.py | 1 + src/swell/tasks/clean_cycle.py | 3 ++- src/swell/tasks/clone_geos.py | 3 ++- src/swell/tasks/clone_geos_mksi.py | 3 ++- src/swell/tasks/clone_gmao_perllib.py | 3 ++- src/swell/tasks/clone_jedi.py | 1 + src/swell/tasks/eva_comparison_increment.py | 3 ++- src/swell/tasks/eva_comparison_jedi_log.py | 3 ++- src/swell/tasks/eva_increment.py | 3 ++- src/swell/tasks/eva_jedi_log.py | 3 ++- src/swell/tasks/eva_observations.py | 3 ++- src/swell/tasks/eva_timeseries.py | 3 ++- src/swell/tasks/generate_b_climatology.py | 3 ++- src/swell/tasks/generate_b_climatology_by_linking.py | 3 ++- src/swell/tasks/generate_observing_system_records.py | 3 ++- src/swell/tasks/get_background.py | 3 ++- src/swell/tasks/get_background_geos_experiment.py | 3 ++- src/swell/tasks/get_bufr.py | 3 ++- src/swell/tasks/get_ensemble.py | 3 ++- src/swell/tasks/get_ensemble_geos_experiment.py | 3 ++- src/swell/tasks/get_geos_adas_background.py | 3 ++- src/swell/tasks/get_geos_restart.py | 3 ++- src/swell/tasks/get_geovals.py | 3 ++- src/swell/tasks/get_gsi_bc.py | 3 ++- src/swell/tasks/get_gsi_ncdiag.py | 3 ++- src/swell/tasks/get_ncdiags.py | 3 ++- src/swell/tasks/get_obs_not_in_r2d2.py | 3 ++- src/swell/tasks/get_observations.py | 3 ++- src/swell/tasks/gsi_bc_to_ioda.py | 3 ++- src/swell/tasks/gsi_ncdiag_to_ioda.py | 3 ++- src/swell/tasks/jedi_log_comparison.py | 3 ++- src/swell/tasks/jedi_oops_log_parser.py | 3 ++- src/swell/tasks/link_geos_output.py | 3 ++- src/swell/tasks/move_da_restart.py | 3 ++- src/swell/tasks/move_forecast_restart.py | 3 ++- src/swell/tasks/prep_geos_run_dir.py | 3 ++- src/swell/tasks/prepare_analysis.py | 3 ++- src/swell/tasks/remove_forecast_dir.py | 3 ++- src/swell/tasks/render_jedi_observations.py | 3 ++- src/swell/tasks/run_geos_executable.py | 3 ++- src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py | 3 ++- src/swell/tasks/run_jedi_ensemble_mean_variance.py | 3 ++- src/swell/tasks/run_jedi_fgat_executable.py | 3 ++- src/swell/tasks/run_jedi_hofx_ensemble_executable.py | 3 ++- src/swell/tasks/run_jedi_hofx_executable.py | 3 ++- src/swell/tasks/run_jedi_local_ensemble_da_executable.py | 3 ++- src/swell/tasks/run_jedi_obsfilters_executable.py | 3 ++- src/swell/tasks/run_jedi_ufo_tests_executable.py | 3 ++- src/swell/tasks/run_jedi_variational_executable.py | 3 ++- src/swell/tasks/save_obs_diags.py | 3 ++- src/swell/tasks/save_restart.py | 3 ++- src/swell/tasks/stage_jedi.py | 3 ++- src/swell/tasks/store_background.py | 3 ++- 56 files changed, 110 insertions(+), 54 deletions(-) diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index 9c0ee9547..2081d0202 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -12,6 +12,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import build_and_source_dirs from swell.utilities.shell_commands import run_subprocess, create_executable_file @@ -21,7 +22,7 @@ task_name = 'BuildGeos' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index f1265d553..95dbe2b64 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -12,6 +12,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import build_and_source_dirs, link_path @@ -20,7 +21,7 @@ task_name = 'BuildGeosByLinking' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index 9ee68b9c4..78a455314 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -14,6 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs @@ -21,7 +22,7 @@ task_name = 'BuildJedi' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index 42739efa7..1b452ca19 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -12,6 +12,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import build_and_source_dirs, link_path from swell.tasks.base.task_attributes import task_attributes diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index 488900569..322131b98 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -10,6 +10,7 @@ import os from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats from datetime import datetime as dt @@ -19,7 +20,7 @@ task_name = 'CleanCycle' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 1f791064a..48b49f627 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -12,6 +12,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import build_and_source_dirs, link_path from swell.utilities.git_utils import git_clone @@ -21,7 +22,7 @@ task_name = 'CloneGeos' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index 8dcffcfd1..0eb4da03c 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -11,6 +11,7 @@ import os from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.build import link_path @@ -18,7 +19,7 @@ task_name = 'CloneGeosMksi' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/clone_gmao_perllib.py b/src/swell/tasks/clone_gmao_perllib.py index b36be7941..1bc49df5a 100644 --- a/src/swell/tasks/clone_gmao_perllib.py +++ b/src/swell/tasks/clone_gmao_perllib.py @@ -13,13 +13,14 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- task_name = 'CloneGmaoPerllib' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index 25795d74b..bca392c0b 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -13,6 +13,7 @@ from swell.utilities.build import link_path from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.pinned_versions.check_hashes import check_hashes from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index b14c9c6e2..790136fd4 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -14,6 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.data_assimilation_window_params import DataAssimilationWindowParams @@ -22,7 +23,7 @@ task_name = 'EvaComparisonIncrement' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index c996abc99..25c01a9c9 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -13,6 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.jinja2 import template_string_jinja2 @@ -21,7 +22,7 @@ task_name = 'EvaComparisonJediLog' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index d7fc5b866..aeb881a6c 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -13,6 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.jinja2 import template_string_jinja2 @@ -20,7 +21,7 @@ task_name = 'EvaIncrement' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index 530570a02..0d2900508 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -13,6 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.jinja2 import template_string_jinja2 @@ -20,7 +21,7 @@ task_name = 'EvaJediLog' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index a4911d02b..c2ad8c502 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -15,6 +15,7 @@ from swell.deployment.platforms.platforms import login_or_compute from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 @@ -34,7 +35,7 @@ def run_eva(eva_dict: dict): task_name = 'EvaObservations' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/eva_timeseries.py b/src/swell/tasks/eva_timeseries.py index 77690a96b..9c60a417b 100644 --- a/src/swell/tasks/eva_timeseries.py +++ b/src/swell/tasks/eva_timeseries.py @@ -17,6 +17,7 @@ from swell.deployment.platforms.platforms import login_or_compute from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary @@ -37,7 +38,7 @@ def run_eva(eva_dict: dict): task_name = 'EvaTimeseries' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index cfd217fcd..a1ea635c1 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -10,6 +10,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.shell_commands import run_track_log_subprocess from swell.utilities.file_system_operations import check_if_files_exist_in_path @@ -18,7 +19,7 @@ task_name = 'GenerateBClimatology' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 9aefba725..f0fa84b98 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -9,6 +9,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import link_all_files_from_first_in_hierarchy_of_sources @@ -17,7 +18,7 @@ task_name = 'GenerateBClimatologyByLinking' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index 5164ae6d8..d6d7048f9 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -12,6 +12,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.observing_system_records import ObservingSystemRecords @@ -19,7 +20,7 @@ task_name = 'GenerateObservingSystemRecords' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 8044bd067..2961a8149 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -10,6 +10,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd import isodate @@ -27,7 +28,7 @@ task_name = 'GetBackground' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index cd81ecbb8..580940ffd 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -15,6 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats @@ -22,7 +23,7 @@ task_name = 'GetBackgroundGeosExperiment' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_bufr.py b/src/swell/tasks/get_bufr.py index 86b5894fb..129840e8c 100644 --- a/src/swell/tasks/get_bufr.py +++ b/src/swell/tasks/get_bufr.py @@ -15,13 +15,14 @@ from swell.utilities.datetime_util import datetime_formats from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- task_name = 'GetBufr' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_ensemble.py b/src/swell/tasks/get_ensemble.py index 3a66eeb34..84d0f2e2f 100644 --- a/src/swell/tasks/get_ensemble.py +++ b/src/swell/tasks/get_ensemble.py @@ -13,6 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd @@ -20,7 +21,7 @@ task_name = 'GetEnsemble' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_ensemble_geos_experiment.py b/src/swell/tasks/get_ensemble_geos_experiment.py index a023e5368..449f9f3b0 100644 --- a/src/swell/tasks/get_ensemble_geos_experiment.py +++ b/src/swell/tasks/get_ensemble_geos_experiment.py @@ -14,6 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats @@ -21,7 +22,7 @@ task_name = 'GetEnsembleGeosExperiment' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py index d273bc3df..3dc544110 100644 --- a/src/swell/tasks/get_geos_adas_background.py +++ b/src/swell/tasks/get_geos_adas_background.py @@ -15,6 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd @@ -22,7 +23,7 @@ task_name = 'GetGeosAdasBackground' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_geos_restart.py b/src/swell/tasks/get_geos_restart.py index 39e0363b7..76d3fc586 100644 --- a/src/swell/tasks/get_geos_restart.py +++ b/src/swell/tasks/get_geos_restart.py @@ -12,6 +12,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import copy_to_dst_dir, check_if_files_exist_in_path @@ -19,7 +20,7 @@ task_name = 'GetGeosRestart' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index cd4cf8eb4..a61f58c73 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -12,6 +12,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.r2d2 import create_r2d2_config @@ -20,7 +21,7 @@ task_name = 'GetGeovals' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index a4ec7d2d8..b340625e5 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -16,6 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd @@ -23,7 +24,7 @@ task_name = 'GetGsiBc' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index d169bcb14..5509cb752 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -13,6 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd @@ -20,7 +21,7 @@ task_name = 'GetGsiNcdiag' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index a69ecb3b3..49db493f2 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -10,13 +10,14 @@ import os from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- task_name = 'GetNcdiags' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py index c19289d45..fbc696600 100644 --- a/src/swell/tasks/get_obs_not_in_r2d2.py +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -14,6 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd @@ -21,7 +22,7 @@ task_name = 'GetObsNotInR2d2' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index aa6682b20..0bd67aaa3 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -15,6 +15,7 @@ from datetime import timedelta, datetime as dt from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.r2d2 import create_r2d2_config from swell.utilities.datetime_util import datetime_formats @@ -32,7 +33,7 @@ task_name = 'GetObservations' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index ac418e9d4..15e9846ae 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -14,6 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.dictionary import write_dict_to_yaml from swell.utilities.shell_commands import run_track_log_subprocess @@ -23,7 +24,7 @@ task_name = 'GsiBcToIoda' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 108ba05ba..c4eb9d509 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -16,6 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.shell_commands import run_subprocess, create_executable_file @@ -25,7 +26,7 @@ task_name = 'GsiNcdiagToIoda' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py index 5589d941c..40b3f1986 100644 --- a/src/swell/tasks/jedi_log_comparison.py +++ b/src/swell/tasks/jedi_log_comparison.py @@ -15,6 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- @@ -25,7 +26,7 @@ task_name = 'JediLogComparison' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/jedi_oops_log_parser.py b/src/swell/tasks/jedi_oops_log_parser.py index 640049966..3147db8a6 100644 --- a/src/swell/tasks/jedi_oops_log_parser.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -13,13 +13,14 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- task_name = 'JediOopsLogParser' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index 3b5b4b21d..dd2504cc8 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -17,13 +17,14 @@ from swell.utilities.datetime_util import datetime_formats from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- task_name = 'LinkGeosOutput' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/move_da_restart.py b/src/swell/tasks/move_da_restart.py index 5cc6b2411..772ec4ac7 100644 --- a/src/swell/tasks/move_da_restart.py +++ b/src/swell/tasks/move_da_restart.py @@ -14,6 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import move_files @@ -21,7 +22,7 @@ task_name = 'MoveDaRestart' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/move_forecast_restart.py b/src/swell/tasks/move_forecast_restart.py index 64b8748b7..80e9955da 100644 --- a/src/swell/tasks/move_forecast_restart.py +++ b/src/swell/tasks/move_forecast_restart.py @@ -13,6 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import move_files @@ -20,7 +21,7 @@ task_name = 'MoveForecastRestart' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/prep_geos_run_dir.py b/src/swell/tasks/prep_geos_run_dir.py index 896685656..b3e86c081 100644 --- a/src/swell/tasks/prep_geos_run_dir.py +++ b/src/swell/tasks/prep_geos_run_dir.py @@ -16,6 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import copy_to_dst_dir, check_if_files_exist_in_path @@ -23,7 +24,7 @@ task_name = 'PrepGeosRunDir' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index 7340c4a5a..aa5d63312 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -16,13 +16,14 @@ from swell.utilities.shell_commands import run_subprocess from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- task_name = 'PrepareAnalysis' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/remove_forecast_dir.py b/src/swell/tasks/remove_forecast_dir.py index 66271fd0a..e93e7e9e9 100644 --- a/src/swell/tasks/remove_forecast_dir.py +++ b/src/swell/tasks/remove_forecast_dir.py @@ -11,12 +11,13 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes # -------------------------------------------------------------------------------------------------- task_name = 'RemoveForecastDir' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index ade6113d1..230dacfe0 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -13,6 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import check_obs @@ -20,7 +21,7 @@ task_name = 'RenderJediObservations' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/run_geos_executable.py b/src/swell/tasks/run_geos_executable.py index 128feccf0..580ddf521 100644 --- a/src/swell/tasks/run_geos_executable.py +++ b/src/swell/tasks/run_geos_executable.py @@ -12,13 +12,14 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.shell_commands import run_track_log_subprocess # -------------------------------------------------------------------------------------------------- task_name = 'RunGeosExecutable' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py index 7c032ab65..088cb6b5e 100644 --- a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py +++ b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py @@ -13,6 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable @@ -21,7 +22,7 @@ task_name = 'RunJediConvertStateSoca2ciceExecutable' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/run_jedi_ensemble_mean_variance.py b/src/swell/tasks/run_jedi_ensemble_mean_variance.py index 0fc525c7f..a4e785de9 100644 --- a/src/swell/tasks/run_jedi_ensemble_mean_variance.py +++ b/src/swell/tasks/run_jedi_ensemble_mean_variance.py @@ -13,6 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable @@ -21,7 +22,7 @@ task_name = 'RunJediEnsembleMeanVariance' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/run_jedi_fgat_executable.py b/src/swell/tasks/run_jedi_fgat_executable.py index 428c2c0e6..62a68c08b 100644 --- a/src/swell/tasks/run_jedi_fgat_executable.py +++ b/src/swell/tasks/run_jedi_fgat_executable.py @@ -12,6 +12,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable @@ -20,7 +21,7 @@ task_name = 'RunJediFgatExecutable' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py index ec5dfa91f..3af6c1b40 100644 --- a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py +++ b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py @@ -13,6 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable from swell.tasks.run_jedi_hofx_executable import RunJediHofxExecutable @@ -21,7 +22,7 @@ task_name = 'RunJediHofxEnsembleExecutable' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index deb166bf1..1716ea3e6 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -15,6 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.netcdf_files import combine_files_without_groups from swell.utilities.run_jedi_executables import run_executable @@ -24,7 +25,7 @@ task_name = 'RunJediHofxExecutable' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index ea9746275..5d93bebe5 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -14,6 +14,7 @@ from swell.swell_path import get_swell_path from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable @@ -40,7 +41,7 @@ def replace_key(obj, old_key, new_key): task_name = 'RunJediLocalEnsembleDaExecutable' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index a07e3d447..dc05aa594 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -14,6 +14,7 @@ import random from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable @@ -21,7 +22,7 @@ task_name = 'RunJediObsfiltersExecutable' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index cc2a08803..c8792ed65 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -14,6 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.dictionary import update_dict from swell.utilities.run_jedi_executables import run_executable @@ -23,7 +24,7 @@ task_name = 'RunJediUfoTestsExecutable' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 6da894c41..dc25e40fa 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -12,6 +12,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable @@ -20,7 +21,7 @@ task_name = 'RunJediVariationalExecutable' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 8d8ed0675..3f15fc202 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -11,6 +11,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.r2d2 import create_r2d2_config from swell.utilities.run_jedi_executables import check_obs @@ -19,7 +20,7 @@ task_name = 'SaveObsDiags' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index c51d4d741..f3e22f8a6 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -13,6 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.file_system_operations import copy_to_dst_dir @@ -21,7 +22,7 @@ task_name = 'SaveRestart' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index d820d9eaf..b5d27db91 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -13,6 +13,7 @@ from swell.swell_path import get_swell_path from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.filehandler import get_file_handler from swell.utilities.exceptions import SwellError @@ -23,7 +24,7 @@ task_name = 'StageJedi' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index 1530c646e..425a08ac6 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -15,6 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats @@ -23,7 +24,7 @@ task_name = 'StoreBackground' - +@task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name From 8d34b1584e854e28d0595acb1d08bd938655ad51 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 13:41:51 -0500 Subject: [PATCH 171/299] fix stage_jedi_cycle --- src/swell/tasks/base/task_attributes.py | 12 +----------- src/swell/tasks/stage_jedi.py | 9 +++++++++ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index 99afb9254..49f65631c 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -15,8 +15,6 @@ from swell.swell_path import get_swell_path from swell.utilities.case_switching import snake_case_to_camel_case from swell.tasks.base.task_setup import TaskSetup -from swell.tasks.stage_jedi import Setup as StageJedi -import swell.tasks # -------------------------------------------------------------------------------------------------- @@ -30,15 +28,6 @@ def set_attributes(self): 'config': '$CYLC_SUITE_DEF_PATH/experiment.yaml'})] # noqa -class StageJediCycle(StageJedi): - def set_attributes(self): - super().set_attributes() - self.base_name = "StageJedi" - self.scheduling_name = "StageJediCycle-{model}" - self.is_cycling = True - self.is_model = True - - class sync_point(TaskSetup): def set_attributes(self): self.script = "true" @@ -75,6 +64,7 @@ def get(self, task_name): task_attributes = TaskAttributes() +import swell.tasks discover_plugins(swell.tasks) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index b5d27db91..6b9a468d0 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -39,6 +39,15 @@ def set_attributes(self): qd.vertical_resolution() ] +@task_attributes.register(task_name) +class StageJediCycle(Setup): + def set_attributes(self): + super().set_attributes() + self.base_name = "StageJedi" + self.scheduling_name = "StageJediCycle-{model}" + self.is_cycling = True + self.is_model = True + # -------------------------------------------------------------------------------------------------- From c0839a7e371a70786d17867da16fe0e19c08292d Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 14:08:18 -0500 Subject: [PATCH 172/299] add registry to all tasks --- src/swell/tasks/base/task_attributes.py | 8 ++------ src/swell/tasks/bufr_to_ioda.py | 1 + src/swell/tasks/build_geos.py | 1 + src/swell/tasks/build_geos_by_linking.py | 1 + src/swell/tasks/build_jedi.py | 1 + src/swell/tasks/build_jedi_by_linking.py | 1 + src/swell/tasks/clean_cycle.py | 1 + src/swell/tasks/clone_geos.py | 1 + src/swell/tasks/clone_geos_mksi.py | 1 + src/swell/tasks/clone_gmao_perllib.py | 1 + src/swell/tasks/clone_jedi.py | 1 + src/swell/tasks/eva_comparison_increment.py | 1 + src/swell/tasks/eva_comparison_jedi_log.py | 1 + src/swell/tasks/eva_increment.py | 1 + src/swell/tasks/eva_jedi_log.py | 1 + src/swell/tasks/eva_observations.py | 1 + src/swell/tasks/eva_timeseries.py | 1 + src/swell/tasks/generate_b_climatology.py | 1 + src/swell/tasks/generate_b_climatology_by_linking.py | 1 + src/swell/tasks/generate_observing_system_records.py | 1 + src/swell/tasks/get_background.py | 1 + src/swell/tasks/get_background_geos_experiment.py | 1 + src/swell/tasks/get_bufr.py | 1 + src/swell/tasks/get_ensemble.py | 1 + src/swell/tasks/get_ensemble_geos_experiment.py | 1 + src/swell/tasks/get_geos_adas_background.py | 1 + src/swell/tasks/get_geos_restart.py | 1 + src/swell/tasks/get_geovals.py | 1 + src/swell/tasks/get_gsi_bc.py | 1 + src/swell/tasks/get_gsi_ncdiag.py | 1 + src/swell/tasks/get_ncdiags.py | 1 + src/swell/tasks/get_obs_not_in_r2d2.py | 1 + src/swell/tasks/get_observations.py | 1 + src/swell/tasks/gsi_bc_to_ioda.py | 1 + src/swell/tasks/gsi_ncdiag_to_ioda.py | 1 + src/swell/tasks/jedi_log_comparison.py | 1 + src/swell/tasks/jedi_oops_log_parser.py | 1 + src/swell/tasks/link_geos_output.py | 1 + src/swell/tasks/move_da_restart.py | 1 + src/swell/tasks/move_forecast_restart.py | 1 + src/swell/tasks/prep_geos_run_dir.py | 1 + src/swell/tasks/prepare_analysis.py | 1 + src/swell/tasks/remove_forecast_dir.py | 1 + src/swell/tasks/render_jedi_observations.py | 1 + src/swell/tasks/run_geos_executable.py | 1 + .../tasks/run_jedi_convert_state_soca2cice_executable.py | 1 + src/swell/tasks/run_jedi_ensemble_mean_variance.py | 1 + src/swell/tasks/run_jedi_fgat_executable.py | 1 + src/swell/tasks/run_jedi_hofx_ensemble_executable.py | 1 + src/swell/tasks/run_jedi_hofx_executable.py | 1 + src/swell/tasks/run_jedi_local_ensemble_da_executable.py | 1 + src/swell/tasks/run_jedi_obsfilters_executable.py | 1 + src/swell/tasks/run_jedi_ufo_tests_executable.py | 1 + src/swell/tasks/run_jedi_variational_executable.py | 1 + src/swell/tasks/save_obs_diags.py | 1 + src/swell/tasks/save_restart.py | 1 + src/swell/tasks/stage_jedi.py | 4 +++- src/swell/tasks/store_background.py | 1 + 58 files changed, 61 insertions(+), 7 deletions(-) diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index 49f65631c..f502dc0c4 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -7,13 +7,10 @@ # -------------------------------------------------------------------------------------------------- -import os -import glob import importlib import pkgutil -from swell.swell_path import get_swell_path -from swell.utilities.case_switching import snake_case_to_camel_case +import swell.tasks from swell.tasks.base.task_setup import TaskSetup # -------------------------------------------------------------------------------------------------- @@ -44,10 +41,10 @@ def discover_plugins(package): # -------------------------------------------------------------------------------------------------- + class TaskAttributes(): def __init__(self) -> None: setattr(self, 'root', root) - setattr(self, 'StageJediCycle', StageJediCycle) setattr(self, 'sync_point', sync_point) def register(self, name): @@ -64,7 +61,6 @@ def get(self, task_name): task_attributes = TaskAttributes() -import swell.tasks discover_plugins(swell.tasks) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 6cfd52afe..16bb2340d 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -38,6 +38,7 @@ task_name = 'BufrToIoda' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index 2081d0202..a636c55e2 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -22,6 +22,7 @@ task_name = 'BuildGeos' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index 95dbe2b64..b56c201cc 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -21,6 +21,7 @@ task_name = 'BuildGeosByLinking' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index 78a455314..e2039182a 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -22,6 +22,7 @@ task_name = 'BuildJedi' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index 1b452ca19..e67606f41 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -21,6 +21,7 @@ task_name = 'BuildJediByLinking' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index 322131b98..f991faac7 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -20,6 +20,7 @@ task_name = 'CleanCycle' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 48b49f627..ce7f59781 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -22,6 +22,7 @@ task_name = 'CloneGeos' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index 0eb4da03c..b5a08a583 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -19,6 +19,7 @@ task_name = 'CloneGeosMksi' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/clone_gmao_perllib.py b/src/swell/tasks/clone_gmao_perllib.py index 1bc49df5a..c73eea362 100644 --- a/src/swell/tasks/clone_gmao_perllib.py +++ b/src/swell/tasks/clone_gmao_perllib.py @@ -20,6 +20,7 @@ task_name = 'CloneGmaoPerllib' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index bca392c0b..7021a3f4b 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -24,6 +24,7 @@ task_name = 'CloneJedi' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index 790136fd4..fbe22d8c6 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -23,6 +23,7 @@ task_name = 'EvaComparisonIncrement' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 25c01a9c9..94c9dc127 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -22,6 +22,7 @@ task_name = 'EvaComparisonJediLog' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index aeb881a6c..269a197fb 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -21,6 +21,7 @@ task_name = 'EvaIncrement' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index 0d2900508..d6aac9ac5 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -21,6 +21,7 @@ task_name = 'EvaJediLog' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index c2ad8c502..4265b0ba3 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -35,6 +35,7 @@ def run_eva(eva_dict: dict): task_name = 'EvaObservations' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/eva_timeseries.py b/src/swell/tasks/eva_timeseries.py index 9c60a417b..9342abe18 100644 --- a/src/swell/tasks/eva_timeseries.py +++ b/src/swell/tasks/eva_timeseries.py @@ -38,6 +38,7 @@ def run_eva(eva_dict: dict): task_name = 'EvaTimeseries' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index a1ea635c1..55476dd19 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -19,6 +19,7 @@ task_name = 'GenerateBClimatology' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index f0fa84b98..941fa814c 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -18,6 +18,7 @@ task_name = 'GenerateBClimatologyByLinking' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index d6d7048f9..bbeb93b99 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -20,6 +20,7 @@ task_name = 'GenerateObservingSystemRecords' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 2961a8149..fa8fa63fe 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -28,6 +28,7 @@ task_name = 'GetBackground' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index 580940ffd..eb09b30c5 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -23,6 +23,7 @@ task_name = 'GetBackgroundGeosExperiment' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_bufr.py b/src/swell/tasks/get_bufr.py index 129840e8c..800f4623d 100644 --- a/src/swell/tasks/get_bufr.py +++ b/src/swell/tasks/get_bufr.py @@ -22,6 +22,7 @@ task_name = 'GetBufr' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_ensemble.py b/src/swell/tasks/get_ensemble.py index 84d0f2e2f..9302d8c74 100644 --- a/src/swell/tasks/get_ensemble.py +++ b/src/swell/tasks/get_ensemble.py @@ -21,6 +21,7 @@ task_name = 'GetEnsemble' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_ensemble_geos_experiment.py b/src/swell/tasks/get_ensemble_geos_experiment.py index 449f9f3b0..72c60a5ef 100644 --- a/src/swell/tasks/get_ensemble_geos_experiment.py +++ b/src/swell/tasks/get_ensemble_geos_experiment.py @@ -22,6 +22,7 @@ task_name = 'GetEnsembleGeosExperiment' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py index 3dc544110..1d83c832c 100644 --- a/src/swell/tasks/get_geos_adas_background.py +++ b/src/swell/tasks/get_geos_adas_background.py @@ -23,6 +23,7 @@ task_name = 'GetGeosAdasBackground' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_geos_restart.py b/src/swell/tasks/get_geos_restart.py index 76d3fc586..80bb37116 100644 --- a/src/swell/tasks/get_geos_restart.py +++ b/src/swell/tasks/get_geos_restart.py @@ -20,6 +20,7 @@ task_name = 'GetGeosRestart' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index a61f58c73..80207b05d 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -21,6 +21,7 @@ task_name = 'GetGeovals' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index b340625e5..3b8e5ce43 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -24,6 +24,7 @@ task_name = 'GetGsiBc' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index 5509cb752..5f499c4e5 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -21,6 +21,7 @@ task_name = 'GetGsiNcdiag' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index 49db493f2..10a508a51 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -17,6 +17,7 @@ task_name = 'GetNcdiags' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py index fbc696600..5dd5bf425 100644 --- a/src/swell/tasks/get_obs_not_in_r2d2.py +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -22,6 +22,7 @@ task_name = 'GetObsNotInR2d2' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 0bd67aaa3..309df7aa1 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -33,6 +33,7 @@ task_name = 'GetObservations' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 15e9846ae..0dac3e626 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -24,6 +24,7 @@ task_name = 'GsiBcToIoda' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index c4eb9d509..930bc22a1 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -26,6 +26,7 @@ task_name = 'GsiNcdiagToIoda' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py index 40b3f1986..99a1e6acd 100644 --- a/src/swell/tasks/jedi_log_comparison.py +++ b/src/swell/tasks/jedi_log_comparison.py @@ -26,6 +26,7 @@ task_name = 'JediLogComparison' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/jedi_oops_log_parser.py b/src/swell/tasks/jedi_oops_log_parser.py index 3147db8a6..dc001d315 100644 --- a/src/swell/tasks/jedi_oops_log_parser.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -20,6 +20,7 @@ task_name = 'JediOopsLogParser' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index dd2504cc8..d5d32ec0b 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -24,6 +24,7 @@ task_name = 'LinkGeosOutput' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/move_da_restart.py b/src/swell/tasks/move_da_restart.py index 772ec4ac7..db370ee1c 100644 --- a/src/swell/tasks/move_da_restart.py +++ b/src/swell/tasks/move_da_restart.py @@ -22,6 +22,7 @@ task_name = 'MoveDaRestart' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/move_forecast_restart.py b/src/swell/tasks/move_forecast_restart.py index 80e9955da..1d3515604 100644 --- a/src/swell/tasks/move_forecast_restart.py +++ b/src/swell/tasks/move_forecast_restart.py @@ -21,6 +21,7 @@ task_name = 'MoveForecastRestart' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/prep_geos_run_dir.py b/src/swell/tasks/prep_geos_run_dir.py index b3e86c081..549d0e05b 100644 --- a/src/swell/tasks/prep_geos_run_dir.py +++ b/src/swell/tasks/prep_geos_run_dir.py @@ -24,6 +24,7 @@ task_name = 'PrepGeosRunDir' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index aa5d63312..3349bc94d 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -23,6 +23,7 @@ task_name = 'PrepareAnalysis' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/remove_forecast_dir.py b/src/swell/tasks/remove_forecast_dir.py index e93e7e9e9..5e8ee390e 100644 --- a/src/swell/tasks/remove_forecast_dir.py +++ b/src/swell/tasks/remove_forecast_dir.py @@ -17,6 +17,7 @@ task_name = 'RemoveForecastDir' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index 230dacfe0..02d0766ae 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -21,6 +21,7 @@ task_name = 'RenderJediObservations' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/run_geos_executable.py b/src/swell/tasks/run_geos_executable.py index 580ddf521..0ed4e6d2f 100644 --- a/src/swell/tasks/run_geos_executable.py +++ b/src/swell/tasks/run_geos_executable.py @@ -19,6 +19,7 @@ task_name = 'RunGeosExecutable' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py index 088cb6b5e..449198c17 100644 --- a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py +++ b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py @@ -22,6 +22,7 @@ task_name = 'RunJediConvertStateSoca2ciceExecutable' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/run_jedi_ensemble_mean_variance.py b/src/swell/tasks/run_jedi_ensemble_mean_variance.py index a4e785de9..eeaa74f64 100644 --- a/src/swell/tasks/run_jedi_ensemble_mean_variance.py +++ b/src/swell/tasks/run_jedi_ensemble_mean_variance.py @@ -22,6 +22,7 @@ task_name = 'RunJediEnsembleMeanVariance' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/run_jedi_fgat_executable.py b/src/swell/tasks/run_jedi_fgat_executable.py index 62a68c08b..d2bf89aea 100644 --- a/src/swell/tasks/run_jedi_fgat_executable.py +++ b/src/swell/tasks/run_jedi_fgat_executable.py @@ -21,6 +21,7 @@ task_name = 'RunJediFgatExecutable' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py index 3af6c1b40..d970e6025 100644 --- a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py +++ b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py @@ -22,6 +22,7 @@ task_name = 'RunJediHofxEnsembleExecutable' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 1716ea3e6..3562137f9 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -25,6 +25,7 @@ task_name = 'RunJediHofxExecutable' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 5d93bebe5..e8d402610 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -41,6 +41,7 @@ def replace_key(obj, old_key, new_key): task_name = 'RunJediLocalEnsembleDaExecutable' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index dc05aa594..cf9ab843a 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -22,6 +22,7 @@ task_name = 'RunJediObsfiltersExecutable' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index c8792ed65..c46151e6e 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -24,6 +24,7 @@ task_name = 'RunJediUfoTestsExecutable' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index dc25e40fa..8ec002924 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -21,6 +21,7 @@ task_name = 'RunJediVariationalExecutable' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 3f15fc202..8d0acf605 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -20,6 +20,7 @@ task_name = 'SaveObsDiags' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index f3e22f8a6..c8ace09f8 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -22,6 +22,7 @@ task_name = 'SaveRestart' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 6b9a468d0..95e002aa1 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -24,6 +24,7 @@ task_name = 'StageJedi' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): @@ -39,7 +40,8 @@ def set_attributes(self): qd.vertical_resolution() ] -@task_attributes.register(task_name) + +@task_attributes.register('StageJediCycle') class StageJediCycle(Setup): def set_attributes(self): super().set_attributes() diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index 425a08ac6..70d5491a1 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -24,6 +24,7 @@ task_name = 'StoreBackground' + @task_attributes.register(task_name) class Setup(TaskSetup): def set_attributes(self): From c3346598715c788a99c23eec784700e8987107d0 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 15:08:04 -0500 Subject: [PATCH 173/299] Add docs --- docs/examples/templating_workflows.md | 27 ++++++++++++++++++++----- src/swell/tasks/base/task_attributes.py | 9 +++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/examples/templating_workflows.md b/docs/examples/templating_workflows.md index 6929216ff..593ffe6ed 100644 --- a/docs/examples/templating_workflows.md +++ b/docs/examples/templating_workflows.md @@ -8,30 +8,47 @@ The `flow.cylc` that is generated under this method is not much different from t ## Tasks and the runtime section -Swell will parse the graph section, which is constructed first, to obtain the tasks which are used by the experiment. It will then build the runtime section by consulting `src/swell/tasks/base/task_attributes.py`. Since swell tasks broadly fall into only a few categories (model-dependent or independent, cycling or non-cycling) that do not differ much between suites, they are easily abstracted into a `TaskSetup` class. This class will dynamically set attributes such as messaging parameters and slurm settings. +Swell will parse the graph section, which is constructed first, to obtain the tasks which are used by the experiment. It will then build the runtime section by consulting task setup objects. Since swell tasks broadly fall into only a few categories (model-dependent or independent, cycling or non-cycling) that do not differ much between suites, they are easily abstracted into a `TaskSetup` class. This class will dynamically set attributes such as messaging parameters and slurm settings. Each task has an associated `TaskSetup` class, which is defined in the main task file, and registered into the `TaskAttributes` container. For example, the following displays the `TaskSetup` class for `CloneJedi`, located in `src/swell/tasks/clone_jedi.py`: ```python -class CloneJedi(TaskSetup): + +task_name = 'CloneJedi' + + +@task_attributes.register(task_name) +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.questions = [ qd.bundles(), qd.existing_jedi_source_directory(), qd.existing_jedi_source_directory_pinned(), qd.jedi_build_method() ] +``` +Other tasks have different requirements, such as `EvaObservations`: -class EvaObservations(TaskSetup): +```python +task_name = 'EvaObservations' + + +@task_attributes.register(task_name) +class Setup(TaskSetup): def set_attributes(self): + self.base_name = task_name self.time_limit = True self.is_cycling = True self.is_model = True self.slurm = {} self.questions = [ - background_crtm_obs, + qd.background_time_offset(), + qd.crtm_coeff_dir(), + qd.observations(), + qd.observing_system_records_path(), qd.marine_models(), qd.observing_system_records_path(), - qd.window_offset(), + qd.window_length(), qd.marine_models(), ] ``` diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index f502dc0c4..a8ea6aa8d 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -33,7 +33,11 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- def discover_plugins(package): + '''Walk through packages to trigger any hooks. + Parameters: + package: Python package + ''' for loader, module_name, is_pkg in pkgutil.walk_packages(package.__path__): full_module_name = f"{package.__name__}.{module_name}" @@ -48,6 +52,11 @@ def __init__(self) -> None: setattr(self, 'sync_point', sync_point) def register(self, name): + '''Provides wrapper to register class using . + + Parameters: + name: Name to refer to Setup object + ''' def wrapper(cls): setattr(self, name, cls) return cls From d329a390c6653985cdf96a8467ce3ed62e45bf7f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 15:16:56 -0500 Subject: [PATCH 174/299] Move jedi_bundle import --- src/swell/tasks/build_jedi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index e2039182a..4287c709b 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -10,8 +10,6 @@ import os -from jedi_bundle.bin.jedi_bundle import execute_tasks, get_bundles - from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes @@ -41,6 +39,8 @@ class BuildJedi(taskBase): def execute(self) -> None: + from jedi_bundle.bin.jedi_bundle import execute_tasks, get_bundles + # Get the experiment/jedi_bundle directory # ---------------------------------------- swell_exp_path = self.experiment_path() From 47462f593decc69b2966481cb5a0ad883637de26 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 16:58:26 -0500 Subject: [PATCH 175/299] fix import --- src/swell/tasks/build_geos.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index a636c55e2..fac942d73 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -14,7 +14,6 @@ from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.utilities.build import build_and_source_dirs from swell.utilities.shell_commands import run_subprocess, create_executable_file @@ -43,6 +42,8 @@ def execute(self) -> None: swell_exp_path = self.experiment_path() geos_gcm_path = os.path.join(swell_exp_path, 'GEOSgcm') + from swell.utilities.build import build_and_source_dirs + # Get paths to build and source # ----------------------------- geos_gcm_build_path, geos_gcm_source_path = build_and_source_dirs(geos_gcm_path) From 2fc2e189c21a953b912e384a43b3d309af5d3f2b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 17:25:34 -0500 Subject: [PATCH 176/299] Move build imports --- src/swell/tasks/build_geos_by_linking.py | 3 ++- src/swell/tasks/build_jedi.py | 3 ++- src/swell/tasks/build_jedi_by_linking.py | 3 ++- src/swell/tasks/clone_geos.py | 3 ++- src/swell/tasks/clone_geos_mksi.py | 3 ++- src/swell/tasks/clone_jedi.py | 3 +-- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index b56c201cc..9f6989535 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -14,7 +14,6 @@ from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.utilities.build import build_and_source_dirs, link_path # -------------------------------------------------------------------------------------------------- @@ -44,6 +43,8 @@ def execute(self) -> None: swell_exp_path = self.experiment_path() geos_gcm_path = os.path.join(swell_exp_path, 'GEOSgcm') + from swell.utilities.build import build_and_source_dirs, link_path + # Get paths to build and source # ----------------------------- geos_gcm_build_path, geos_gcm_source_path = build_and_source_dirs(geos_gcm_path) diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index 4287c709b..dc720ea82 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -14,7 +14,6 @@ from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs # -------------------------------------------------------------------------------------------------- @@ -46,6 +45,8 @@ def execute(self) -> None: swell_exp_path = self.experiment_path() jedi_bundle_path = os.path.join(swell_exp_path, 'jedi_bundle') + from swell.utilities.build import build_and_source_dirs, link_path + # Get paths to build and source # ----------------------------- jedi_bundle_build_path, jedi_bundle_source_path = build_and_source_dirs(jedi_bundle_path) diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index e67606f41..24c744d87 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -14,7 +14,6 @@ from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.utilities.build import build_and_source_dirs, link_path from swell.tasks.base.task_attributes import task_attributes # -------------------------------------------------------------------------------------------------- @@ -50,6 +49,8 @@ def execute(self) -> None: swell_exp_path = self.experiment_path() jedi_bundle_path = os.path.join(swell_exp_path, 'jedi_bundle') + from swell.utilities.build import build_and_source_dirs, link_path + # Get paths to build and source jedi_bundle_build_path, jedi_bundle_source_path = build_and_source_dirs(jedi_bundle_path) diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index ce7f59781..3bfd15846 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -14,7 +14,6 @@ from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.utilities.build import build_and_source_dirs, link_path from swell.utilities.git_utils import git_clone from swell.utilities.shell_commands import run_subprocess @@ -45,6 +44,8 @@ def execute(self) -> None: swell_exp_path = self.experiment_path() geos_gcm_path = os.path.join(swell_exp_path, 'GEOSgcm') + from swell.utilities.build import build_and_source_dirs, link_path + # Get paths to build and source # ----------------------------- geos_gcm_build_path, geos_gcm_source_path = build_and_source_dirs(geos_gcm_path) diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index b5a08a583..5cfc63476 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -13,7 +13,6 @@ from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.utilities.build import link_path # -------------------------------------------------------------------------------------------------- @@ -41,6 +40,8 @@ def execute(self) -> None: Generate the satellite channel record from GEOSmksi files """ + from swell.utilities.build import link_path + # This task should only execute for geos_atmosphere # ------------------------------------------------- if self.get_model() != 'geos_atmosphere': diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index 7021a3f4b..43fc985fe 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -10,13 +10,11 @@ import os -from swell.utilities.build import link_path from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd from swell.utilities.pinned_versions.check_hashes import check_hashes -from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs from swell.tasks.base.task_attributes import task_attributes @@ -46,6 +44,7 @@ def execute(self) -> None: # Import JEDI modules # ------------------- from jedi_bundle.bin.jedi_bundle import execute_tasks, get_bundles + from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs, link_path # Get the experiment/jedi_bundle directory # ---------------------------------------- From a9007b2a029b18e45c51ae4e760b821dfadf19c6 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 17:28:44 -0500 Subject: [PATCH 177/299] code test fix --- src/swell/tasks/run_jedi_hofx_executable.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 3562137f9..f7821fed7 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -17,7 +17,6 @@ from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.utilities.netcdf_files import combine_files_without_groups from swell.utilities.run_jedi_executables import run_executable @@ -68,6 +67,8 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # --------------------- jedi_application = 'hofx' + from swell.utilities.netcdf_files import combine_files_without_groups + # Parse configuration # ------------------- window_type = self.config.window_type() From 8cb8012ad6fa8a96f728b3fea75e1e241183f819 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 17:31:02 -0500 Subject: [PATCH 178/299] Code test fix --- src/swell/tasks/build_jedi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index dc720ea82..e7f6b957d 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -45,7 +45,7 @@ def execute(self) -> None: swell_exp_path = self.experiment_path() jedi_bundle_path = os.path.join(swell_exp_path, 'jedi_bundle') - from swell.utilities.build import build_and_source_dirs, link_path + from swell.utilities.build import build_and_source_dirs # Get paths to build and source # ----------------------------- From 901bb5ae7249574787cda7d1282ce759e693637f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 20 Jan 2026 17:52:47 -0500 Subject: [PATCH 179/299] fix for jedi_bundle --- src/swell/tasks/build_jedi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index e7f6b957d..ce9aa6e29 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -45,7 +45,7 @@ def execute(self) -> None: swell_exp_path = self.experiment_path() jedi_bundle_path = os.path.join(swell_exp_path, 'jedi_bundle') - from swell.utilities.build import build_and_source_dirs + from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs # Get paths to build and source # ----------------------------- From c20689bfc6f7d1bf9c0ece6c00becc126f9abb0f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 21 Jan 2026 17:29:42 -0500 Subject: [PATCH 180/299] Increase time limit for build_jedi --- src/swell/tasks/build_jedi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index ce9aa6e29..0e8a675ee 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -24,7 +24,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name - self.time_limit = True + self.time_limit = 'PT3H' self.slurm = {} self.questions = [ qd.bundles(), From 691444b4492487325193128d88cf5be645474f35 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 27 Jan 2026 13:46:58 -0500 Subject: [PATCH 181/299] cast to list --- src/swell/suites/base/all_suites.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/suites/base/all_suites.py b/src/swell/suites/base/all_suites.py index a0ead6f54..e723d4012 100644 --- a/src/swell/suites/base/all_suites.py +++ b/src/swell/suites/base/all_suites.py @@ -92,7 +92,7 @@ def get_config(self, name: str) -> QuestionList: # -------------------------------------------------------------------------------------------------- def all_configs(self) -> list: - return self.config_dict.keys() + return list(self.config_dict.keys()) # -------------------------------------------------------------------------------------------------- From 275647b25928dc2dd5ab393dc21166285a2b1b17 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 27 Jan 2026 14:09:09 -0500 Subject: [PATCH 182/299] Fix override bug and use ruamel --- src/swell/deployment/create_task_config.py | 26 ++++++++++++++++------ src/swell/suites/base/all_suites.py | 2 +- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/swell/deployment/create_task_config.py b/src/swell/deployment/create_task_config.py index 828887d19..149b7934b 100644 --- a/src/swell/deployment/create_task_config.py +++ b/src/swell/deployment/create_task_config.py @@ -7,12 +7,13 @@ # -------------------------------------------------------------------------------------------------- +import io import os from typing import Optional -import yaml +from ruamel.yaml import YAML import isodate -from swell.tasks.base.task_attributes import TaskAttributes +from swell.tasks.base.task_attributes import task_attributes from swell.utilities.logger import get_logger from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import \ PrepareExperimentConfigAndSuite @@ -60,7 +61,7 @@ def task_config_wrapper(task_name: str, logger.info(f'Generating config for task {task_name}') # Get the task attributes for the class - task_attr_class = getattr(TaskAttributes, task_name) + task_attr_class = getattr(task_attributes, task_name) task = task_attr_class(model=model, platform=platform) # Check that model is specified for the task @@ -73,9 +74,16 @@ def task_config_wrapper(task_name: str, logger.abort('Task requires datetime (e.g. 20231010T000000Z)' ' but none was specified at the command line.') + yaml = YAML(typ='safe') + # Construct overrides - if override is None: + if isinstance(override, str): + with open(override, 'r') as f: + override = yaml.load(f) + elif override is None: override = {} + elif not isinstance(override, dict): + raise TypeError('Specified override is not a string filepath or dictionary.') if model is not None: override['model_components'] = [model] @@ -125,12 +133,16 @@ def task_config_wrapper(task_name: str, experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() # Expand all environment vars in the dictionary - experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) + output = io.StringIO() + yaml.dump(experiment_dict, output) + experiment_dict_string = output.getvalue() experiment_dict_string = os.path.expandvars(experiment_dict_string) - experiment_dict = yaml.safe_load(experiment_dict_string) + experiment_dict = yaml.load(experiment_dict_string) # Add comments to dictionary - experiment_dict_string = yaml.dump(experiment_dict, default_flow_style=False, sort_keys=False) + output = io.StringIO() + yaml.dump(experiment_dict, output) + experiment_dict_string = output.getvalue() experiment_dict_string_comments = add_comments_to_dictionary(logger, experiment_dict_string, comment_dict) diff --git a/src/swell/suites/base/all_suites.py b/src/swell/suites/base/all_suites.py index e723d4012..8ecc70eca 100644 --- a/src/swell/suites/base/all_suites.py +++ b/src/swell/suites/base/all_suites.py @@ -43,7 +43,7 @@ def __init__(self) -> None: self.workflow_dict = workflow_dict - def get_workflow(self, suite: str) -> CylcWorkflow: + def get_workflow(self, suite: str) -> type[CylcWorkflow]: return self.workflow_dict[suite] def all_workflows(self) -> list: From f4a27c7e83c5ddcf1bc81da1d158333516763027 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 27 Jan 2026 14:16:27 -0500 Subject: [PATCH 183/299] Type hint fixes --- .../prepare_config_and_suite/prepare_config_and_suite.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 8ff7d1dfa..9f036195c 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -100,6 +100,11 @@ def __init__( # ------------------------------- self.possible_model_components = get_model_components() + # Initialize task trackers + # ------------------------ + self.model_dependent_tasks = [] + self.model_independent_tasks = {} + # Start initializing the suite questions first # -------------------------------------------- self.prepare_suite_question_dictionary() @@ -109,7 +114,7 @@ def __init__( # ---------------------------------------------------------------------------------------------- - def configure_and_ask_task_questions(self) -> None: + def configure_and_ask_task_questions(self) -> tuple[dict, dict]: # Finalize the experiment config with task questions self.prepare_task_question_dictionary() @@ -385,7 +390,7 @@ def get_questions_of_type(self, # ---------------------------------------------------------------------------------------------- - def ask_questions_and_configure(self, suite_task: QuestionType) -> Tuple[dict, dict]: + def ask_questions_and_configure(self, suite_task: QuestionType) -> None: # Handle asking questions for either suites or tasks if self.config_client.__class__.__name__ == 'GetAnswerCli' and ( From 613e31dea13c8f8e82f0a7f15a444ae4f7cee241 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 27 Jan 2026 14:38:58 -0500 Subject: [PATCH 184/299] Add docstrings --- src/swell/tasks/base/task_attributes.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index a8ea6aa8d..f16e81645 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -15,9 +15,31 @@ # -------------------------------------------------------------------------------------------------- +''' +The TaskAttributes class provides tracking of TaskSetup classes, which should be defined in each +task's file. It handles this by providing a wrapper method to register each class. + +Attributes: +root and sync_point: Setup for tasks used swell-wide that don't require separate files +discover_plugins: Handles discovery of packages, which then run the register hooks when imported +TaskAttributes: class that registers TaskSetup classes in each task file + +Example for task registry: + + +from swell.tasks.base.task_attributes import task_attributes + +@task_attributes.register('Example') +class Setup(TaskSetup): + def __init__(self): + pass +''' + +# -------------------------------------------------------------------------------------------------- class root(TaskSetup): def set_attributes(self): + # root is a precursor to all tasks, it runs the pre-script before any task's script self.script = False self.pre_script = "source $CYLC_SUITE_DEF_PATH/modules" self.additional_sections = [self.create_new_section('environment', @@ -26,6 +48,8 @@ def set_attributes(self): class sync_point(TaskSetup): + # placeholder task to check run dependencies in cylc graph + # The command "true" is run in the shell as a placeholder def set_attributes(self): self.script = "true" From 9ebf6fd79a74a366594484f97f2672197564ca90 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 27 Jan 2026 14:39:28 -0500 Subject: [PATCH 185/299] Remove flow.cylc --- src/swell/suites/localensembleda/flow.cylc | 238 --------------------- 1 file changed, 238 deletions(-) delete mode 100644 src/swell/suites/localensembleda/flow.cylc diff --git a/src/swell/suites/localensembleda/flow.cylc b/src/swell/suites/localensembleda/flow.cylc deleted file mode 100644 index 8675bccba..000000000 --- a/src/swell/suites/localensembleda/flow.cylc +++ /dev/null @@ -1,238 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing JEDI-based LocalEnsembleDA Algorithm - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - runahead limit = {{runahead_limit}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - {% for model_component in model_components %} - # Clone geos ana for generating observing system records - CloneGeosMksi-{{model_component}} - {% endfor %} - """ - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - {% for model_component in model_components %} - {% if cycle_time[model_component] %} - # Task triggers for: {{model_component}} - # ------------------ - - # Perform staging that is cycle dependent - BuildJediByLinking[^]? | BuildJedi[^] => StageJediCycle-{{model_component}} => sync_point - - GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} - - GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} - - RenderJediObservations-{{model_component}} => sync_point - - CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} => sync_point - - GetEnsembleGeosExperiment-{{model_component}} => sync_point - - sync_point => RunJediObsfiltersExecutable-{{model_component}} - {% if skip_ensemble_hofx %} - sync_point => RunJediObsfiltersExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} - {% else %} - # Run hofx for ensemble members according to strategy - {% if ensemble_hofx_strategy == 'serial' %} - sync_point => RunJediEnsembleMeanVariance-{{model_component}} => RunJediHofxEnsembleExecutable-{{model_component}} - RunJediHofxEnsembleExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} - - {% elif ensemble_hofx_strategy == 'parallel' %} - {% for packet in range(ensemble_hofx_packets) %} - # When strategy is parallel, only proceed if all RunJediHofxEnsembleExecutable completes successfully for each packet - - # There is a need for a task to combine all hofx observations together, compute node preferred, put here as placeholder - # RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunEnsembleHofxCombiner-{{model_component}} - # RunEnsembleHofxCombiner-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} - - sync_point => RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} - RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}} => RunJediLocalEnsembleDaExecutable-{{model_component}} - {% endfor %} - {% endif %} - {% endif %} - - - # EvaIncrement - RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaIncrement-{{model_component}} - - # EvaObservations - # RunJediLocalEnsembleDaExecutable-{{model_component}} => EvaObservations-{{model_component}} - - # Save observations - # RunJediLocalEnsembleDaExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - - # Clean up large files - # EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} & - EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} - - {% endif %} - {% endfor %} - """ - {% endfor %} - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% for model_component in model_components %} - - [[CloneGeosMksi-{{model_component}}]] - script = "swell task CloneGeosMksi $config -m {{model_component}}" - - [[GenerateObservingSystemRecords-{{model_component}}]] - script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" - - [[StageJediCycle-{{model_component}}]] - script = "swell task StageJedi $config -d $datetime -m {{model_component}}" - - [[ GetBackground-{{model_component}} ]] - script = "swell task GetBackground $config -d $datetime -m {{model_component}}" - - [[GetEnsembleGeosExperiment-{{model_component}}]] - script = "swell task GetEnsembleGeosExperiment $config -d $datetime -m {{model_component}}" - - [[RenderJediObservations-geos_atmosphere]] - script = "swell task RenderJediObservations $config -d $datetime -m {{model_component}}" - - [[RunJediObsfiltersExecutable-{{model_component}}]] - script = "swell task RunJediObsfiltersExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediObsfiltersExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediObsfiltersExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[RunJediEnsembleMeanVariance-{{model_component}}]] - script = "swell task RunJediEnsembleMeanVariance $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediEnsembleMeanVariance"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediEnsembleMeanVariance"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[GetObservations-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - - [[GetObsNotInR2d2-{{model_component}}]] - script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" - - {% if not skip_ensemble_hofx %} - {% if ensemble_hofx_strategy == 'serial' %} - [[RunJediHofxEnsembleExecutable-{{model_component}}]] - script = "swell task RunJediHofxEnsembleExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediHofxEnsembleExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediHofxEnsembleExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - {% elif ensemble_hofx_strategy == 'parallel' %} - {% for packet in range(ensemble_hofx_packets) %} - [[RunJediHofxEnsembleExecutable-{{model_component}}_pack{{packet}}]] - script = "swell task RunJediHofxEnsembleExecutable $config -d $datetime -m {{model_component}} -p {{packet}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediHofxEnsembleExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediHofxEnsembleExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% endfor %} - {% endif %} - {% endif %} - - [[RunJediLocalEnsembleDaExecutable-{{model_component}}]] - script = "swell task RunJediLocalEnsembleDaExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediLocalEnsembleDaExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediLocalEnsembleDaExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[EvaIncrement-{{model_component}}]] - script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" - - [[EvaObservations-{{model_component}}]] - script = true -# EnKF not ready to use Eva -# script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" -# platform = {{platform}} -# execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} -# [[[directives]]] -# {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} -# --{{key}} = {{value}} -# {%- endfor %} - - [[SaveObsDiags-{{model_component}}]] - script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" - - [[CleanCycle-{{model_component}}]] - script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" - {% endfor %} - - - [[sync_point]] - script = true -# -------------------------------------------------------------------------------------------------- From f745458dafc33fd4cb7e63725cdb872c0d5ea9e3 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 27 Jan 2026 14:56:04 -0500 Subject: [PATCH 186/299] Add type annotations and remove registry --- src/swell/tasks/base/task_setup.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index 91b5e1f5c..de04d287a 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -41,7 +41,24 @@ class TaskSetup(ABC): additional_sections: list of additional CylcSection objects to append to the runtime section ''' - _registry = {} + model: str | None + platform: str | None + + base_name: str | None + scheduling_name: str | None + + is_cycling: bool + is_model: bool + + pre_script: bool | str | None + script: bool | str | None + retry: str | None + time_limit: str | dict | None + slurm: dict | None + + mail_events: list + questions: list + additional_sections: list def __init__(self, model: str | None = None, platform: str | None = None) -> None: @@ -71,13 +88,6 @@ def __init__(self, model: str | None = None, platform: str | None = None) -> Non # -------------------------------------------------------------------------------------------------- - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) - - cls._registry[cls.__name__.lower()] = cls - - # -------------------------------------------------------------------------------------------------- - @abstractmethod def set_attributes(self) -> None: '''Abstract method to be overridden by each task in order to set attributes. From cb3753c23cfe63aedf7c63f19f5b0f60f702adf3 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 27 Jan 2026 15:19:59 -0500 Subject: [PATCH 187/299] add comments for local imports --- src/swell/tasks/eva_comparison_increment.py | 3 ++- src/swell/tasks/eva_comparison_jedi_log.py | 1 + src/swell/tasks/eva_increment.py | 1 + src/swell/tasks/eva_jedi_log.py | 1 + src/swell/tasks/eva_observations.py | 2 ++ src/swell/tasks/eva_timeseries.py | 1 + src/swell/tasks/get_background.py | 1 + src/swell/tasks/get_observations.py | 2 +- src/swell/tasks/save_obs_diags.py | 1 + 9 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index fbe22d8c6..6ad0f334f 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -54,7 +54,8 @@ def window_info_from_config(self, path: str): return window_type, window_length def execute(self) -> None: - + + # Local import because module is not loaded until experiment launch from eva.eva_driver import eva model = self.get_model() diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 94c9dc127..9a034bd70 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -40,6 +40,7 @@ class EvaComparisonJediLog(taskBase): def execute(self) -> None: + # Local import because module is not loaded until experiment launch from eva.eva_driver import eva # Get the model diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index 269a197fb..223ade8e1 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -41,6 +41,7 @@ class EvaIncrement(taskBase): def execute(self) -> None: + # Local import because module is not loaded until experiment launch from eva.eva_driver import eva # Get the model and window type diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index d6aac9ac5..d8193d6c6 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -36,6 +36,7 @@ class EvaJediLog(taskBase): def execute(self) -> None: + # Local import because module is not loaded until experiment launch from eva.eva_driver import eva # Get the model diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 4265b0ba3..8e867b741 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -27,6 +27,8 @@ # Pass through to avoid confusion with optional logger argument inside eva def run_eva(eva_dict: dict): + + # Local import because module is not loaded until experiment launch from eva.eva_driver import eva eva(eva_dict) diff --git a/src/swell/tasks/eva_timeseries.py b/src/swell/tasks/eva_timeseries.py index 9342abe18..723ab0ecc 100644 --- a/src/swell/tasks/eva_timeseries.py +++ b/src/swell/tasks/eva_timeseries.py @@ -30,6 +30,7 @@ # Pass through to avoid confusion with optional logger argument inside eva def run_eva(eva_dict: dict): + # Local import because module is not loaded until experiment launch from eva.eva_driver import eva eva(eva_dict) diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index fa8fa63fe..312da0491 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -60,6 +60,7 @@ def execute(self) -> None: See the taskBase constructor for more information. """ + # Local import because module is not loaded until experiment launch from swell.utilities.r2d2 import create_r2d2_config import r2d2 diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 309df7aa1..555321147 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -121,7 +121,7 @@ def execute(self) -> None: "tlapse" files need to be fetched. """ - # Import modules + # # Local import because module is not loaded until experiment launch # -------------- import r2d2 diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 8d0acf605..0d7579292 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -48,6 +48,7 @@ class SaveObsDiags(taskBase): def execute(self) -> None: + # Local import because module is not loaded until experiment launch import r2d2 # Parse config From 537afb644fcab0ba59172a2241954b610543a6da Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 27 Jan 2026 15:46:44 -0500 Subject: [PATCH 188/299] Update adding a suite docs --- docs/adding_a_suite.md | 62 ++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/docs/adding_a_suite.md b/docs/adding_a_suite.md index b3d255f24..a9b925810 100644 --- a/docs/adding_a_suite.md +++ b/docs/adding_a_suite.md @@ -122,7 +122,7 @@ The `root` section defines actions and variables shared by all tasks. Note that ## How the experiment is created -When an experiment is created using `swell create `, a dictionary of questions is pieced together from questions associated with the suite and its member tasks. Answers for these questions are set either from default configurations, from user input on the command line, or overridden from a specified file. In a complex process, the answers provided are then used to generate the `experiment.yaml` and the experiment's `flow.cylc file. +When an experiment is created using `swell create `, a dictionary of questions is pieced together from questions associated with the suite and its member tasks. Answers for these questions are set either from default configurations, from user input on the command line, or overridden from a specified file. In a complex process, the answers provided are then used to generate the `experiment.yaml` and the experiment's `workflow.py` file. # Creating a Suite @@ -145,19 +145,20 @@ Creating visualizations such as flowcharts may help in designing workflows. In practice, there are three major steps towards creating a suite. Completing all of these steps is necessary to make the suite work, so these steps will likely be done iteratively/non-linearly: 1. Write the tasks. -2. Create the `flow.cylc` file. +2. Create the `workflow.py` file. 3. Add the appropriate suite and task question lists. More detailed instructions and examples for these steps follows in this section. ### Writing tasks -Swell has a variety of tasks, many of which are shared across suites. Tasks in Swell are defined as classes which extend the `taskBase` parent class, which has many helpful functions and attributes. When a task is run by swell, it calls the `execute` function. +Swell has a variety of tasks, many of which are shared across suites. Tasks in Swell are defined as classes which extend the `taskBase` parent class, which has many helpful functions and attributes. When a task is run by swell, it calls the `execute` function. Information on how to run the task and the parameters the task needs are specified in the `TaskSetup` class. Calls to parameters are made using either functions of `taskBase`, for more common parameters, or using `self.config.`. ### Example Swell Task ```python + class CloneGeosMksi(taskBase): def execute(self) -> None: @@ -179,32 +180,34 @@ class CloneGeosMksi(taskBase): ``` This example shows the basics of writing a task, including task definition and the execute function. The current model is accessed by the `self.get_model()` function, inherited from `taskBase`. The variables `path_to_geos_mksi`, and `tag`, are pulled from the experiment configuration, which is sourced from the `experiment.yaml`. -Tasks that have a slurm requirement need to be specified in `src/swell/utilities/slurm.py`. +The `TaskSetup` class informs swell how to construct the task's cylc parameters, as well as the questions that are used by the task. The `TaskSetup` class for `CloneGeosMksi` looks like the following: -For debugging purposes, it may be easier to first create and test some tasks outside of Swell, and then port them to Swell by changing relevant variables and path specifications. Alternatively, `experiment.yaml` can be populated manually and tested using `swell task experiment.yaml`. +```python -### Creating the flow.cylc template +task_name = 'CloneGeosMksi' +@task_attributes.register(task_name) +class Setup(TaskSetup): + def set_attributes(self): + self.base_name = task_name + self.is_model = True + self.questions = [ + qd.observing_system_records_mksi_path(), + qd.observing_system_records_mksi_path_tag() + ] -For more detailed information on cylc workflows, see the [cylc documentation](https://cylc.github.io/cylc-doc/latest/html/index.html). Existing Swell suite workflows can also provide useful examples to consider. +``` -Suite workflows are stored in `src/swell/suites/`. +The `set_attributes` abstract method is used to set values for the class. The `self.questions` list sets needed question parameters for the config. Slurm requirements are also set in the the `TaskSetup`. For more information, see documentation in `src/swell/tasks/base/task_setup.py`. -The experiment `flow.cylc` file is generated from a suite template using a `jinja2` process. For example, here is part of a suite template, versus a filled-in experiment `flow.cylc`. During creation, specified questions are used to fill in the template: +For debugging purposes, it may be easier to first create and test some tasks outside of Swell, and then port them to Swell by changing relevant variables and path specifications. Alternatively, `experiment.yaml` can be populated manually and tested using `swell task experiment.yaml`. -``` -[scheduling] +### Creating the flow.cylc template - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - runahead limit = {{runahead_limit}} -``` -``` -[scheduling] +For more detailed information on cylc workflows, see the [cylc documentation](https://cylc.github.io/cylc-doc/latest/html/index.html). Existing Swell suite workflows can also provide useful examples to consider. - initial cycle point = 2021-07-01T12:00:00Z - final cycle point = 2021-07-01T12:00:00Z - runahead limit = P4 -``` +Suite workflows are stored in `src/swell/suites//workflow.py`. + +The experiment `flow.cylc` file is generated from a suite template. This is handled through the `CylcWorkflow` task and generally consists of two steps, setting the graph and iterating through the tasks to generate. For more information, see the documentation for `CylcWorkflow` under `src/swell/utilities/cylc_workflow.py` and example suites. For initial development/testing purposes, it may be easier to create a `flow.cylc` using hard-coded values, then replace these with `jinja2` templated values as the suite nears completion. @@ -274,22 +277,11 @@ question = existing_jedi_build_directory(options=['example1', 'example2']) Each individual suite and most tasks have an associated list of questions which are used to create the experiment. Suite question lists are stored in `src/swell/suites//suite_config.py` -Task question lists are stored in `src/swell/tasks/task_questions.py` +Task question lists are stored in `src/swell/tasks/.py` -`QuestionList` objects store and handle questions in an object-oriented manner. They can store questions directly, or store other lists to use their questions. Here is an example of a question list for a task: - -```python - BuildJediByLinking = QuestionList( - list_name="BuildJediByLinking", - questions=[ - qd.existing_jedi_build_directory(), - qd.existing_jedi_build_directory_pinned(), - qd.jedi_build_method() - ] - ) -``` +`QuestionList` objects store and handle questions in an object-oriented manner. They can store questions directly, or store other lists to use their questions. -During experiment creation, Swell scans the suite's `flow.cylc` file to find all of the tasks used in the workflow. It then finds the corresponding task lists in `src/swell/tasks/task_questions.py`, and fits together a list of uniquely named questions from all of the lists. Questions have a priority depending on order. In the case of duplicate questions, those further DOWN the list take priority. For this reason, it is NOT RECOMMENDED to set different default values for tasks in `task_questions.py`, since questions may be overridden by a questions in a different task. +During experiment creation, Swell consults the suite's `workflow.py` file to find all of the tasks used in the workflow. It then finds the corresponding task lists in the `TaskSetup` object located in `src/swell/tasks/.py`, and fits together a list of uniquely named questions from all of the lists. Questions have a priority depending on order. In the case of duplicate questions, those further DOWN the list take priority. For this reason, it is NOT RECOMMENDED to set different default values for questions listed in tasks, since questions may be overridden by a questions in a different task. In this question infrastructure, **suites take priority over tasks**. Any question specified in a suite configuration will override the default value for a question in one of its member tasks. This allows for easily setting different configurations for suites without having to specify redundant questions. For ease of use, model-dependent questions can be assigned directly in their respective lists. From 2b15bd44d32832227c91c960d26e50b52746b83b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 27 Jan 2026 16:12:52 -0500 Subject: [PATCH 189/299] Resolve override dictionary --- src/swell/deployment/create_task_config.py | 15 +++------------ src/swell/swell.py | 9 ++++++++- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/swell/deployment/create_task_config.py b/src/swell/deployment/create_task_config.py index 149b7934b..149db5869 100644 --- a/src/swell/deployment/create_task_config.py +++ b/src/swell/deployment/create_task_config.py @@ -51,7 +51,7 @@ def task_config_wrapper(task_name: str, datetime: Optional[str], model: Optional[str], input_method: str, - override: str, + override: dict, slurm: str, cwd: bool) -> None: @@ -74,17 +74,6 @@ def task_config_wrapper(task_name: str, logger.abort('Task requires datetime (e.g. 20231010T000000Z)' ' but none was specified at the command line.') - yaml = YAML(typ='safe') - - # Construct overrides - if isinstance(override, str): - with open(override, 'r') as f: - override = yaml.load(f) - elif override is None: - override = {} - elif not isinstance(override, dict): - raise TypeError('Specified override is not a string filepath or dictionary.') - if model is not None: override['model_components'] = [model] else: @@ -132,6 +121,8 @@ def task_config_wrapper(task_name: str, # Configure and ask all questions experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() + yaml = YAML(typ='safe') + # Expand all environment vars in the dictionary output = io.StringIO() yaml.dump(experiment_dict, output) diff --git a/src/swell/swell.py b/src/swell/swell.py index 376dd43c8..083d722bb 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -9,6 +9,7 @@ import click +from ruamel.yaml import YAML from typing import Union, Optional, Literal from swell.deployment.platforms.platforms import get_platforms @@ -160,7 +161,13 @@ def create_task_config( task (str): Name of the task to execute.\n """ - task_config_wrapper(task, platform, datetime, model, input_method, override, slurm, cwd) + if override is not None: + yaml = YAML(typ='safe') + with open(override, 'r') as f: + override_dict = yaml.load(f) + else: + override_dict = {} + task_config_wrapper(task, platform, datetime, model, input_method, override_dict, slurm, cwd) # -------------------------------------------------------------------------------------------------- From 0a92f44cb59a8f32e3de1b3721147a1de6af569b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 27 Jan 2026 16:17:24 -0500 Subject: [PATCH 190/299] Code test fixes --- .../prepare_config_and_suite/prepare_config_and_suite.py | 2 +- src/swell/tasks/base/task_attributes.py | 1 + src/swell/tasks/eva_comparison_increment.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 9f036195c..a072f8233 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -11,7 +11,7 @@ import os import yaml from collections.abc import Mapping -from typing import Union, Tuple, Optional +from typing import Union, Optional import datetime from swell.swell_path import get_swell_path diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index f16e81645..b48959a90 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -37,6 +37,7 @@ def __init__(self): # -------------------------------------------------------------------------------------------------- + class root(TaskSetup): def set_attributes(self): # root is a precursor to all tasks, it runs the pre-script before any task's script diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index 6ad0f334f..ef72aacac 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -54,7 +54,7 @@ def window_info_from_config(self, path: str): return window_type, window_length def execute(self) -> None: - + # Local import because module is not loaded until experiment launch from eva.eva_driver import eva From 90ded9a176be7dfe3199196fbc2283268cf1740f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 27 Jan 2026 16:47:37 -0500 Subject: [PATCH 191/299] Code test fixes --- src/swell/tasks/get_observations.py | 1 - src/swell/tasks/save_obs_diags.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 9fd75d303..6c1e81bdc 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -8,7 +8,6 @@ # -------------------------------------------------------------------------------------------------- import isodate -import netCDF4 as nc import numpy as np import os import shutil diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 586ac4a0d..7ffbdad7b 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -7,8 +7,6 @@ # -------------------------------------------------------------------------------------------------- -import os - from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes From d094ac10740edcfd3d183580f9a9b5666d470cf3 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 5 Feb 2026 15:14:54 -0500 Subject: [PATCH 192/299] Fix for ensemble --- src/swell/suites/localensembleda/workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 6709335ac..e882e660a 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -73,7 +73,7 @@ GetEnsembleGeosExperiment-{{model_component}} => sync_point sync_point => RunJediObsfiltersExecutable-{{model_component}} - {% if skip_ensemble_hofx %} + {% if models[model_component]['skip_ensemble_hofx'] %} sync_point => RunJediObsfiltersExecutable-{{model_component}} => RunJediLocalEnsembleDaExecutable-{{model_component}} {% else %} # Run hofx for ensemble members according to strategy From d72b2756e875e6c9965645ad8e3b4d9c387a6be0 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 6 Feb 2026 13:55:54 -0500 Subject: [PATCH 193/299] Refactor names of time_limit and is_model --- docs/adding_a_suite.md | 2 +- docs/examples/templating_workflows.md | 16 +++++----- src/swell/deployment/create_task_config.py | 10 +++---- src/swell/tasks/base/task_setup.py | 30 +++++++++---------- src/swell/tasks/bufr_to_ioda.py | 2 +- src/swell/tasks/build_jedi.py | 2 +- src/swell/tasks/clean_cycle.py | 2 +- src/swell/tasks/clone_geos_mksi.py | 2 +- src/swell/tasks/eva_comparison_increment.py | 2 +- src/swell/tasks/eva_comparison_jedi_log.py | 2 +- src/swell/tasks/eva_increment.py | 2 +- src/swell/tasks/eva_jedi_log.py | 2 +- src/swell/tasks/eva_observations.py | 4 +-- src/swell/tasks/eva_timeseries.py | 4 +-- src/swell/tasks/generate_b_climatology.py | 4 +-- .../generate_b_climatology_by_linking.py | 2 +- .../generate_observing_system_records.py | 2 +- src/swell/tasks/get_background.py | 2 +- .../tasks/get_background_geos_experiment.py | 2 +- src/swell/tasks/get_bufr.py | 2 +- .../tasks/get_ensemble_geos_experiment.py | 2 +- src/swell/tasks/get_geos_adas_background.py | 2 +- src/swell/tasks/get_geovals.py | 2 +- src/swell/tasks/get_gsi_bc.py | 2 +- src/swell/tasks/get_gsi_ncdiag.py | 2 +- src/swell/tasks/get_ncdiags.py | 2 +- src/swell/tasks/get_obs_not_in_r2d2.py | 2 +- src/swell/tasks/get_observations.py | 2 +- src/swell/tasks/gsi_bc_to_ioda.py | 2 +- src/swell/tasks/gsi_ncdiag_to_ioda.py | 2 +- src/swell/tasks/jedi_log_comparison.py | 2 +- src/swell/tasks/jedi_oops_log_parser.py | 2 +- src/swell/tasks/link_geos_output.py | 2 +- src/swell/tasks/move_da_restart.py | 2 +- src/swell/tasks/prepare_analysis.py | 2 +- src/swell/tasks/render_jedi_observations.py | 2 +- ...jedi_convert_state_soca2cice_executable.py | 4 +-- .../tasks/run_jedi_ensemble_mean_variance.py | 4 +-- src/swell/tasks/run_jedi_fgat_executable.py | 4 +-- .../run_jedi_hofx_ensemble_executable.py | 4 +-- src/swell/tasks/run_jedi_hofx_executable.py | 4 +-- .../run_jedi_local_ensemble_da_executable.py | 4 +-- .../tasks/run_jedi_obsfilters_executable.py | 4 +-- .../tasks/run_jedi_ufo_tests_executable.py | 4 +-- .../tasks/run_jedi_variational_executable.py | 4 +-- src/swell/tasks/save_obs_diags.py | 2 +- src/swell/tasks/save_restart.py | 2 +- src/swell/tasks/stage_jedi.py | 4 +-- src/swell/tasks/store_background.py | 2 +- 49 files changed, 87 insertions(+), 87 deletions(-) diff --git a/docs/adding_a_suite.md b/docs/adding_a_suite.md index a9b925810..631680a89 100644 --- a/docs/adding_a_suite.md +++ b/docs/adding_a_suite.md @@ -189,7 +189,7 @@ task_name = 'CloneGeosMksi' class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name - self.is_model = True + self.model_dep = True self.questions = [ qd.observing_system_records_mksi_path(), qd.observing_system_records_mksi_path_tag() diff --git a/docs/examples/templating_workflows.md b/docs/examples/templating_workflows.md index 593ffe6ed..3f3213588 100644 --- a/docs/examples/templating_workflows.md +++ b/docs/examples/templating_workflows.md @@ -37,9 +37,9 @@ task_name = 'EvaObservations' class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name - self.time_limit = True + self.task_time_limit = True self.is_cycling = True - self.is_model = True + self.model_dep = True self.slurm = {} self.questions = [ qd.background_time_offset(), @@ -55,10 +55,10 @@ class Setup(TaskSetup): Attributes are set by override the `set_attributes` method in `TaskSetup`. This has been combined with the previously-used `task_questions.py` for simplicity. -The tags `is_cycling` and `is_model` (both `False` by default) modify the script command (`swell task $config`): +The tags `is_cycling` and `model_dep` (both `False` by default) modify the script command (`swell task $config`): - `is_cycling = True` adds `-d $datetime` for cycling tasks -- `is_model = True` adds `-m {model}` to indicate model-specific tasks. +- `model_dep = True` adds `-m {model}` to indicate model-specific tasks. The `slurm` attribute determines where or not the task requires Slurm and provides a way to set task-specific overrides: @@ -89,8 +89,8 @@ This can be used to set task-specific defaults in `task_attributes.py`, rather t class RunJediConvertStateSoca2ciceExecutable(TaskSetup): def set_attributes(self): self.is_cycling = True - self.is_model = True - self.time_limit = True + self.model_dep = True + self.task_time_limit = True self.slurm = {'nodes': 1} ``` @@ -100,8 +100,8 @@ This supports setting platform-specific overrides, for example: class RunJediConvertStateSoca2ciceExecutable(TaskSetup): def set_attributes(self): self.is_cycling = True - self.is_model = True - self.time_limit = True + self.model_dep = True + self.task_time_limit = True self.slurm = {'all': 1, 'nccs_discover_cascade': 2} ``` diff --git a/src/swell/deployment/create_task_config.py b/src/swell/deployment/create_task_config.py index 149db5869..e3e679b69 100644 --- a/src/swell/deployment/create_task_config.py +++ b/src/swell/deployment/create_task_config.py @@ -65,7 +65,7 @@ def task_config_wrapper(task_name: str, task = task_attr_class(model=model, platform=platform) # Check that model is specified for the task - if task.is_model and model is None: + if task.model_dep and model is None: logger.abort('Task requires model (e.g. geos_marine, geos_atmsophere)' ' but none was specified at the command line.') @@ -145,10 +145,10 @@ def task_config_wrapper(task_name: str, slurm_external_dict = prepare_slurm_defaults_and_overrides(logger, platform, slurm) task_slurm_dict = task.generate_task_slurm_dict(slurm_external_dict) - time_limit = task.time_limit - if time_limit is not None: - time_limit_dto = isodate.parse_duration(time_limit) - task_slurm_dict['time'] = isodate.strftime(time_limit_dto, '%H:%M:%S') + task_time_limit = task.task_time_limit + if task_time_limit is not None: + task_time_limit_dto = isodate.parse_duration(task_time_limit) + task_slurm_dict['time'] = isodate.strftime(task_time_limit_dto, '%H:%M:%S') # Determine the path for task results experiment_root = experiment_dict['experiment_root'] diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index de04d287a..736819309 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -30,11 +30,11 @@ class TaskSetup(ABC): base_name: basic name of the task within Swell scheduling_name: name for the task within cylc is_cycling: boolean for whether the task is run on cycles - is_model: boolean for whether the task is run on a certain model + model_dep: boolean for whether the task is run on a certain model pre_script: cylc setting for scripts run before the main script script: string of shell code to be run by cylc for the task retry: times * time interval cylc should retry the task, e.g. 2*PT10s - time_limit: execution time limit for slurm + task_time_limit: execution time limit for slurm slurm: dictionary of slurm parameters mail events: list of events for email messaging through cylc question_list: list of questions keys used by the task @@ -48,12 +48,12 @@ class TaskSetup(ABC): scheduling_name: str | None is_cycling: bool - is_model: bool + model_dep: bool pre_script: bool | str | None script: bool | str | None retry: str | None - time_limit: str | dict | None + task_time_limit: str | dict | None slurm: dict | None mail_events: list @@ -69,13 +69,13 @@ def __init__(self, model: str | None = None, platform: str | None = None) -> Non self.scheduling_name = None self.is_cycling = False - self.is_model = False + self.model_dep = False self.pre_script = False self.script = None self.retry = None - self.time_limit = None + self.task_time_limit = None self.slurm = None self.mail_events = ['failed', 'submit-failed'] @@ -106,7 +106,7 @@ def post_init(self): if self.scheduling_name is None: self.scheduling_name = self.base_name - if self.is_model and self.model is not None: + if self.model_dep and self.model is not None: self.scheduling_name += f'-{self.model}' if self.script is None: @@ -115,10 +115,10 @@ def post_init(self): if self.is_cycling: self.script += ' -d $datetime' - if self.is_model and self.model is not None: + if self.model_dep and self.model is not None: self.script += ' -m {model}' - if self.is_model and self.model is not None: + if self.model_dep and self.model is not None: self.script = self.script.format(model=self.model) self.scheduling_name = self.scheduling_name.format(model=self.model) @@ -129,10 +129,10 @@ def post_init(self): self.retry = self.match_platform(self.retry) # Set time limit defaults - if self.time_limit is True: - self.time_limit = 'PT1H' - elif self.time_limit: - self.time_limit = self.match_platform(self.time_limit) + if self.task_time_limit is True: + self.task_time_limit = 'PT1H' + elif self.task_time_limit: + self.task_time_limit = self.match_platform(self.task_time_limit) # Convert questions list into object self.question_list = QuestionList(self.questions) @@ -310,8 +310,8 @@ def runtime_string(self, experiment_dict: Mapping, slurm_external: Mapping) -> s if self.slurm is not None: runtime_dict['platform'] = platform - if self.time_limit is not None: - runtime_dict['execution time limit'] = self.time_limit + if self.task_time_limit is not None: + runtime_dict['execution time limit'] = self.task_time_limit # Set the retry if this task needs it if self.retry: diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 271c1cc6a..39c43f424 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -44,7 +44,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index 0e8a675ee..19514af8e 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -24,7 +24,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name - self.time_limit = 'PT3H' + self.task_time_limit = 'PT3H' self.slurm = {} self.questions = [ qd.bundles(), diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index f991faac7..d01f7238c 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -26,7 +26,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.clean_patterns() ] diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index 5cfc63476..dac2f1ed7 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -23,7 +23,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name - self.is_model = True + self.model_dep = True self.questions = [ qd.observing_system_records_mksi_path(), qd.observing_system_records_mksi_path_tag() diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index 6bdc3c8bf..d13848ea0 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -29,7 +29,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.marine_models(), qd.window_length(), diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 789ec4cc2..c3f12579c 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -28,7 +28,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.comparison_log_type() ] diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index 9ae420804..864ef352d 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -27,7 +27,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.marine_models(), qd.window_length(), diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index 16947960b..e1bbd8314 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -27,7 +27,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 50321a84f..b026aa2f3 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -42,9 +42,9 @@ def run_eva(eva_dict: dict): class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name - self.time_limit = True + self.task_time_limit = True self.is_cycling = True - self.is_model = True + self.model_dep = True self.slurm = {} self.questions = [ qd.background_time_offset(), diff --git a/src/swell/tasks/eva_timeseries.py b/src/swell/tasks/eva_timeseries.py index 886ae20a6..d6c770519 100644 --- a/src/swell/tasks/eva_timeseries.py +++ b/src/swell/tasks/eva_timeseries.py @@ -44,9 +44,9 @@ def run_eva(eva_dict: dict): class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name - self.time_limit = True + self.task_time_limit = True self.is_cycling = True - self.is_model = True + self.model_dep = True self.slurm = {} self.questions = [ qd.background_time_offset(), diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index 3f9487f21..c3634f60e 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -24,9 +24,9 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name - self.time_limit = True + self.task_time_limit = True self.is_cycling = True - self.is_model = True + self.model_dep = True self.retry = '2*PT1M' self.slurm = {} self.questions = [ diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 941fa814c..40362f3f4 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -24,7 +24,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.swell_static_files(), qd.swell_static_files_user(), diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index bbeb93b99..4f6d436a3 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -26,7 +26,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.observations(), qd.observing_system_records_mksi_path(), diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 312da0491..3e6f9572d 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -34,7 +34,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.window_length(), qd.window_type(), diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index eb09b30c5..b21c9868b 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -29,7 +29,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.mail_events = ['submit-failed'] self.questions = [ qd.horizontal_resolution(), diff --git a/src/swell/tasks/get_bufr.py b/src/swell/tasks/get_bufr.py index 800f4623d..4fbaae6ba 100644 --- a/src/swell/tasks/get_bufr.py +++ b/src/swell/tasks/get_bufr.py @@ -28,7 +28,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.bufr_obs_classes() ] diff --git a/src/swell/tasks/get_ensemble_geos_experiment.py b/src/swell/tasks/get_ensemble_geos_experiment.py index 72c60a5ef..f6dff38f9 100644 --- a/src/swell/tasks/get_ensemble_geos_experiment.py +++ b/src/swell/tasks/get_ensemble_geos_experiment.py @@ -28,7 +28,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.background_experiment(), qd.background_time_offset(), diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py index 1d83c832c..675024e06 100644 --- a/src/swell/tasks/get_geos_adas_background.py +++ b/src/swell/tasks/get_geos_adas_background.py @@ -29,7 +29,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.path_to_geos_adas_background() ] diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index 80207b05d..afbc15a38 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -27,7 +27,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.background_time_offset(), qd.crtm_coeff_dir(), diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index 3b8e5ce43..d61cb1140 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -30,7 +30,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.path_to_gsi_bc_coefficients(), qd.window_length() diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index 5f499c4e5..03249fbb8 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -27,7 +27,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.path_to_gsi_nc_diags() ] diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index 10a508a51..96689b6dc 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -23,7 +23,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.background_time_offset(), qd.crtm_coeff_dir(), diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py index 5dd5bf425..cccda4746 100644 --- a/src/swell/tasks/get_obs_not_in_r2d2.py +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -28,7 +28,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.mail_events = ['submit-failed'] self.questions = [ qd.ioda_locations_not_in_r2d2(), diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 6c1e81bdc..18c818773 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -40,7 +40,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.background_time_offset(), qd.crtm_coeff_dir(), diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 0dac3e626..871232496 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -30,7 +30,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.path_to_gsi_bc_coefficients(), qd.window_length() diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 930bc22a1..8a64682ea 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -32,7 +32,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.observations(), qd.produce_geovals(), diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py index 7eb61ddcd..1b87e3694 100644 --- a/src/swell/tasks/jedi_log_comparison.py +++ b/src/swell/tasks/jedi_log_comparison.py @@ -31,7 +31,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name - self.is_model = True + self.model_dep = True self.questions = [ qd.number_of_iterations(), qd.comparison_log_type(), diff --git a/src/swell/tasks/jedi_oops_log_parser.py b/src/swell/tasks/jedi_oops_log_parser.py index dc001d315..b1831b92d 100644 --- a/src/swell/tasks/jedi_oops_log_parser.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -26,7 +26,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.parser_options(), qd.comparison_log_type() diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index d5d32ec0b..248a9a314 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -30,7 +30,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.window_length(), qd.window_type(), diff --git a/src/swell/tasks/move_da_restart.py b/src/swell/tasks/move_da_restart.py index db370ee1c..1250407d4 100644 --- a/src/swell/tasks/move_da_restart.py +++ b/src/swell/tasks/move_da_restart.py @@ -28,7 +28,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.mom6_iau(), qd.window_length() diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index 3349bc94d..0f29d65fc 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -29,7 +29,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.analysis_variables(), qd.mom6_iau(), diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index bf4d1aa5b..fc4e9f942 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -27,7 +27,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.check_for_obs(), qd.crtm_coeff_dir(), diff --git a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py index 8ce617fb0..a3fdbc301 100644 --- a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py +++ b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py @@ -28,8 +28,8 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True - self.time_limit = True + self.model_dep = True + self.task_time_limit = True self.slurm = {'nodes': 1} self.questions = [ qd.analysis_variables(), diff --git a/src/swell/tasks/run_jedi_ensemble_mean_variance.py b/src/swell/tasks/run_jedi_ensemble_mean_variance.py index 3aa829708..afbeb7f5a 100644 --- a/src/swell/tasks/run_jedi_ensemble_mean_variance.py +++ b/src/swell/tasks/run_jedi_ensemble_mean_variance.py @@ -28,8 +28,8 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True - self.time_limit = True + self.model_dep = True + self.task_time_limit = True self.slurm = {} self.questions = [ qd.npx_proc(), diff --git a/src/swell/tasks/run_jedi_fgat_executable.py b/src/swell/tasks/run_jedi_fgat_executable.py index ce86ad391..30f8ecf2c 100644 --- a/src/swell/tasks/run_jedi_fgat_executable.py +++ b/src/swell/tasks/run_jedi_fgat_executable.py @@ -27,8 +27,8 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True - self.time_limit = True + self.model_dep = True + self.task_time_limit = True self.slurm = {} self.questions = [ qd.background_time_offset(), diff --git a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py index 5038cac07..613407c92 100644 --- a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py +++ b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py @@ -28,8 +28,8 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True - self.time_limit = True + self.model_dep = True + self.task_time_limit = True self.slurm = {} self.questions = [ qd.npx_proc(), diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 91d1cd0f5..5070087da 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -30,8 +30,8 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True - self.time_limit = True + self.model_dep = True + self.task_time_limit = True self.slurm = {} self.questions = [ qd.npx_proc(), diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 2cb94b573..5f9e515e9 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -47,8 +47,8 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True - self.time_limit = True + self.model_dep = True + self.task_time_limit = True self.slurm = {} self.questions = [ qd.npx_proc(), diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index 5bd80f6e5..a2251edd9 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -30,8 +30,8 @@ def set_attributes(self): self.script = ("swell task RunJediObsfiltersExecutable $config" " -d $datetime -m geos_atmosphere") self.is_cycling = True - self.is_model = True - self.time_limit = True + self.model_dep = True + self.task_time_limit = True self.slurm = {} self.questions = [ qd.npx_proc(), diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 62c53d2b3..40643ea4c 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -29,9 +29,9 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name - self.time_limit = True + self.task_time_limit = True self.is_cycling = True - self.is_model = True + self.model_dep = True self.slurm = {'ntasks-per-node': 1} self.questions = [ qd.background_time_offset(), diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index e0eff9da8..78ea88018 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -26,9 +26,9 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name - self.time_limit = True + self.task_time_limit = True self.is_cycling = True - self.is_model = True + self.model_dep = True self.slurm = {'nodes': 3} self.questions = [ qd.npx_proc(), diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 7ffbdad7b..98ee8fff8 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -24,7 +24,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.background_time_offset(), qd.crtm_coeff_dir(), diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index c8ace09f8..dfa0693d7 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -28,7 +28,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.window_length(), qd.window_type(), diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 95e002aa1..836abf2d7 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -29,7 +29,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name - self.is_model = True + self.model_dep = True self.questions = [ qd.swell_static_files(), qd.swell_static_files_user(), @@ -48,7 +48,7 @@ def set_attributes(self): self.base_name = "StageJedi" self.scheduling_name = "StageJediCycle-{model}" self.is_cycling = True - self.is_model = True + self.model_dep = True # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index 70d5491a1..b13540619 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -30,7 +30,7 @@ class Setup(TaskSetup): def set_attributes(self): self.base_name = task_name self.is_cycling = True - self.is_model = True + self.model_dep = True self.questions = [ qd.window_length(), qd.window_type(), From a2324742b78a5d115abf01868c4b235fac6d1cd2 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 18 Feb 2026 17:23:04 -0500 Subject: [PATCH 194/299] Code test fixes --- src/swell/utilities/question_defaults.py | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 01b21255d..db4e58362 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -1012,55 +1012,55 @@ class ncdiag_experiments(TaskQuestion): # -------------------------------------------------------------------------------------------------- @dataclass - class npx_proc(TaskQuestion): + class npx(TaskQuestion): default_value: str = "defer_to_model" - question_name: str = "npx_proc" + question_name: str = "npx" ask_question: bool = True models: List[str] = mutable_field([ - "geos_atmosphere", "geos_cf" ]) - prompt: str = "What number of processors do you wish to use in the x-direction?" + prompt: str = "What is the number of grid points in the x-direction on each cube face?" widget_type: WType = WType.INTEGER # -------------------------------------------------------------------------------------------------- @dataclass - class npy_proc(TaskQuestion): + class npx_proc(TaskQuestion): default_value: str = "defer_to_model" - question_name: str = "npy_proc" + question_name: str = "npx_proc" ask_question: bool = True models: List[str] = mutable_field([ "geos_atmosphere", "geos_cf" ]) - prompt: str = "What number of processors do you wish to use in the y-direction?" + prompt: str = "What number of processors do you wish to use in the x-direction?" widget_type: WType = WType.INTEGER # -------------------------------------------------------------------------------------------------- @dataclass - class npx(TaskQuestion): + class npy(TaskQuestion): default_value: str = "defer_to_model" - question_name: str = "npx" + question_name: str = "npy" ask_question: bool = True models: List[str] = mutable_field([ "geos_cf" ]) - prompt: str = "What is the number of grid points in the x-direction on each cube face?" + prompt: str = "What is the number of grid points in the y-direction on each cube face?" widget_type: WType = WType.INTEGER # -------------------------------------------------------------------------------------------------- @dataclass - class npy(TaskQuestion): + class npy_proc(TaskQuestion): default_value: str = "defer_to_model" - question_name: str = "npy" + question_name: str = "npy_proc" ask_question: bool = True models: List[str] = mutable_field([ + "geos_atmosphere", "geos_cf" ]) - prompt: str = "What is the number of grid points in the y-direction on each cube face?" + prompt: str = "What number of processors do you wish to use in the y-direction?" widget_type: WType = WType.INTEGER # -------------------------------------------------------------------------------------------------- From d490b7c3bd286efad3a040531ea2e58b9231d526 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 18 Feb 2026 17:29:03 -0500 Subject: [PATCH 195/299] pycodestyle --- src/swell/utilities/question_defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index db4e58362..263dcb59b 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -1039,7 +1039,7 @@ class npx_proc(TaskQuestion): # -------------------------------------------------------------------------------------------------- @dataclass - class npy(TaskQuestion): + class npy(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "npy" ask_question: bool = True From 4f48070cd2f6feabae24a095b4fbc1a4311704b0 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 24 Feb 2026 16:40:26 -0500 Subject: [PATCH 196/299] Add experiment ID for R2D2 --- .../prepare_config_and_suite.py | 51 ++++++++++++++++++- src/swell/suites/suite_questions.py | 3 +- src/swell/tasks/save_obs_diags.py | 2 +- src/swell/tasks/save_restart.py | 4 +- src/swell/utilities/question_defaults.py | 9 ++++ 5 files changed, 63 insertions(+), 6 deletions(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 7ba60dd75..5d0542fae 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -1,5 +1,4 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. + # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -13,6 +12,7 @@ from ruamel.yaml import YAML from collections.abc import Mapping from typing import Union, Tuple, Optional +import random from swell.swell_path import get_swell_path from swell.deployment.prepare_config_and_suite.question_and_answer_cli import GetAnswerCli @@ -23,6 +23,7 @@ from swell.utilities.dictionary import update_dict from swell.tasks.task_questions import TaskQuestions as task_questions from swell.suites.all_suites import AllSuites +from swell.utilities.r2d2 import load_r2d2_credentials # -------------------------------------------------------------------------------------------------- @@ -326,6 +327,7 @@ def override_with_defaults(self) -> None: # Look for defer_to_code in the model_ind dictionary # -------------------------------------------------- for key, val in self.question_dictionary_model_ind.items(): + if key == 'model_components': if val['default_value'] == 'defer_to_code': val['default_value'] = self.possible_model_components @@ -335,6 +337,13 @@ def override_with_defaults(self) -> None: if key == 'experiment_id' and val['default_value'] == 'defer_to_code': val['default_value'] = f'swell-{self.suite}' + if key == 'r2d2_experiment_id' and val['default_value'] == 'defer_to_code': + swell_id = self.question_dictionary_model_ind['experiment_id']['default_value'] + if swell_id == 'defer_to_code': + swell_id = f'swell-{self.suite}' + self.question_dictionary_model_ind['experiment_id']['default_value'] = swell_id + val['default_value'] = self.create_r2d2_id(swell_id) + # ---------------------------------------------------------------------------------------------- def override_with_external(self) -> None: @@ -689,4 +698,42 @@ def get_dynamic_tasks(self, question_list: list) -> list: return tasks + # ---------------------------------------------------------------------------------------------- + + def random_hex_id(self, swell_id: str, length: int = 8): + return f"{swell_id}-{random.randrange(16**length):0{length}x}" + + # ---------------------------------------------------------------------------------------------- + + def create_r2d2_id(self, swell_id: str) -> str: + + # Load credentials to allow search + load_r2d2_credentials(self.logger, self.platform) + + import r2d2 + + self.logger.info('Generating Experiment ID for R2D2') + + # Only try this 10 times + for i in range(10): + temp_id = self.random_hex_id(swell_id, length=8) + try: + r2d2.get(item='experiment', name=temp_id) + except Exception as e: + + if '400 Client Error' in str(e): + user = r2d2.get_client_user() + host = r2d2.get_client_host() + compiler = r2d2.get_client_compiler() + + r2d2.register(item='experiment', + name=temp_id, + user=user, + compute_host=f'{host}-{compiler}', + lifetime='debug') + + return temp_id + + raise Exception('Could not find a valid experiment_id for R2D2') + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index 247bee869..a4da99431 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -40,7 +40,8 @@ class SuiteQuestions(QuestionContainer, Enum): qd.start_cycle_point(), qd.final_cycle_point(), qd.model_components(), - qd.runahead_limit() + qd.runahead_limit(), + qd.r2d2_experiment_id() ] ) diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index d03c6ba0c..0cd489434 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -81,7 +81,7 @@ def execute(self) -> None: try: r2d2.store( item='feedback', - experiment=self.experiment_id(), + experiment=self.config.r2d2_experiment_id(), observation_type=name, file_extension=obs_path_file.split('.')[-1], window_length='PT6H', diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index c251cebe1..91445adb6 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -68,7 +68,7 @@ def execute(self): step=window_length, resolution=self.config.horizontal_resolution(), type='fc', - experiment=self.experiment_id()) + experiment=self.config.r2d2_experiment_id()) # Loop over an for an in r2d2_dict['store']['an']: @@ -79,7 +79,7 @@ def execute(self): fc_date_rendering='analysis', resolution=self.config.horizontal_resolution(), type='an', - experiment=self.experiment_id()) + experiment=self.config.r2d2_experiment_id()) # Oceanstats needs special handling from the forecast folder. It is produced at the end of # the forecast and could be saved as a good metric. We are replicating the same structure as diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index bf11de750..a63faa2c6 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -152,6 +152,15 @@ class parser_options(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class r2d2_experiment_id(SuiteQuestion): + default_value: str = "defer_to_code" + question_name: str = "r2d2_experiment_id" + prompt: str = "What experiment_id should r2d2 reference for experiment?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + @dataclass class runahead_limit(SuiteQuestion): default_value: str = "P4" From d32c3b9133c1bdf7fdbcc6f2760ca027fddb1b20 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 24 Feb 2026 16:56:30 -0500 Subject: [PATCH 197/299] Change location that registry happens in --- src/swell/deployment/create_experiment.py | 19 +++++++++++++++++++ .../prepare_config_and_suite.py | 10 ---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 1c2105e9f..c565c019a 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -237,6 +237,25 @@ def create_experiment_directory( with open(os.path.join(exp_suite_path, 'experiment.yaml'), 'w') as file: file.write(experiment_dict_str) + # Register the experiment in R2D2 + # ------------------------------- + + import r2d2 + from swell.utilities.r2d2 import load_r2d2_credentials + + r2d2_id = experiment_dict['r2d2_experiment_id'] + + load_r2d2_credentials(logger, platform) + user = r2d2.get_client_user() + host = r2d2.get_client_host() + compiler = r2d2.get_client_compiler() + + r2d2.register(item='experiment', + name=r2d2_id, + user=user, + compute_host=f'{host}-{compiler}', + lifetime='debug') + # At this point we need to write the complete suite file with all templates resolved. Call the # function to build the scheduling dictionary, combine with the experiment dictionary, # resolve the templates and write the suite file to the experiment suite directory. diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 5d0542fae..db5b0e870 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -722,16 +722,6 @@ def create_r2d2_id(self, swell_id: str) -> str: except Exception as e: if '400 Client Error' in str(e): - user = r2d2.get_client_user() - host = r2d2.get_client_host() - compiler = r2d2.get_client_compiler() - - r2d2.register(item='experiment', - name=temp_id, - user=user, - compute_host=f'{host}-{compiler}', - lifetime='debug') - return temp_id raise Exception('Could not find a valid experiment_id for R2D2') From 28be5ba804576df852637e76c600412fa2e45f26 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 24 Feb 2026 17:03:25 -0500 Subject: [PATCH 198/299] add key for experiment lifetime --- src/swell/deployment/create_experiment.py | 31 ++++++++++++----------- src/swell/suites/suite_questions.py | 3 ++- src/swell/utilities/question_defaults.py | 10 ++++++++ 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index c565c019a..14dede765 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -239,22 +239,23 @@ def create_experiment_directory( # Register the experiment in R2D2 # ------------------------------- - - import r2d2 - from swell.utilities.r2d2 import load_r2d2_credentials + if 'r2d2_experiment_id' in experiment_dict: + import r2d2 + from swell.utilities.r2d2 import load_r2d2_credentials - r2d2_id = experiment_dict['r2d2_experiment_id'] - - load_r2d2_credentials(logger, platform) - user = r2d2.get_client_user() - host = r2d2.get_client_host() - compiler = r2d2.get_client_compiler() - - r2d2.register(item='experiment', - name=r2d2_id, - user=user, - compute_host=f'{host}-{compiler}', - lifetime='debug') + r2d2_id = experiment_dict['r2d2_experiment_id'] + r2d2_lifetime = experiment_dict['r2d2_experiment_lifetime'] + + load_r2d2_credentials(logger, platform) + user = r2d2.get_client_user() + host = r2d2.get_client_host() + compiler = r2d2.get_client_compiler() + + r2d2.register(item='experiment', + name=r2d2_id, + user=user, + compute_host=f'{host}-{compiler}', + lifetime=r2d2_lifetime) # At this point we need to write the complete suite file with all templates resolved. Call the # function to build the scheduling dictionary, combine with the experiment dictionary, diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index a4da99431..2073e69b9 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -41,7 +41,8 @@ class SuiteQuestions(QuestionContainer, Enum): qd.final_cycle_point(), qd.model_components(), qd.runahead_limit(), - qd.r2d2_experiment_id() + qd.r2d2_experiment_id(), + qd.r2d2_experiment_lifetime() ] ) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index a63faa2c6..521220ae5 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -161,6 +161,16 @@ class r2d2_experiment_id(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class r2d2_experiment_lifetime(SuiteQuestion): + default_value: str = "debug" + question_name: str = "r2d2_experiment_lifetime" + options: list = mutable_field(['debug', 'science', 'publication', 'release']) + prompt: str = "What lifetime should the experiment have in R2D2?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + @dataclass class runahead_limit(SuiteQuestion): default_value: str = "P4" From 3d571cf7bd5dc41ae07496d9f1ad4cc09659292e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 24 Feb 2026 17:13:58 -0500 Subject: [PATCH 199/299] pycodestyle --- src/swell/deployment/create_experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 14dede765..286d22939 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -242,7 +242,7 @@ def create_experiment_directory( if 'r2d2_experiment_id' in experiment_dict: import r2d2 from swell.utilities.r2d2 import load_r2d2_credentials - + r2d2_id = experiment_dict['r2d2_experiment_id'] r2d2_lifetime = experiment_dict['r2d2_experiment_lifetime'] From 1e65b35ff797cb4b6c2ff9804f435dc88c597087 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 25 Feb 2026 10:22:00 -0500 Subject: [PATCH 200/299] Refactor --- src/swell/deployment/create_experiment.py | 45 +++++++++-------- .../prepare_config_and_suite.py | 33 +------------ src/swell/utilities/r2d2.py | 48 ++++++++++++++++++- 3 files changed, 73 insertions(+), 53 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 286d22939..b73ee23ca 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -163,6 +163,31 @@ def prepare_config( logger.abort(f'SLURM file contains invalid keys: {slurm_invalid_keys}') experiment_dict = {**experiment_dict, **slurm_dict} + # Register the experiment in R2D2 + # ------------------------------- + if 'r2d2_experiment_id' in experiment_dict: + + import r2d2 + from swell.utilities.r2d2 import load_r2d2_credentials, unique_r2d2_id + + r2d2_id = experiment_dict['r2d2_experiment_id'] + + unique_id = unique_r2d2_id(r2d2_id, platform) + experiment_dict['r2d2_experiment_id'] = unique_id + + r2d2_lifetime = experiment_dict['r2d2_experiment_lifetime'] + + load_r2d2_credentials(logger, platform) + user = r2d2.get_client_user() + host = r2d2.get_client_host() + compiler = r2d2.get_client_compiler() + + r2d2.register(item='experiment', + name=unique_id, + user=user, + compute_host=f'{host}-{compiler}', + lifetime=r2d2_lifetime) + # Expand all environment vars in the dictionary # --------------------------------------------- output = io.StringIO() @@ -237,26 +262,6 @@ def create_experiment_directory( with open(os.path.join(exp_suite_path, 'experiment.yaml'), 'w') as file: file.write(experiment_dict_str) - # Register the experiment in R2D2 - # ------------------------------- - if 'r2d2_experiment_id' in experiment_dict: - import r2d2 - from swell.utilities.r2d2 import load_r2d2_credentials - - r2d2_id = experiment_dict['r2d2_experiment_id'] - r2d2_lifetime = experiment_dict['r2d2_experiment_lifetime'] - - load_r2d2_credentials(logger, platform) - user = r2d2.get_client_user() - host = r2d2.get_client_host() - compiler = r2d2.get_client_compiler() - - r2d2.register(item='experiment', - name=r2d2_id, - user=user, - compute_host=f'{host}-{compiler}', - lifetime=r2d2_lifetime) - # At this point we need to write the complete suite file with all templates resolved. Call the # function to build the scheduling dictionary, combine with the experiment dictionary, # resolve the templates and write the suite file to the experiment suite directory. diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index db5b0e870..e66f2110e 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -12,7 +12,6 @@ from ruamel.yaml import YAML from collections.abc import Mapping from typing import Union, Tuple, Optional -import random from swell.swell_path import get_swell_path from swell.deployment.prepare_config_and_suite.question_and_answer_cli import GetAnswerCli @@ -23,8 +22,6 @@ from swell.utilities.dictionary import update_dict from swell.tasks.task_questions import TaskQuestions as task_questions from swell.suites.all_suites import AllSuites -from swell.utilities.r2d2 import load_r2d2_credentials - # -------------------------------------------------------------------------------------------------- @@ -342,7 +339,7 @@ def override_with_defaults(self) -> None: if swell_id == 'defer_to_code': swell_id = f'swell-{self.suite}' self.question_dictionary_model_ind['experiment_id']['default_value'] = swell_id - val['default_value'] = self.create_r2d2_id(swell_id) + val['default_value'] = swell_id # ---------------------------------------------------------------------------------------------- @@ -698,32 +695,4 @@ def get_dynamic_tasks(self, question_list: list) -> list: return tasks - # ---------------------------------------------------------------------------------------------- - - def random_hex_id(self, swell_id: str, length: int = 8): - return f"{swell_id}-{random.randrange(16**length):0{length}x}" - - # ---------------------------------------------------------------------------------------------- - - def create_r2d2_id(self, swell_id: str) -> str: - - # Load credentials to allow search - load_r2d2_credentials(self.logger, self.platform) - - import r2d2 - - self.logger.info('Generating Experiment ID for R2D2') - - # Only try this 10 times - for i in range(10): - temp_id = self.random_hex_id(swell_id, length=8) - try: - r2d2.get(item='experiment', name=temp_id) - except Exception as e: - - if '400 Client Error' in str(e): - return temp_id - - raise Exception('Could not find a valid experiment_id for R2D2') - # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/r2d2.py b/src/swell/utilities/r2d2.py index 5810f4b7f..963632a0d 100644 --- a/src/swell/utilities/r2d2.py +++ b/src/swell/utilities/r2d2.py @@ -9,10 +9,11 @@ import os from ruamel.yaml import YAML +import random from swell.swell_path import get_swell_path from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.logger import Logger +from swell.utilities.logger import get_logger, Logger # -------------------------------------------------------------------------------------------------- @@ -59,6 +60,8 @@ def create_r2d2_config( with open(r2d2_config_file, 'w') as f: f.write(r2d2_config_file_template_str) +# -------------------------------------------------------------------------------------------------- + def _get_platform_r2d2_config(logger: Logger, platform: str = None) -> tuple: if not platform: @@ -95,6 +98,8 @@ def _get_platform_r2d2_config(logger: Logger, platform: str = None) -> tuple: logger.warning(f"Unknown platform '{platform}', cannot determine R2D2 host/compiler") return None, None +# -------------------------------------------------------------------------------------------------- + def load_r2d2_credentials( logger: Logger, @@ -161,5 +166,46 @@ def load_r2d2_credentials( logger.info("R2D2 v3 credentials loaded successfully") +# ---------------------------------------------------------------------------------------------- + + +def random_hex_id(swell_id: str, length: int = 8): + return f"{swell_id}-{random.randrange(16**length):0{length}x}" + +# ---------------------------------------------------------------------------------------------- + + +def experiment_exists(r2d2_id: str): + import r2d2 + + try: + r2d2.get(item='experiment', name=r2d2_id) + except Exception as e: + if '400 Client Error' in str(e): + return False + + return True # ---------------------------------------------------------------------------------------------- + + +def unique_r2d2_id(swell_id: str, platform: str) -> str: + logger = get_logger('CreateR2D2ID') + + # Load credentials to allow search + load_r2d2_credentials(logger, platform) + + # Just use the ID if it doesn't exist + if not experiment_exists(swell_id): + return swell_id + + # If not, append an unused hex id + # Only try this 10 times + for i in range(10): + temp_id = random_hex_id(swell_id, length=8) + if not experiment_exists(temp_id): + return temp_id + + raise Exception('Could not find a valid experiment_id for R2D2') + +# -------------------------------------------------------------------------------------------------- From f69dbe55feeebfc4a6e5eaa08732a50cc4b09e4a Mon Sep 17 00:00:00 2001 From: ftgoktas Date: Wed, 25 Feb 2026 12:44:51 -0500 Subject: [PATCH 201/299] Load r2d2 modules during experiment creation (#701). --- src/swell/deployment/create_experiment.py | 7 ++-- src/swell/swell.py | 6 ++++ src/swell/utilities/r2d2.py | 43 ++++++++++++++++++++++- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index b73ee23ca..bf8196017 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -167,8 +167,12 @@ def prepare_config( # ------------------------------- if 'r2d2_experiment_id' in experiment_dict: + from swell.utilities.r2d2 import load_r2d2_credentials, load_r2d2_module, unique_r2d2_id + + load_r2d2_credentials(logger, platform) + load_r2d2_module(logger, platform) + import r2d2 - from swell.utilities.r2d2 import load_r2d2_credentials, unique_r2d2_id r2d2_id = experiment_dict['r2d2_experiment_id'] @@ -177,7 +181,6 @@ def prepare_config( r2d2_lifetime = experiment_dict['r2d2_experiment_lifetime'] - load_r2d2_credentials(logger, platform) user = r2d2.get_client_user() host = r2d2.get_client_host() compiler = r2d2.get_client_compiler() diff --git a/src/swell/swell.py b/src/swell/swell.py index 944a43779..923ab779b 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -112,6 +112,12 @@ def create( suite (str): Name of the suite you wish to run. \n """ + # Load R2D2 credentials + from swell.utilities.r2d2 import load_r2d2_credentials + from swell.utilities.logger import get_logger + logger = get_logger("Swell Create") + load_r2d2_credentials(logger, platform) + # Create the experiment directory create_experiment_directory(suite, input_method, platform, override, advanced, slurm) diff --git a/src/swell/utilities/r2d2.py b/src/swell/utilities/r2d2.py index 963632a0d..b3d60e558 100644 --- a/src/swell/utilities/r2d2.py +++ b/src/swell/utilities/r2d2.py @@ -16,7 +16,48 @@ from swell.utilities.logger import get_logger, Logger # -------------------------------------------------------------------------------------------------- - +import subprocess + +# Platform-specific R2D2 module config +_R2D2_MODULE_CONFIG = { + 'nccs_discover_sles15': { + 'module_path': '/discover/nobackup/projects/gmao/advda/JediOpt/modulefiles/core', + 'module_name': 'r2d2-client/112025', + }, + 'nccs_discover_cascade': { + 'module_path': '/discover/nobackup/projects/gmao/advda/JediOpt/modulefiles/core', + 'module_name': 'r2d2-client/112025', + }, +} + +def load_r2d2_module(logger: Logger, platform: str) -> None: + """Load R2D2 module via bash, capture env, apply to current process.""" + if platform not in _R2D2_MODULE_CONFIG: + return + config = _R2D2_MODULE_CONFIG[platform] + cmd = ( + f'source /usr/share/lmod/lmod/init/bash && ' + f'module use -a {config["module_path"]} && ' + f'module load {config["module_name"]} && env' + ) + try: + result = subprocess.run(['bash', '-c', cmd], capture_output=True, text=True, timeout=30) + if result.returncode != 0: + logger.warning(f'Failed to load R2D2 module: {result.stderr}') + return + for line in result.stdout.strip().split('\n'): + if '=' in line: + key, _, value = line.partition('=') + os.environ[key] = value + # PYTHONPATH needs to be added to sys.path for import to work + if key == 'PYTHONPATH': + import sys + for p in value.split(':'): + if p and p not in sys.path: + sys.path.insert(0, p) + logger.info(f'Loaded R2D2 module: {config["module_name"]}') + except Exception as e: + logger.warning(f'Could not load R2D2 module: {e}') def create_r2d2_config( logger: Logger, From 7041a44a3d729913f5aab597ba6dd840b25f9365 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 25 Feb 2026 15:33:21 -0500 Subject: [PATCH 202/299] few touches --- src/swell/swell.py | 5 ----- src/swell/utilities/r2d2.py | 8 +++++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/swell/swell.py b/src/swell/swell.py index 923ab779b..94675da52 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -112,11 +112,6 @@ def create( suite (str): Name of the suite you wish to run. \n """ - # Load R2D2 credentials - from swell.utilities.r2d2 import load_r2d2_credentials - from swell.utilities.logger import get_logger - logger = get_logger("Swell Create") - load_r2d2_credentials(logger, platform) # Create the experiment directory create_experiment_directory(suite, input_method, platform, override, advanced, slurm) diff --git a/src/swell/utilities/r2d2.py b/src/swell/utilities/r2d2.py index b3d60e558..bc4f08c2d 100644 --- a/src/swell/utilities/r2d2.py +++ b/src/swell/utilities/r2d2.py @@ -10,13 +10,13 @@ import os from ruamel.yaml import YAML import random +import subprocess from swell.swell_path import get_swell_path from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.logger import get_logger, Logger # -------------------------------------------------------------------------------------------------- -import subprocess # Platform-specific R2D2 module config _R2D2_MODULE_CONFIG = { @@ -30,6 +30,9 @@ }, } +# -------------------------------------------------------------------------------------------------- + + def load_r2d2_module(logger: Logger, platform: str) -> None: """Load R2D2 module via bash, capture env, apply to current process.""" if platform not in _R2D2_MODULE_CONFIG: @@ -59,6 +62,9 @@ def load_r2d2_module(logger: Logger, platform: str) -> None: except Exception as e: logger.warning(f'Could not load R2D2 module: {e}') +# ---------------------------------------------------------------------------------------------- + + def create_r2d2_config( logger: Logger, platform: str, From 07714eab8bed721f83c78e1e9b055dab4152fe80 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 26 Feb 2026 13:02:22 -0500 Subject: [PATCH 203/299] fix mistake --- .../prepare_config_and_suite/prepare_config_and_suite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index e66f2110e..0b230f845 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -1,4 +1,5 @@ - +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. From 07add942f54cd29d1a4146606706d2acb663bc3c Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 26 Feb 2026 13:10:34 -0500 Subject: [PATCH 204/299] Hard-code debug --- src/swell/deployment/create_experiment.py | 4 +--- src/swell/suites/suite_questions.py | 1 - src/swell/utilities/question_defaults.py | 10 ---------- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index bf8196017..dc57f9dad 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -179,8 +179,6 @@ def prepare_config( unique_id = unique_r2d2_id(r2d2_id, platform) experiment_dict['r2d2_experiment_id'] = unique_id - r2d2_lifetime = experiment_dict['r2d2_experiment_lifetime'] - user = r2d2.get_client_user() host = r2d2.get_client_host() compiler = r2d2.get_client_compiler() @@ -189,7 +187,7 @@ def prepare_config( name=unique_id, user=user, compute_host=f'{host}-{compiler}', - lifetime=r2d2_lifetime) + lifetime='debug') # Expand all environment vars in the dictionary # --------------------------------------------- diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index 2073e69b9..cc2824591 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -42,7 +42,6 @@ class SuiteQuestions(QuestionContainer, Enum): qd.model_components(), qd.runahead_limit(), qd.r2d2_experiment_id(), - qd.r2d2_experiment_lifetime() ] ) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 521220ae5..a63faa2c6 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -161,16 +161,6 @@ class r2d2_experiment_id(SuiteQuestion): # -------------------------------------------------------------------------------------------------- - @dataclass - class r2d2_experiment_lifetime(SuiteQuestion): - default_value: str = "debug" - question_name: str = "r2d2_experiment_lifetime" - options: list = mutable_field(['debug', 'science', 'publication', 'release']) - prompt: str = "What lifetime should the experiment have in R2D2?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - @dataclass class runahead_limit(SuiteQuestion): default_value: str = "P4" From d3abccac6da703287588b7a9e15bcaf4ff6ad92a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 26 Feb 2026 17:28:30 -0500 Subject: [PATCH 205/299] Add option to skip R2D2 --- src/swell/deployment/create_experiment.py | 36 +++++++++++-- .../prepare_config_and_suite.py | 50 +++++++------------ src/swell/suites/3dfgat_atmos/flow.cylc | 6 ++- src/swell/suites/3dfgat_cycle/flow.cylc | 6 ++- src/swell/suites/3dvar/flow.cylc | 8 +-- src/swell/suites/3dvar_atmos/flow.cylc | 7 +-- src/swell/suites/3dvar_cycle/flow.cylc | 7 +-- src/swell/suites/hofx/flow.cylc | 7 +-- src/swell/suites/hofx_cf/flow.cylc | 9 ++-- src/swell/suites/suite_questions.py | 1 + src/swell/swell.py | 8 ++- src/swell/utilities/question_defaults.py | 9 ++++ 12 files changed, 94 insertions(+), 60 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index dc57f9dad..23e883d14 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -31,6 +31,19 @@ # -------------------------------------------------------------------------------------------------- +def read_override_file(override_path: str | None) -> dict: + + yaml = YAML(typ='safe') + + if override_path is None: + return {} + else: + with open(override_path, 'r') as f: + return yaml.load(f) + +# -------------------------------------------------------------------------------------------------- + + def clone_config( configuration: str, experiment_id: str, @@ -74,9 +87,9 @@ def prepare_config( suite_config: str, method: str, platform: str, - override: Union[dict, str, None], + override: dict, advanced: bool, - slurm: str + slurm: str, ) -> str: # Create a logger @@ -165,7 +178,8 @@ def prepare_config( # Register the experiment in R2D2 # ------------------------------- - if 'r2d2_experiment_id' in experiment_dict: + if 'r2d2_experiment_id' in experiment_dict and 'skip_r2d2' in experiment_dict \ + and not experiment_dict['skip_r2d2']: from swell.utilities.r2d2 import load_r2d2_credentials, load_r2d2_module, unique_r2d2_id @@ -221,7 +235,8 @@ def create_experiment_directory( platform: str, override: str, advanced: bool, - slurm: Optional[str] + slurm: str | None, + skip_r2d2: bool ) -> None: # Get the base name of the suite @@ -232,10 +247,21 @@ def create_experiment_directory( # --------------- logger = get_logger('SwellCreateExperiment') + # Read override file + # ------------------ + override_dict = read_override_file(override) + + # Specify whether to skip registering and storing in R2D2 + # ------------------------------------------------------- + if skip_r2d2: + + # Only override this if it is true, otherwise let the suite decide + override_dict['skip_r2d2'] = skip_r2d2 + # Call the experiment config and suite generation # ------------------------------------------------ experiment_dict_str = prepare_config(suite, suite_config, method, platform, - override, advanced, slurm) + override_dict, advanced, slurm) # Load the string using yaml # -------------------------- diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 0b230f845..571a69484 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -58,7 +58,7 @@ def __init__( suite_config: str, platform: str, config_client: str, - override: Union[str, dict, None] + override: dict ) -> None: # Store local copy of the inputs @@ -346,40 +346,24 @@ def override_with_defaults(self) -> None: def override_with_external(self) -> None: - # Append with any user provide overrides - if self.override is not None: + # In this case the user is sending in a dictionary that looks like the experiment + # dictionary that they will ultimately be looking at. This means the dictionary does + # not contain default_value or options and the override cannot be performed. - # Create an override dictionary - override_dict = {} - - if isinstance(self.override, Mapping): - override_dict.update_dict(override_dict, self.override) + # Iterate over the model_ind dictionary and override + # -------------------------------------------------- + for key, val in self.question_dictionary_model_ind.items(): + if key in self.override: + val['default_value'] = self.override[key] - elif isinstance(self.override, str): - yaml = YAML(typ='safe') - with open(self.override, 'r') as ymlfile: - override_dict = update_dict(override_dict, yaml.load(ymlfile)) - else: - self.logger.abort(f'Override must be a dictionary or a path to a yaml file.') - - # In this case the user is sending in a dictionary that looks like the experiment - # dictionary that they will ultimately be looking at. This means the dictionary does - # not contain default_value or options and the override cannot be performed. - - # Iterate over the model_ind dictionary and override - # -------------------------------------------------- - for key, val in self.question_dictionary_model_ind.items(): - if key in override_dict: - val['default_value'] = override_dict[key] - - # Iterate over the model_dep dictionary and override - # -------------------------------------------------- - if self.suite_needs_model_components and 'models' in override_dict.keys(): - for model, model_dict in self.question_dictionary_model_dep.items(): - for key, val in model_dict.items(): - if model in override_dict['models']: - if key in override_dict['models'][model]: - val['default_value'] = override_dict['models'][model][key] + # Iterate over the model_dep dictionary and override + # -------------------------------------------------- + if self.suite_needs_model_components and 'models' in self.override.keys(): + for model, model_dict in self.question_dictionary_model_dep.items(): + for key, val in model_dict.items(): + if model in self.override['models']: + if key in self.override['models'][model]: + val['default_value'] = self.override['models'][model][key] # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_atmos/flow.cylc b/src/swell/suites/3dfgat_atmos/flow.cylc index 372a6faa2..73e4e89b0 100644 --- a/src/swell/suites/3dfgat_atmos/flow.cylc +++ b/src/swell/suites/3dfgat_atmos/flow.cylc @@ -89,12 +89,14 @@ # EvaIncrement RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} + {% if not skip_r2d2 %} # Save observations RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} + {% endif %} # Clean up large files - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} + EvaObservations-{{model_component}} => CleanCycle-{{model_component}} {% endif %} {% endfor %} diff --git a/src/swell/suites/3dfgat_cycle/flow.cylc b/src/swell/suites/3dfgat_cycle/flow.cylc index db6b339ba..86600410f 100644 --- a/src/swell/suites/3dfgat_cycle/flow.cylc +++ b/src/swell/suites/3dfgat_cycle/flow.cylc @@ -104,9 +104,11 @@ # Move restart to next cycle # SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} + {% if not save_r2d2 %} # Save analysis output # RunJediFgatExecutable-{{model_component}} => SaveAnalysis-{{model_component}} - RunJediFgatExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + RunJediFgatExecutable-{{model_component}} => SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} + {% endif %} # Save model output # MoveBackground-{{model_component}} => StoreBackground-{{model_component}} @@ -116,7 +118,7 @@ MoveDaRestart-{{model_component}} => RemoveForecastDir # Clean up large files - EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & SaveObsDiags-{{model_component}} => + EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} {% endfor %} """ diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index 00e18ee8c..0bf10070c 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -71,13 +71,15 @@ # EvaIncrement RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - + + {% if not skip_r2d2 %} # Save observations RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} + {% endif %} # Clean up large files - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} + EvaObservations-{{model_component}} => CleanCycle-{{model_component}} {% endif %} {% endfor %} diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc index fab25dcfe..48c84f0fd 100644 --- a/src/swell/suites/3dvar_atmos/flow.cylc +++ b/src/swell/suites/3dvar_atmos/flow.cylc @@ -86,12 +86,13 @@ # EvaIncrement RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} + {% if not save_r2d2 %} # Save observations - RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} + {% endif %} # Clean up large files - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} + EvaObservations-{{model_component}} => CleanCycle-{{model_component}} {% endif %} {% endfor %} diff --git a/src/swell/suites/3dvar_cycle/flow.cylc b/src/swell/suites/3dvar_cycle/flow.cylc index 8db2144fd..9aadcb786 100644 --- a/src/swell/suites/3dvar_cycle/flow.cylc +++ b/src/swell/suites/3dvar_cycle/flow.cylc @@ -102,9 +102,11 @@ # Move restart to next cycle # SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} + {% if skip_r2d2 %} # Save analysis output # RunJediVariationalExecutable-{{model_component}} => SaveAnalysis-{{model_component}} - RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} + {% endif %} # Save model output # MoveBackground-{{model_component}} => StoreBackground-{{model_component}} @@ -115,8 +117,7 @@ # Clean up large files # EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & SaveObsDiags-{{model_component}} & RemoveForecastDir => - EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} + EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} {% endfor %} """ {% endfor %} diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc index 666399b9d..b465c6fe1 100644 --- a/src/swell/suites/hofx/flow.cylc +++ b/src/swell/suites/hofx/flow.cylc @@ -72,12 +72,13 @@ # EvaObservations RunJediHofxExecutable-{{model_component}} => EvaObservations-{{model_component}} + {% if not skip_r2d2 %} # Save observations - RunJediHofxExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + RunJediHofxExecutable-{{model_component}} => SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} + {% endif %} # Clean up large files - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} + EvaObservations-{{model_component}} => CleanCycle-{{model_component}} {% endif %} {% endfor %} diff --git a/src/swell/suites/hofx_cf/flow.cylc b/src/swell/suites/hofx_cf/flow.cylc index 562482ecf..2129dafd0 100644 --- a/src/swell/suites/hofx_cf/flow.cylc +++ b/src/swell/suites/hofx_cf/flow.cylc @@ -65,12 +65,13 @@ # EvaObservations RunJediHofxExecutable-{{model_component}} => EvaObservations-{{model_component}} - # Save feedback - RunJediHofxExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + {% if not skip_r2d2 %} + # Save observations + RunJediHofxExecutable-{{model_component}} => SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} + {% endif %} # Clean up large files - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} + EvaObservations-{{model_component}} => CleanCycle-{{model_component}} {% endif %} {% endfor %} diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index cc2824591..ee08b6d08 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -42,6 +42,7 @@ class SuiteQuestions(QuestionContainer, Enum): qd.model_components(), qd.runahead_limit(), qd.r2d2_experiment_id(), + qd.skip_r2d2(), ] ) diff --git a/src/swell/swell.py b/src/swell/swell.py index 94675da52..9b6b6d12c 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -82,6 +82,8 @@ def swell_driver() -> None: or for task-model combinations. """ +skip_r2d2_help = """Skip registering this experiment and storing products in R2D2.""" + # -------------------------------------------------------------------------------------------------- @@ -95,13 +97,15 @@ def swell_driver() -> None: @click.option('-o', '--override', 'override', default=None, help=override_help) @click.option('-a', '--advanced', 'advanced', default=False, help=advanced_help) @click.option('-s', '--slurm', 'slurm', default=None, help=slurm_help) +@click.option('-k', '--skip-r2d2', 'skip_r2d2', is_flag=True, default=False, help=skip_r2d2_help) def create( suite: str, input_method: str, platform: str, override: Union[dict, str, None], advanced: bool, - slurm: str + slurm: str, + skip_r2d2: bool ) -> None: """ Create a new experiment @@ -114,7 +118,7 @@ def create( """ # Create the experiment directory - create_experiment_directory(suite, input_method, platform, override, advanced, slurm) + create_experiment_directory(suite, input_method, platform, override, advanced, slurm, skip_r2d2) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index a63faa2c6..ffcf0fa46 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -184,6 +184,15 @@ class skip_ensemble_hofx(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class skip_r2d2(SuiteQuestion): + default_value: bool = False + question_name: str = "skip_r2d2" + prompt: str = "Skip registering and storing results of this experiment in R2D2?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + @dataclass class start_cycle_point(SuiteQuestion): default_value: str = "2023-10-10T00:00:00Z" From fd523808c285ea2e78951e9c0ca4222493ef6bfe Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 27 Feb 2026 17:11:45 -0500 Subject: [PATCH 206/299] Refactor question_defaults to configuration --- src/swell/suites/3dfgat_atmos/suite_config.py | 2 +- src/swell/suites/3dfgat_cycle/suite_config.py | 2 +- src/swell/suites/3dvar/suite_config.py | 2 +- src/swell/suites/3dvar_atmos/suite_config.py | 2 +- src/swell/suites/3dvar_cycle/suite_config.py | 2 +- src/swell/suites/base/suite_questions.py | 2 +- src/swell/suites/compare/suite_config.py | 2 +- src/swell/suites/convert_bufr/suite_config.py | 2 +- .../suites/convert_ncdiags/suite_config.py | 2 +- .../suites/eva_capabilities/suite_config.py | 2 +- .../suites/forecast_geos/suite_config.py | 2 +- src/swell/suites/geosadas/suite_config.py | 2 +- src/swell/suites/hofx/suite_config.py | 2 +- src/swell/suites/hofx_cf/suite_config.py | 2 +- src/swell/suites/ingest_obs/suite_config.py | 2 +- .../suites/localensembleda/suite_config.py | 2 +- src/swell/suites/ufo_testing/suite_config.py | 2 +- src/swell/tasks/build_geos.py | 2 +- src/swell/tasks/build_geos_by_linking.py | 2 +- src/swell/tasks/build_jedi.py | 2 +- src/swell/tasks/build_jedi_by_linking.py | 2 +- src/swell/tasks/clean_cycle.py | 2 +- src/swell/tasks/clone_geos.py | 2 +- src/swell/tasks/clone_geos_mksi.py | 2 +- src/swell/tasks/clone_gmao_perllib.py | 2 +- src/swell/tasks/clone_jedi.py | 2 +- src/swell/tasks/eva_comparison_increment.py | 2 +- src/swell/tasks/eva_comparison_jedi_log.py | 2 +- .../tasks/eva_comparison_observations.py | 2 +- src/swell/tasks/eva_increment.py | 2 +- src/swell/tasks/eva_observations.py | 2 +- src/swell/tasks/eva_timeseries.py | 2 +- src/swell/tasks/generate_b_climatology.py | 2 +- .../generate_b_climatology_by_linking.py | 2 +- .../generate_observing_system_records.py | 2 +- src/swell/tasks/get_background.py | 2 +- .../tasks/get_background_geos_experiment.py | 2 +- src/swell/tasks/get_bufr.py | 2 +- src/swell/tasks/get_ensemble.py | 2 +- .../tasks/get_ensemble_geos_experiment.py | 2 +- src/swell/tasks/get_geos_adas_background.py | 2 +- src/swell/tasks/get_geos_restart.py | 2 +- src/swell/tasks/get_geovals.py | 2 +- src/swell/tasks/get_gsi_bc.py | 2 +- src/swell/tasks/get_gsi_ncdiag.py | 2 +- src/swell/tasks/get_ncdiags.py | 2 +- src/swell/tasks/get_obs_not_in_r2d2.py | 2 +- src/swell/tasks/get_observations.py | 2 +- src/swell/tasks/gsi_bc_to_ioda.py | 2 +- src/swell/tasks/gsi_ncdiag_to_ioda.py | 2 +- src/swell/tasks/ingest_obs.py | 2 +- src/swell/tasks/jedi_log_comparison.py | 2 +- src/swell/tasks/jedi_oops_log_parser.py | 2 +- src/swell/tasks/link_geos_output.py | 2 +- src/swell/tasks/move_da_restart.py | 2 +- src/swell/tasks/move_forecast_restart.py | 2 +- src/swell/tasks/prep_geos_run_dir.py | 2 +- src/swell/tasks/prepare_analysis.py | 2 +- src/swell/tasks/render_jedi_observations.py | 2 +- ...jedi_convert_state_soca2cice_executable.py | 2 +- .../tasks/run_jedi_ensemble_mean_variance.py | 2 +- src/swell/tasks/run_jedi_fgat_executable.py | 2 +- .../run_jedi_hofx_ensemble_executable.py | 2 +- src/swell/tasks/run_jedi_hofx_executable.py | 2 +- .../run_jedi_local_ensemble_da_executable.py | 2 +- .../tasks/run_jedi_obsfilters_executable.py | 2 +- .../tasks/run_jedi_ufo_tests_executable.py | 2 +- .../tasks/run_jedi_variational_executable.py | 2 +- src/swell/tasks/save_obs_diags.py | 2 +- src/swell/tasks/save_restart.py | 2 +- src/swell/tasks/stage_jedi.py | 2 +- src/swell/tasks/store_background.py | 2 +- src/swell/tasks/task_questions.py | 2 +- src/swell/utilities/question_defaults.py | 1505 ----------------- 74 files changed, 73 insertions(+), 1578 deletions(-) delete mode 100644 src/swell/utilities/question_defaults.py diff --git a/src/swell/suites/3dfgat_atmos/suite_config.py b/src/swell/suites/3dfgat_atmos/suite_config.py index 74d195eed..5ba417a0f 100644 --- a/src/swell/suites/3dfgat_atmos/suite_config.py +++ b/src/swell/suites/3dfgat_atmos/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dfgat_cycle/suite_config.py b/src/swell/suites/3dfgat_cycle/suite_config.py index 28c3d9844..33ee9da0d 100644 --- a/src/swell/suites/3dfgat_cycle/suite_config.py +++ b/src/swell/suites/3dfgat_cycle/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dvar/suite_config.py b/src/swell/suites/3dvar/suite_config.py index df33a9ec5..ef0f29521 100644 --- a/src/swell/suites/3dvar/suite_config.py +++ b/src/swell/suites/3dvar/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dvar_atmos/suite_config.py b/src/swell/suites/3dvar_atmos/suite_config.py index 711f225b8..153d7de0b 100644 --- a/src/swell/suites/3dvar_atmos/suite_config.py +++ b/src/swell/suites/3dvar_atmos/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dvar_cycle/suite_config.py b/src/swell/suites/3dvar_cycle/suite_config.py index 49ef97202..c4d381c6f 100644 --- a/src/swell/suites/3dvar_cycle/suite_config.py +++ b/src/swell/suites/3dvar_cycle/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/base/suite_questions.py b/src/swell/suites/base/suite_questions.py index 5cd1aa02d..e9b5f3d2c 100644 --- a/src/swell/suites/base/suite_questions.py +++ b/src/swell/suites/base/suite_questions.py @@ -11,7 +11,7 @@ from enum import Enum from swell.utilities.swell_questions import QuestionList, QuestionContainer -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare/suite_config.py b/src/swell/suites/compare/suite_config.py index a41df7f40..28dea3650 100644 --- a/src/swell/suites/compare/suite_config.py +++ b/src/swell/suites/compare/suite_config.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.swell_questions import QuestionContainer, QuestionList, WidgetType -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/convert_bufr/suite_config.py b/src/swell/suites/convert_bufr/suite_config.py index c74e0b701..fb40eeceb 100644 --- a/src/swell/suites/convert_bufr/suite_config.py +++ b/src/swell/suites/convert_bufr/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/convert_ncdiags/suite_config.py b/src/swell/suites/convert_ncdiags/suite_config.py index a758b4faf..cd0aba0ad 100644 --- a/src/swell/suites/convert_ncdiags/suite_config.py +++ b/src/swell/suites/convert_ncdiags/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/eva_capabilities/suite_config.py b/src/swell/suites/eva_capabilities/suite_config.py index edd7b7369..41e20c785 100644 --- a/src/swell/suites/eva_capabilities/suite_config.py +++ b/src/swell/suites/eva_capabilities/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/forecast_geos/suite_config.py b/src/swell/suites/forecast_geos/suite_config.py index 246550c07..69733fc2a 100644 --- a/src/swell/suites/forecast_geos/suite_config.py +++ b/src/swell/suites/forecast_geos/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/geosadas/suite_config.py b/src/swell/suites/geosadas/suite_config.py index daf3a6bc8..bbf1a142f 100644 --- a/src/swell/suites/geosadas/suite_config.py +++ b/src/swell/suites/geosadas/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/hofx/suite_config.py b/src/swell/suites/hofx/suite_config.py index 46d486482..d115beaa8 100644 --- a/src/swell/suites/hofx/suite_config.py +++ b/src/swell/suites/hofx/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/hofx_cf/suite_config.py b/src/swell/suites/hofx_cf/suite_config.py index 42e8c3359..7bdc44d5b 100644 --- a/src/swell/suites/hofx_cf/suite_config.py +++ b/src/swell/suites/hofx_cf/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/ingest_obs/suite_config.py b/src/swell/suites/ingest_obs/suite_config.py index 25c66a95a..c315a50bf 100644 --- a/src/swell/suites/ingest_obs/suite_config.py +++ b/src/swell/suites/ingest_obs/suite_config.py @@ -6,7 +6,7 @@ """ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index fc881d1d9..6c0b697b9 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/ufo_testing/suite_config.py b/src/swell/suites/ufo_testing/suite_config.py index fd04b66a9..99f025bf6 100644 --- a/src/swell/suites/ufo_testing/suite_config.py +++ b/src/swell/suites/ufo_testing/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index fac942d73..11193c22d 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.shell_commands import run_subprocess, create_executable_file diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index 9f6989535..3dfa06d6d 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index 19514af8e..9607b9241 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index 24c744d87..ceaa4cf55 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.tasks.base.task_attributes import task_attributes # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index d01f7238c..2721193e4 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -11,7 +11,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats from datetime import datetime as dt import glob diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 3bfd15846..40a1c2dbe 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.git_utils import git_clone from swell.utilities.shell_commands import run_subprocess diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index dac2f1ed7..c8a528ba1 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -12,7 +12,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/clone_gmao_perllib.py b/src/swell/tasks/clone_gmao_perllib.py index c73eea362..6424d204a 100644 --- a/src/swell/tasks/clone_gmao_perllib.py +++ b/src/swell/tasks/clone_gmao_perllib.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index 43fc985fe..edc91879d 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.pinned_versions.check_hashes import check_hashes from swell.tasks.base.task_attributes import task_attributes diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index 2d2238230..521761a23 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.data_assimilation_window_params import DataAssimilationWindowParams from swell.utilities.comparisons import comparison_tags, experiment_ids diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 56d23ec46..7222110ed 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.comparisons import comparison_tags diff --git a/src/swell/tasks/eva_comparison_observations.py b/src/swell/tasks/eva_comparison_observations.py index 81c123b79..6af82e25a 100644 --- a/src/swell/tasks/eva_comparison_observations.py +++ b/src/swell/tasks/eva_comparison_observations.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.observations import ioda_name_to_long_name diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index 864ef352d..538f5cf9e 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index b026aa2f3..db4883f0d 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.observations import ioda_name_to_long_name diff --git a/src/swell/tasks/eva_timeseries.py b/src/swell/tasks/eva_timeseries.py index d6c770519..bacc4d514 100644 --- a/src/swell/tasks/eva_timeseries.py +++ b/src/swell/tasks/eva_timeseries.py @@ -18,7 +18,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index c3634f60e..5c54a533a 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -11,7 +11,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.shell_commands import run_track_log_subprocess from swell.utilities.file_system_operations import check_if_files_exist_in_path diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 40362f3f4..516706bf7 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -10,7 +10,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import link_all_files_from_first_in_hierarchy_of_sources diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index 4f6d436a3..39b837ca1 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.observing_system_records import ObservingSystemRecords # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 0c06c1f73..db6cdc8b5 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -11,7 +11,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd import isodate import os diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index b21c9868b..c5573152c 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_bufr.py b/src/swell/tasks/get_bufr.py index 4fbaae6ba..e04661676 100644 --- a/src/swell/tasks/get_bufr.py +++ b/src/swell/tasks/get_bufr.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_ensemble.py b/src/swell/tasks/get_ensemble.py index 9302d8c74..17216b5a6 100644 --- a/src/swell/tasks/get_ensemble.py +++ b/src/swell/tasks/get_ensemble.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_ensemble_geos_experiment.py b/src/swell/tasks/get_ensemble_geos_experiment.py index f6dff38f9..249f9bb61 100644 --- a/src/swell/tasks/get_ensemble_geos_experiment.py +++ b/src/swell/tasks/get_ensemble_geos_experiment.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py index 675024e06..76233f4fc 100644 --- a/src/swell/tasks/get_geos_adas_background.py +++ b/src/swell/tasks/get_geos_adas_background.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_geos_restart.py b/src/swell/tasks/get_geos_restart.py index 80bb37116..51ca638b1 100644 --- a/src/swell/tasks/get_geos_restart.py +++ b/src/swell/tasks/get_geos_restart.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import copy_to_dst_dir, check_if_files_exist_in_path # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index afbc15a38..ef3e15189 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.r2d2 import create_r2d2_config diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index d61cb1140..67419c6bd 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index 03249fbb8..059134ff0 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index 96689b6dc..cdd95bf09 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -11,7 +11,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py index cccda4746..fc8282a27 100644 --- a/src/swell/tasks/get_obs_not_in_r2d2.py +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 18c818773..c02249e73 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.r2d2 import create_r2d2_config from swell.utilities.datetime_util import datetime_formats from swell.utilities.observations import get_ioda_names_list, get_provider_for_observation diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 871232496..71d8dfd07 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.dictionary import write_dict_to_yaml from swell.utilities.shell_commands import run_track_log_subprocess diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 8a64682ea..b94b49053 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.shell_commands import run_subprocess, create_executable_file diff --git a/src/swell/tasks/ingest_obs.py b/src/swell/tasks/ingest_obs.py index a035eb0da..6d8a9ac33 100644 --- a/src/swell/tasks/ingest_obs.py +++ b/src/swell/tasks/ingest_obs.py @@ -20,7 +20,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.r2d2 import create_r2d2_config from swell.utilities.observations import get_ioda_names_list, get_provider_for_observation import r2d2 diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py index 1007f2662..1d0c89e3b 100644 --- a/src/swell/tasks/jedi_log_comparison.py +++ b/src/swell/tasks/jedi_log_comparison.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.comparisons import comparison_tags # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/jedi_oops_log_parser.py b/src/swell/tasks/jedi_oops_log_parser.py index b1831b92d..45b3d4951 100644 --- a/src/swell/tasks/jedi_oops_log_parser.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index 248a9a314..0376d9fa2 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -18,7 +18,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/move_da_restart.py b/src/swell/tasks/move_da_restart.py index 1250407d4..ed72f4d8f 100644 --- a/src/swell/tasks/move_da_restart.py +++ b/src/swell/tasks/move_da_restart.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import move_files # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/move_forecast_restart.py b/src/swell/tasks/move_forecast_restart.py index 1d3515604..2db5d85df 100644 --- a/src/swell/tasks/move_forecast_restart.py +++ b/src/swell/tasks/move_forecast_restart.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import move_files # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/prep_geos_run_dir.py b/src/swell/tasks/prep_geos_run_dir.py index 220e6f79e..79a3e38c2 100644 --- a/src/swell/tasks/prep_geos_run_dir.py +++ b/src/swell/tasks/prep_geos_run_dir.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.file_system_operations import copy_to_dst_dir, check_if_files_exist_in_path # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index 0f29d65fc..10cc2431f 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index fc4e9f942..ab35a1d47 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import check_obs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py index a3fdbc301..3e231fd8a 100644 --- a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py +++ b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable diff --git a/src/swell/tasks/run_jedi_ensemble_mean_variance.py b/src/swell/tasks/run_jedi_ensemble_mean_variance.py index afbeb7f5a..7ca66e1aa 100644 --- a/src/swell/tasks/run_jedi_ensemble_mean_variance.py +++ b/src/swell/tasks/run_jedi_ensemble_mean_variance.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable diff --git a/src/swell/tasks/run_jedi_fgat_executable.py b/src/swell/tasks/run_jedi_fgat_executable.py index 30f8ecf2c..a2a1daae9 100644 --- a/src/swell/tasks/run_jedi_fgat_executable.py +++ b/src/swell/tasks/run_jedi_fgat_executable.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable diff --git a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py index 613407c92..9454772a5 100644 --- a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py +++ b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable from swell.tasks.run_jedi_hofx_executable import RunJediHofxExecutable diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 22a8bcc94..90e9504f1 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 5f9e515e9..74b85c66e 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index a2251edd9..3fca16d5b 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 40643ea4c..11b5f5e66 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.dictionary import update_dict from swell.utilities.run_jedi_executables import run_executable diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 78ea88018..c36dc5b30 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.run_jedi_executables import run_executable diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 98ee8fff8..31f76f06c 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -10,7 +10,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.r2d2 import create_r2d2_config from swell.utilities.run_jedi_executables import check_obs diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index dfa0693d7..7e990e5d0 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.file_system_operations import copy_to_dst_dir diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 836abf2d7..332344cdc 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.filehandler import get_file_handler from swell.utilities.exceptions import SwellError from swell.utilities.file_system_operations import check_if_files_exist_in_path diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index b13540619..77f7f5397 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd from swell.utilities.datetime_util import datetime_formats diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 09868a61e..7e410ba8d 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -12,7 +12,7 @@ from enum import Enum from swell.utilities.swell_questions import QuestionList, QuestionContainer -from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.configuration.question_defaults import QuestionDefaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py deleted file mode 100644 index 921835a3b..000000000 --- a/src/swell/utilities/question_defaults.py +++ /dev/null @@ -1,1505 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -from dataclasses import dataclass -from typing import List, Dict, Union, Literal - -from swell.utilities.swell_questions import SuiteQuestion, TaskQuestion -from swell.utilities.swell_questions import WidgetType as WType -from swell.utilities.dataclass_utils import mutable_field - - -# -------------------------------------------------------------------------------------------------- - -class QuestionDefaults(): - - # -------------------------------------------------------------------------------------------------- - # Suite question defaults go here - # -------------------------------------------------------------------------------------------------- - - @dataclass - class comparison_experiment_paths(SuiteQuestion): - default_value: list = mutable_field([]) - question_name: str = "comparison_experiment_paths" - ask_question: bool = True - prompt: str = "Provide paths to two experiments to run comparison tests on." - widget_type: WType = WType.STRING_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class cycle_times(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "cycle_times" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the cycle times for this model." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class cycling_varbc(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "cycling_varbc" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Do you want to use cycling VarBC option?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class email_address(SuiteQuestion): - default_value: str = "defer_to_user" - question_name: str = "email_address" - prompt: str = "What email address should cylc messages be sent to?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_packets(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_packets" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the number of ensemble packets." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_strategy(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_strategy" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the ensemble hofx strategy." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class experiment_id(SuiteQuestion): - default_value: str = "defer_to_code" - question_name: str = "experiment_id" - ask_question: bool = True - prompt: str = "What is the experiment id?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class experiment_root(SuiteQuestion): - default_value: str = "defer_to_platform" - question_name: str = "experiment_root" - ask_question: bool = True - prompt: str = ("What is the experiment root (the directory where the " - "experiment will be stored)?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class final_cycle_point(SuiteQuestion): - default_value: str = "2023-10-10T06:00:00Z" - question_name: str = "final_cycle_point" - ask_question: bool = True - prompt: str = "What is the time of the final cycle (middle of the window)?" - widget_type: WType = WType.ISO_DATETIME - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class marine_models(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "marine_models" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_marine" - ]) - prompt: str = "Select the active SOCA models for this model." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class model_components(SuiteQuestion): - default_value: str = "defer_to_code" - question_name: str = "model_components" - ask_question: bool = True - options: str = "defer_to_code" - prompt: str = "Enter the model components for this model." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class parser_options(SuiteQuestion): - default_value: list = mutable_field(['fgrep_residual_norm']) - question_name: str = "parser_options" - ask_question: bool = True - options: list = mutable_field(['fgrep_residual_norm']) - prompt: str = "List the test types to run on the JEDI oops log." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class runahead_limit(SuiteQuestion): - default_value: str = "P4" - question_name: str = "runahead_limit" - ask_question: bool = True - prompt: str = ("Since this suite is non-cycling choose how " - "many hours the workflow can run ahead?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class skip_ensemble_hofx(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "skip_ensemble_hofx" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter if skip ensemble hofx." - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class start_cycle_point(SuiteQuestion): - default_value: str = "2023-10-10T00:00:00Z" - question_name: str = "start_cycle_point" - ask_question: bool = True - prompt: str = "What is the time of the first cycle (middle of the window)?" - widget_type: WType = WType.ISO_DATETIME - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class use_cycle_dir(SuiteQuestion): - default_value: bool = True - question_name: str = "use_cycle_dir" - ask_question: bool = False - prompt: str = ("For cycling tasks, send results to the experiment cycle directory?" - " If false, results will be stored in the current working directory.") - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class window_type(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "window_type" - options: List[str] = mutable_field([ - "3D", - "4D" - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the window type for this model." - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - # Task question defaults go here - # -------------------------------------------------------------------------------------------------- - - @dataclass - class analysis_variables(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "analysis_variables" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What are the analysis variables?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_error_model(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "background_error_model" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which background error model do you want to use?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_experiment(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "background_experiment" - ask_question: bool = True - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the name of the name of the experiment providing the backgrounds?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_frequency(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "background_frequency" - models: List[str] = mutable_field([ - "all_models" - ]) - depends: Dict = mutable_field({ - "window_type": "4D" - }) - prompt: str = "What is the frequency of the background files?" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_time_offset(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "background_time_offset" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = ("How long before the middle of the analysis window did" - " the background providing forecast begin?") - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - @dataclass - class bufr_obs_classes(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "bufr_obs_classes" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What BUFR observation classes will be used?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class bundles(TaskQuestion): - default_value: List[str] = mutable_field([ - "fv3-jedi", - "soca", - "iodaconv", - "ufo" - ]) - question_name: str = "bundles" - ask_question: bool = True - options: List[str] = mutable_field([ - "fv3-jedi", - "soca", - "iodaconv", - "ufo", - "ioda", - "oops", - "saber" - ]) - depends: Dict = mutable_field({ - "jedi_build_method": "create" - }) - prompt: str = "Which JEDI bundles do you wish to build?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class check_for_obs(TaskQuestion): - default_value: bool = True - question_name: str = "check_for_obs" - options: List[bool] = mutable_field([True, False]) - models: List[str] = mutable_field([ - 'all_models' - ]) - prompt: str = "Perform check for observations? Set to false for debugging purposes." - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class clean_patterns(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "clean_patterns" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Provide a list of patterns that you wish to remove from the cycle directory." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class comparison_log_type(TaskQuestion): - default_value: str = "variational" - question_name: str = "comparison_log_type" - options: List[str] = mutable_field([ - 'variational', - 'fgat', - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Provide the log naming convention (e.g. 'variational', 'fgat')." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class crtm_coeff_dir(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "crtm_coeff_dir" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to the CRTM coefficient files?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class dry_run(TaskQuestion): - default_value: bool = True - question_name: str = "dry_run" - ask_question: bool = False - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Dry-run mode: preview what would be ingested before storing to R2D2" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_packets(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_packets" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Enter number of packets in which ensemble observers should be computed." - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_strategy(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_strategy" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Enter hofx strategy." - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_num_members(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_num_members" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "How many members comprise the ensemble?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensmean_only(TaskQuestion): - default_value: bool = False - question_name: str = "ensmean_only" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Calculate ensemble mean only?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensmeanvariance_only(TaskQuestion): - default_value: bool = False - question_name: str = "ensmeanvariance_only" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Calculate ensemble mean and variance only?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_geos_gcm_build_path(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_geos_gcm_build_path" - ask_question: bool = True - depends: Dict = mutable_field({ - "geos_build_method": "use_existing" - }) - prompt: str = "What is the path to the existing GEOS build directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_geos_gcm_source_path(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_geos_gcm_source_path" - ask_question: bool = True - depends: Dict = mutable_field({ - "geos_build_method": "use_existing" - }) - prompt: str = "What is the path to the existing GEOS source code directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_jedi_build_directory(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_jedi_build_directory" - ask_question: bool = True - depends: Dict = mutable_field({ - "jedi_build_method": "use_existing" - }) - prompt: str = "What is the path to the existing JEDI build directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_jedi_build_directory_pinned(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_jedi_build_directory_pinned" - ask_question: bool = True - depends: Dict = mutable_field({ - "jedi_build_method": "use_pinned_existing" - }) - prompt: str = "What is the path to the existing pinned JEDI build directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_jedi_source_directory(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_jedi_source_directory" - ask_question: bool = True - depends: Dict = mutable_field({ - "jedi_build_method": "use_existing" - }) - prompt: str = "What is the path to the existing JEDI source code directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_jedi_source_directory_pinned(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_jedi_source_directory_pinned" - ask_question: bool = True - depends: Dict = mutable_field({ - "jedi_build_method": "use_pinned_existing" - }) - prompt: str = "What is the path to the existing pinned JEDI source code directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_perllib_path(TaskQuestion): - default_value: str = 'defer_to_platform' - question_name: str = 'existing_perllib_path' - prompt: str = "Provide a path to an existing location for GMAO_perllib." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class forecast_duration(TaskQuestion): - default_value: str = "PT12H" - question_name: str = "forecast_duration" - ask_question: bool = True - prompt: str = "GEOS forecast duration" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class generate_yaml_and_exit(TaskQuestion): - default_value: bool = False - question_name: str = "generate_yaml_and_exit" - prompt: str = "Generate JEDI executable YAML and exit?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_build_method(TaskQuestion): - default_value: str = "create" - question_name: str = "geos_build_method" - ask_question: bool = True - options: List[str] = mutable_field([ - "use_existing", - "create" - ]) - prompt: str = "Do you want to use an existing GEOS build or create a new build?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_experiment_directory(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "geos_experiment_directory" - ask_question: bool = True - prompt: str = "What is the path to the GEOS restarts directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_gcm_tag(TaskQuestion): - default_value: str = "v11.6.0" - question_name: str = "geos_gcm_tag" - ask_question: bool = True - prompt: str = "Which GEOS tag do you wish to clone?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_restarts_directory(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "geos_restarts_directory" - ask_question: bool = True - prompt: str = "What is the path to the GEOS restarts directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_x_background_directory(TaskQuestion): - default_value: str = "/dev/null/" - question_name: str = "geos_x_background_directory" - ask_question: bool = True - options: List[str] = mutable_field([ - "/dev/null/", - "/discover/nobackup/projects/gmao/dadev/rtodling/archive/Restarts/JEDI/541x" - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the path to the GEOS X-backgrounds directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_x_ensemble_directory(TaskQuestion): - default_value: str = "/dev/null/" - question_name: str = "geos_x_ensemble_directory" - ask_question: bool = True - options: List[str] = mutable_field([ - "/dev/null/", - "/gpfsm/dnb05/projects/p139/rtodling/archive/" - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to the GEOS X-backgrounds directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geovals_experiment(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "geovals_experiment" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the name of the R2D2 experiment providing the GeoVaLs?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geovals_provider(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "geovals_provider" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the name of the R2D2 database providing the GeoVaLs?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gmao_perllib_tag(TaskQuestion): - default_value: str = 'g1.0.1' - question_name: str = 'gmao_perllib_tag' - prompt: str = "Specify the tag at which GMAO_perllib should be cloned." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gradient_norm_reduction(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "gradient_norm_reduction" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What value of gradient norm reduction for convergence?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gsibec_configuration(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "gsibec_configuration" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which GSIBEC climatological or hybrid?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gsibec_nlats(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "gsibec_nlats" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "How many number of latutides in GSIBEC grid?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gsibec_nlons(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "gsibec_nlons" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "How many number of longitudes in GSIBEC grid?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class horizontal_localization_lengthscale(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "horizontal_localization_lengthscale" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the length scale for horizontal covariance localization?" - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class horizontal_localization_max_nobs(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "horizontal_localization_max_nobs" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("What is the maximum number of observations to consider" - " for horizontal covariance localization?") - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class horizontal_localization_method(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "horizontal_localization_method" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which localization scheme should be applied in the horizontal?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class horizontal_resolution(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "horizontal_resolution" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the horizontal resolution for the forecast model and backgrounds?" - widget_type: WType = WType.STRING_DROP_LIST - - # ------------------------------------------------------------------------------------------------ - - @dataclass - class obs_to_ingest(TaskQuestion): - default_value: list = mutable_field([]) - question_name: str = "obs_to_ingest" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which observations do you want to ingest to R2D2?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ioda_locations_not_in_r2d2(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "ioda_locations_not_in_r2d2" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ( - "Provide a path that contains observation files not in r2d2.") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class jedi_build_method(TaskQuestion): - default_value: str = "create" - question_name: str = "jedi_build_method" - ask_question: bool = True - options: List[str] = mutable_field([ - "use_existing", - "use_pinned_existing", - "create", - "pinned_create" - ]) - prompt: str = "Do you want to use an existing JEDI build or create a new build?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class jedi_forecast_model(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "jedi_forecast_model" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - depends: Dict = mutable_field({ - "window_type": "4D" - }) - prompt: str = "What forecast model should be used within JEDI for 4D window propagation?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_inflation_mult(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_inflation_mult" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Specify the multiplicative prior inflation coefficient (0 inf]." - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_inflation_rtpp(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_inflation_rtpp" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Specify the Relaxation To Prior Perturbation (RTPP) coefficient (0 1]." - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_inflation_rtps(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_inflation_rtps" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Specify the Relaxation To Prior Spread (RTPS) coefficient (0 1]." - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_save_posterior_ensemble(TaskQuestion): - default_value: bool = False - question_name: str = "local_ensemble_save_posterior_ensemble" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Save the posterior ensemble members?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_save_posterior_ensemble_increments(TaskQuestion): - default_value: bool = False - question_name: str = "local_ensemble_save_posterior_ensemble_increments" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Save the posterior ensemble member increments?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_save_posterior_mean(TaskQuestion): - default_value: bool = False - question_name: str = "local_ensemble_save_posterior_mean" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Save the posterior ensemble mean?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_save_posterior_mean_increment(TaskQuestion): - default_value: bool = True - question_name: str = "local_ensemble_save_posterior_mean_increment" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Save the posterior ensemble mean increment?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_solver(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_solver" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which local ensemble solver type should be implemented?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_use_linear_observer(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_use_linear_observer" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which local ensemble solver type should be implemented?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class minimizer(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "minimizer" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which data assimilation minimizer do you wish to use?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class mom6_iau(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "mom6_iau" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_marine", - ]) - prompt: str = "Do you wish to use IAU for MOM6?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class mom6_iau_nhours(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "mom6_iau_nhours" - options: List[str] = mutable_field([ - 'PT3H', - 'PT12H' - ]) - depends: dict = mutable_field({'mom6_iau': True}) - models: List[str] = mutable_field([ - "geos_marine", - ]) - prompt: str = "What is the IAU length (ODA_INCUPD_NHOURS) for MOM6?" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ncdiag_experiments(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ncdiag_experiments" - options: List[str] = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which previously run experiments do you wish to use for the NCdiag?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class npx(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "npx" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What is the number of grid points in the x-direction on each cube face?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class npx_proc(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "npx_proc" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere", - "geos_cf" - ]) - prompt: str = "What number of processors do you wish to use in the x-direction?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class npy(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "npy" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What is the number of grid points in the y-direction on each cube face?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class npy_proc(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "npy_proc" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere", - "geos_cf" - ]) - prompt: str = "What number of processors do you wish to use in the y-direction?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class number_of_iterations(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "number_of_iterations" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = ( - "What number of iterations do you wish to use for each outer loop?" - " Provide a list of integers the same length as the number of outer loops.") - widget_type: WType = WType.INTEGER_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class obs_experiment(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "obs_experiment" - ask_question: bool = True - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the database providing the observations?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class obs_thinning_rej_fraction(TaskQuestion): - default_value: float = 0.75 - question_name: str = "obs_thinning_rej_fraction" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the rejection fraction for obs thinning?" - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observations(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "observations" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which observations do you want to include?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observing_system_records_mksi_path(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "observing_system_records_mksi_path" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to the GSI formatted observing system records?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observing_system_records_mksi_path_tag(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "observing_system_records_mksi_path_tag" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the GSI formatted observing system records tag?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observing_system_records_path(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "observing_system_records_path" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to the Swell formatted observing system records?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class path_to_ensemble(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "path_to_ensemble" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere", - "geos_marine" - ]) - prompt: str = "What is the path to where ensemble members are stored?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class path_to_geos_adas_background(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "path_to_geos_adas_background" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ( - "What is the path for the GEOSadas cubed sphere backgrounds?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class path_to_gsi_bc_coefficients(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "path_to_gsi_bc_coefficients" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the location where GSI bias correction files can be found?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class path_to_gsi_nc_diags(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "path_to_gsi_nc_diags" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to where the GSI ncdiags are stored?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class pause_on_tasks(TaskQuestion): - default_value: list = mutable_field([]) - question_name: str = "pause_on_tasks" - ask_question: bool = False - prompt: str = ("Specify any tasks that the workflow should pause on " - "(for development purposes).") - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class perhost(TaskQuestion): - default_value: str = None - question_name: str = "perhost" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the number of processors per host?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class produce_geovals(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "produce_geovals" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("When running the ncdiag to ioda converted do you " - "want to produce GeoVaLs files?") - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class r2d2_local_path(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "r2d2_local_path" - prompt: str = "What is the path to the R2D2 local directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class save_geovals(TaskQuestion): - default_value: bool = False - question_name: str = "save_geovals" - options: List[bool] = mutable_field([ - True, - False - ]) - prompt: str = "When running hofx do you want to output the GeoVaLs?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class set_obs_as_local(TaskQuestion): - default_value: bool = False - question_name: str = "set_obs_as_local" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - 'all_models' - ]) - prompt: str = "Treat observations as 'local' to the directory?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class single_observations(TaskQuestion): - default_value: bool = False - question_name: str = "single_observations" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Is it a single-observation test?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class swell_static_files(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "swell_static_files" - prompt: str = "What is the path to the Swell Static files directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class swell_static_files_user(TaskQuestion): - default_value: str = "None" - question_name: str = "swell_static_files_user" - prompt: str = "What is the path to the user provided Swell Static Files directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class task_email_parameters(TaskQuestion): - default_value: Union[Literal["auto"], dict] = "auto" - question_name: str = "task_email_parameters" - prompt: str = ("Provide a dictionary mapping tasks to cylc event statuses, or 'auto' to " - "automatically configure these based on the graph.") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class total_processors(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "total_processors" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_marine", - ]) - prompt: str = "What is the number of processors for JEDI?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_apply_log_transform(TaskQuestion): - default_value: bool = True - question_name: str = "vertical_localization_apply_log_transform" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("Should a log (base 10) transformation be applied " - "to vertical coordinate when " - "constructing vertical localization?") - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_function(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_function" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which localization scheme should be applied in the vertical?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_ioda_vertical_coord(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_ioda_vertical_coord" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which coordinate should be used in constructing vertical localization?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_ioda_vertical_coord_group(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_ioda_vertical_coord_group" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("Which vertical coordinate group should be used " - "in constructing vertical localization?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_lengthscale(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_lengthscale" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the length scale for vertical covariance localization?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_method(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_method" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("What localization scheme should be applied in " - "constructing a vertical localization?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_resolution(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_resolution" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the vertical resolution for the forecast model and background?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class window_length(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "window_length" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the duration for the data assimilation window?" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class window_type(TaskQuestion): - question_name: str = "window_type" - default_value: str = "defer_to_model" - ask_question: bool = True - options: List[str] = mutable_field([ - "3D", - "4D" - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Do you want to use a 3D or 4D (including FGAT) window?" - widget_type: WType = WType.STRING_DROP_LIST - -# -------------------------------------------------------------------------------------------------- From fad4afae502431625de3aef209c6e1fc535faae5 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 27 Feb 2026 17:15:32 -0500 Subject: [PATCH 207/299] add configuration question defaults --- src/swell/configuration/question_defaults.py | 1505 ++++++++++++++++++ 1 file changed, 1505 insertions(+) create mode 100644 src/swell/configuration/question_defaults.py diff --git a/src/swell/configuration/question_defaults.py b/src/swell/configuration/question_defaults.py new file mode 100644 index 000000000..921835a3b --- /dev/null +++ b/src/swell/configuration/question_defaults.py @@ -0,0 +1,1505 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +from dataclasses import dataclass +from typing import List, Dict, Union, Literal + +from swell.utilities.swell_questions import SuiteQuestion, TaskQuestion +from swell.utilities.swell_questions import WidgetType as WType +from swell.utilities.dataclass_utils import mutable_field + + +# -------------------------------------------------------------------------------------------------- + +class QuestionDefaults(): + + # -------------------------------------------------------------------------------------------------- + # Suite question defaults go here + # -------------------------------------------------------------------------------------------------- + + @dataclass + class comparison_experiment_paths(SuiteQuestion): + default_value: list = mutable_field([]) + question_name: str = "comparison_experiment_paths" + ask_question: bool = True + prompt: str = "Provide paths to two experiments to run comparison tests on." + widget_type: WType = WType.STRING_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class cycle_times(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "cycle_times" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter the cycle times for this model." + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class cycling_varbc(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "cycling_varbc" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Do you want to use cycling VarBC option?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class email_address(SuiteQuestion): + default_value: str = "defer_to_user" + question_name: str = "email_address" + prompt: str = "What email address should cylc messages be sent to?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class ensemble_hofx_packets(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "ensemble_hofx_packets" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter the number of ensemble packets." + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class ensemble_hofx_strategy(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "ensemble_hofx_strategy" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter the ensemble hofx strategy." + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class experiment_id(SuiteQuestion): + default_value: str = "defer_to_code" + question_name: str = "experiment_id" + ask_question: bool = True + prompt: str = "What is the experiment id?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class experiment_root(SuiteQuestion): + default_value: str = "defer_to_platform" + question_name: str = "experiment_root" + ask_question: bool = True + prompt: str = ("What is the experiment root (the directory where the " + "experiment will be stored)?") + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class final_cycle_point(SuiteQuestion): + default_value: str = "2023-10-10T06:00:00Z" + question_name: str = "final_cycle_point" + ask_question: bool = True + prompt: str = "What is the time of the final cycle (middle of the window)?" + widget_type: WType = WType.ISO_DATETIME + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class marine_models(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "marine_models" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_marine" + ]) + prompt: str = "Select the active SOCA models for this model." + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class model_components(SuiteQuestion): + default_value: str = "defer_to_code" + question_name: str = "model_components" + ask_question: bool = True + options: str = "defer_to_code" + prompt: str = "Enter the model components for this model." + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class parser_options(SuiteQuestion): + default_value: list = mutable_field(['fgrep_residual_norm']) + question_name: str = "parser_options" + ask_question: bool = True + options: list = mutable_field(['fgrep_residual_norm']) + prompt: str = "List the test types to run on the JEDI oops log." + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class runahead_limit(SuiteQuestion): + default_value: str = "P4" + question_name: str = "runahead_limit" + ask_question: bool = True + prompt: str = ("Since this suite is non-cycling choose how " + "many hours the workflow can run ahead?") + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class skip_ensemble_hofx(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "skip_ensemble_hofx" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter if skip ensemble hofx." + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class start_cycle_point(SuiteQuestion): + default_value: str = "2023-10-10T00:00:00Z" + question_name: str = "start_cycle_point" + ask_question: bool = True + prompt: str = "What is the time of the first cycle (middle of the window)?" + widget_type: WType = WType.ISO_DATETIME + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class use_cycle_dir(SuiteQuestion): + default_value: bool = True + question_name: str = "use_cycle_dir" + ask_question: bool = False + prompt: str = ("For cycling tasks, send results to the experiment cycle directory?" + " If false, results will be stored in the current working directory.") + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class window_type(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "window_type" + options: List[str] = mutable_field([ + "3D", + "4D" + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter the window type for this model." + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + # Task question defaults go here + # -------------------------------------------------------------------------------------------------- + + @dataclass + class analysis_variables(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "analysis_variables" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What are the analysis variables?" + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class background_error_model(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "background_error_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which background error model do you want to use?" + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class background_experiment(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "background_experiment" + ask_question: bool = True + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the name of the name of the experiment providing the backgrounds?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class background_frequency(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "background_frequency" + models: List[str] = mutable_field([ + "all_models" + ]) + depends: Dict = mutable_field({ + "window_type": "4D" + }) + prompt: str = "What is the frequency of the background files?" + widget_type: WType = WType.ISO_DURATION + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class background_time_offset(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "background_time_offset" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = ("How long before the middle of the analysis window did" + " the background providing forecast begin?") + widget_type: WType = WType.ISO_DURATION + + # -------------------------------------------------------------------------------------------------- + @dataclass + class bufr_obs_classes(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "bufr_obs_classes" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What BUFR observation classes will be used?" + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class bundles(TaskQuestion): + default_value: List[str] = mutable_field([ + "fv3-jedi", + "soca", + "iodaconv", + "ufo" + ]) + question_name: str = "bundles" + ask_question: bool = True + options: List[str] = mutable_field([ + "fv3-jedi", + "soca", + "iodaconv", + "ufo", + "ioda", + "oops", + "saber" + ]) + depends: Dict = mutable_field({ + "jedi_build_method": "create" + }) + prompt: str = "Which JEDI bundles do you wish to build?" + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class check_for_obs(TaskQuestion): + default_value: bool = True + question_name: str = "check_for_obs" + options: List[bool] = mutable_field([True, False]) + models: List[str] = mutable_field([ + 'all_models' + ]) + prompt: str = "Perform check for observations? Set to false for debugging purposes." + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class clean_patterns(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "clean_patterns" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Provide a list of patterns that you wish to remove from the cycle directory." + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class comparison_log_type(TaskQuestion): + default_value: str = "variational" + question_name: str = "comparison_log_type" + options: List[str] = mutable_field([ + 'variational', + 'fgat', + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Provide the log naming convention (e.g. 'variational', 'fgat')." + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class crtm_coeff_dir(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "crtm_coeff_dir" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to the CRTM coefficient files?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class dry_run(TaskQuestion): + default_value: bool = True + question_name: str = "dry_run" + ask_question: bool = False + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Dry-run mode: preview what would be ingested before storing to R2D2" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class ensemble_hofx_packets(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "ensemble_hofx_packets" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Enter number of packets in which ensemble observers should be computed." + widget_type: WType = WType.INTEGER + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class ensemble_hofx_strategy(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "ensemble_hofx_strategy" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Enter hofx strategy." + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class ensemble_num_members(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "ensemble_num_members" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "How many members comprise the ensemble?" + widget_type: WType = WType.INTEGER + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class ensmean_only(TaskQuestion): + default_value: bool = False + question_name: str = "ensmean_only" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Calculate ensemble mean only?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class ensmeanvariance_only(TaskQuestion): + default_value: bool = False + question_name: str = "ensmeanvariance_only" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Calculate ensemble mean and variance only?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class existing_geos_gcm_build_path(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "existing_geos_gcm_build_path" + ask_question: bool = True + depends: Dict = mutable_field({ + "geos_build_method": "use_existing" + }) + prompt: str = "What is the path to the existing GEOS build directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class existing_geos_gcm_source_path(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "existing_geos_gcm_source_path" + ask_question: bool = True + depends: Dict = mutable_field({ + "geos_build_method": "use_existing" + }) + prompt: str = "What is the path to the existing GEOS source code directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class existing_jedi_build_directory(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "existing_jedi_build_directory" + ask_question: bool = True + depends: Dict = mutable_field({ + "jedi_build_method": "use_existing" + }) + prompt: str = "What is the path to the existing JEDI build directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class existing_jedi_build_directory_pinned(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "existing_jedi_build_directory_pinned" + ask_question: bool = True + depends: Dict = mutable_field({ + "jedi_build_method": "use_pinned_existing" + }) + prompt: str = "What is the path to the existing pinned JEDI build directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class existing_jedi_source_directory(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "existing_jedi_source_directory" + ask_question: bool = True + depends: Dict = mutable_field({ + "jedi_build_method": "use_existing" + }) + prompt: str = "What is the path to the existing JEDI source code directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class existing_jedi_source_directory_pinned(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "existing_jedi_source_directory_pinned" + ask_question: bool = True + depends: Dict = mutable_field({ + "jedi_build_method": "use_pinned_existing" + }) + prompt: str = "What is the path to the existing pinned JEDI source code directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class existing_perllib_path(TaskQuestion): + default_value: str = 'defer_to_platform' + question_name: str = 'existing_perllib_path' + prompt: str = "Provide a path to an existing location for GMAO_perllib." + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class forecast_duration(TaskQuestion): + default_value: str = "PT12H" + question_name: str = "forecast_duration" + ask_question: bool = True + prompt: str = "GEOS forecast duration" + widget_type: WType = WType.ISO_DURATION + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class generate_yaml_and_exit(TaskQuestion): + default_value: bool = False + question_name: str = "generate_yaml_and_exit" + prompt: str = "Generate JEDI executable YAML and exit?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class geos_build_method(TaskQuestion): + default_value: str = "create" + question_name: str = "geos_build_method" + ask_question: bool = True + options: List[str] = mutable_field([ + "use_existing", + "create" + ]) + prompt: str = "Do you want to use an existing GEOS build or create a new build?" + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class geos_experiment_directory(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "geos_experiment_directory" + ask_question: bool = True + prompt: str = "What is the path to the GEOS restarts directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class geos_gcm_tag(TaskQuestion): + default_value: str = "v11.6.0" + question_name: str = "geos_gcm_tag" + ask_question: bool = True + prompt: str = "Which GEOS tag do you wish to clone?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class geos_restarts_directory(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "geos_restarts_directory" + ask_question: bool = True + prompt: str = "What is the path to the GEOS restarts directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class geos_x_background_directory(TaskQuestion): + default_value: str = "/dev/null/" + question_name: str = "geos_x_background_directory" + ask_question: bool = True + options: List[str] = mutable_field([ + "/dev/null/", + "/discover/nobackup/projects/gmao/dadev/rtodling/archive/Restarts/JEDI/541x" + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the path to the GEOS X-backgrounds directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class geos_x_ensemble_directory(TaskQuestion): + default_value: str = "/dev/null/" + question_name: str = "geos_x_ensemble_directory" + ask_question: bool = True + options: List[str] = mutable_field([ + "/dev/null/", + "/gpfsm/dnb05/projects/p139/rtodling/archive/" + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to the GEOS X-backgrounds directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class geovals_experiment(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "geovals_experiment" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the name of the R2D2 experiment providing the GeoVaLs?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class geovals_provider(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "geovals_provider" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the name of the R2D2 database providing the GeoVaLs?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class gmao_perllib_tag(TaskQuestion): + default_value: str = 'g1.0.1' + question_name: str = 'gmao_perllib_tag' + prompt: str = "Specify the tag at which GMAO_perllib should be cloned." + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class gradient_norm_reduction(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "gradient_norm_reduction" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What value of gradient norm reduction for convergence?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class gsibec_configuration(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "gsibec_configuration" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which GSIBEC climatological or hybrid?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class gsibec_nlats(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "gsibec_nlats" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "How many number of latutides in GSIBEC grid?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class gsibec_nlons(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "gsibec_nlons" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "How many number of longitudes in GSIBEC grid?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class horizontal_localization_lengthscale(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "horizontal_localization_lengthscale" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the length scale for horizontal covariance localization?" + widget_type: WType = WType.FLOAT + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class horizontal_localization_max_nobs(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "horizontal_localization_max_nobs" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("What is the maximum number of observations to consider" + " for horizontal covariance localization?") + widget_type: WType = WType.INTEGER + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class horizontal_localization_method(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "horizontal_localization_method" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which localization scheme should be applied in the horizontal?" + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class horizontal_resolution(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "horizontal_resolution" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the horizontal resolution for the forecast model and backgrounds?" + widget_type: WType = WType.STRING_DROP_LIST + + # ------------------------------------------------------------------------------------------------ + + @dataclass + class obs_to_ingest(TaskQuestion): + default_value: list = mutable_field([]) + question_name: str = "obs_to_ingest" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which observations do you want to ingest to R2D2?" + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class ioda_locations_not_in_r2d2(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "ioda_locations_not_in_r2d2" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ( + "Provide a path that contains observation files not in r2d2.") + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class jedi_build_method(TaskQuestion): + default_value: str = "create" + question_name: str = "jedi_build_method" + ask_question: bool = True + options: List[str] = mutable_field([ + "use_existing", + "use_pinned_existing", + "create", + "pinned_create" + ]) + prompt: str = "Do you want to use an existing JEDI build or create a new build?" + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class jedi_forecast_model(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "jedi_forecast_model" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + depends: Dict = mutable_field({ + "window_type": "4D" + }) + prompt: str = "What forecast model should be used within JEDI for 4D window propagation?" + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class local_ensemble_inflation_mult(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "local_ensemble_inflation_mult" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Specify the multiplicative prior inflation coefficient (0 inf]." + widget_type: WType = WType.FLOAT + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class local_ensemble_inflation_rtpp(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "local_ensemble_inflation_rtpp" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Specify the Relaxation To Prior Perturbation (RTPP) coefficient (0 1]." + widget_type: WType = WType.FLOAT + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class local_ensemble_inflation_rtps(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "local_ensemble_inflation_rtps" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Specify the Relaxation To Prior Spread (RTPS) coefficient (0 1]." + widget_type: WType = WType.FLOAT + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class local_ensemble_save_posterior_ensemble(TaskQuestion): + default_value: bool = False + question_name: str = "local_ensemble_save_posterior_ensemble" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Save the posterior ensemble members?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class local_ensemble_save_posterior_ensemble_increments(TaskQuestion): + default_value: bool = False + question_name: str = "local_ensemble_save_posterior_ensemble_increments" + ask_question: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Save the posterior ensemble member increments?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class local_ensemble_save_posterior_mean(TaskQuestion): + default_value: bool = False + question_name: str = "local_ensemble_save_posterior_mean" + ask_question: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Save the posterior ensemble mean?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class local_ensemble_save_posterior_mean_increment(TaskQuestion): + default_value: bool = True + question_name: str = "local_ensemble_save_posterior_mean_increment" + ask_question: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Save the posterior ensemble mean increment?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class local_ensemble_solver(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "local_ensemble_solver" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which local ensemble solver type should be implemented?" + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class local_ensemble_use_linear_observer(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "local_ensemble_use_linear_observer" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which local ensemble solver type should be implemented?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class minimizer(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "minimizer" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which data assimilation minimizer do you wish to use?" + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class mom6_iau(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "mom6_iau" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_marine", + ]) + prompt: str = "Do you wish to use IAU for MOM6?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class mom6_iau_nhours(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "mom6_iau_nhours" + options: List[str] = mutable_field([ + 'PT3H', + 'PT12H' + ]) + depends: dict = mutable_field({'mom6_iau': True}) + models: List[str] = mutable_field([ + "geos_marine", + ]) + prompt: str = "What is the IAU length (ODA_INCUPD_NHOURS) for MOM6?" + widget_type: WType = WType.ISO_DURATION + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class ncdiag_experiments(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "ncdiag_experiments" + options: List[str] = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which previously run experiments do you wish to use for the NCdiag?" + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class npx(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "npx" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What is the number of grid points in the x-direction on each cube face?" + widget_type: WType = WType.INTEGER + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class npx_proc(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "npx_proc" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere", + "geos_cf" + ]) + prompt: str = "What number of processors do you wish to use in the x-direction?" + widget_type: WType = WType.INTEGER + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class npy(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "npy" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What is the number of grid points in the y-direction on each cube face?" + widget_type: WType = WType.INTEGER + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class npy_proc(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "npy_proc" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere", + "geos_cf" + ]) + prompt: str = "What number of processors do you wish to use in the y-direction?" + widget_type: WType = WType.INTEGER + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class number_of_iterations(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "number_of_iterations" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = ( + "What number of iterations do you wish to use for each outer loop?" + " Provide a list of integers the same length as the number of outer loops.") + widget_type: WType = WType.INTEGER_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class obs_experiment(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "obs_experiment" + ask_question: bool = True + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the database providing the observations?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class obs_thinning_rej_fraction(TaskQuestion): + default_value: float = 0.75 + question_name: str = "obs_thinning_rej_fraction" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the rejection fraction for obs thinning?" + widget_type: WType = WType.FLOAT + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class observations(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "observations" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which observations do you want to include?" + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class observing_system_records_mksi_path(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "observing_system_records_mksi_path" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to the GSI formatted observing system records?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class observing_system_records_mksi_path_tag(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "observing_system_records_mksi_path_tag" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the GSI formatted observing system records tag?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class observing_system_records_path(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "observing_system_records_path" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to the Swell formatted observing system records?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class path_to_ensemble(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "path_to_ensemble" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere", + "geos_marine" + ]) + prompt: str = "What is the path to where ensemble members are stored?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class path_to_geos_adas_background(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "path_to_geos_adas_background" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ( + "What is the path for the GEOSadas cubed sphere backgrounds?") + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class path_to_gsi_bc_coefficients(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "path_to_gsi_bc_coefficients" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the location where GSI bias correction files can be found?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class path_to_gsi_nc_diags(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "path_to_gsi_nc_diags" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to where the GSI ncdiags are stored?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class pause_on_tasks(TaskQuestion): + default_value: list = mutable_field([]) + question_name: str = "pause_on_tasks" + ask_question: bool = False + prompt: str = ("Specify any tasks that the workflow should pause on " + "(for development purposes).") + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class perhost(TaskQuestion): + default_value: str = None + question_name: str = "perhost" + ask_question: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the number of processors per host?" + widget_type: WType = WType.INTEGER + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class produce_geovals(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "produce_geovals" + ask_question: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("When running the ncdiag to ioda converted do you " + "want to produce GeoVaLs files?") + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class r2d2_local_path(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "r2d2_local_path" + prompt: str = "What is the path to the R2D2 local directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class save_geovals(TaskQuestion): + default_value: bool = False + question_name: str = "save_geovals" + options: List[bool] = mutable_field([ + True, + False + ]) + prompt: str = "When running hofx do you want to output the GeoVaLs?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class set_obs_as_local(TaskQuestion): + default_value: bool = False + question_name: str = "set_obs_as_local" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + 'all_models' + ]) + prompt: str = "Treat observations as 'local' to the directory?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class single_observations(TaskQuestion): + default_value: bool = False + question_name: str = "single_observations" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Is it a single-observation test?" + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class swell_static_files(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "swell_static_files" + prompt: str = "What is the path to the Swell Static files directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class swell_static_files_user(TaskQuestion): + default_value: str = "None" + question_name: str = "swell_static_files_user" + prompt: str = "What is the path to the user provided Swell Static Files directory?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class task_email_parameters(TaskQuestion): + default_value: Union[Literal["auto"], dict] = "auto" + question_name: str = "task_email_parameters" + prompt: str = ("Provide a dictionary mapping tasks to cylc event statuses, or 'auto' to " + "automatically configure these based on the graph.") + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class total_processors(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "total_processors" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_marine", + ]) + prompt: str = "What is the number of processors for JEDI?" + widget_type: WType = WType.INTEGER + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class vertical_localization_apply_log_transform(TaskQuestion): + default_value: bool = True + question_name: str = "vertical_localization_apply_log_transform" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("Should a log (base 10) transformation be applied " + "to vertical coordinate when " + "constructing vertical localization?") + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class vertical_localization_function(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_localization_function" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which localization scheme should be applied in the vertical?" + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class vertical_localization_ioda_vertical_coord(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_localization_ioda_vertical_coord" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which coordinate should be used in constructing vertical localization?" + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class vertical_localization_ioda_vertical_coord_group(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_localization_ioda_vertical_coord_group" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("Which vertical coordinate group should be used " + "in constructing vertical localization?") + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class vertical_localization_lengthscale(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_localization_lengthscale" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the length scale for vertical covariance localization?" + widget_type: WType = WType.INTEGER + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class vertical_localization_method(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_localization_method" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("What localization scheme should be applied in " + "constructing a vertical localization?") + widget_type: WType = WType.STRING + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class vertical_resolution(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_resolution" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the vertical resolution for the forecast model and background?" + widget_type: WType = WType.STRING_DROP_LIST + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class window_length(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "window_length" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the duration for the data assimilation window?" + widget_type: WType = WType.ISO_DURATION + + # -------------------------------------------------------------------------------------------------- + + @dataclass + class window_type(TaskQuestion): + question_name: str = "window_type" + default_value: str = "defer_to_model" + ask_question: bool = True + options: List[str] = mutable_field([ + "3D", + "4D" + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Do you want to use a 3D or 4D (including FGAT) window?" + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- From 5f7c031afea49a4e4cc63d59049918fd89f696fe Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 27 Feb 2026 17:21:47 -0500 Subject: [PATCH 208/299] move discover_plugins --- src/swell/tasks/base/task_attributes.py | 17 +---------------- src/swell/utilities/plugins.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 src/swell/utilities/plugins.py diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index b48959a90..17d1da65d 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -7,11 +7,9 @@ # -------------------------------------------------------------------------------------------------- -import importlib -import pkgutil - import swell.tasks from swell.tasks.base.task_setup import TaskSetup +from swell.utilities.plugins import discover_plugins # -------------------------------------------------------------------------------------------------- @@ -57,19 +55,6 @@ def set_attributes(self): # -------------------------------------------------------------------------------------------------- -def discover_plugins(package): - '''Walk through packages to trigger any hooks. - - Parameters: - package: Python package - ''' - for loader, module_name, is_pkg in pkgutil.walk_packages(package.__path__): - full_module_name = f"{package.__name__}.{module_name}" - - importlib.import_module(full_module_name) - -# -------------------------------------------------------------------------------------------------- - class TaskAttributes(): def __init__(self) -> None: diff --git a/src/swell/utilities/plugins.py b/src/swell/utilities/plugins.py new file mode 100644 index 000000000..5bf9eb7c2 --- /dev/null +++ b/src/swell/utilities/plugins.py @@ -0,0 +1,25 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +import importlib +import pkgutil + +# -------------------------------------------------------------------------------------------------- + +def discover_plugins(package): + '''Walk through packages to trigger any hooks. + + Parameters: + package: Python package + ''' + for loader, module_name, is_pkg in pkgutil.walk_packages(package.__path__): + full_module_name = f"{package.__name__}.{module_name}" + + importlib.import_module(full_module_name) + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file From 7cde879c6dd40f817305fa963496892eef0270b5 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 2 Mar 2026 11:37:53 -0500 Subject: [PATCH 209/299] Refactor task defaults to allow suite override --- src/swell/tasks/base/task_attributes.py | 4 +- src/swell/tasks/base/task_setup.py | 59 ++++++++++++++++++- src/swell/tasks/bufr_to_ioda.py | 2 +- src/swell/tasks/build_geos.py | 2 +- src/swell/tasks/build_geos_by_linking.py | 2 +- src/swell/tasks/build_jedi.py | 2 +- src/swell/tasks/build_jedi_by_linking.py | 2 +- src/swell/tasks/clean_cycle.py | 2 +- src/swell/tasks/clone_geos.py | 2 +- src/swell/tasks/clone_geos_mksi.py | 2 +- src/swell/tasks/clone_gmao_perllib.py | 2 +- src/swell/tasks/clone_jedi.py | 2 +- src/swell/tasks/eva_comparison_increment.py | 2 +- src/swell/tasks/eva_comparison_jedi_log.py | 2 +- .../tasks/eva_comparison_observations.py | 2 +- src/swell/tasks/eva_increment.py | 2 +- src/swell/tasks/eva_jedi_log.py | 2 +- src/swell/tasks/eva_observations.py | 2 +- src/swell/tasks/eva_timeseries.py | 2 +- src/swell/tasks/generate_b_climatology.py | 2 +- .../generate_b_climatology_by_linking.py | 2 +- .../generate_observing_system_records.py | 2 +- src/swell/tasks/get_background.py | 2 +- .../tasks/get_background_geos_experiment.py | 2 +- src/swell/tasks/get_bufr.py | 2 +- src/swell/tasks/get_ensemble.py | 2 +- .../tasks/get_ensemble_geos_experiment.py | 2 +- src/swell/tasks/get_geos_adas_background.py | 2 +- src/swell/tasks/get_geos_restart.py | 2 +- src/swell/tasks/get_geovals.py | 2 +- src/swell/tasks/get_gsi_bc.py | 2 +- src/swell/tasks/get_gsi_ncdiag.py | 2 +- src/swell/tasks/get_ncdiags.py | 2 +- src/swell/tasks/get_obs_not_in_r2d2.py | 2 +- src/swell/tasks/get_observations.py | 2 +- src/swell/tasks/gsi_bc_to_ioda.py | 2 +- src/swell/tasks/gsi_ncdiag_to_ioda.py | 2 +- src/swell/tasks/ingest_obs.py | 2 +- src/swell/tasks/jedi_log_comparison.py | 2 +- src/swell/tasks/jedi_oops_log_parser.py | 2 +- src/swell/tasks/link_geos_output.py | 2 +- src/swell/tasks/move_da_restart.py | 2 +- src/swell/tasks/move_forecast_restart.py | 2 +- src/swell/tasks/prep_geos_run_dir.py | 2 +- src/swell/tasks/prepare_analysis.py | 2 +- src/swell/tasks/remove_forecast_dir.py | 2 +- src/swell/tasks/render_jedi_observations.py | 2 +- src/swell/tasks/run_geos_executable.py | 2 +- ...jedi_convert_state_soca2cice_executable.py | 2 +- .../tasks/run_jedi_ensemble_mean_variance.py | 2 +- src/swell/tasks/run_jedi_fgat_executable.py | 2 +- .../run_jedi_hofx_ensemble_executable.py | 2 +- src/swell/tasks/run_jedi_hofx_executable.py | 2 +- .../run_jedi_local_ensemble_da_executable.py | 2 +- .../tasks/run_jedi_obsfilters_executable.py | 2 +- .../tasks/run_jedi_ufo_tests_executable.py | 2 +- .../tasks/run_jedi_variational_executable.py | 2 +- src/swell/tasks/save_obs_diags.py | 2 +- src/swell/tasks/save_restart.py | 2 +- src/swell/tasks/stage_jedi.py | 6 +- src/swell/tasks/store_background.py | 2 +- 61 files changed, 119 insertions(+), 66 deletions(-) diff --git a/src/swell/tasks/base/task_attributes.py b/src/swell/tasks/base/task_attributes.py index 17d1da65d..d1d7dcd87 100644 --- a/src/swell/tasks/base/task_attributes.py +++ b/src/swell/tasks/base/task_attributes.py @@ -37,7 +37,7 @@ def __init__(self): class root(TaskSetup): - def set_attributes(self): + def set_defaults(self): # root is a precursor to all tasks, it runs the pre-script before any task's script self.script = False self.pre_script = "source $CYLC_SUITE_DEF_PATH/modules" @@ -49,7 +49,7 @@ def set_attributes(self): class sync_point(TaskSetup): # placeholder task to check run dependencies in cylc graph # The command "true" is run in the shell as a placeholder - def set_attributes(self): + def set_defaults(self): self.script = "true" diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index 736819309..24ba5b872 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -9,6 +9,7 @@ from collections.abc import Mapping from abc import abstractmethod, ABC +from typing import Literal from swell.utilities.cylc_formatting import CylcSection, indent_lines from swell.utilities.suite_utils import get_model_components @@ -17,6 +18,7 @@ # -------------------------------------------------------------------------------------------------- +blank_spec = 'BLANKSPEC' class TaskSetup(ABC): @@ -60,7 +62,21 @@ class TaskSetup(ABC): questions: list additional_sections: list - def __init__(self, model: str | None = None, platform: str | None = None) -> None: + def __init__(self, model: str | None = None, + platform: str | None = None, + base_name: str = blank_spec, + scheduling_name: str = blank_spec, + is_cycling: str | bool = blank_spec, + model_dep: str | bool = blank_spec, + pre_script: str | Literal[False] = blank_spec, + script: str | Literal[False] | None = blank_spec, + retry: str | None = blank_spec, + task_time_limit: str | dict | None = blank_spec, + slurm: Literal['BLANKSPEC'] | dict | None = blank_spec, + mail_events: list | None = None, + questions: list | None = None, + additional_sections: list | None = None + ) -> None: self.model = model self.platform = platform @@ -83,13 +99,50 @@ def __init__(self, model: str | None = None, platform: str | None = None) -> Non self.questions = [] self.additional_sections = [] - self.set_attributes() + self.set_defaults() + + if base_name != blank_spec: + self.base_name = base_name + + if scheduling_name != blank_spec: + self.scheduling_name = scheduling_name + + if is_cycling != blank_spec: + self.is_cycling = is_cycling + + if model_dep != blank_spec: + self.model_dep = model_dep + + if pre_script != blank_spec: + self.pre_script = pre_script + + if script != blank_spec: + self.script = pre_script + + if retry != blank_spec: + self.retry = retry + + if task_time_limit != blank_spec: + self.task_time_limit = task_time_limit + + if slurm != blank_spec: + self.slurm = slurm + + if mail_events is not None: + self.mail_events = mail_events + + if questions is not None: + self.questions = questions + + if additional_sections is not None: + self.additional_sections = additional_sections + self.post_init() # -------------------------------------------------------------------------------------------------- @abstractmethod - def set_attributes(self) -> None: + def set_defaults(self) -> None: '''Abstract method to be overridden by each task in order to set attributes. ''' pass diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 39c43f424..8ca95abfa 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -41,7 +41,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index 11193c22d..5465ad65b 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -24,7 +24,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.questions = [ qd.geos_build_method() diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index 3dfa06d6d..f5e91a5f6 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -23,7 +23,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.mail_events = ['submit-failed'] self.questions = [ diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index 9607b9241..b5e71c183 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -22,7 +22,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.task_time_limit = 'PT3H' self.slurm = {} diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index ceaa4cf55..cc870100c 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -23,7 +23,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.mail_events = ['submit-failed'] self.questions = [ diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index 2721193e4..8f250d822 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -23,7 +23,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 40a1c2dbe..3820ab617 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -24,7 +24,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.questions = [ qd.existing_geos_gcm_source_path(), diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index c8a528ba1..edafe2558 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -21,7 +21,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.model_dep = True self.questions = [ diff --git a/src/swell/tasks/clone_gmao_perllib.py b/src/swell/tasks/clone_gmao_perllib.py index 6424d204a..1e80ed483 100644 --- a/src/swell/tasks/clone_gmao_perllib.py +++ b/src/swell/tasks/clone_gmao_perllib.py @@ -23,7 +23,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.questions = [ qd.existing_perllib_path(), diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index edc91879d..a73a49804 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -25,7 +25,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.questions = [ qd.bundles(), diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index 521761a23..396ee989d 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -27,7 +27,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 7222110ed..e4d69a6e1 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -26,7 +26,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/eva_comparison_observations.py b/src/swell/tasks/eva_comparison_observations.py index 6af82e25a..f0383d12c 100644 --- a/src/swell/tasks/eva_comparison_observations.py +++ b/src/swell/tasks/eva_comparison_observations.py @@ -32,7 +32,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.task_time_limit = True self.is_cycling = True diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index 538f5cf9e..f45ded4c6 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -24,7 +24,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index e1bbd8314..1f98dc031 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -24,7 +24,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index db4883f0d..24ce02a10 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -40,7 +40,7 @@ def run_eva(eva_dict: dict): @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.task_time_limit = True self.is_cycling = True diff --git a/src/swell/tasks/eva_timeseries.py b/src/swell/tasks/eva_timeseries.py index bacc4d514..8d013aa40 100644 --- a/src/swell/tasks/eva_timeseries.py +++ b/src/swell/tasks/eva_timeseries.py @@ -42,7 +42,7 @@ def run_eva(eva_dict: dict): @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.task_time_limit = True self.is_cycling = True diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index 5c54a533a..54160aa11 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -22,7 +22,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.task_time_limit = True self.is_cycling = True diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 516706bf7..0afd240a4 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -21,7 +21,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index 39b837ca1..ba876a0c0 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -23,7 +23,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index db6cdc8b5..5701172fe 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -32,7 +32,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index c5573152c..f885dcf00 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -26,7 +26,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/get_bufr.py b/src/swell/tasks/get_bufr.py index e04661676..f2ef5b1be 100644 --- a/src/swell/tasks/get_bufr.py +++ b/src/swell/tasks/get_bufr.py @@ -25,7 +25,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/get_ensemble.py b/src/swell/tasks/get_ensemble.py index 17216b5a6..abb09d615 100644 --- a/src/swell/tasks/get_ensemble.py +++ b/src/swell/tasks/get_ensemble.py @@ -24,7 +24,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.questions = [ qd.path_to_ensemble() diff --git a/src/swell/tasks/get_ensemble_geos_experiment.py b/src/swell/tasks/get_ensemble_geos_experiment.py index 249f9bb61..ca02d1974 100644 --- a/src/swell/tasks/get_ensemble_geos_experiment.py +++ b/src/swell/tasks/get_ensemble_geos_experiment.py @@ -25,7 +25,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py index 76233f4fc..13cd57691 100644 --- a/src/swell/tasks/get_geos_adas_background.py +++ b/src/swell/tasks/get_geos_adas_background.py @@ -26,7 +26,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/get_geos_restart.py b/src/swell/tasks/get_geos_restart.py index 51ca638b1..3c68cf357 100644 --- a/src/swell/tasks/get_geos_restart.py +++ b/src/swell/tasks/get_geos_restart.py @@ -23,7 +23,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.questions = [ diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index ef3e15189..ca4da51c8 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -24,7 +24,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index 67419c6bd..ff2547137 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -27,7 +27,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index 059134ff0..8465b7c1f 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -24,7 +24,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index cdd95bf09..b1682c60e 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -20,7 +20,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py index fc8282a27..33046392e 100644 --- a/src/swell/tasks/get_obs_not_in_r2d2.py +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -25,7 +25,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index c02249e73..a67d77d52 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -37,7 +37,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 71d8dfd07..9738ce29f 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -27,7 +27,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index b94b49053..851d0b5de 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -29,7 +29,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/ingest_obs.py b/src/swell/tasks/ingest_obs.py index 6d8a9ac33..be8912f50 100644 --- a/src/swell/tasks/ingest_obs.py +++ b/src/swell/tasks/ingest_obs.py @@ -32,7 +32,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py index 1d0c89e3b..ec6a93aee 100644 --- a/src/swell/tasks/jedi_log_comparison.py +++ b/src/swell/tasks/jedi_log_comparison.py @@ -30,7 +30,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.model_dep = True self.questions = [ diff --git a/src/swell/tasks/jedi_oops_log_parser.py b/src/swell/tasks/jedi_oops_log_parser.py index 45b3d4951..2a90145ca 100644 --- a/src/swell/tasks/jedi_oops_log_parser.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -23,7 +23,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index 0376d9fa2..0febac9b1 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -27,7 +27,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/move_da_restart.py b/src/swell/tasks/move_da_restart.py index ed72f4d8f..d8b986177 100644 --- a/src/swell/tasks/move_da_restart.py +++ b/src/swell/tasks/move_da_restart.py @@ -25,7 +25,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/move_forecast_restart.py b/src/swell/tasks/move_forecast_restart.py index 2db5d85df..494ee9dcc 100644 --- a/src/swell/tasks/move_forecast_restart.py +++ b/src/swell/tasks/move_forecast_restart.py @@ -24,7 +24,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.questions = [ diff --git a/src/swell/tasks/prep_geos_run_dir.py b/src/swell/tasks/prep_geos_run_dir.py index 79a3e38c2..c93b6a892 100644 --- a/src/swell/tasks/prep_geos_run_dir.py +++ b/src/swell/tasks/prep_geos_run_dir.py @@ -27,7 +27,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.questions = [ diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index 10cc2431f..7ef1da8ee 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -26,7 +26,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/remove_forecast_dir.py b/src/swell/tasks/remove_forecast_dir.py index 5e8ee390e..05e968dc4 100644 --- a/src/swell/tasks/remove_forecast_dir.py +++ b/src/swell/tasks/remove_forecast_dir.py @@ -20,7 +20,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index ab35a1d47..90eca8c47 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -24,7 +24,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/run_geos_executable.py b/src/swell/tasks/run_geos_executable.py index 0ed4e6d2f..a2b84f64b 100644 --- a/src/swell/tasks/run_geos_executable.py +++ b/src/swell/tasks/run_geos_executable.py @@ -22,7 +22,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True diff --git a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py index 3e231fd8a..fcf2be0d2 100644 --- a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py +++ b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py @@ -25,7 +25,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/run_jedi_ensemble_mean_variance.py b/src/swell/tasks/run_jedi_ensemble_mean_variance.py index 7ca66e1aa..7a62678c0 100644 --- a/src/swell/tasks/run_jedi_ensemble_mean_variance.py +++ b/src/swell/tasks/run_jedi_ensemble_mean_variance.py @@ -25,7 +25,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/run_jedi_fgat_executable.py b/src/swell/tasks/run_jedi_fgat_executable.py index a2a1daae9..22595e44d 100644 --- a/src/swell/tasks/run_jedi_fgat_executable.py +++ b/src/swell/tasks/run_jedi_fgat_executable.py @@ -24,7 +24,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py index 9454772a5..db7a801cd 100644 --- a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py +++ b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py @@ -25,7 +25,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 90e9504f1..a13aa57ad 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -27,7 +27,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 74b85c66e..2c71ef2f0 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -44,7 +44,7 @@ def replace_key(obj, old_key, new_key): @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index 3fca16d5b..c85202727 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -25,7 +25,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.script = ("swell task RunJediObsfiltersExecutable $config" " -d $datetime -m geos_atmosphere") diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 11b5f5e66..439878b65 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -27,7 +27,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.task_time_limit = True self.is_cycling = True diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index c36dc5b30..e390da069 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -24,7 +24,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.task_time_limit = True self.is_cycling = True diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 31f76f06c..1bf83a640 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -21,7 +21,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index 7e990e5d0..bf104bceb 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -25,7 +25,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 332344cdc..62b99a147 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -27,7 +27,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.model_dep = True self.questions = [ @@ -43,8 +43,8 @@ def set_attributes(self): @task_attributes.register('StageJediCycle') class StageJediCycle(Setup): - def set_attributes(self): - super().set_attributes() + def set_defaults(self): + super().set_defaults() self.base_name = "StageJedi" self.scheduling_name = "StageJediCycle-{model}" self.is_cycling = True diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index 77f7f5397..8b14fe446 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -27,7 +27,7 @@ @task_attributes.register(task_name) class Setup(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = task_name self.is_cycling = True self.model_dep = True From ade560020cb73c44f131d38d0ea7f11cf541ae5b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 2 Mar 2026 11:41:54 -0500 Subject: [PATCH 210/299] Update docs for new locations --- docs/adding_a_suite.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/adding_a_suite.md b/docs/adding_a_suite.md index 631680a89..42a39a9a1 100644 --- a/docs/adding_a_suite.md +++ b/docs/adding_a_suite.md @@ -215,7 +215,7 @@ For initial development/testing purposes, it may be easier to create a `flow.cyl ### Question Objects -Questions for swell are stored as dataclass instances, in the file `src/swell/utilities/question_defaults.py`. Dataclasses allow for simple declaration of data fields, and powerful type checking capabilities. Each question is an extension of the `SuiteQuestion` or `TaskQuestion` class, which are extensions of the `SwellQuestion` parent: +Questions for swell are stored as dataclass instances, in the file `src/swell/configuration/question_defaults.py`. Dataclasses allow for simple declaration of data fields, and powerful type checking capabilities. Each question is an extension of the `SuiteQuestion` or `TaskQuestion` class, which are extensions of the `SwellQuestion` parent: ```python @dataclass @@ -288,8 +288,8 @@ In this question infrastructure, **suites take priority over tasks**. Any questi Consider the following example of suite questions for `3dvar` (in python, variable names cannot begin with digits): ```python -from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq +from swell.configuration.question_defaults import QuestionDefaults as qd +from swell.suites.base.suite_questions import SuiteQuestions as sq class SuiteQuestions(QuestionContainer, Enum): @@ -383,4 +383,4 @@ class SuiteConfig(QuestionContainer, Enum): ``` The class `SuiteQuestions` contains lists of questions which are common to many suites. This avoids the need for redundantly setting the same questions for every suite. -`_3dvar_base` is responsible for establishing the baseline for questions used by the suite. The 'base' list should be used to associate all questions used by the suite. This list will be populated with the questions that match the defaults in `QuestionDefaults` (`src/swell/utilities/question_defaults.py`). However, in many cases, those defaults will not be ideal defaults for the individual suite. Thus, `_3dvar_tier1` sets different default values which override the question defaults. If desired, other configurations can then inherit question defaults from `_3dvar_tier1`, and set their own defaults on top of the existing ones. +`_3dvar_base` is responsible for establishing the baseline for questions used by the suite. The 'base' list should be used to associate all questions used by the suite. This list will be populated with the questions that match the defaults in `QuestionDefaults` (`src/swell/configuration/question_defaults.py`). However, in many cases, those defaults will not be ideal defaults for the individual suite. Thus, `_3dvar_tier1` sets different default values which override the question defaults. If desired, other configurations can then inherit question defaults from `_3dvar_tier1`, and set their own defaults on top of the existing ones. From dcd9caf4ae7764c20053edf09f92b4b35b194953 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 2 Mar 2026 11:44:13 -0500 Subject: [PATCH 211/299] add docstrings --- src/swell/tasks/base/task_setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index 24ba5b872..fbd7a2571 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -78,6 +78,7 @@ def __init__(self, model: str | None = None, additional_sections: list | None = None ) -> None: + # Set the base defaults needed by the class self.model = model self.platform = platform @@ -99,8 +100,10 @@ def __init__(self, model: str | None = None, self.questions = [] self.additional_sections = [] + # Set the defaults for the individual task self.set_defaults() + # Override the task defaults with the defaults being provided by the suite if base_name != blank_spec: self.base_name = base_name From f11c84eb2f597e69d081e779d0a796b21c8a5bce Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 2 Mar 2026 16:20:49 -0500 Subject: [PATCH 212/299] Switch suite config method --- src/swell/suites/base/all_suites.py | 67 +++++------- src/swell/suites/base/suite_questions.py | 132 +++++++++++------------ 2 files changed, 94 insertions(+), 105 deletions(-) diff --git a/src/swell/suites/base/all_suites.py b/src/swell/suites/base/all_suites.py index 8ecc70eca..f7f0eb022 100644 --- a/src/swell/suites/base/all_suites.py +++ b/src/swell/suites/base/all_suites.py @@ -16,6 +16,8 @@ from swell.suites.base.suite_questions import SuiteQuestions from swell.suites.base.cylc_workflow import CylcWorkflow from swell.utilities.swell_questions import QuestionList +import swell.suites +from swell.utilities.plugins import discover_plugins # -------------------------------------------------------------------------------------------------- @@ -49,61 +51,48 @@ def get_workflow(self, suite: str) -> type[CylcWorkflow]: def all_workflows(self) -> list: return self.workflow_dict.keys() - # -------------------------------------------------------------------------------------------------- +workflows = Workflows() + +# -------------------------------------------------------------------------------------------------- class SuiteConfigs(): - # Maps suite configuration objects def __init__(self) -> None: - # Dictionary used to create the enum - config_dict = {} - # Map of config names to their parent suites - config_map = {} - - # Find all of the suite configs - for suite in get_suites(): - config_path = os.path.join(get_swell_path(), 'suites', suite, 'suite_config.py') - if os.path.exists(config_path): - suite_container = getattr( - import_module(f'swell.suites.{suite}.suite_config'), 'SuiteConfig') - suite_configs = suite_container.get_all() - - for config in suite_configs: - config_dict[format_suite_name(config)] = getattr(suite_container, config) - config_map[format_suite_name(config)] = suite - else: - config_dict[suite] = SuiteQuestions.all_suites - config_map[suite] = suite + # Dictionary tracking configs under each suite + self.__suites_to_configs_map__ = {} + self.__configs_to_suites_map__ = {} - config_dict['task_minimum'] = SuiteQuestions.task_minimum - config_map['task_minimum'] = 'task_minimum' + # Dictionary tracking the suite for each config + self.__config_map__ = {} + + def register(self, base_suite: str, config_name: str, question_list: QuestionList) -> None: + + if base_suite not in self.__suite_map__: + self.__suites_to_configs_map__[base_suite] = [] - self.config_dict = config_dict - self.__config_map__ = config_map + self.__suites_to_configs_map__[base_suite].append(config_name) - # -------------------------------------------------------------------------------------------------- + self.__configs_to_suites_map__[config_name] = base_suite - def get_config(self, name: str) -> QuestionList: - return self.config_dict[name].value + self.__config_map__[config_name] = question_list - # -------------------------------------------------------------------------------------------------- - - def all_configs(self) -> list: - return list(self.config_dict.keys()) - - # -------------------------------------------------------------------------------------------------- - - def base_suite(self, config: str) -> str: - return self.__config_map__[config] + def get_config(self, config_name: str) -> QuestionList: + return self.__config_map__[config_name] + + def base_suite(self, config_name: str) -> str: + return self.__configs_to_suites_map__[config_name] + + def all_configs(self) -> str: + return list(self.__configs_to_suites_map__.keys()) # -------------------------------------------------------------------------------------------------- - # Objects to reference in imports suite_configs = SuiteConfigs() -workflows = Workflows() + +discover_plugins(swell.suites) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/base/suite_questions.py b/src/swell/suites/base/suite_questions.py index e9b5f3d2c..b246a2828 100644 --- a/src/swell/suites/base/suite_questions.py +++ b/src/swell/suites/base/suite_questions.py @@ -12,73 +12,73 @@ from swell.utilities.swell_questions import QuestionList, QuestionContainer from swell.configuration.question_defaults import QuestionDefaults as qd +from swell.suites.base.all_suites import suite_configs +# -------------------------------------------------------------------------------------------------- +# Shared groups of questions across suites +# -------------------------------------------------------------------------------------------------- + +all_suites = QuestionList( + list_name="all_suites", + questions=[ + qd.experiment_id(), + qd.experiment_root(), + qd.pause_on_tasks(), + qd.task_email_parameters(), + qd.email_address() + ] +) + +suite_configs.register('AllSuites', 'AllSuites', all_suites) + +# -------------------------------------------------------------------------------------------------- + +common = QuestionList( + list_name="common", + questions=[ + all_suites, + qd.cycle_times(), + qd.start_cycle_point(), + qd.final_cycle_point(), + qd.model_components(), + qd.runahead_limit() + ] +) + +# -------------------------------------------------------------------------------------------------- + +marine = QuestionList( + list_name="marine", + questions=[ + common, + qd.marine_models() + ] +) + +# -------------------------------------------------------------------------------------------------- + +compare = QuestionList( + list_name="compare", + questions=[ + all_suites, + qd.comparison_experiment_paths() + ] +) # -------------------------------------------------------------------------------------------------- -class SuiteQuestions(QuestionContainer, Enum): - - # -------------------------------------------------------------------------------------------------- - # Shared groups of questions across suites - # -------------------------------------------------------------------------------------------------- - - all_suites = QuestionList( - list_name="all_suites", - questions=[ - qd.experiment_id(), - qd.experiment_root(), - qd.pause_on_tasks(), - qd.task_email_parameters(), - qd.email_address() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - common = QuestionList( - list_name="common", - questions=[ - all_suites, - qd.cycle_times(), - qd.start_cycle_point(), - qd.final_cycle_point(), - qd.model_components(), - qd.runahead_limit() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - marine = QuestionList( - list_name="marine", - questions=[ - common, - qd.marine_models() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - compare = QuestionList( - list_name="compare", - questions=[ - all_suites, - qd.comparison_experiment_paths() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - task_minimum = QuestionList( - list_name="task_minimum", - questions=[ - qd.experiment_id(), - qd.experiment_root(), - qd.comparison_experiment_paths(), - qd.model_components(), - qd.marine_models(), - qd.use_cycle_dir(), - ] - ) - - # -------------------------------------------------------------------------------------------------- +task_minimum = QuestionList( + list_name="task_minimum", + questions=[ + qd.experiment_id(), + qd.experiment_root(), + qd.comparison_experiment_paths(), + qd.model_components(), + qd.marine_models(), + qd.use_cycle_dir(), + ] +) + +suite_configs.register('TaskMinimum', 'TaskMinimum', task_minimum) + +# -------------------------------------------------------------------------------------------------- From 4c318656346a3aee5c302de4be18d0a9093b5e52 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 2 Mar 2026 16:55:54 -0500 Subject: [PATCH 213/299] Refactor suite configs --- src/swell/suites/base/all_suites.py | 49 +++++++++++++++++++---------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/src/swell/suites/base/all_suites.py b/src/swell/suites/base/all_suites.py index f7f0eb022..f481f4e6b 100644 --- a/src/swell/suites/base/all_suites.py +++ b/src/swell/suites/base/all_suites.py @@ -13,7 +13,6 @@ from swell.swell_path import get_swell_path from swell.utilities.suite_utils import get_suites -from swell.suites.base.suite_questions import SuiteQuestions from swell.suites.base.cylc_workflow import CylcWorkflow from swell.utilities.swell_questions import QuestionList import swell.suites @@ -61,32 +60,50 @@ class SuiteConfigs(): def __init__(self) -> None: - # Dictionary tracking configs under each suite - self.__suites_to_configs_map__ = {} - self.__configs_to_suites_map__ = {} - # Dictionary tracking the suite for each config self.__config_map__ = {} - def register(self, base_suite: str, config_name: str, question_list: QuestionList) -> None: + # -------------------------------------------------------------------------------------------------- + + def register(self, + base_suite: str, + config_name: str, + question_list: QuestionList) -> None: - if base_suite not in self.__suite_map__: - self.__suites_to_configs_map__[base_suite] = [] - - self.__suites_to_configs_map__[base_suite].append(config_name) - - self.__configs_to_suites_map__[config_name] = base_suite + self.__config_map__[config_name] = sub_dict = {} - self.__config_map__[config_name] = question_list + sub_dict[config_name]['suite'] = base_suite + sub_dict['list'] = question_list + + # -------------------------------------------------------------------------------------------------- def get_config(self, config_name: str) -> QuestionList: - return self.__config_map__[config_name] + return self.__config_map__[config_name]['list'] + + # -------------------------------------------------------------------------------------------------- def base_suite(self, config_name: str) -> str: - return self.__configs_to_suites_map__[config_name] + return self.__config_map__[config_name]['suite'] + + # -------------------------------------------------------------------------------------------------- def all_configs(self) -> str: - return list(self.__configs_to_suites_map__.keys()) + return list(self.__config_map__.keys()) + + # -------------------------------------------------------------------------------------------------- + + def configs_under_suites(self) -> dict: + suite_map = {} + + for config_name, config_dict in self.__config_map__.items(): + suite_name = config_dict['suite'] + + if suite_name not in suite_map: + suite_map[suite_name] = [] + + suite_map[suite_name].append(config_name) + + return suite_map # -------------------------------------------------------------------------------------------------- From 66ef0d08d3cb309f600c220c7567047e648381b5 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 2 Mar 2026 17:04:08 -0500 Subject: [PATCH 214/299] Refactor 3dvar --- src/swell/suites/3dvar/suite_config.py | 99 +++++++++++++------------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/src/swell/suites/3dvar/suite_config.py b/src/swell/suites/3dvar/suite_config.py index ef0f29521..96ac19d88 100644 --- a/src/swell/suites/3dvar/suite_config.py +++ b/src/swell/suites/3dvar/suite_config.py @@ -10,63 +10,64 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq +from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_questions import marine from enum import Enum # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): +_3dvar_tier1 = QuestionList( + list_name="3dvar", + questions=[ + marine, + qd.cycling_varbc(), + qd.start_cycle_point("2021-07-01T12:00:00Z"), + qd.final_cycle_point("2021-07-01T12:00:00Z"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_marine']), + qd.parser_options(), + ], + geos_marine=[ + qd.cycle_times(['T12']), + qd.marine_models(['mom6']), + qd.window_length("P1D"), + qd.horizontal_resolution("72x36"), + qd.vertical_resolution("50"), + qd.total_processors(6), + qd.obs_experiment("s2s_v1"), + qd.observations([ + "adt_cryosat2n", + "adt_jason3", + "adt_saral", + "adt_sentinel3a", + "adt_sentinel3b", + "insitu_profile_argo", + "sst_ostia", + "sss_smos", + "sss_smapv5", + "sst_abi_g16_l3c", + "sst_gmi_l3u", + "sst_viirs_n20_l3u", + "temp_profile_xbt" + ]), + qd.background_time_offset("PT18H"), + qd.clean_patterns(['*.nc4', '*.txt']), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register('3dvar_tier1', '3dvar', _3dvar_tier1) - _3dvar_tier1 = QuestionList( - list_name="3dvar", - questions=[ - sq.marine, - qd.cycling_varbc(), - qd.start_cycle_point("2021-07-01T12:00:00Z"), - qd.final_cycle_point("2021-07-01T12:00:00Z"), - qd.jedi_build_method("use_existing"), - qd.model_components(['geos_marine']), - qd.parser_options(), - ], - geos_marine=[ - qd.cycle_times(['T12']), - qd.marine_models(['mom6']), - qd.window_length("P1D"), - qd.horizontal_resolution("72x36"), - qd.vertical_resolution("50"), - qd.total_processors(6), - qd.obs_experiment("s2s_v1"), - qd.observations([ - "adt_cryosat2n", - "adt_jason3", - "adt_saral", - "adt_sentinel3a", - "adt_sentinel3b", - "insitu_profile_argo", - "sst_ostia", - "sss_smos", - "sss_smapv5", - "sst_abi_g16_l3c", - "sst_gmi_l3u", - "sst_viirs_n20_l3u", - "temp_profile_xbt" - ]), - qd.background_time_offset("PT18H"), - qd.clean_patterns(['*.nc4', '*.txt']), - ] - ) +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +_3dvar = QuestionList( + list_name="3dvar", + questions=[ + _3dvar_tier1 + ] +) - _3dvar = QuestionList( - list_name="3dvar", - questions=[ - _3dvar_tier1 - ] - ) +suite_configs.register('3dvar', '3dvar', _3dvar) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- From 2d1cf08c098e8bf96bd21dbb8bbb6d29acbeb45c Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 2 Mar 2026 17:45:09 -0500 Subject: [PATCH 215/299] Refactor suite configs --- src/swell/suites/3dfgat_atmos/__init__.py | 0 src/swell/suites/3dfgat_cycle/__init__.py | 0 src/swell/suites/3dvar/__init__.py | 12 ++++++++++++ src/swell/suites/3dvar_atmos/__init__.py | 0 src/swell/suites/3dvar_cycle/__init__.py | 0 src/swell/suites/base/__init__.py | 0 src/swell/suites/base/all_suites.py | 6 +++--- src/swell/suites/build_geos/__init__.py | 0 src/swell/suites/build_jedi/__init__.py | 0 src/swell/suites/compare/__init__.py | 0 src/swell/suites/convert_bufr/__init__.py | 0 src/swell/suites/convert_ncdiags/__init__.py | 0 src/swell/suites/eva_capabilities/__init__.py | 0 src/swell/suites/forecast_geos/__init__.py | 0 src/swell/suites/geosadas/__init__.py | 0 src/swell/suites/hofx/__init__.py | 0 src/swell/suites/hofx_cf/__init__.py | 0 src/swell/suites/ingest_obs/__init__.py | 0 src/swell/suites/localensembleda/__init__.py | 0 src/swell/suites/ufo_testing/__init__.py | 0 src/swell/utilities/plugins.py | 6 ++++-- 21 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 src/swell/suites/3dfgat_atmos/__init__.py create mode 100644 src/swell/suites/3dfgat_cycle/__init__.py create mode 100644 src/swell/suites/3dvar/__init__.py create mode 100644 src/swell/suites/3dvar_atmos/__init__.py create mode 100644 src/swell/suites/3dvar_cycle/__init__.py create mode 100644 src/swell/suites/base/__init__.py create mode 100644 src/swell/suites/build_geos/__init__.py create mode 100644 src/swell/suites/build_jedi/__init__.py create mode 100644 src/swell/suites/compare/__init__.py create mode 100644 src/swell/suites/convert_bufr/__init__.py create mode 100644 src/swell/suites/convert_ncdiags/__init__.py create mode 100644 src/swell/suites/eva_capabilities/__init__.py create mode 100644 src/swell/suites/forecast_geos/__init__.py create mode 100644 src/swell/suites/geosadas/__init__.py create mode 100644 src/swell/suites/hofx/__init__.py create mode 100644 src/swell/suites/hofx_cf/__init__.py create mode 100644 src/swell/suites/ingest_obs/__init__.py create mode 100644 src/swell/suites/localensembleda/__init__.py create mode 100644 src/swell/suites/ufo_testing/__init__.py diff --git a/src/swell/suites/3dfgat_atmos/__init__.py b/src/swell/suites/3dfgat_atmos/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/3dfgat_cycle/__init__.py b/src/swell/suites/3dfgat_cycle/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/3dvar/__init__.py b/src/swell/suites/3dvar/__init__.py new file mode 100644 index 000000000..d02359e0e --- /dev/null +++ b/src/swell/suites/3dvar/__init__.py @@ -0,0 +1,12 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import os + +repo_directory = os.path.dirname(__file__) + +# Set the version for swell +__version__ = '1.20.0' diff --git a/src/swell/suites/3dvar_atmos/__init__.py b/src/swell/suites/3dvar_atmos/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/3dvar_cycle/__init__.py b/src/swell/suites/3dvar_cycle/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/base/__init__.py b/src/swell/suites/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/base/all_suites.py b/src/swell/suites/base/all_suites.py index f481f4e6b..1160d7050 100644 --- a/src/swell/suites/base/all_suites.py +++ b/src/swell/suites/base/all_suites.py @@ -72,7 +72,7 @@ def register(self, self.__config_map__[config_name] = sub_dict = {} - sub_dict[config_name]['suite'] = base_suite + sub_dict['suite'] = base_suite sub_dict['list'] = question_list # -------------------------------------------------------------------------------------------------- @@ -109,7 +109,7 @@ def configs_under_suites(self) -> dict: # Objects to reference in imports suite_configs = SuiteConfigs() - +print('huh') discover_plugins(swell.suites) - +print(suite_configs.all_configs()) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/__init__.py b/src/swell/suites/build_geos/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/build_jedi/__init__.py b/src/swell/suites/build_jedi/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/compare/__init__.py b/src/swell/suites/compare/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/convert_bufr/__init__.py b/src/swell/suites/convert_bufr/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/convert_ncdiags/__init__.py b/src/swell/suites/convert_ncdiags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/eva_capabilities/__init__.py b/src/swell/suites/eva_capabilities/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/forecast_geos/__init__.py b/src/swell/suites/forecast_geos/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/geosadas/__init__.py b/src/swell/suites/geosadas/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/hofx/__init__.py b/src/swell/suites/hofx/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/hofx_cf/__init__.py b/src/swell/suites/hofx_cf/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/ingest_obs/__init__.py b/src/swell/suites/ingest_obs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/localensembleda/__init__.py b/src/swell/suites/localensembleda/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/ufo_testing/__init__.py b/src/swell/suites/ufo_testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/utilities/plugins.py b/src/swell/utilities/plugins.py index 5bf9eb7c2..7e07b1ac9 100644 --- a/src/swell/utilities/plugins.py +++ b/src/swell/utilities/plugins.py @@ -19,7 +19,9 @@ def discover_plugins(package): ''' for loader, module_name, is_pkg in pkgutil.walk_packages(package.__path__): full_module_name = f"{package.__name__}.{module_name}" + module = importlib.import_module(full_module_name) - importlib.import_module(full_module_name) + if is_pkg: + discover_plugins(module) -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- From 13674b6e172c356698ce6335e4cc499dbe79bccd Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 10:49:51 -0500 Subject: [PATCH 216/299] Refactor suite configs to be more straightforward --- src/swell/suites/3dfgat_atmos/suite_config.py | 176 ++++++------- src/swell/suites/3dfgat_cycle/suite_config.py | 154 +++++------ src/swell/suites/3dvar/suite_config.py | 11 +- src/swell/suites/3dvar_atmos/suite_config.py | 158 ++++++------ src/swell/suites/3dvar_cycle/suite_config.py | 128 +++++----- src/swell/suites/base/suite_questions.py | 2 +- src/swell/suites/build_geos/suite_config.py | 26 +- src/swell/suites/build_jedi/suite_config.py | 26 +- src/swell/suites/compare/suite_config.py | 96 +++---- src/swell/suites/convert_bufr/suite_config.py | 57 ++--- .../suites/convert_ncdiags/suite_config.py | 151 +++++------ .../suites/eva_capabilities/suite_config.py | 174 ++++++------- .../suites/forecast_geos/suite_config.py | 74 +++--- src/swell/suites/geosadas/suite_config.py | 123 ++++----- src/swell/suites/hofx/suite_config.py | 139 +++++----- src/swell/suites/hofx_cf/suite_config.py | 61 +++-- src/swell/suites/ingest_obs/suite_config.py | 77 +++--- .../suites/localensembleda/suite_config.py | 240 +++++++++--------- src/swell/suites/ufo_testing/suite_config.py | 158 ++++++------ src/swell/utilities/swell_questions.py | 15 -- 20 files changed, 1029 insertions(+), 1017 deletions(-) diff --git a/src/swell/suites/3dfgat_atmos/suite_config.py b/src/swell/suites/3dfgat_atmos/suite_config.py index 5ba417a0f..3ae15e47e 100644 --- a/src/swell/suites/3dfgat_atmos/suite_config.py +++ b/src/swell/suites/3dfgat_atmos/suite_config.py @@ -8,102 +8,104 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.suites.base.suite_questions import common +from swell.suites.base.all_suites import suite_configs +suite_name = '3dfgat_atmos' # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): +_3dfgat_atmos_tier1 = QuestionList( + list_name="3dfgat_atmos", + questions=[ + common, + qd.start_cycle_point("2023-10-10T00:00:00Z"), + qd.final_cycle_point("2023-10-10T06:00:00Z"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_atmosphere']), + qd.runahead_limit("P2"), + qd.cycling_varbc() + ], + geos_atmosphere=[ + qd.cycle_times([ + "T00", + "T06", + "T12", + "T18" + ]), + qd.horizontal_resolution("91"), + qd.geos_x_background_directory("/discover/nobackup/projects/gmao/" + "dadev/rtodling/archive/Restarts/JEDI/541x"), + qd.window_type("4D"), + qd.observations([ + "aircraft_temperature", + "aircraft_wind", + "airs_aqua", + "amsr2_gcom-w1", + "amsua_aqua", + "amsua_metop-b", + "amsua_metop-c", + "amsua_n15", + "amsua_n18", + "amsua_n19", + "atms_n20", + "atms_npp", + "avhrr3_metop-b", + "avhrr3_n18", + "avhrr3_n19", + "cris-fsr_n20", + "cris-fsr_npp", + "gmi_gpm", + "gps", + "iasi_metop-b", + "iasi_metop-c", + "mhs_metop-b", + "mhs_metop-c", + "mhs_n19", + "mls55_aura", + "omi_aura", + "ompsnm_npp", + "pibal", + "satwind", + "scatwind", + "sfcship", + "sfc", + "sondes", + "ssmis_f17" + ]), + qd.gradient_norm_reduction("1e-3"), + qd.number_of_iterations([10]), + qd.clean_patterns(['*.txt', '*.csv']), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, '3dfgat_atmos_tier1', _3dfgat_atmos_tier1) - _3dfgat_atmos_tier1 = QuestionList( - list_name="3dfgat_atmos", - questions=[ - sq.common, - qd.start_cycle_point("2023-10-10T00:00:00Z"), - qd.final_cycle_point("2023-10-10T06:00:00Z"), - qd.jedi_build_method("use_existing"), - qd.model_components(['geos_atmosphere']), - qd.runahead_limit("P2"), - qd.cycling_varbc() - ], - geos_atmosphere=[ - qd.cycle_times([ - "T00", - "T06", - "T12", - "T18" - ]), - qd.horizontal_resolution("91"), - qd.geos_x_background_directory("/discover/nobackup/projects/gmao/" - "dadev/rtodling/archive/Restarts/JEDI/541x"), - qd.window_type("4D"), - qd.observations([ - "aircraft_temperature", - "aircraft_wind", - "airs_aqua", - "amsr2_gcom-w1", - "amsua_aqua", - "amsua_metop-b", - "amsua_metop-c", - "amsua_n15", - "amsua_n18", - "amsua_n19", - "atms_n20", - "atms_npp", - "avhrr3_metop-b", - "avhrr3_n18", - "avhrr3_n19", - "cris-fsr_n20", - "cris-fsr_npp", - "gmi_gpm", - "gps", - "iasi_metop-b", - "iasi_metop-c", - "mhs_metop-b", - "mhs_metop-c", - "mhs_n19", - "mls55_aura", - "omi_aura", - "ompsnm_npp", - "pibal", - "satwind", - "scatwind", - "sfcship", - "sfc", - "sondes", - "ssmis_f17" - ]), - qd.gradient_norm_reduction("1e-3"), - qd.number_of_iterations([10]), - qd.clean_patterns(['*.txt', '*.csv']), - ] - ) +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +_3dfgat_atmos = QuestionList( + list_name="3dfgat_atmos", + questions=[ + _3dfgat_atmos_tier1 + ] +) - _3dfgat_atmos = QuestionList( - list_name="3dfgat_atmos", - questions=[ - _3dfgat_atmos_tier1 - ] - ) +suite_configs.register(suite_name, '3dfgat_atmos', _3dfgat_atmos) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- - _3dfgat_atmos_tier2 = QuestionList( - list_name="3dfgat_atmos_tier2", - questions=[ - _3dfgat_atmos_tier1, - ], - geos_atmosphere=[ - qd.number_of_iterations([100]), - ] - ) +_3dfgat_atmos_tier2 = QuestionList( + list_name="3dfgat_atmos_tier2", + questions=[ + _3dfgat_atmos_tier1, + ], + geos_atmosphere=[ + qd.number_of_iterations([100]), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, '3dfgat_atmos_tier2', _3dfgat_atmos_tier2) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/suite_config.py b/src/swell/suites/3dfgat_cycle/suite_config.py index 33ee9da0d..05ff9a449 100644 --- a/src/swell/suites/3dfgat_cycle/suite_config.py +++ b/src/swell/suites/3dfgat_cycle/suite_config.py @@ -8,90 +8,90 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.suites.base.suite_questions import marine +from swell.suites.base.all_suites import suite_configs +suite_name = '3dfgat_cycle' # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): +_3dfgat_cycle_tier1 = QuestionList( + list_name="3dfgat_cycle", + questions=[ + marine, + qd.cycling_varbc(), + qd.start_cycle_point("2021-07-02T06:00:00Z"), + qd.final_cycle_point("2021-07-02T12:00:00Z"), + qd.runahead_limit("P2"), + qd.jedi_build_method("use_existing"), + qd.geos_build_method("use_existing"), + qd.model_components(['geos_marine']), + qd.comparison_log_type('fgat'), + ], + geos_marine=[ + qd.cycle_times([ + "T00", + "T06", + "T12", + "T18" + ]), + qd.analysis_variables([ + "sea_water_salinity", + "sea_water_potential_temperature", + "sea_surface_height_above_geoid", + "sea_water_cell_thickness", + "sea_ice_area_fraction", + "sea_ice_thickness", + "sea_ice_snow_thickness" + ]), + qd.window_length("PT6H"), + qd.window_type("4D"), + qd.horizontal_resolution("72x36"), + qd.vertical_resolution("50"), + qd.total_processors(6), + qd.observations([ + "adt_cryosat2n", + "adt_jason3", + "adt_saral", + "adt_sentinel3a", + "adt_sentinel3b", + "insitu_profile_argo", + "icec_amsr2_north", + "icec_amsr2_south", + "icec_nsidc_nh", + "icec_nsidc_sh", + "sst_ostia", + "sss_smos", + "sss_smapv5", + "sst_abi_g16_l3c", + "sst_gmi_l3u", + "sst_viirs_n20_l3u", + "temp_profile_xbt" + ]), + qd.number_of_iterations([10]), + qd.mom6_iau(True), + qd.background_time_offset("PT9H"), + qd.clean_patterns([ + "*.txt", + "*.rc", + "*.bin" + ]), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, '3dfgat_cycle_tier1', _3dfgat_cycle_tier1) - _3dfgat_cycle_tier1 = QuestionList( - list_name="3dfgat_cycle", - questions=[ - sq.marine, - qd.cycling_varbc(), - qd.start_cycle_point("2021-07-02T06:00:00Z"), - qd.final_cycle_point("2021-07-02T12:00:00Z"), - qd.runahead_limit("P2"), - qd.jedi_build_method("use_existing"), - qd.geos_build_method("use_existing"), - qd.model_components(['geos_marine']), - qd.comparison_log_type('fgat'), - ], - geos_marine=[ - qd.cycle_times([ - "T00", - "T06", - "T12", - "T18" - ]), - qd.analysis_variables([ - "sea_water_salinity", - "sea_water_potential_temperature", - "sea_surface_height_above_geoid", - "sea_water_cell_thickness", - "sea_ice_area_fraction", - "sea_ice_thickness", - "sea_ice_snow_thickness" - ]), - qd.window_length("PT6H"), - qd.window_type("4D"), - qd.horizontal_resolution("72x36"), - qd.vertical_resolution("50"), - qd.total_processors(6), - qd.observations([ - "adt_cryosat2n", - "adt_jason3", - "adt_saral", - "adt_sentinel3a", - "adt_sentinel3b", - "insitu_profile_argo", - "icec_amsr2_north", - "icec_amsr2_south", - "icec_nsidc_nh", - "icec_nsidc_sh", - "sst_ostia", - "sss_smos", - "sss_smapv5", - "sst_abi_g16_l3c", - "sst_gmi_l3u", - "sst_viirs_n20_l3u", - "temp_profile_xbt" - ]), - qd.number_of_iterations([10]), - qd.mom6_iau(True), - qd.background_time_offset("PT9H"), - qd.clean_patterns([ - "*.txt", - "*.rc", - "*.bin" - ]), - ] - ) +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +_3dfgat_cycle = QuestionList( + list_name="3dfgat_cycle", + questions=[ + _3dfgat_cycle_tier1 + ] +) - _3dfgat_cycle = QuestionList( - list_name="3dfgat_cycle", - questions=[ - _3dfgat_cycle_tier1 - ] - ) +suite_configs.register(suite_name, '3dfgat_cycle', _3dfgat_cycle) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/suite_config.py b/src/swell/suites/3dvar/suite_config.py index 96ac19d88..a6f396a02 100644 --- a/src/swell/suites/3dvar/suite_config.py +++ b/src/swell/suites/3dvar/suite_config.py @@ -8,16 +8,15 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.all_suites import suite_configs from swell.suites.base.suite_questions import marine -from enum import Enum - - # -------------------------------------------------------------------------------------------------- +suite_name = '3dvar' + _3dvar_tier1 = QuestionList( list_name="3dvar", questions=[ @@ -57,7 +56,7 @@ ] ) -suite_configs.register('3dvar_tier1', '3dvar', _3dvar_tier1) +suite_configs.register(suite_name, '3dvar_tier1', _3dvar_tier1) # -------------------------------------------------------------------------------------------------- @@ -68,6 +67,6 @@ ] ) -suite_configs.register('3dvar', '3dvar', _3dvar) +suite_configs.register(suite_name, '3dvar', _3dvar) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/suite_config.py b/src/swell/suites/3dvar_atmos/suite_config.py index 153d7de0b..1b739e452 100644 --- a/src/swell/suites/3dvar_atmos/suite_config.py +++ b/src/swell/suites/3dvar_atmos/suite_config.py @@ -8,92 +8,92 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_questions import common +suite_name = '3dvar_atmos' # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): +_3dvar_atmos_tier1 = QuestionList( + list_name="3dvar_atmos", + questions=[ + common, + qd.start_cycle_point("2023-10-10T00:00:00Z"), + qd.final_cycle_point("2023-10-10T06:00:00Z"), + qd.runahead_limit("P2"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_atmosphere']), + qd.cycling_varbc(), + ], + geos_atmosphere=[ + qd.cycle_times([ + "T00", + "T06", + "T12", + "T18" + ]), + qd.geos_x_background_directory("/discover/nobackup/projects/gmao/" + "dadev/rtodling/archive/Restarts/JEDI/541x"), + qd.window_length("PT6H"), + qd.window_type("3D"), + qd.horizontal_resolution("91"), + qd.gsibec_nlats("91"), + qd.gsibec_nlons("144"), + qd.vertical_resolution("72"), + qd.observations([ + "aircraft_temperature", + "aircraft_wind", + "airs_aqua", + "amsr2_gcom-w1", + "amsua_aqua", + "amsua_metop-b", + "amsua_metop-c", + "amsua_n15", + "amsua_n18", + "amsua_n19", + "atms_n20", + "atms_npp", + "avhrr3_metop-b", + "avhrr3_n18", + "avhrr3_n19", + "cris-fsr_n20", + "cris-fsr_npp", + "gmi_gpm", + "gps", + "iasi_metop-b", + "iasi_metop-c", + "mhs_metop-b", + "mhs_metop-c", + "mhs_n19", + "mls55_aura", + "omi_aura", + "ompsnm_npp", + "pibal", + "satwind", + "scatwind", + "sfcship", + "sfc", + "sondes", + "ssmis_f17" + ]), + qd.clean_patterns(['*.txt', '*.csv']), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, '3dvar_atmos_tier1', _3dvar_atmos_tier1) - _3dvar_atmos_tier1 = QuestionList( - list_name="3dvar_atmos", - questions=[ - sq.common, - qd.start_cycle_point("2023-10-10T00:00:00Z"), - qd.final_cycle_point("2023-10-10T06:00:00Z"), - qd.runahead_limit("P2"), - qd.jedi_build_method("use_existing"), - qd.model_components(['geos_atmosphere']), - qd.cycling_varbc(), - ], - geos_atmosphere=[ - qd.cycle_times([ - "T00", - "T06", - "T12", - "T18" - ]), - qd.geos_x_background_directory("/discover/nobackup/projects/gmao/" - "dadev/rtodling/archive/Restarts/JEDI/541x"), - qd.window_length("PT6H"), - qd.window_type("3D"), - qd.horizontal_resolution("91"), - qd.gsibec_nlats("91"), - qd.gsibec_nlons("144"), - qd.vertical_resolution("72"), - qd.observations([ - "aircraft_temperature", - "aircraft_wind", - "airs_aqua", - "amsr2_gcom-w1", - "amsua_aqua", - "amsua_metop-b", - "amsua_metop-c", - "amsua_n15", - "amsua_n18", - "amsua_n19", - "atms_n20", - "atms_npp", - "avhrr3_metop-b", - "avhrr3_n18", - "avhrr3_n19", - "cris-fsr_n20", - "cris-fsr_npp", - "gmi_gpm", - "gps", - "iasi_metop-b", - "iasi_metop-c", - "mhs_metop-b", - "mhs_metop-c", - "mhs_n19", - "mls55_aura", - "omi_aura", - "ompsnm_npp", - "pibal", - "satwind", - "scatwind", - "sfcship", - "sfc", - "sondes", - "ssmis_f17" - ]), - qd.clean_patterns(['*.txt', '*.csv']), - ] - ) +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +_3dvar_atmos = QuestionList( + list_name="3dvar_atmos", + questions=[ + _3dvar_atmos_tier1 + ] +) - _3dvar_atmos = QuestionList( - list_name="3dvar_atmos", - questions=[ - _3dvar_atmos_tier1 - ] - ) +suite_configs.register(suite_name, '3dvar_atmos', _3dvar_atmos) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/suite_config.py b/src/swell/suites/3dvar_cycle/suite_config.py index c4d381c6f..60329fbb3 100644 --- a/src/swell/suites/3dvar_cycle/suite_config.py +++ b/src/swell/suites/3dvar_cycle/suite_config.py @@ -8,77 +8,77 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.suites.base.suite_questions import marine +from swell.suites.base.all_suites import suite_configs +suite_name = '3dvar_cycle' # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): +_3dvar_cycle_tier1 = QuestionList( + list_name="3dvar_cycle", + questions=[ + marine, + qd.cycling_varbc(), + qd.start_cycle_point("2021-07-02T06:00:00Z"), + qd.final_cycle_point("2021-07-02T12:00:00Z"), + qd.runahead_limit("P2"), + qd.jedi_build_method("use_existing"), + qd.geos_build_method("use_existing"), + qd.model_components(['geos_marine']), + ], + geos_marine=[ + qd.cycle_times([ + "T00", + "T06", + "T12", + "T18", + ]), + qd.window_length("PT6H"), + qd.horizontal_resolution("72x36"), + qd.vertical_resolution("50"), + qd.total_processors(6), + qd.observations([ + "adt_cryosat2n", + "adt_jason3", + "adt_saral", + "adt_sentinel3a", + "adt_sentinel3b", + "insitu_profile_argo", + "sst_ostia", + "sss_smos", + "sss_smapv5", + "sst_abi_g16_l3c", + "sst_gmi_l3u", + "sst_viirs_n20_l3u", + "temp_profile_xbt" + ]), + qd.number_of_iterations([10]), + qd.mom6_iau(True), + qd.marine_models(['mom6']), + qd.background_time_offset("PT9H"), + qd.clean_patterns([ + "*.nc4", + "*.txt", + "*.rc", + "*.bin" + ]), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, '3dvar_cycle_tier1', _3dvar_cycle_tier1) - _3dvar_cycle_tier1 = QuestionList( - list_name="3dvar_cycle", - questions=[ - sq.marine, - qd.cycling_varbc(), - qd.start_cycle_point("2021-07-02T06:00:00Z"), - qd.final_cycle_point("2021-07-02T12:00:00Z"), - qd.runahead_limit("P2"), - qd.jedi_build_method("use_existing"), - qd.geos_build_method("use_existing"), - qd.model_components(['geos_marine']), - ], - geos_marine=[ - qd.cycle_times([ - "T00", - "T06", - "T12", - "T18", - ]), - qd.window_length("PT6H"), - qd.horizontal_resolution("72x36"), - qd.vertical_resolution("50"), - qd.total_processors(6), - qd.observations([ - "adt_cryosat2n", - "adt_jason3", - "adt_saral", - "adt_sentinel3a", - "adt_sentinel3b", - "insitu_profile_argo", - "sst_ostia", - "sss_smos", - "sss_smapv5", - "sst_abi_g16_l3c", - "sst_gmi_l3u", - "sst_viirs_n20_l3u", - "temp_profile_xbt" - ]), - qd.number_of_iterations([10]), - qd.mom6_iau(True), - qd.marine_models(['mom6']), - qd.background_time_offset("PT9H"), - qd.clean_patterns([ - "*.nc4", - "*.txt", - "*.rc", - "*.bin" - ]), - ] - ) +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +_3dvar_cycle = QuestionList( + list_name="3dvar_cycle", + questions=[ + _3dvar_cycle_tier1 + ] +) - _3dvar_cycle = QuestionList( - list_name="3dvar_cycle", - questions=[ - _3dvar_cycle_tier1 - ] - ) +suite_configs.register(suite_name, '3dvar_cycle', _3dvar_cycle) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/base/suite_questions.py b/src/swell/suites/base/suite_questions.py index b246a2828..a0d408e88 100644 --- a/src/swell/suites/base/suite_questions.py +++ b/src/swell/suites/base/suite_questions.py @@ -10,7 +10,7 @@ from enum import Enum -from swell.utilities.swell_questions import QuestionList, QuestionContainer +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/build_geos/suite_config.py b/src/swell/suites/build_geos/suite_config.py index 4327d1eb0..db0fe567b 100644 --- a/src/swell/suites/build_geos/suite_config.py +++ b/src/swell/suites/build_geos/suite_config.py @@ -8,23 +8,21 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.suites.base.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.utilities.swell_questions import QuestionList +from swell.suites.base.suite_questions import all_suites +from swell.suites.base.all_suites import suite_configs +suite_name = 'build_geos' # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): - - # -------------------------------------------------------------------------------------------------- +build_geos = QuestionList( + list_name="build_geos", + questions=[ + all_suites + ] +) - build_geos = QuestionList( - list_name="build_geos", - questions=[ - sq.all_suites - ] - ) +suite_configs.register(suite_name, 'build_geos', build_geos) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/suite_config.py b/src/swell/suites/build_jedi/suite_config.py index 65ae82a9e..c37cd4ebf 100644 --- a/src/swell/suites/build_jedi/suite_config.py +++ b/src/swell/suites/build_jedi/suite_config.py @@ -8,23 +8,21 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.suites.base.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.utilities.swell_questions import QuestionList +from swell.suites.base.suite_questions import all_suites +from swell.suites.base.all_suites import suite_configs +suite_name = 'build_jedi' # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): - - # -------------------------------------------------------------------------------------------------- +build_jedi = QuestionList( + list_name="build_jedi", + questions=[ + all_suites + ] +) - build_jedi = QuestionList( - list_name="build_jedi", - questions=[ - sq.all_suites - ] - ) +suite_configs.register(suite_name, 'build_jedi', build_jedi) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare/suite_config.py b/src/swell/suites/compare/suite_config.py index 28dea3650..d3b43da02 100644 --- a/src/swell/suites/compare/suite_config.py +++ b/src/swell/suites/compare/suite_config.py @@ -7,63 +7,67 @@ # # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList, WidgetType +from swell.utilities.swell_questions import QuestionList, WidgetType from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_questions import all_suites # -------------------------------------------------------------------------------------------------- +suite_name = 'compare' -class SuiteConfig(QuestionContainer, Enum): +compare = QuestionList( + list_name="compare", + questions=[ + all_suites, + qd.comparison_experiment_paths(), + qd.start_cycle_point(default_value=None, widget_type=WidgetType.STRING), + qd.final_cycle_point(default_value=None, widget_type=WidgetType.STRING), + qd.cycle_times(default_value=[None], widget_type=WidgetType.STRING_CHECK_LIST), + qd.model_components(), + qd.runahead_limit(), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, 'compare', compare) - compare = QuestionList( - list_name="compare", - questions=[ - sq.all_suites, - qd.comparison_experiment_paths(), - qd.start_cycle_point(default_value=None, widget_type=WidgetType.STRING), - qd.final_cycle_point(default_value=None, widget_type=WidgetType.STRING), - qd.cycle_times(default_value=[None], widget_type=WidgetType.STRING_CHECK_LIST), - qd.model_components(), - qd.runahead_limit(), - ] - ) +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +compare_variational_marine = QuestionList( + list_name="compare_variational_marine", + questions=[ + compare, + qd.comparison_log_type('variational'), + qd.model_components(['geos_marine']), + ] +) - compare_variational_marine = QuestionList( - list_name="compare_variational_marine", - questions=[ - compare, - qd.comparison_log_type('variational'), - qd.model_components(['geos_marine']), - ] - ) +suite_configs.register(suite_name, 'compare_variational_marine', compare_variational_marine) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- - compare_variational_atmosphere = QuestionList( - list_name="compare_variational_atmosphere", - questions=[ - compare, - qd.comparison_log_type('variational'), - qd.model_components(['geos_atmosphere']), - ] - ) +compare_variational_atmosphere = QuestionList( + list_name="compare_variational_atmosphere", + questions=[ + compare, + qd.comparison_log_type('variational'), + qd.model_components(['geos_atmosphere']), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, 'compare_variational_atmosphere', compare_variational_atmosphere) - compare_fgat_marine = QuestionList( - list_name="compare_fgat_marine", - questions=[ - compare, - qd.comparison_log_type('fgat'), - qd.model_components(['geos_marine']), - ] - ) +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +compare_fgat_marine = QuestionList( + list_name="compare_fgat_marine", + questions=[ + compare, + qd.comparison_log_type('fgat'), + qd.model_components(['geos_marine']), + ] +) + +suite_configs.register(suite_name, 'compare_fgat_marine', compare_fgat_marine) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_bufr/suite_config.py b/src/swell/suites/convert_bufr/suite_config.py index fb40eeceb..6df7a15ad 100644 --- a/src/swell/suites/convert_bufr/suite_config.py +++ b/src/swell/suites/convert_bufr/suite_config.py @@ -8,39 +8,38 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.suites.base.suite_questions import common +from swell.suites.base.all_suites import suite_configs # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): - - # -------------------------------------------------------------------------------------------------- - - convert_bufr = QuestionList( - list_name="convert_bufr", - questions=[ - sq.common, - qd.start_cycle_point("2023-10-10T00:00:00Z"), - qd.final_cycle_point("2023-10-10T06:00:00Z"), - qd.jedi_build_method("use_existing"), - qd.model_components(['geos_atmosphere']), - ], - geos_atmosphere=[ - qd.cycle_times(['T00', 'T06', 'T12', 'T18']), - qd.clean_patterns([ - "gsi_bcs/*.nc4", - "gsi_bcs/*.txt", - ]), - qd.bufr_obs_classes([ - "ncep_1bamua_bufr", - "ncep_mtiasi_bufr", - ]), - ] - ) +suite_name = 'convert_bufr' + +convert_bufr = QuestionList( + list_name="convert_bufr", + questions=[ + common, + qd.start_cycle_point("2023-10-10T00:00:00Z"), + qd.final_cycle_point("2023-10-10T06:00:00Z"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_atmosphere']), + ], + geos_atmosphere=[ + qd.cycle_times(['T00', 'T06', 'T12', 'T18']), + qd.clean_patterns([ + "gsi_bcs/*.nc4", + "gsi_bcs/*.txt", + ]), + qd.bufr_obs_classes([ + "ncep_1bamua_bufr", + "ncep_mtiasi_bufr", + ]), + ] +) + +suite_configs.register(suite_name, 'convert_bufr', convert_bufr) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/suite_config.py b/src/swell/suites/convert_ncdiags/suite_config.py index cd0aba0ad..8d561d095 100644 --- a/src/swell/suites/convert_ncdiags/suite_config.py +++ b/src/swell/suites/convert_ncdiags/suite_config.py @@ -8,88 +8,89 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.suites.base.suite_questions import common +from swell.suites.base.all_suites import suite_configs # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): +suite_name = 'convert_ncdiags' + +convert_ncdiags_tier1 = QuestionList( + list_name="convert_ncdiags", + questions=[ + common, + qd.start_cycle_point("2021-12-12T00:00:00Z"), + qd.final_cycle_point("2021-12-12T06:00:00Z"), + qd.jedi_build_method("use_existing"), + qd.bundles("REMOVE"), + qd.model_components(['geos_atmosphere']), + ], + geos_atmosphere=[ + qd.cycle_times(['T00', 'T06']), + qd.clean_patterns([ + "gsi_bcs/*.nc4", + "gsi_bcs/*.txt", + "gsi_bcs/*.yaml", + "gsi_bcs", + "gsi_ncdiags/*.nc4", + "gsi_ncdiags/aircraft/*.nc4", + "gsi_ncdiags/aircraft", + "gsi_ncdiags" + ]), + qd.observations([ + "aircraft", + "airs_aqua", + "amsr2_gcom-w1", + "amsua_aqua", + "amsua_metop-b", + "amsua_metop-c", + "amsua_n15", + "amsua_n18", + "amsua_n19", + "atms_n20", + "atms_npp", + "avhrr3_metop-b", + "avhrr3_n18", + "avhrr3_n19", + "cris-fsr_n20", + "cris-fsr_npp", + "gmi_gpm", + "gps", + "iasi_metop-b", + "iasi_metop-c", + "mhs_metop-b", + "mhs_metop-c", + "mhs_n19", + "mls55_aura", + "omi_aura", + "ompsnm_npp", + "pibal", + "satwind", + "scatwind", + "sfcship", + "sfc", + "sondes", + "ssmis_f17" + ]), + qd.path_to_gsi_nc_diags("/discover/nobackup/projects/gmao/advda/SwellTestData/" + "ufo_testing/ncdiagv2/%Y%m%d%H"), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, 'convert_ncdiags_tier1', convert_ncdiags_tier1) - convert_ncdiags_tier1 = QuestionList( - list_name="convert_ncdiags", - questions=[ - sq.common, - qd.start_cycle_point("2021-12-12T00:00:00Z"), - qd.final_cycle_point("2021-12-12T06:00:00Z"), - qd.jedi_build_method("use_existing"), - qd.bundles("REMOVE"), - qd.model_components(['geos_atmosphere']), - ], - geos_atmosphere=[ - qd.cycle_times(['T00', 'T06']), - qd.clean_patterns([ - "gsi_bcs/*.nc4", - "gsi_bcs/*.txt", - "gsi_bcs/*.yaml", - "gsi_bcs", - "gsi_ncdiags/*.nc4", - "gsi_ncdiags/aircraft/*.nc4", - "gsi_ncdiags/aircraft", - "gsi_ncdiags" - ]), - qd.observations([ - "aircraft", - "airs_aqua", - "amsr2_gcom-w1", - "amsua_aqua", - "amsua_metop-b", - "amsua_metop-c", - "amsua_n15", - "amsua_n18", - "amsua_n19", - "atms_n20", - "atms_npp", - "avhrr3_metop-b", - "avhrr3_n18", - "avhrr3_n19", - "cris-fsr_n20", - "cris-fsr_npp", - "gmi_gpm", - "gps", - "iasi_metop-b", - "iasi_metop-c", - "mhs_metop-b", - "mhs_metop-c", - "mhs_n19", - "mls55_aura", - "omi_aura", - "ompsnm_npp", - "pibal", - "satwind", - "scatwind", - "sfcship", - "sfc", - "sondes", - "ssmis_f17" - ]), - qd.path_to_gsi_nc_diags("/discover/nobackup/projects/gmao/advda/SwellTestData/" - "ufo_testing/ncdiagv2/%Y%m%d%H"), - ] - ) +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +convert_ncdiags = QuestionList( + list_name="convert_ncdiags", + questions=[ + convert_ncdiags_tier1 + ] +) - convert_ncdiags = QuestionList( - list_name="convert_ncdiags", - questions=[ - convert_ncdiags_tier1 - ] - ) +suite_configs.register(suite_name, 'convert_bufr', convert_ncdiags) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eva_capabilities/suite_config.py b/src/swell/suites/eva_capabilities/suite_config.py index 41e20c785..941f440df 100644 --- a/src/swell/suites/eva_capabilities/suite_config.py +++ b/src/swell/suites/eva_capabilities/suite_config.py @@ -8,98 +8,100 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import marine +from swell.suites.base.all_suites import suite_configs -from enum import Enum +# -------------------------------------------------------------------------------------------------- +suite_name = 'eva_capabilities' -# -------------------------------------------------------------------------------------------------- +eva_capabilities = QuestionList( + list_name="eva_capabilities", + questions=[ + marine, + qd.start_cycle_point("2021-07-02T06:00:00Z"), + qd.final_cycle_point("2021-07-03T06:00:00Z"), + qd.model_components(['geos_marine']), + ], + geos_marine=[ + qd.cycle_times(['T00', 'T06', 'T12', 'T18']), + qd.window_length("PT6H"), + qd.observations([ + "adt_cryosat2n", + "adt_jason3", + "adt_saral", + "adt_sentinel3a", + "adt_sentinel3b", + "insitu_profile_argo", + "sss_smos", + "sss_smapv5", + "sst_abi_g16_l3c", + "sst_gmi_l3u", + "sst_viirs_n20_l3u", + "temp_profile_xbt" + ]), + qd.ncdiag_experiments(['fgat_jra55_01']), + qd.clean_patterns(['*.nc4', '*.txt']), + ] +) -class SuiteConfig(QuestionContainer, Enum): +suite_configs.register(suite_name, 'eva_capabilities', eva_capabilities) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- - eva_capabilities = QuestionList( - list_name="eva_capabilities", - questions=[ - sq.marine, - qd.start_cycle_point("2021-07-02T06:00:00Z"), - qd.final_cycle_point("2021-07-03T06:00:00Z"), - qd.model_components(['geos_marine']), - ], - geos_marine=[ - qd.cycle_times(['T00', 'T06', 'T12', 'T18']), - qd.window_length("PT6H"), - qd.observations([ - "adt_cryosat2n", - "adt_jason3", - "adt_saral", - "adt_sentinel3a", - "adt_sentinel3b", - "insitu_profile_argo", - "sss_smos", - "sss_smapv5", - "sst_abi_g16_l3c", - "sst_gmi_l3u", - "sst_viirs_n20_l3u", - "temp_profile_xbt" - ]), - qd.ncdiag_experiments(['fgat_jra55_01']), - qd.clean_patterns(['*.nc4', '*.txt']), - ] - ) +eva_capabilities_atmosphere = QuestionList( + list_name="eva_capabilities_atmosphere", + questions=[ + eva_capabilities, + qd.start_cycle_point("2023-10-10T00:00:00Z"), + qd.final_cycle_point("2023-10-10T06:00:00Z"), + qd.model_components(['geos_atmosphere']), + ], + geos_atmosphere=[ + qd.cycle_times(['T00', 'T06', 'T12', 'T18']), + qd.observations([ + # "aircraft_temperature", + # "aircraft_wind", + "airs_aqua", + "amsr2_gcom-w1", + "amsua_aqua", + "amsua_metop-b", + "amsua_metop-c", + "amsua_n15", + # "amsua_n18", + # "amsua_n19", + "atms_n20", + "atms_npp", + "avhrr3_metop-b", + # "avhrr3_n18", + "avhrr3_n19", + "cris-fsr_n20", + "cris-fsr_npp", + "gmi_gpm", + "gps", + "iasi_metop-b", + "iasi_metop-c", + "mhs_metop-b", + "mhs_metop-c", + "mhs_n19", + # "mls55_aura", + # "omi_aura", + # "ompsnm_npp", + # "pibal", + "satwind", + "scatwind", + "sfcship", + "sfc", + "sondes", + "ssmis_f17" + ]), + qd.ncdiag_experiments(['x0050_fgat']), + qd.clean_patterns(['*.txt', '*.csv']), + ] +) - eva_capabilities_atmosphere = QuestionList( - list_name="eva_capabilities_atmosphere", - questions=[ - eva_capabilities, - qd.start_cycle_point("2023-10-10T00:00:00Z"), - qd.final_cycle_point("2023-10-10T06:00:00Z"), - qd.model_components(['geos_atmosphere']), - ], - geos_atmosphere=[ - qd.cycle_times(['T00', 'T06', 'T12', 'T18']), - qd.observations([ - # "aircraft_temperature", - # "aircraft_wind", - "airs_aqua", - "amsr2_gcom-w1", - "amsua_aqua", - "amsua_metop-b", - "amsua_metop-c", - "amsua_n15", - # "amsua_n18", - # "amsua_n19", - "atms_n20", - "atms_npp", - "avhrr3_metop-b", - # "avhrr3_n18", - "avhrr3_n19", - "cris-fsr_n20", - "cris-fsr_npp", - "gmi_gpm", - "gps", - "iasi_metop-b", - "iasi_metop-c", - "mhs_metop-b", - "mhs_metop-c", - "mhs_n19", - # "mls55_aura", - # "omi_aura", - # "ompsnm_npp", - # "pibal", - "satwind", - "scatwind", - "sfcship", - "sfc", - "sondes", - "ssmis_f17" - ]), - qd.ncdiag_experiments(['x0050_fgat']), - qd.clean_patterns(['*.txt', '*.csv']), - ] - ) +suite_configs.register(suite_name, 'eva_capabilities_atmosphere', eva_capabilities_atmosphere) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/suite_config.py b/src/swell/suites/forecast_geos/suite_config.py index 69733fc2a..89e33ae26 100644 --- a/src/swell/suites/forecast_geos/suite_config.py +++ b/src/swell/suites/forecast_geos/suite_config.py @@ -8,46 +8,46 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import all_suites +from swell.suites.base.all_suites import suite_configs -from enum import Enum +# -------------------------------------------------------------------------------------------------- +suite_name = 'forecast_geos' + +forecast_geos_tier1 = QuestionList( + list_name="forecast_geos", + questions=[ + all_suites, + qd.cycle_times(), + qd.final_cycle_point(), + qd.start_cycle_point(), + qd.start_cycle_point("2021-06-20T00:00:00Z"), + qd.final_cycle_point("2021-06-21T00:00:00Z"), + qd.cycle_times([ + "T00", + "T06", + "T12", + "T18" + ]), + qd.geos_build_method("use_existing"), + qd.forecast_duration("PT6H"), + ], +) + +suite_configs.register(suite_name, 'forecast_geos_tier1', forecast_geos_tier1) # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): - - # -------------------------------------------------------------------------------------------------- - - forecast_geos_tier1 = QuestionList( - list_name="forecast_geos", - questions=[ - sq.all_suites, - qd.cycle_times(), - qd.final_cycle_point(), - qd.start_cycle_point(), - qd.start_cycle_point("2021-06-20T00:00:00Z"), - qd.final_cycle_point("2021-06-21T00:00:00Z"), - qd.cycle_times([ - "T00", - "T06", - "T12", - "T18" - ]), - qd.geos_build_method("use_existing"), - qd.forecast_duration("PT6H"), - ], - ) - - # -------------------------------------------------------------------------------------------------- - - forecast_geos = QuestionList( - list_name="forecast_geos", - questions=[ - forecast_geos_tier1 - ] - ) - - # -------------------------------------------------------------------------------------------------- +forecast_geos = QuestionList( + list_name="forecast_geos", + questions=[ + forecast_geos_tier1 + ] +) + +suite_configs.register(suite_name, 'forecast_geos', forecast_geos) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/suite_config.py b/src/swell/suites/geosadas/suite_config.py index bbf1a142f..1405b31c8 100644 --- a/src/swell/suites/geosadas/suite_config.py +++ b/src/swell/suites/geosadas/suite_config.py @@ -8,74 +8,75 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.suites.base.suite_questions import all_suites +from swell.suites.base.all_suites import suite_configs # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): +suite_name = 'geosadas' + +geosadas_tier1 = QuestionList( + list_name="geosadas", + questions=[ + all_suites, + qd.jedi_build_method("use_existing"), + qd.bundles("REMOVE"), + qd.model_components(['geos_atmosphere']), + ], + geos_atmosphere=[ + qd.horizontal_resolution("13"), + qd.observations([ + "aircraft_temperature", + "aircraft_wind", + "airs_aqua", + "amsr2_gcom-w1", + "amsua_aqua", + "amsua_metop-b", + "amsua_metop-c", + "amsua_n15", + "amsua_n18", + "amsua_n19", + "atms_n20", + "atms_npp", + "avhrr3_metop-b", + "avhrr3_n18", + "avhrr3_n19", + "cris-fsr_n20", + "cris-fsr_npp", + "gmi_gpm", + "iasi_metop-b", + "iasi_metop-c", + "mhs_metop-b", + "mhs_metop-c", + "mhs_n19", + "mls55_aura", + "omi_aura", + "ompsnm_npp", + "satwind", + "scatwind", + "ssmis_f17" + ]), + qd.produce_geovals(False), + qd.window_type("3D"), + qd.gradient_norm_reduction("1e-6"), + qd.number_of_iterations([5]), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, 'geosadas_tier1', geosadas_tier1) - geosadas_tier1 = QuestionList( - list_name="geosadas", - questions=[ - sq.all_suites, - qd.jedi_build_method("use_existing"), - qd.bundles("REMOVE"), - qd.model_components(['geos_atmosphere']), - ], - geos_atmosphere=[ - qd.horizontal_resolution("13"), - qd.observations([ - "aircraft_temperature", - "aircraft_wind", - "airs_aqua", - "amsr2_gcom-w1", - "amsua_aqua", - "amsua_metop-b", - "amsua_metop-c", - "amsua_n15", - "amsua_n18", - "amsua_n19", - "atms_n20", - "atms_npp", - "avhrr3_metop-b", - "avhrr3_n18", - "avhrr3_n19", - "cris-fsr_n20", - "cris-fsr_npp", - "gmi_gpm", - "iasi_metop-b", - "iasi_metop-c", - "mhs_metop-b", - "mhs_metop-c", - "mhs_n19", - "mls55_aura", - "omi_aura", - "ompsnm_npp", - "satwind", - "scatwind", - "ssmis_f17" - ]), - qd.produce_geovals(False), - qd.window_type("3D"), - qd.gradient_norm_reduction("1e-6"), - qd.number_of_iterations([5]), - ] - ) +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +geosadas = QuestionList( + list_name="geosadas", + questions=[ + geosadas_tier1 + ] +) - geosadas = QuestionList( - list_name="geosadas", - questions=[ - geosadas_tier1 - ] - ) +suite_configs.register(suite_name, 'geosadas', geosadas) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/suite_config.py b/src/swell/suites/hofx/suite_config.py index d115beaa8..b7a92c74a 100644 --- a/src/swell/suites/hofx/suite_config.py +++ b/src/swell/suites/hofx/suite_config.py @@ -8,82 +8,83 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.suites.base.suite_questions import marine +from swell.suites.base.all_suites import suite_configs # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): +suite_name = 'hofx' + +hofx_tier1 = QuestionList( + list_name="hofx", + questions=[ + marine, + qd.cycling_varbc(), + qd.window_type(), + qd.jedi_build_method("use_existing"), + qd.save_geovals(True), + qd.model_components(['geos_atmosphere']), + ], + geos_atmosphere=[ + qd.horizontal_resolution("91"), + qd.geos_x_background_directory("/discover/nobackup/projects/gmao/dadev/" + "rtodling/archive/Restarts/JEDI/541x"), + qd.npx_proc(2), + qd.npy_proc(2), + qd.observations([ + "aircraft_temperature", + "aircraft_wind", + "airs_aqua", + "amsr2_gcom-w1", + "amsua_aqua", + "amsua_metop-b", + "amsua_metop-c", + "amsua_n15", + "amsua_n18", + "amsua_n19", + "atms_n20", + "atms_npp", + "avhrr3_metop-b", + "avhrr3_n18", + "avhrr3_n19", + "cris-fsr_n20", + "cris-fsr_npp", + "gmi_gpm", + "gps", + "iasi_metop-b", + "iasi_metop-c", + "mhs_metop-b", + "mhs_metop-c", + "mhs_n19", + "mls55_aura", + "omi_aura", + "ompsnm_npp", + "pibal", + "satwind", + "scatwind", + "sfcship", + "sfc", + "sondes", + "ssmis_f17" + ]), + qd.clean_patterns([]), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, 'hofx_tier1', hofx_tier1) - hofx_tier1 = QuestionList( - list_name="hofx", - questions=[ - sq.marine, - qd.cycling_varbc(), - qd.window_type(), - qd.jedi_build_method("use_existing"), - qd.save_geovals(True), - qd.model_components(['geos_atmosphere']), - ], - geos_atmosphere=[ - qd.horizontal_resolution("91"), - qd.geos_x_background_directory("/discover/nobackup/projects/gmao/dadev/" - "rtodling/archive/Restarts/JEDI/541x"), - qd.npx_proc(2), - qd.npy_proc(2), - qd.observations([ - "aircraft_temperature", - "aircraft_wind", - "airs_aqua", - "amsr2_gcom-w1", - "amsua_aqua", - "amsua_metop-b", - "amsua_metop-c", - "amsua_n15", - "amsua_n18", - "amsua_n19", - "atms_n20", - "atms_npp", - "avhrr3_metop-b", - "avhrr3_n18", - "avhrr3_n19", - "cris-fsr_n20", - "cris-fsr_npp", - "gmi_gpm", - "gps", - "iasi_metop-b", - "iasi_metop-c", - "mhs_metop-b", - "mhs_metop-c", - "mhs_n19", - "mls55_aura", - "omi_aura", - "ompsnm_npp", - "pibal", - "satwind", - "scatwind", - "sfcship", - "sfc", - "sondes", - "ssmis_f17" - ]), - qd.clean_patterns([]), - ] - ) +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +hofx = QuestionList( + list_name="hofx", + questions=[ + hofx_tier1 + ] +) - hofx = QuestionList( - list_name="hofx", - questions=[ - hofx_tier1 - ] - ) +suite_configs.register(suite_name, 'hofx', hofx) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx_cf/suite_config.py b/src/swell/suites/hofx_cf/suite_config.py index 7bdc44d5b..621930554 100644 --- a/src/swell/suites/hofx_cf/suite_config.py +++ b/src/swell/suites/hofx_cf/suite_config.py @@ -8,37 +8,42 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import common +from swell.suites.base.all_suites import suite_configs -from enum import Enum +# -------------------------------------------------------------------------------------------------- + +suite_name = 'hofx_cf' + +hofx_cf = QuestionList( + list_name="hofx_cf", + questions=[ + common, + qd.start_cycle_point("2023-08-05T18:00:00Z"), + qd.final_cycle_point("2023-08-05T18:00:00Z"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_cf']), + qd.check_for_obs(False) # don't check empty for empty obs + ], + + geos_cf=[ + ] +) + +suite_configs.register(suite_name, 'hofx_cf', hofx_cf) # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): - - # -------------------------------------------------------------------------------------------------- - - hofx_cf = QuestionList( - list_name="hofx_cf", - questions=[ - sq.common, - qd.start_cycle_point("2023-08-05T18:00:00Z"), - qd.final_cycle_point("2023-08-05T18:00:00Z"), - qd.jedi_build_method("use_existing"), - qd.model_components(['geos_cf']), - qd.check_for_obs(False) # don't check empty for empty obs - ], - - geos_cf=[ - ] - ) - - hofx_cf_tier1 = QuestionList( - list_name="hofx_cf_tier1", - questions=[ - hofx_cf - ] - ) +hofx_cf_tier1 = QuestionList( + list_name="hofx_cf_tier1", + questions=[ + hofx_cf + ] +) + +suite_configs.register(suite_name, 'hofx_cf_tier1', hofx_cf_tier1) + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/suites/ingest_obs/suite_config.py b/src/swell/suites/ingest_obs/suite_config.py index c315a50bf..ce82e29e9 100644 --- a/src/swell/suites/ingest_obs/suite_config.py +++ b/src/swell/suites/ingest_obs/suite_config.py @@ -5,36 +5,49 @@ """ -from swell.utilities.swell_questions import QuestionContainer, QuestionList +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq -from enum import Enum - - -class SuiteConfig(QuestionContainer, Enum): - - ingest_obs = QuestionList( - list_name="ingest_obs", - questions=[ - sq.common, - ], - ) - # This name should be unique and not conflict with other suites - # (otherwise it might get overwritten) - ingest_obs_marine = QuestionList( - list_name="ingest_obs_marine", - questions=[ - ingest_obs, - sq.marine, - qd.start_cycle_point("2021-07-02T06:00:00Z"), - qd.final_cycle_point("2021-07-03T06:00:00Z"), - qd.model_components(['geos_marine']), - qd.runahead_limit("P5"), - ], - geos_marine=[ - qd.window_length("PT6H"), - qd.cycle_times(['T00', 'T06', 'T12', 'T18']), - qd.obs_to_ingest(['adt_cryosat2n']), # List of obs names - qd.dry_run(True), - ] - ) +from swell.suites.base.suite_questions import common, marine +from swell.suites.base.all_suites import suite_configs + + +# -------------------------------------------------------------------------------------------------- + +suite_name = 'ingest_obs' + +ingest_obs = QuestionList( + list_name="ingest_obs", + questions=[ + common, + ], +) + +suite_configs.register(suite_name, 'ingest_obs', ingest_obs) + +# -------------------------------------------------------------------------------------------------- + +# This name should be unique and not conflict with other suites +# (otherwise it might get overwritten) +ingest_obs_marine = QuestionList( + list_name="ingest_obs_marine", + questions=[ + ingest_obs, + marine, + qd.start_cycle_point("2021-07-02T06:00:00Z"), + qd.final_cycle_point("2021-07-03T06:00:00Z"), + qd.model_components(['geos_marine']), + qd.runahead_limit("P5"), + ], + geos_marine=[ + qd.window_length("PT6H"), + qd.cycle_times(['T00', 'T06', 'T12', 'T18']), + qd.obs_to_ingest(['adt_cryosat2n']), # List of obs names + qd.dry_run(True), + ] +) + +suite_configs.register(suite_name, 'ingest_obs_marine', ingest_obs_marine) + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index 6c0b697b9..f91bb9fec 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -8,132 +8,136 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq +from swell.suites.base.suite_questions import marine +from swell.suites.base.all_suites import suite_configs -from enum import Enum +# -------------------------------------------------------------------------------------------------- +suite_name = 'localensembleda' -# -------------------------------------------------------------------------------------------------- +localensembleda_tier1 = QuestionList( + list_name="localensembleda", + questions=[ + marine, + qd.cycling_varbc(), + qd.ensemble_hofx_packets(), + qd.ensemble_hofx_strategy(), + qd.skip_ensemble_hofx(), + qd.final_cycle_point("2023-10-10T12:00:00Z"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_atmosphere']), + ], + geos_atmosphere=[ + qd.horizontal_resolution('91'), + qd.background_experiment('x0050'), + qd.geos_x_background_directory('/discover/nobackup/projects/gmao/dadev/' + 'rtodling/archive/Restarts/JEDI/541x'), + qd.geos_x_ensemble_directory('/discover/nobackup/projects/gmao/dadev/' + 'rtodling/archive/541/Milan'), + qd.npx_proc(3), + qd.npy_proc(3), + qd.cycle_times(['T00']), + qd.ensemble_num_members(3), + qd.skip_ensemble_hofx(True), + qd.local_ensemble_solver("Deterministic GETKF"), + qd.local_ensemble_use_linear_observer(False), + qd.ensmean_only(False), + qd.local_ensemble_save_posterior_mean(True), + qd.local_ensemble_save_posterior_mean_increment(True), + qd.local_ensemble_save_posterior_ensemble(False), + qd.local_ensemble_save_posterior_ensemble_increments(False), + qd.obs_thinning_rej_fraction(0.75), + qd.observations([ + "atms_n20", + ]), + qd.window_type("4D"), + qd.clean_patterns(['*.txt']) + ] +) + +suite_configs.register(suite_name, 'localensembleda_tier1', localensembleda_tier1) -class SuiteConfig(QuestionContainer, Enum): +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +localensembleda_tier2 = QuestionList( + list_name="localensembleda", + questions=[ + marine, + qd.ensemble_hofx_packets(), + qd.ensemble_hofx_strategy(), + qd.skip_ensemble_hofx(), + qd.final_cycle_point("2023-10-10T12:00:00Z"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_atmosphere']), + ], + geos_atmosphere=[ + qd.horizontal_resolution('91'), + qd.background_experiment('x0050'), + qd.geos_x_background_directory('/discover/nobackup/projects/gmao/dadev/' + 'rtodling/archive/Restarts/JEDI/541x'), + qd.geos_x_ensemble_directory('/discover/nobackup/projects/gmao/dadev/' + 'rtodling/archive/541/Milan'), + qd.npx_proc(4), + qd.npy_proc(4), + # qd.perhost(32), + qd.cycle_times(['T00']), + qd.ensemble_num_members(16), + qd.skip_ensemble_hofx(True), + qd.local_ensemble_solver("Deterministic GETKF"), + qd.local_ensemble_use_linear_observer(True), + qd.ensmean_only(False), + qd.local_ensemble_save_posterior_mean(True), + qd.local_ensemble_save_posterior_mean_increment(True), + qd.local_ensemble_save_posterior_ensemble(False), + qd.local_ensemble_save_posterior_ensemble_increments(False), + qd.obs_thinning_rej_fraction(0.75), + qd.observations([ + "aircraft_temperature", + "aircraft_wind", + "sondes", + "gps", + "amsua_aqua", + "amsua_n15", + "amsua_n18", + "amsua_n19", + "amsr2_gcom-w1", + "atms_n20", + "atms_npp", + "avhrr3_metop-b", + "avhrr3_n18", + "avhrr3_n19", + "scatwind", + "sfcship", + "sfc", + "mhs_metop-b", + "mhs_metop-c", + "mhs_n19", + "mls55_aura", + "omi_aura", + "ompsnm_npp", + "pibal", + "ssmis_f17", + "amsua_metop-b", + "amsua_metop-c" + ]), + qd.window_type("4D"), + qd.clean_patterns(['*.txt']) + ] +) - localensembleda_tier1 = QuestionList( - list_name="localensembleda", - questions=[ - sq.marine, - qd.cycling_varbc(), - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.skip_ensemble_hofx(), - qd.final_cycle_point("2023-10-10T12:00:00Z"), - qd.jedi_build_method("use_existing"), - qd.model_components(['geos_atmosphere']), - ], - geos_atmosphere=[ - qd.horizontal_resolution('91'), - qd.background_experiment('x0050'), - qd.geos_x_background_directory('/discover/nobackup/projects/gmao/dadev/' - 'rtodling/archive/Restarts/JEDI/541x'), - qd.geos_x_ensemble_directory('/discover/nobackup/projects/gmao/dadev/' - 'rtodling/archive/541/Milan'), - qd.npx_proc(3), - qd.npy_proc(3), - qd.cycle_times(['T00']), - qd.ensemble_num_members(3), - qd.skip_ensemble_hofx(True), - qd.local_ensemble_solver("Deterministic GETKF"), - qd.local_ensemble_use_linear_observer(False), - qd.ensmean_only(False), - qd.local_ensemble_save_posterior_mean(True), - qd.local_ensemble_save_posterior_mean_increment(True), - qd.local_ensemble_save_posterior_ensemble(False), - qd.local_ensemble_save_posterior_ensemble_increments(False), - qd.obs_thinning_rej_fraction(0.75), - qd.observations([ - "atms_n20", - ]), - qd.window_type("4D"), - qd.clean_patterns(['*.txt']) - ] - ) +suite_configs.register(suite_name, 'localensembleda_tier2', localensembleda_tier2) - localensembleda_tier2 = QuestionList( - list_name="localensembleda", - questions=[ - sq.marine, - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.skip_ensemble_hofx(), - qd.final_cycle_point("2023-10-10T12:00:00Z"), - qd.jedi_build_method("use_existing"), - qd.model_components(['geos_atmosphere']), - ], - geos_atmosphere=[ - qd.horizontal_resolution('91'), - qd.background_experiment('x0050'), - qd.geos_x_background_directory('/discover/nobackup/projects/gmao/dadev/' - 'rtodling/archive/Restarts/JEDI/541x'), - qd.geos_x_ensemble_directory('/discover/nobackup/projects/gmao/dadev/' - 'rtodling/archive/541/Milan'), - qd.npx_proc(4), - qd.npy_proc(4), - # qd.perhost(32), - qd.cycle_times(['T00']), - qd.ensemble_num_members(16), - qd.skip_ensemble_hofx(True), - qd.local_ensemble_solver("Deterministic GETKF"), - qd.local_ensemble_use_linear_observer(True), - qd.ensmean_only(False), - qd.local_ensemble_save_posterior_mean(True), - qd.local_ensemble_save_posterior_mean_increment(True), - qd.local_ensemble_save_posterior_ensemble(False), - qd.local_ensemble_save_posterior_ensemble_increments(False), - qd.obs_thinning_rej_fraction(0.75), - qd.observations([ - "aircraft_temperature", - "aircraft_wind", - "sondes", - "gps", - "amsua_aqua", - "amsua_n15", - "amsua_n18", - "amsua_n19", - "amsr2_gcom-w1", - "atms_n20", - "atms_npp", - "avhrr3_metop-b", - "avhrr3_n18", - "avhrr3_n19", - "scatwind", - "sfcship", - "sfc", - "mhs_metop-b", - "mhs_metop-c", - "mhs_n19", - "mls55_aura", - "omi_aura", - "ompsnm_npp", - "pibal", - "ssmis_f17", - "amsua_metop-b", - "amsua_metop-c" - ]), - qd.window_type("4D"), - qd.clean_patterns(['*.txt']) - ] - ) +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +localensembleda = QuestionList( + list_name="localensembleda", + questions=[ + localensembleda_tier2 + ] +) - localensembleda = QuestionList( - list_name="localensembleda", - questions=[ - localensembleda_tier2 - ] - ) +suite_configs.register(suite_name, 'localensembleda', localensembleda) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/suite_config.py b/src/swell/suites/ufo_testing/suite_config.py index 99f025bf6..d71fa437e 100644 --- a/src/swell/suites/ufo_testing/suite_config.py +++ b/src/swell/suites/ufo_testing/suite_config.py @@ -8,91 +8,91 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq - -from enum import Enum - +from swell.suites.base.suite_questions import common +from swell.suites.base.all_suites import suite_configs # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): +suite_name = 'ufo_testing' - # -------------------------------------------------------------------------------------------------- +ufo_testing_tier1 = QuestionList( + list_name="ufo_testing", + questions=[ + common, + qd.final_cycle_point("2023-10-10T00:00:00Z"), + qd.jedi_build_method("use_existing"), + qd.bundles("REMOVE"), + qd.model_components(['geos_atmosphere']), + ], + geos_atmosphere=[ + qd.cycle_times(['T00']), + qd.observations([ + "aircraft_temperature", + "aircraft_wind", + "airs_aqua", + "amsr2_gcom-w1", + "amsua_aqua", + "amsua_metop-b", + "amsua_metop-c", + "amsua_n15", + "amsua_n19", + "atms_n20", + "atms_npp", + "avhrr3_metop-b", + "avhrr3_n19", + "cris-fsr_n20", + "cris-fsr_npp", + "gmi_gpm", + "gps", + "iasi_metop-b", + "iasi_metop-c", + "mhs_metop-b", + "mhs_metop-c", + "mhs_n19", + "pibal", + "satwind", + "scatwind", + "sfc", + "sfcship", + "sondes", + "ssmis_f17" + ]), + qd.produce_geovals(False), + qd.clean_patterns([ + "*.txt", + "*.log", + "*.yaml", + "*.csv", + "gsi_bcs/*.nc4", + "gsi_bcs/*.txt", + "gsi_bcs/*.yaml", + "gsi_bcs", + "gsi_ncdiags/*.nc4", + "gsi_ncdiags/aircraft/*.nc4", + "gsi_ncdiags/aircraft", + "gsi_ncdiags" + ]), + qd.path_to_gsi_bc_coefficients("/discover/nobackup/projects/gmao/dadev/rtodling/" + "archive/541/Milan/x0050/ana/Y%Y/M%m/" + "*bias*%Y%m%d_%Hz.txt"), + qd.path_to_gsi_nc_diags("/discover/nobackup/projects/gmao/dadev/rtodling/archive/" + "541/Milan/x0050/obs/Y%Y/M%m/D%d/H%H/"), + ] +) - ufo_testing_tier1 = QuestionList( - list_name="ufo_testing", - questions=[ - sq.common, - qd.final_cycle_point("2023-10-10T00:00:00Z"), - qd.jedi_build_method("use_existing"), - qd.bundles("REMOVE"), - qd.model_components(['geos_atmosphere']), - ], - geos_atmosphere=[ - qd.cycle_times(['T00']), - qd.observations([ - "aircraft_temperature", - "aircraft_wind", - "airs_aqua", - "amsr2_gcom-w1", - "amsua_aqua", - "amsua_metop-b", - "amsua_metop-c", - "amsua_n15", - "amsua_n19", - "atms_n20", - "atms_npp", - "avhrr3_metop-b", - "avhrr3_n19", - "cris-fsr_n20", - "cris-fsr_npp", - "gmi_gpm", - "gps", - "iasi_metop-b", - "iasi_metop-c", - "mhs_metop-b", - "mhs_metop-c", - "mhs_n19", - "pibal", - "satwind", - "scatwind", - "sfc", - "sfcship", - "sondes", - "ssmis_f17" - ]), - qd.produce_geovals(False), - qd.clean_patterns([ - "*.txt", - "*.log", - "*.yaml", - "*.csv", - "gsi_bcs/*.nc4", - "gsi_bcs/*.txt", - "gsi_bcs/*.yaml", - "gsi_bcs", - "gsi_ncdiags/*.nc4", - "gsi_ncdiags/aircraft/*.nc4", - "gsi_ncdiags/aircraft", - "gsi_ncdiags" - ]), - qd.path_to_gsi_bc_coefficients("/discover/nobackup/projects/gmao/dadev/rtodling/" - "archive/541/Milan/x0050/ana/Y%Y/M%m/" - "*bias*%Y%m%d_%Hz.txt"), - qd.path_to_gsi_nc_diags("/discover/nobackup/projects/gmao/dadev/rtodling/archive/" - "541/Milan/x0050/obs/Y%Y/M%m/D%d/H%H/"), - ] - ) +suite_configs.register(suite_name, 'ufo_testing_tier1', ufo_testing_tier1) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- - ufo_testing = QuestionList( - list_name="ufo_testing", - questions=[ - ufo_testing_tier1 - ] - ) +ufo_testing = QuestionList( + list_name="ufo_testing", + questions=[ + ufo_testing_tier1 + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, 'ufo_testing', ufo_testing) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index 39f34667b..afc19d7fa 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -116,21 +116,6 @@ class SwellQuestion: # -------------------------------------------------------------------------------------------------- - -class QuestionContainer: - """ Class to extend question lists for suites and tasks, use with Enum """ - - def __init__(self, *args): - arg_dict = asdict(args[0]) - setattr(self, arg_dict['list_name'], args[0]) - - @classmethod - def get_all(cls): - return cls._member_names_ - -# -------------------------------------------------------------------------------------------------- - - @dataclass class QuestionList: """Basic dataclass containing a list of questions for each model, suite, task""" From 06e4c8626b8064726603151dc7b1e21a26980d74 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 11:02:43 -0500 Subject: [PATCH 217/299] Add files for merge with develop --- src/swell/suites/ingest_obs/flow.cylc | 32 - src/swell/suites/ingest_obs/workflow.py | 65 ++ src/swell/tasks/task_questions.py | 772 ------------------------ 3 files changed, 65 insertions(+), 804 deletions(-) delete mode 100644 src/swell/suites/ingest_obs/flow.cylc create mode 100644 src/swell/suites/ingest_obs/workflow.py delete mode 100644 src/swell/tasks/task_questions.py diff --git a/src/swell/suites/ingest_obs/flow.cylc b/src/swell/suites/ingest_obs/flow.cylc deleted file mode 100644 index dc999598f..000000000 --- a/src/swell/suites/ingest_obs/flow.cylc +++ /dev/null @@ -1,32 +0,0 @@ -[scheduler] - UTC mode = True - allow implicit tasks = False - -[scheduling] - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - - [[graph]] - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - {% for model_component in model_components %} - IngestObs-{{model_component}} - {% endfor %} - """ - {% endfor %} - -[runtime] - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - {% for model_component in model_components %} - [[IngestObs-{{model_component}}]] - script = "swell task IngestObs $config -d $datetime -m {{model_component}}" - execution time limit = PT10M - - {% endfor %} \ No newline at end of file diff --git a/src/swell/suites/ingest_obs/workflow.py b/src/swell/suites/ingest_obs/workflow.py new file mode 100644 index 000000000..1803650d6 --- /dev/null +++ b/src/swell/suites/ingest_obs/workflow.py @@ -0,0 +1,65 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.jinja2 import template_string_jinja2 +from swell.suites.base.cylc_workflow import CylcWorkflow +from swell.tasks.base.task_attributes import task_attributes as ta + +# -------------------------------------------------------------------------------------------------- + +template_str = ''' + +[scheduler] + UTC mode = True + allow implicit tasks = False + +[scheduling] + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + + [[graph]] + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + IngestObs-{{model_component}} + {% endfor %} + """ + {% endfor %} + +[runtime] + +''' # noqa + +# -------------------------------------------------------------------------------------------------- + + +class Workflow_ingest_obs(CylcWorkflow): + + def get_workflow_string(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + for task in self.tasks: + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) + + return workflow_str + + def set_tasks(self) -> list: + + self.tasks.append(ta.root()) + + for model in self.experiment_dict['model_components']: + self.tasks.append(ta.IngestObs(model=model)) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py deleted file mode 100644 index 7e410ba8d..000000000 --- a/src/swell/tasks/task_questions.py +++ /dev/null @@ -1,772 +0,0 @@ -# -------------------------------------------------------------------------------------------------- -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -from enum import Enum - -from swell.utilities.swell_questions import QuestionList, QuestionContainer -from swell.configuration.question_defaults import QuestionDefaults as qd - - -# -------------------------------------------------------------------------------------------------- - -class TaskQuestions(QuestionContainer, Enum): - - # -------------------------------------------------------------------------------------------------- - # Helper question lists used by multiple tasks (in order of use) - # -------------------------------------------------------------------------------------------------- - - background_crtm_obs = QuestionList( - list_name="background_crtm_obs", - questions=[ - qd.background_time_offset(), - qd.crtm_coeff_dir(), - qd.observations(), - qd.observing_system_records_path() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - np_proc_resolution = QuestionList( - list_name="np_resolution", - questions=[ - qd.npx_proc(), - qd.npy_proc(), - qd.npx(), - qd.npy(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - window_questions = QuestionList( - list_name="window_questions", - questions=[ - qd.window_length(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - run_jedi_executable = QuestionList( - list_name="run_jedi_executable", - questions=[ - background_crtm_obs, - np_proc_resolution, - window_questions, - qd.analysis_variables(), - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.gradient_norm_reduction(), - qd.gsibec_configuration(), - qd.jedi_forecast_model(), - qd.minimizer(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.number_of_iterations(), - qd.total_processors(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - swell_static_file_questions = QuestionList( - list_name="swell_static_file_questions", - questions=[ - qd.swell_static_files(), - qd.swell_static_files_user() - ] - ) - - # -------------------------------------------------------------------------------------------------- - # Task-specific question lists (in alphabetical order) - # -------------------------------------------------------------------------------------------------- - - BuildGeos = QuestionList( - list_name="BuildGeos", - questions=[ - qd.geos_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - BuildGeosByLinking = QuestionList( - list_name="BuildGeosByLinking", - questions=[ - qd.existing_geos_gcm_build_path(), - qd.geos_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - BuildJedi = QuestionList( - list_name="BuildJedi", - questions=[ - qd.bundles(), - qd.jedi_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - BuildJediByLinking = QuestionList( - list_name="BuildJediByLinking", - questions=[ - qd.existing_jedi_build_directory(), - qd.existing_jedi_build_directory_pinned(), - qd.jedi_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CleanCycle = QuestionList( - list_name="CleanCycle", - questions=[ - qd.clean_patterns() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CloneGeos = QuestionList( - list_name="CloneGeos", - questions=[ - qd.existing_geos_gcm_source_path(), - qd.geos_build_method(), - qd.geos_gcm_tag() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CloneGeosMksi = QuestionList( - list_name="CloneGeosMksi", - questions=[ - qd.observing_system_records_mksi_path(), - qd.observing_system_records_mksi_path_tag() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CloneGmaoPerllib = QuestionList( - list_name="CloneGmaoPerllib", - questions=[ - qd.existing_perllib_path(), - qd.gmao_perllib_tag() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - CloneJedi = QuestionList( - list_name="CloneJedi", - questions=[ - qd.bundles(), - qd.existing_jedi_source_directory(), - qd.existing_jedi_source_directory_pinned(), - qd.jedi_build_method() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaComparisonJediLog = QuestionList( - list_name="EvaJediLog", - questions=[ - qd.comparison_log_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaComparisonObservations = QuestionList( - list_name="EvaComparisonObservations", - questions=[ - qd.comparison_log_type(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaIncrement = QuestionList( - list_name="EvaIncrement", - questions=[ - qd.marine_models(), - qd.window_length(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaObservations = QuestionList( - list_name="EvaObservations", - questions=[ - background_crtm_obs, - qd.marine_models(), - qd.observing_system_records_path(), - qd.window_length(), - qd.marine_models(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - EvaTimeseries = QuestionList( - list_name="EvaTimeseries", - questions=[ - background_crtm_obs, - qd.window_length(), - qd.ncdiag_experiments(), - qd.marine_models(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GenerateBClimatology = QuestionList( - list_name="GenerateBClimatology", - questions=[ - np_proc_resolution, - swell_static_file_questions, - qd.analysis_variables(), - qd.background_error_model(), - qd.generate_yaml_and_exit(), - qd.gradient_norm_reduction(), - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.jedi_forecast_model(), - qd.marine_models(), - qd.minimizer(), - qd.number_of_iterations(), - qd.observing_system_records_path(), - qd.total_processors(), - qd.window_length(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GenerateBClimatologyByLinking = QuestionList( - list_name="GenerateBClimatologyByLinking", - questions=[ - swell_static_file_questions, - qd.background_error_model(), - qd.horizontal_resolution(), - qd.vertical_resolution(), - qd.window_type() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GenerateObservingSystemRecords = QuestionList( - list_name="GenerateObservingSystemRecords", - questions=[ - qd.observations(), - qd.observing_system_records_mksi_path(), - qd.observing_system_records_path() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetBackground = QuestionList( - list_name="GetBackground", - questions=[ - window_questions, - qd.background_experiment(), - qd.background_frequency(), - qd.horizontal_resolution(), - qd.marine_models(), - qd.r2d2_local_path(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetBackgroundGeosExperiment = QuestionList( - list_name="GetBackgroundGeosExperiment", - questions=[ - qd.horizontal_resolution(), - qd.background_experiment(), - qd.background_time_offset(), - qd.geos_x_background_directory() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetBufr = QuestionList( - list_name="GetBufr", - questions=[ - qd.bufr_obs_classes() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetEnsemble = QuestionList( - list_name="GetEnsemble", - questions=[ - qd.path_to_ensemble() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetEnsembleGeosExperiment = QuestionList( - list_name="GetEnsembleGeosExperiment", - questions=[ - qd.background_experiment(), - qd.background_time_offset(), - qd.geos_x_ensemble_directory() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGeovals = QuestionList( - list_name="GetGeovals", - questions=[ - background_crtm_obs, - qd.geovals_experiment(), - qd.geovals_provider(), - qd.r2d2_local_path(), - qd.window_length(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGeosAdasBackground = QuestionList( - list_name="GetGeosAdasBackground", - questions=[ - qd.path_to_geos_adas_background() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGeosRestart = QuestionList( - list_name="GetGeosRestart", - questions=[ - swell_static_file_questions, - qd.geos_restarts_directory() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGsiBc = QuestionList( - list_name="GetGsiBc", - questions=[ - qd.path_to_gsi_bc_coefficients(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetGsiNcdiag = QuestionList( - list_name="GetGsiNcdiag", - questions=[ - qd.path_to_gsi_nc_diags() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetNcdiags = QuestionList( - list_name="GetNcdiags", - questions=[ - background_crtm_obs, - qd.ncdiag_experiments(), - qd.marine_models(), - qd.r2d2_local_path(), - qd.window_length(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetObservations = QuestionList( - list_name="GetObservations", - questions=[ - background_crtm_obs, - qd.cycling_varbc(), - qd.obs_experiment(), - qd.observing_system_records_path(), - qd.r2d2_local_path(), - qd.window_length(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GetObsNotInR2d2 = QuestionList( - list_name="GetExistingObservations", - questions=[ - qd.ioda_locations_not_in_r2d2(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GsiBcToIoda = QuestionList( - list_name="GsiBcToIoda", - questions=[ - background_crtm_obs, - qd.observing_system_records_path(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - GsiNcdiagToIoda = QuestionList( - list_name="GsiNcdiagToIoda", - questions=[ - qd.observations(), - qd.produce_geovals(), - qd.single_observations(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - IngestObs = QuestionList( - list_name="IngestObs", - questions=[ - qd.dry_run(), - qd.obs_to_ingest(), - qd.r2d2_local_path(), - qd.window_length(), - # qd.window_offset(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - JediLogComparison = QuestionList( - list_name="JediComparisonLog", - questions=[ - qd.comparison_log_type(), - qd.number_of_iterations() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - JediOopsLogParser = QuestionList( - list_name="JediOopsLogParser", - questions=[ - qd.comparison_log_type(), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - LinkGeosOutput = QuestionList( - list_name="LinkGeosOutput", - questions=[ - window_questions, - qd.background_frequency(), - qd.marine_models() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - MoveDaRestart = QuestionList( - list_name="MoveDaRestart", - questions=[ - qd.mom6_iau(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - MoveForecastRestart = QuestionList( - list_name="MoveForecastRestart", - questions=[ - qd.forecast_duration() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - PrepareAnalysis = QuestionList( - list_name="PrepareAnalysis", - questions=[ - qd.analysis_variables(), - qd.mom6_iau(), - qd.total_processors(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - PrepGeosRunDir = QuestionList( - list_name="PrepGeosRunDir", - questions=[ - swell_static_file_questions, - qd.existing_geos_gcm_build_path(), - qd.forecast_duration(), - qd.geos_experiment_directory(), - qd.mom6_iau_nhours() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RenderJediObservations = QuestionList( - list_name="RenderJediObservations", - questions=[ - qd.check_for_obs(), - qd.crtm_coeff_dir(), - qd.background_time_offset(), - qd.observing_system_records_path(), - qd.observations(), - qd.set_obs_as_local(), - qd.window_length() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediConvertStateSoca2ciceExecutable = QuestionList( - list_name="RunJediConvertStateSoca2ciceExecutable", - questions=[ - qd.analysis_variables(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.marine_models(), - qd.observations(), - qd.total_processors(), - qd.window_length(), - qd.window_type(), - qd.comparison_log_type('convert_state_soca2cice'), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediEnsembleMeanVariance = QuestionList( - list_name="RunJediEnsembleMeanVariance", - questions=[ - np_proc_resolution, - window_questions, - qd.analysis_variables(), - qd.ensemble_num_members(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.observations(), - qd.observing_system_records_path(), - qd.comparison_log_type('ensmeanvariance'), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediFgatExecutable = QuestionList( - list_name="RunJediFgatExecutable", - questions=[ - run_jedi_executable, - qd.marine_models(), - qd.comparison_log_type('fgat') - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediHofxEnsembleExecutable = QuestionList( - list_name="RunJediHofxEnsembleExecutable", - questions=[ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.ensemble_num_members(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.total_processors(), - qd.comparison_log_type('hofx') - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediHofxExecutable = QuestionList( - list_name="RunJediHofxExecutable", - questions=[ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.save_geovals(), - qd.total_processors(), - qd.comparison_log_type('ensemblehofx'), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediLocalEnsembleDaExecutable = QuestionList( - list_name="RunJediLocalEnsembleDaExecutable", - questions=[ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.ensemble_hofx_packets(), - qd.ensemble_hofx_strategy(), - qd.ensemble_num_members(), - qd.ensmean_only(), - qd.ensmeanvariance_only(), - qd.generate_yaml_and_exit(), - qd.horizontal_localization_lengthscale(), - qd.horizontal_localization_max_nobs(), - qd.horizontal_localization_method(), - qd.jedi_forecast_model(), - qd.local_ensemble_inflation_mult(), - qd.local_ensemble_inflation_rtpp(), - qd.local_ensemble_inflation_rtps(), - qd.local_ensemble_save_posterior_ensemble(), - qd.local_ensemble_save_posterior_ensemble_increments(), - qd.local_ensemble_save_posterior_mean(), - qd.local_ensemble_save_posterior_mean_increment(), - qd.local_ensemble_solver(), - qd.local_ensemble_use_linear_observer(), - qd.skip_ensemble_hofx(), - qd.total_processors(), - qd.vertical_localization_apply_log_transform(), - qd.vertical_localization_function(), - qd.vertical_localization_ioda_vertical_coord(), - qd.vertical_localization_ioda_vertical_coord_group(), - qd.vertical_localization_lengthscale(), - qd.vertical_localization_method(), - qd.perhost(), - qd.comparison_log_type('localensembleda'), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediObsfiltersExecutable = QuestionList( - list_name="RunJediObsfiltersExecutable", - questions=[ - np_proc_resolution, - window_questions, - background_crtm_obs, - qd.background_frequency(), - qd.generate_yaml_and_exit(), - qd.jedi_forecast_model(), - qd.observing_system_records_path(), - qd.total_processors(), - qd.obs_thinning_rej_fraction(), - qd.comparison_log_type('obsfilters') - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediUfoTestsExecutable = QuestionList( - list_name="RunJediUfoTestsExecutable", - questions=[ - background_crtm_obs, - qd.generate_yaml_and_exit(), - qd.single_observations(), - qd.window_length(), - qd.comparison_log_type('ufo_tests'), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - RunJediVariationalExecutable = QuestionList( - list_name="RunJediVariationalExecutable", - questions=[ - run_jedi_executable, - qd.perhost(), - qd.comparison_log_type('variational'), - ] - ) - - # -------------------------------------------------------------------------------------------------- - - SaveObsDiags = QuestionList( - list_name="SaveObsDiags", - questions=[ - background_crtm_obs, - qd.window_length(), - qd.r2d2_local_path(), - qd.marine_models() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - SaveRestart = QuestionList( - list_name="SaveRestart", - questions=[ - window_questions, - qd.background_time_offset(), - qd.forecast_duration(), - qd.horizontal_resolution(), - qd.marine_models(), - qd.r2d2_local_path() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - StageJedi = QuestionList( - list_name="StageJedi", - questions=[ - swell_static_file_questions, - qd.gsibec_configuration(), - qd.gsibec_nlats(), - qd.gsibec_nlons(), - qd.horizontal_resolution(), - qd.vertical_resolution() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - StoreBackground = QuestionList( - list_name="StoreBackground", - questions=[ - window_questions, - qd.background_experiment(), - qd.background_frequency(), - qd.horizontal_resolution(), - qd.r2d2_local_path(), - ] - ) - - # -------------------------------------------------------------------------------------------------- From 8e1b8f35e132faef024ad00cfa8d4da3dcc428e3 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 11:32:13 -0500 Subject: [PATCH 218/299] Register workflows --- src/swell/deployment/create_experiment.py | 2 +- src/swell/suites/3dfgat_atmos/workflow.py | 2 + src/swell/suites/3dfgat_cycle/workflow.py | 2 + src/swell/suites/3dvar/workflow.py | 2 + src/swell/suites/3dvar_atmos/workflow.py | 2 + src/swell/suites/3dvar_cycle/workflow.py | 2 + src/swell/suites/base/all_suites.py | 40 ++++++++----------- src/swell/suites/build_geos/workflow.py | 2 + src/swell/suites/build_jedi/workflow.py | 2 + src/swell/suites/compare/workflow.py | 2 + src/swell/suites/convert_bufr/workflow.py | 2 + src/swell/suites/convert_ncdiags/workflow.py | 2 + src/swell/suites/eva_capabilities/workflow.py | 2 + src/swell/suites/forecast_geos/workflow.py | 2 + src/swell/suites/geosadas/workflow.py | 2 + src/swell/suites/hofx/workflow.py | 2 + src/swell/suites/hofx_cf/workflow.py | 2 + src/swell/suites/ingest_obs/workflow.py | 2 + src/swell/suites/localensembleda/workflow.py | 2 + src/swell/suites/ufo_testing/workflow.py | 2 + 20 files changed, 53 insertions(+), 25 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 93b5e26b5..48bd77cc5 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -165,7 +165,7 @@ def prepare_config( # Initialize the workflow # ----------------------- - workflow_class = workflows.get_workflow(suite) + workflow_class = workflows.get(suite) workflow = workflow_class(suite_dict, slurm_dict) # Get the list of tasks from the workflow's graph diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 82f36a088..28621aa48 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -126,6 +127,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('3dfgat_atmos') class Workflow_3dfgat_atmos(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index 38201c2e4..1ac12059e 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -144,6 +145,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('3dfgat_cycle') class Workflow_3dfgat_cycle(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index f28bf5089..5518c1ad9 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -106,6 +107,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('3dvar') class Workflow_3dvar(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index 674b379c6..9536aac67 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -120,6 +121,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('3dvar_atmos') class Workflow_3dvar_atmos(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index c0842d5b6..74d759a23 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -142,6 +143,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('3dvar_cycle') class Workflow_3dvar_cycle(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/base/all_suites.py b/src/swell/suites/base/all_suites.py index 1160d7050..bec2a0f05 100644 --- a/src/swell/suites/base/all_suites.py +++ b/src/swell/suites/base/all_suites.py @@ -27,34 +27,26 @@ def format_suite_name(suite_name): # -------------------------------------------------------------------------------------------------- - class Workflows(): - # Maps suites to workflow objects def __init__(self) -> None: - workflow_dict = {} - - for suite in get_suites(): - workflow_path = os.path.join(get_swell_path(), 'suites', suite, 'workflow.py') - if os.path.exists(workflow_path): - workflow = getattr( - import_module(f'swell.suites.{suite}.workflow'), f'Workflow_{suite}') - - workflow_dict[suite] = workflow - - self.workflow_dict = workflow_dict - - def get_workflow(self, suite: str) -> type[CylcWorkflow]: - return self.workflow_dict[suite] - - def all_workflows(self) -> list: - return self.workflow_dict.keys() + self.__workflow_names__ = [] + + def register(self, name: str) -> None: + self.__workflow_names__.append(name) + def wrapper(cls): + setattr(self, name, cls) + return cls + return wrapper + + def get(self, name: str) -> type[CylcWorkflow]: + return getattr(self, name) + + def all(self) -> list: + return self.__workflow_names__ # -------------------------------------------------------------------------------------------------- -workflows = Workflows() - -# -------------------------------------------------------------------------------------------------- class SuiteConfigs(): @@ -109,7 +101,7 @@ def configs_under_suites(self) -> dict: # Objects to reference in imports suite_configs = SuiteConfigs() -print('huh') +workflows = Workflows() discover_plugins(swell.suites) -print(suite_configs.all_configs()) + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py index e770703ad..e444b6a16 100644 --- a/src/swell/suites/build_geos/workflow.py +++ b/src/swell/suites/build_geos/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -46,6 +47,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('build_geos') class Workflow_build_geos(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index 186971a49..1e7b62406 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -46,6 +47,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('build_jedi') class Workflow_build_jedi(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/compare/workflow.py b/src/swell/suites/compare/workflow.py index b32fb1bf7..ae1895c81 100644 --- a/src/swell/suites/compare/workflow.py +++ b/src/swell/suites/compare/workflow.py @@ -12,6 +12,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -66,6 +67,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('compare') class Workflow_compare(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/convert_bufr/workflow.py b/src/swell/suites/convert_bufr/workflow.py index 1823efc4c..d17e9e563 100644 --- a/src/swell/suites/convert_bufr/workflow.py +++ b/src/swell/suites/convert_bufr/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -87,6 +88,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('convert_bufr') class Workflow_convert_bufr(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index a25c5c736..b86704da6 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -76,6 +77,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('convert_ncdiags') class Workflow_convert_ncdiags(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index a1e1d5e11..0447a7b44 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -72,6 +73,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('eva_capabilities') class Workflow_eva_capabilities(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index 5c4caffb5..cf256d77e 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -81,6 +82,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('forecast_geos') class Workflow_forecast_geos(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index 375892dc4..7282b00d1 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -79,6 +80,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('geosadas') class Workflow_geosadas(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index 412cb0c3a..397da336b 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -106,6 +107,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('hofx') class Workflow_hofx(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/hofx_cf/workflow.py b/src/swell/suites/hofx_cf/workflow.py index 4993578c7..67c6e68ea 100644 --- a/src/swell/suites/hofx_cf/workflow.py +++ b/src/swell/suites/hofx_cf/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -99,6 +100,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('hofx_cf') class Workflow_hofx_cf(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/ingest_obs/workflow.py b/src/swell/suites/ingest_obs/workflow.py index 1803650d6..f7444f569 100644 --- a/src/swell/suites/ingest_obs/workflow.py +++ b/src/swell/suites/ingest_obs/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -40,6 +41,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('ingest_obs') class Workflow_ingest_obs(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index e882e660a..347557805 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -124,6 +125,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('localensembleda') class Workflow_localensembleda(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index 341567f0b..de623a0fe 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -10,6 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.all_suites import workflows # -------------------------------------------------------------------------------------------------- @@ -96,6 +97,7 @@ # -------------------------------------------------------------------------------------------------- +@workflows.register('ufo_testing') class Workflow_ufo_testing(CylcWorkflow): def get_workflow_string(self): From 90c30630138555be849720c381222a73ef95725e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 11:44:00 -0500 Subject: [PATCH 219/299] Fix convert_ncdiags --- src/swell/suites/convert_ncdiags/suite_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/suites/convert_ncdiags/suite_config.py b/src/swell/suites/convert_ncdiags/suite_config.py index 8d561d095..739c7ddf3 100644 --- a/src/swell/suites/convert_ncdiags/suite_config.py +++ b/src/swell/suites/convert_ncdiags/suite_config.py @@ -91,6 +91,6 @@ ] ) -suite_configs.register(suite_name, 'convert_bufr', convert_ncdiags) +suite_configs.register(suite_name, 'convert_ncdiags', convert_ncdiags) # -------------------------------------------------------------------------------------------------- From 02c588597929566be679fefccb99e8d8b169fa65 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 11:52:01 -0500 Subject: [PATCH 220/299] Update docs --- docs/adding_a_suite.md | 143 ++++++++++---------------- docs/examples/templating_workflows.md | 2 +- 2 files changed, 54 insertions(+), 91 deletions(-) diff --git a/docs/adding_a_suite.md b/docs/adding_a_suite.md index 42a39a9a1..d2df71b9e 100644 --- a/docs/adding_a_suite.md +++ b/docs/adding_a_suite.md @@ -287,99 +287,62 @@ In this question infrastructure, **suites take priority over tasks**. Any questi Consider the following example of suite questions for `3dvar` (in python, variable names cannot begin with digits): -```python -from swell.configuration.question_defaults import QuestionDefaults as qd -from swell.suites.base.suite_questions import SuiteQuestions as sq - -class SuiteQuestions(QuestionContainer, Enum): - - # -------------------------------------------------------------------------------------------------- - # Shared groups of questions across suites - # -------------------------------------------------------------------------------------------------- - - all_suites = QuestionList( - list_name="all_suites", - questions=[ - qd.experiment_id(), - qd.experiment_root() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - common = QuestionList( - list_name="common", - questions=[ - all_suites, - qd.cycle_times(), - qd.start_cycle_point(), - qd.final_cycle_point(), - qd.model_components(), - qd.runahead_limit() - ] - ) - - # -------------------------------------------------------------------------------------------------- - - marine = QuestionList( - list_name="marine", - questions=[ - common, - qd.marine_models() - ] - ) -``` - ```python -class SuiteConfig(QuestionContainer, Enum): - _3dvar_base = QuestionList( - list_name="3dvar", - questions=[ - sq.marine - ] - ) - - # -------------------------------------------------------------------------------------------------- - - _3dvar_tier1 = QuestionList( - list_name="3dvar", - questions=[ - _3dvar_base, - qd.start_cycle_point("2021-07-01T12:00:00Z"), - qd.final_cycle_point("2021-07-01T12:00:00Z"), - qd.jedi_build_method("use_existing"), - qd.model_components(['geos_marine']), - ], - geos_marine=[ - qd.cycle_times(['T12']), - qd.marine_models(['mom6']), - qd.window_length("P1D"), - qd.horizontal_resolution("72x36"), - qd.vertical_resolution("50"), - qd.total_processors(6), - qd.obs_experiment("s2s_v1"), - qd.observations([ - "adt_cryosat2n", - "adt_jason3", - "adt_saral", - "adt_sentinel3a", - "adt_sentinel3b", - "insitu_profile_argo", - "sst_ostia", - "sss_smos", - "sss_smapv5", - "sst_abi_g16_l3c", - "sst_gmi_l3u", - "sst_viirs_n20_l3u", - "temp_profile_xbt" - ]), - qd.obs_provider(['odas', 'gdas_marine']), - qd.background_time_offset("PT18H"), - qd.clean_patterns(['*.nc4', '*.txt']), - ] - ) +from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_questions import marine + +_3dvar_base = QuestionList( + list_name="3dvar", + questions=[ + marine + ] +) + +suite_configs.register('3dvar', '3dvar_base', _3dvar_base) + +# -------------------------------------------------------------------------------------------------- + +_3dvar_tier1 = QuestionList( + list_name="3dvar", + questions=[ + _3dvar_base, + qd.start_cycle_point("2021-07-01T12:00:00Z"), + qd.final_cycle_point("2021-07-01T12:00:00Z"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_marine']), + ], + geos_marine=[ + qd.cycle_times(['T12']), + qd.marine_models(['mom6']), + qd.window_length("P1D"), + qd.horizontal_resolution("72x36"), + qd.vertical_resolution("50"), + qd.total_processors(6), + qd.obs_experiment("s2s_v1"), + qd.observations([ + "adt_cryosat2n", + "adt_jason3", + "adt_saral", + "adt_sentinel3a", + "adt_sentinel3b", + "insitu_profile_argo", + "sst_ostia", + "sss_smos", + "sss_smapv5", + "sst_abi_g16_l3c", + "sst_gmi_l3u", + "sst_viirs_n20_l3u", + "temp_profile_xbt" + ]), + qd.obs_provider(['odas', 'gdas_marine']), + qd.background_time_offset("PT18H"), + qd.clean_patterns(['*.nc4', '*.txt']), + ] +) + +suite_configs.register('3dvar', '3dvar_tier1', _3dvar_tier1) ``` The class `SuiteQuestions` contains lists of questions which are common to many suites. This avoids the need for redundantly setting the same questions for every suite. diff --git a/docs/examples/templating_workflows.md b/docs/examples/templating_workflows.md index 3f3213588..0ccc29c18 100644 --- a/docs/examples/templating_workflows.md +++ b/docs/examples/templating_workflows.md @@ -3,7 +3,7 @@ The `flow.cylc` file informs the `Cylc` workflow engine on how to run an experim ## Cylc sections -The `flow.cylc` that is generated under this method is not much different from the one generated before, and users shouldn't notice a difference when it comes to creating an experiment, using overrides, etc. When creating an experiment, `swell` consults a file `src/swell/suites//workflow.py` on how to construct the suite. This file should be an extension of the `CylcWorkflow` class (defined in `src/swell/utilities/cylc_workflow.py`). The method `get_workflow_string` is called to return a string which fills the contents of the `flow.cylc` file. Overriding this method is used to manually specify the contents of the file. Typically, the graph section is templated in `jinja2`, and the runtime sections for each task are generated using swell's `TaskAttribute` class. However, the entire `flow.cylc` file can be templated in `jinja`, if necessary. +The `flow.cylc` that is generated under this method is not much different from the one generated before, and users shouldn't notice a difference when it comes to creating an experiment, using overrides, etc. When creating an experiment, `swell` consults a file `src/swell/suites//workflow.py` on how to construct the suite. This file should be an extension of the `CylcWorkflow` class (defined in `src/swell/suites/base/cylc_workflow.py`). The method `get_workflow_string` is called to return a string which fills the contents of the `flow.cylc` file. Overriding this method is used to manually specify the contents of the file. Typically, the graph section is templated in `jinja2`, and the runtime sections for each task are generated using swell's `TaskSetup` class. However, the entire `flow.cylc` file can be templated in `jinja`, if necessary. ## Tasks and the runtime section From 583f6aa1bd65ae9a39cd7807bdb879e985890db4 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 12:16:43 -0500 Subject: [PATCH 221/299] Fix code tests --- src/swell/configuration/question_defaults.py | 30 +++++++++---------- src/swell/suites/3dfgat_atmos/suite_config.py | 2 +- src/swell/suites/3dvar_atmos/suite_config.py | 2 +- src/swell/suites/base/all_suites.py | 29 +++++++++--------- src/swell/suites/base/suite_questions.py | 2 -- src/swell/suites/hofx/suite_config.py | 2 +- src/swell/suites/hofx_cf/suite_config.py | 2 +- src/swell/suites/ingest_obs/suite_config.py | 2 +- .../suites/localensembleda/suite_config.py | 8 ++--- src/swell/suites/ufo_testing/suite_config.py | 4 +-- src/swell/tasks/base/task_setup.py | 1 + src/swell/utilities/plugins.py | 1 + .../utilities/scripts/check_question_order.py | 2 +- src/swell/utilities/swell_questions.py | 1 + 14 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/swell/configuration/question_defaults.py b/src/swell/configuration/question_defaults.py index 921835a3b..289dac224 100644 --- a/src/swell/configuration/question_defaults.py +++ b/src/swell/configuration/question_defaults.py @@ -394,7 +394,7 @@ class dry_run(TaskQuestion): ]) prompt: str = "Dry-run mode: preview what would be ingested before storing to R2D2" widget_type: WType = WType.BOOLEAN - + # -------------------------------------------------------------------------------------------------- @dataclass @@ -788,20 +788,6 @@ class horizontal_resolution(TaskQuestion): # ------------------------------------------------------------------------------------------------ - @dataclass - class obs_to_ingest(TaskQuestion): - default_value: list = mutable_field([]) - question_name: str = "obs_to_ingest" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which observations do you want to ingest to R2D2?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - @dataclass class ioda_locations_not_in_r2d2(TaskQuestion): default_value: str = "defer_to_platform" @@ -1132,6 +1118,20 @@ class obs_thinning_rej_fraction(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class obs_to_ingest(TaskQuestion): + default_value: list = mutable_field([]) + question_name: str = "obs_to_ingest" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which observations do you want to ingest to R2D2?" + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + @dataclass class observations(TaskQuestion): default_value: str = "defer_to_model" diff --git a/src/swell/suites/3dfgat_atmos/suite_config.py b/src/swell/suites/3dfgat_atmos/suite_config.py index 3ae15e47e..c2f875432 100644 --- a/src/swell/suites/3dfgat_atmos/suite_config.py +++ b/src/swell/suites/3dfgat_atmos/suite_config.py @@ -37,7 +37,7 @@ ]), qd.horizontal_resolution("91"), qd.geos_x_background_directory("/discover/nobackup/projects/gmao/" - "dadev/rtodling/archive/Restarts/JEDI/541x"), + "dadev/rtodling/archive/Restarts/JEDI/541x"), qd.window_type("4D"), qd.observations([ "aircraft_temperature", diff --git a/src/swell/suites/3dvar_atmos/suite_config.py b/src/swell/suites/3dvar_atmos/suite_config.py index 1b739e452..ff00118e2 100644 --- a/src/swell/suites/3dvar_atmos/suite_config.py +++ b/src/swell/suites/3dvar_atmos/suite_config.py @@ -36,7 +36,7 @@ "T18" ]), qd.geos_x_background_directory("/discover/nobackup/projects/gmao/" - "dadev/rtodling/archive/Restarts/JEDI/541x"), + "dadev/rtodling/archive/Restarts/JEDI/541x"), qd.window_length("PT6H"), qd.window_type("3D"), qd.horizontal_resolution("91"), diff --git a/src/swell/suites/base/all_suites.py b/src/swell/suites/base/all_suites.py index bec2a0f05..24a21b27d 100644 --- a/src/swell/suites/base/all_suites.py +++ b/src/swell/suites/base/all_suites.py @@ -8,11 +8,6 @@ # -------------------------------------------------------------------------------------------------- -import os -from importlib import import_module - -from swell.swell_path import get_swell_path -from swell.utilities.suite_utils import get_suites from swell.suites.base.cylc_workflow import CylcWorkflow from swell.utilities.swell_questions import QuestionList import swell.suites @@ -27,6 +22,7 @@ def format_suite_name(suite_name): # -------------------------------------------------------------------------------------------------- + class Workflows(): def __init__(self) -> None: @@ -34,14 +30,15 @@ def __init__(self) -> None: def register(self, name: str) -> None: self.__workflow_names__.append(name) + def wrapper(cls): setattr(self, name, cls) return cls return wrapper - + def get(self, name: str) -> type[CylcWorkflow]: return getattr(self, name) - + def all(self) -> list: return self.__workflow_names__ @@ -54,34 +51,34 @@ def __init__(self) -> None: # Dictionary tracking the suite for each config self.__config_map__ = {} - + # -------------------------------------------------------------------------------------------------- - + def register(self, base_suite: str, config_name: str, question_list: QuestionList) -> None: - + self.__config_map__[config_name] = sub_dict = {} sub_dict['suite'] = base_suite sub_dict['list'] = question_list - + # -------------------------------------------------------------------------------------------------- def get_config(self, config_name: str) -> QuestionList: return self.__config_map__[config_name]['list'] - + # -------------------------------------------------------------------------------------------------- - + def base_suite(self, config_name: str) -> str: return self.__config_map__[config_name]['suite'] # -------------------------------------------------------------------------------------------------- - + def all_configs(self) -> str: return list(self.__config_map__.keys()) - + # -------------------------------------------------------------------------------------------------- def configs_under_suites(self) -> dict: @@ -99,9 +96,11 @@ def configs_under_suites(self) -> dict: # -------------------------------------------------------------------------------------------------- + # Objects to reference in imports suite_configs = SuiteConfigs() workflows = Workflows() + discover_plugins(swell.suites) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/base/suite_questions.py b/src/swell/suites/base/suite_questions.py index a0d408e88..504a48e73 100644 --- a/src/swell/suites/base/suite_questions.py +++ b/src/swell/suites/base/suite_questions.py @@ -8,8 +8,6 @@ # -------------------------------------------------------------------------------------------------- -from enum import Enum - from swell.utilities.swell_questions import QuestionList from swell.configuration.question_defaults import QuestionDefaults as qd from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/hofx/suite_config.py b/src/swell/suites/hofx/suite_config.py index b7a92c74a..eefc0c107 100644 --- a/src/swell/suites/hofx/suite_config.py +++ b/src/swell/suites/hofx/suite_config.py @@ -31,7 +31,7 @@ geos_atmosphere=[ qd.horizontal_resolution("91"), qd.geos_x_background_directory("/discover/nobackup/projects/gmao/dadev/" - "rtodling/archive/Restarts/JEDI/541x"), + "rtodling/archive/Restarts/JEDI/541x"), qd.npx_proc(2), qd.npy_proc(2), qd.observations([ diff --git a/src/swell/suites/hofx_cf/suite_config.py b/src/swell/suites/hofx_cf/suite_config.py index 621930554..37bab4e15 100644 --- a/src/swell/suites/hofx_cf/suite_config.py +++ b/src/swell/suites/hofx_cf/suite_config.py @@ -46,4 +46,4 @@ suite_configs.register(suite_name, 'hofx_cf_tier1', hofx_cf_tier1) -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ingest_obs/suite_config.py b/src/swell/suites/ingest_obs/suite_config.py index ce82e29e9..642a1db83 100644 --- a/src/swell/suites/ingest_obs/suite_config.py +++ b/src/swell/suites/ingest_obs/suite_config.py @@ -50,4 +50,4 @@ suite_configs.register(suite_name, 'ingest_obs_marine', ingest_obs_marine) -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index f91bb9fec..8b6fe7313 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -33,9 +33,9 @@ qd.horizontal_resolution('91'), qd.background_experiment('x0050'), qd.geos_x_background_directory('/discover/nobackup/projects/gmao/dadev/' - 'rtodling/archive/Restarts/JEDI/541x'), + 'rtodling/archive/Restarts/JEDI/541x'), qd.geos_x_ensemble_directory('/discover/nobackup/projects/gmao/dadev/' - 'rtodling/archive/541/Milan'), + 'rtodling/archive/541/Milan'), qd.npx_proc(3), qd.npy_proc(3), qd.cycle_times(['T00']), @@ -76,9 +76,9 @@ qd.horizontal_resolution('91'), qd.background_experiment('x0050'), qd.geos_x_background_directory('/discover/nobackup/projects/gmao/dadev/' - 'rtodling/archive/Restarts/JEDI/541x'), + 'rtodling/archive/Restarts/JEDI/541x'), qd.geos_x_ensemble_directory('/discover/nobackup/projects/gmao/dadev/' - 'rtodling/archive/541/Milan'), + 'rtodling/archive/541/Milan'), qd.npx_proc(4), qd.npy_proc(4), # qd.perhost(32), diff --git a/src/swell/suites/ufo_testing/suite_config.py b/src/swell/suites/ufo_testing/suite_config.py index d71fa437e..d60f74918 100644 --- a/src/swell/suites/ufo_testing/suite_config.py +++ b/src/swell/suites/ufo_testing/suite_config.py @@ -75,8 +75,8 @@ "gsi_ncdiags" ]), qd.path_to_gsi_bc_coefficients("/discover/nobackup/projects/gmao/dadev/rtodling/" - "archive/541/Milan/x0050/ana/Y%Y/M%m/" - "*bias*%Y%m%d_%Hz.txt"), + "archive/541/Milan/x0050/ana/Y%Y/M%m/" + "*bias*%Y%m%d_%Hz.txt"), qd.path_to_gsi_nc_diags("/discover/nobackup/projects/gmao/dadev/rtodling/archive/" "541/Milan/x0050/obs/Y%Y/M%m/D%d/H%H/"), ] diff --git a/src/swell/tasks/base/task_setup.py b/src/swell/tasks/base/task_setup.py index fbd7a2571..1d5996cf4 100644 --- a/src/swell/tasks/base/task_setup.py +++ b/src/swell/tasks/base/task_setup.py @@ -20,6 +20,7 @@ blank_spec = 'BLANKSPEC' + class TaskSetup(ABC): ''' diff --git a/src/swell/utilities/plugins.py b/src/swell/utilities/plugins.py index 7e07b1ac9..9088f5e1f 100644 --- a/src/swell/utilities/plugins.py +++ b/src/swell/utilities/plugins.py @@ -11,6 +11,7 @@ # -------------------------------------------------------------------------------------------------- + def discover_plugins(package): '''Walk through packages to trigger any hooks. diff --git a/src/swell/utilities/scripts/check_question_order.py b/src/swell/utilities/scripts/check_question_order.py index d36ae1e78..bcd1f425a 100644 --- a/src/swell/utilities/scripts/check_question_order.py +++ b/src/swell/utilities/scripts/check_question_order.py @@ -16,7 +16,7 @@ def check_question_order(): - question_file = os.path.join(get_swell_path(), 'utilities', 'question_defaults.py') + question_file = os.path.join(get_swell_path(), 'configuration', 'question_defaults.py') with open(question_file, 'r') as f: lines = f.readlines() diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index afc19d7fa..066453b4a 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -116,6 +116,7 @@ class SwellQuestion: # -------------------------------------------------------------------------------------------------- + @dataclass class QuestionList: """Basic dataclass containing a list of questions for each model, suite, task""" From 36903ef04938c96baa43cffde894ddcb47c8c80a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 12:22:03 -0500 Subject: [PATCH 222/299] Refactor question defaults --- src/swell/configuration/question_defaults.py | 2961 ++++++++--------- src/swell/suites/3dfgat_atmos/suite_config.py | 2 +- src/swell/suites/3dfgat_cycle/suite_config.py | 2 +- src/swell/suites/3dvar/suite_config.py | 2 +- src/swell/suites/3dvar_atmos/suite_config.py | 2 +- src/swell/suites/3dvar_cycle/suite_config.py | 2 +- src/swell/suites/base/suite_questions.py | 2 +- src/swell/suites/compare/suite_config.py | 2 +- src/swell/suites/convert_bufr/suite_config.py | 2 +- .../suites/convert_ncdiags/suite_config.py | 2 +- .../suites/eva_capabilities/suite_config.py | 2 +- .../suites/forecast_geos/suite_config.py | 2 +- src/swell/suites/geosadas/suite_config.py | 2 +- src/swell/suites/hofx/suite_config.py | 2 +- src/swell/suites/hofx_cf/suite_config.py | 2 +- src/swell/suites/ingest_obs/suite_config.py | 2 +- .../suites/localensembleda/suite_config.py | 2 +- src/swell/suites/ufo_testing/suite_config.py | 2 +- src/swell/tasks/build_geos.py | 2 +- src/swell/tasks/build_geos_by_linking.py | 2 +- src/swell/tasks/build_jedi.py | 2 +- src/swell/tasks/build_jedi_by_linking.py | 2 +- src/swell/tasks/clean_cycle.py | 2 +- src/swell/tasks/clone_geos.py | 2 +- src/swell/tasks/clone_geos_mksi.py | 2 +- src/swell/tasks/clone_gmao_perllib.py | 2 +- src/swell/tasks/clone_jedi.py | 2 +- src/swell/tasks/eva_comparison_increment.py | 2 +- src/swell/tasks/eva_comparison_jedi_log.py | 2 +- .../tasks/eva_comparison_observations.py | 2 +- src/swell/tasks/eva_increment.py | 2 +- src/swell/tasks/eva_observations.py | 2 +- src/swell/tasks/eva_timeseries.py | 2 +- src/swell/tasks/generate_b_climatology.py | 2 +- .../generate_b_climatology_by_linking.py | 2 +- .../generate_observing_system_records.py | 2 +- src/swell/tasks/get_background.py | 2 +- .../tasks/get_background_geos_experiment.py | 2 +- src/swell/tasks/get_bufr.py | 2 +- src/swell/tasks/get_ensemble.py | 2 +- .../tasks/get_ensemble_geos_experiment.py | 2 +- src/swell/tasks/get_geos_adas_background.py | 2 +- src/swell/tasks/get_geos_restart.py | 2 +- src/swell/tasks/get_geovals.py | 2 +- src/swell/tasks/get_gsi_bc.py | 2 +- src/swell/tasks/get_gsi_ncdiag.py | 2 +- src/swell/tasks/get_ncdiags.py | 2 +- src/swell/tasks/get_obs_not_in_r2d2.py | 2 +- src/swell/tasks/get_observations.py | 2 +- src/swell/tasks/gsi_bc_to_ioda.py | 2 +- src/swell/tasks/gsi_ncdiag_to_ioda.py | 2 +- src/swell/tasks/ingest_obs.py | 2 +- src/swell/tasks/jedi_log_comparison.py | 2 +- src/swell/tasks/jedi_oops_log_parser.py | 2 +- src/swell/tasks/link_geos_output.py | 2 +- src/swell/tasks/move_da_restart.py | 2 +- src/swell/tasks/move_forecast_restart.py | 2 +- src/swell/tasks/prep_geos_run_dir.py | 2 +- src/swell/tasks/prepare_analysis.py | 2 +- src/swell/tasks/render_jedi_observations.py | 2 +- ...jedi_convert_state_soca2cice_executable.py | 2 +- .../tasks/run_jedi_ensemble_mean_variance.py | 2 +- src/swell/tasks/run_jedi_fgat_executable.py | 2 +- .../run_jedi_hofx_ensemble_executable.py | 2 +- src/swell/tasks/run_jedi_hofx_executable.py | 2 +- .../run_jedi_local_ensemble_da_executable.py | 2 +- .../tasks/run_jedi_obsfilters_executable.py | 2 +- .../tasks/run_jedi_ufo_tests_executable.py | 2 +- .../tasks/run_jedi_variational_executable.py | 2 +- src/swell/tasks/save_obs_diags.py | 2 +- src/swell/tasks/save_restart.py | 2 +- src/swell/tasks/stage_jedi.py | 2 +- src/swell/tasks/store_background.py | 2 +- 73 files changed, 1550 insertions(+), 1555 deletions(-) diff --git a/src/swell/configuration/question_defaults.py b/src/swell/configuration/question_defaults.py index 289dac224..a46720f2c 100644 --- a/src/swell/configuration/question_defaults.py +++ b/src/swell/configuration/question_defaults.py @@ -15,1491 +15,1486 @@ from swell.utilities.swell_questions import WidgetType as WType from swell.utilities.dataclass_utils import mutable_field +# -------------------------------------------------------------------------------------------------- +# Suite question defaults go here +# -------------------------------------------------------------------------------------------------- + +@dataclass +class comparison_experiment_paths(SuiteQuestion): + default_value: list = mutable_field([]) + question_name: str = "comparison_experiment_paths" + ask_question: bool = True + prompt: str = "Provide paths to two experiments to run comparison tests on." + widget_type: WType = WType.STRING_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class cycle_times(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "cycle_times" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter the cycle times for this model." + widget_type: WType = WType.STRING_CHECK_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class cycling_varbc(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "cycling_varbc" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Do you want to use cycling VarBC option?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class email_address(SuiteQuestion): + default_value: str = "defer_to_user" + question_name: str = "email_address" + prompt: str = "What email address should cylc messages be sent to?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensemble_hofx_packets(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "ensemble_hofx_packets" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter the number of ensemble packets." + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensemble_hofx_strategy(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "ensemble_hofx_strategy" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter the ensemble hofx strategy." + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class experiment_id(SuiteQuestion): + default_value: str = "defer_to_code" + question_name: str = "experiment_id" + ask_question: bool = True + prompt: str = "What is the experiment id?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class experiment_root(SuiteQuestion): + default_value: str = "defer_to_platform" + question_name: str = "experiment_root" + ask_question: bool = True + prompt: str = ("What is the experiment root (the directory where the " + "experiment will be stored)?") + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class final_cycle_point(SuiteQuestion): + default_value: str = "2023-10-10T06:00:00Z" + question_name: str = "final_cycle_point" + ask_question: bool = True + prompt: str = "What is the time of the final cycle (middle of the window)?" + widget_type: WType = WType.ISO_DATETIME + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class marine_models(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "marine_models" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_marine" + ]) + prompt: str = "Select the active SOCA models for this model." + widget_type: WType = WType.STRING_CHECK_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class model_components(SuiteQuestion): + default_value: str = "defer_to_code" + question_name: str = "model_components" + ask_question: bool = True + options: str = "defer_to_code" + prompt: str = "Enter the model components for this model." + widget_type: WType = WType.STRING_CHECK_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class parser_options(SuiteQuestion): + default_value: list = mutable_field(['fgrep_residual_norm']) + question_name: str = "parser_options" + ask_question: bool = True + options: list = mutable_field(['fgrep_residual_norm']) + prompt: str = "List the test types to run on the JEDI oops log." + widget_type: WType = WType.STRING_CHECK_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class runahead_limit(SuiteQuestion): + default_value: str = "P4" + question_name: str = "runahead_limit" + ask_question: bool = True + prompt: str = ("Since this suite is non-cycling choose how " + "many hours the workflow can run ahead?") + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class skip_ensemble_hofx(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "skip_ensemble_hofx" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter if skip ensemble hofx." + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class start_cycle_point(SuiteQuestion): + default_value: str = "2023-10-10T00:00:00Z" + question_name: str = "start_cycle_point" + ask_question: bool = True + prompt: str = "What is the time of the first cycle (middle of the window)?" + widget_type: WType = WType.ISO_DATETIME + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class use_cycle_dir(SuiteQuestion): + default_value: bool = True + question_name: str = "use_cycle_dir" + ask_question: bool = False + prompt: str = ("For cycling tasks, send results to the experiment cycle directory?" + " If false, results will be stored in the current working directory.") + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class window_type(SuiteQuestion): + default_value: str = "defer_to_model" + question_name: str = "window_type" + options: List[str] = mutable_field([ + "3D", + "4D" + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter the window type for this model." + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- +# Task question defaults go here +# -------------------------------------------------------------------------------------------------- + +@dataclass +class analysis_variables(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "analysis_variables" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What are the analysis variables?" + widget_type: WType = WType.STRING_CHECK_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class background_error_model(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "background_error_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which background error model do you want to use?" + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class background_experiment(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "background_experiment" + ask_question: bool = True + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the name of the name of the experiment providing the backgrounds?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class background_frequency(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "background_frequency" + models: List[str] = mutable_field([ + "all_models" + ]) + depends: Dict = mutable_field({ + "window_type": "4D" + }) + prompt: str = "What is the frequency of the background files?" + widget_type: WType = WType.ISO_DURATION + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class background_time_offset(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "background_time_offset" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = ("How long before the middle of the analysis window did" + " the background providing forecast begin?") + widget_type: WType = WType.ISO_DURATION + +# -------------------------------------------------------------------------------------------------- +@dataclass +class bufr_obs_classes(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "bufr_obs_classes" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What BUFR observation classes will be used?" + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class bundles(TaskQuestion): + default_value: List[str] = mutable_field([ + "fv3-jedi", + "soca", + "iodaconv", + "ufo" + ]) + question_name: str = "bundles" + ask_question: bool = True + options: List[str] = mutable_field([ + "fv3-jedi", + "soca", + "iodaconv", + "ufo", + "ioda", + "oops", + "saber" + ]) + depends: Dict = mutable_field({ + "jedi_build_method": "create" + }) + prompt: str = "Which JEDI bundles do you wish to build?" + widget_type: WType = WType.STRING_CHECK_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class check_for_obs(TaskQuestion): + default_value: bool = True + question_name: str = "check_for_obs" + options: List[bool] = mutable_field([True, False]) + models: List[str] = mutable_field([ + 'all_models' + ]) + prompt: str = "Perform check for observations? Set to false for debugging purposes." + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class clean_patterns(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "clean_patterns" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Provide a list of patterns that you wish to remove from the cycle directory." + widget_type: WType = WType.STRING_CHECK_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class comparison_log_type(TaskQuestion): + default_value: str = "variational" + question_name: str = "comparison_log_type" + options: List[str] = mutable_field([ + 'variational', + 'fgat', + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Provide the log naming convention (e.g. 'variational', 'fgat')." + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class crtm_coeff_dir(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "crtm_coeff_dir" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to the CRTM coefficient files?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class dry_run(TaskQuestion): + default_value: bool = True + question_name: str = "dry_run" + ask_question: bool = False + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Dry-run mode: preview what would be ingested before storing to R2D2" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensemble_hofx_packets(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "ensemble_hofx_packets" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Enter number of packets in which ensemble observers should be computed." + widget_type: WType = WType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensemble_hofx_strategy(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "ensemble_hofx_strategy" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Enter hofx strategy." + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensemble_num_members(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "ensemble_num_members" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "How many members comprise the ensemble?" + widget_type: WType = WType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensmean_only(TaskQuestion): + default_value: bool = False + question_name: str = "ensmean_only" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Calculate ensemble mean only?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensmeanvariance_only(TaskQuestion): + default_value: bool = False + question_name: str = "ensmeanvariance_only" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Calculate ensemble mean and variance only?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_geos_gcm_build_path(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "existing_geos_gcm_build_path" + ask_question: bool = True + depends: Dict = mutable_field({ + "geos_build_method": "use_existing" + }) + prompt: str = "What is the path to the existing GEOS build directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_geos_gcm_source_path(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "existing_geos_gcm_source_path" + ask_question: bool = True + depends: Dict = mutable_field({ + "geos_build_method": "use_existing" + }) + prompt: str = "What is the path to the existing GEOS source code directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_jedi_build_directory(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "existing_jedi_build_directory" + ask_question: bool = True + depends: Dict = mutable_field({ + "jedi_build_method": "use_existing" + }) + prompt: str = "What is the path to the existing JEDI build directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_jedi_build_directory_pinned(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "existing_jedi_build_directory_pinned" + ask_question: bool = True + depends: Dict = mutable_field({ + "jedi_build_method": "use_pinned_existing" + }) + prompt: str = "What is the path to the existing pinned JEDI build directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_jedi_source_directory(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "existing_jedi_source_directory" + ask_question: bool = True + depends: Dict = mutable_field({ + "jedi_build_method": "use_existing" + }) + prompt: str = "What is the path to the existing JEDI source code directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_jedi_source_directory_pinned(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "existing_jedi_source_directory_pinned" + ask_question: bool = True + depends: Dict = mutable_field({ + "jedi_build_method": "use_pinned_existing" + }) + prompt: str = "What is the path to the existing pinned JEDI source code directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_perllib_path(TaskQuestion): + default_value: str = 'defer_to_platform' + question_name: str = 'existing_perllib_path' + prompt: str = "Provide a path to an existing location for GMAO_perllib." + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class forecast_duration(TaskQuestion): + default_value: str = "PT12H" + question_name: str = "forecast_duration" + ask_question: bool = True + prompt: str = "GEOS forecast duration" + widget_type: WType = WType.ISO_DURATION + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class generate_yaml_and_exit(TaskQuestion): + default_value: bool = False + question_name: str = "generate_yaml_and_exit" + prompt: str = "Generate JEDI executable YAML and exit?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_build_method(TaskQuestion): + default_value: str = "create" + question_name: str = "geos_build_method" + ask_question: bool = True + options: List[str] = mutable_field([ + "use_existing", + "create" + ]) + prompt: str = "Do you want to use an existing GEOS build or create a new build?" + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_experiment_directory(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "geos_experiment_directory" + ask_question: bool = True + prompt: str = "What is the path to the GEOS restarts directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_gcm_tag(TaskQuestion): + default_value: str = "v11.6.0" + question_name: str = "geos_gcm_tag" + ask_question: bool = True + prompt: str = "Which GEOS tag do you wish to clone?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_restarts_directory(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "geos_restarts_directory" + ask_question: bool = True + prompt: str = "What is the path to the GEOS restarts directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_x_background_directory(TaskQuestion): + default_value: str = "/dev/null/" + question_name: str = "geos_x_background_directory" + ask_question: bool = True + options: List[str] = mutable_field([ + "/dev/null/", + "/discover/nobackup/projects/gmao/dadev/rtodling/archive/Restarts/JEDI/541x" + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the path to the GEOS X-backgrounds directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_x_ensemble_directory(TaskQuestion): + default_value: str = "/dev/null/" + question_name: str = "geos_x_ensemble_directory" + ask_question: bool = True + options: List[str] = mutable_field([ + "/dev/null/", + "/gpfsm/dnb05/projects/p139/rtodling/archive/" + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to the GEOS X-backgrounds directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geovals_experiment(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "geovals_experiment" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the name of the R2D2 experiment providing the GeoVaLs?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geovals_provider(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "geovals_provider" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the name of the R2D2 database providing the GeoVaLs?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class gmao_perllib_tag(TaskQuestion): + default_value: str = 'g1.0.1' + question_name: str = 'gmao_perllib_tag' + prompt: str = "Specify the tag at which GMAO_perllib should be cloned." + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class gradient_norm_reduction(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "gradient_norm_reduction" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What value of gradient norm reduction for convergence?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class gsibec_configuration(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "gsibec_configuration" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which GSIBEC climatological or hybrid?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class gsibec_nlats(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "gsibec_nlats" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "How many number of latutides in GSIBEC grid?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class gsibec_nlons(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "gsibec_nlons" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "How many number of longitudes in GSIBEC grid?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class horizontal_localization_lengthscale(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "horizontal_localization_lengthscale" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the length scale for horizontal covariance localization?" + widget_type: WType = WType.FLOAT + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class horizontal_localization_max_nobs(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "horizontal_localization_max_nobs" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("What is the maximum number of observations to consider" + " for horizontal covariance localization?") + widget_type: WType = WType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class horizontal_localization_method(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "horizontal_localization_method" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which localization scheme should be applied in the horizontal?" + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class horizontal_resolution(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "horizontal_resolution" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the horizontal resolution for the forecast model and backgrounds?" + widget_type: WType = WType.STRING_DROP_LIST + +# ------------------------------------------------------------------------------------------------ + +@dataclass +class ioda_locations_not_in_r2d2(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "ioda_locations_not_in_r2d2" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ( + "Provide a path that contains observation files not in r2d2.") + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class jedi_build_method(TaskQuestion): + default_value: str = "create" + question_name: str = "jedi_build_method" + ask_question: bool = True + options: List[str] = mutable_field([ + "use_existing", + "use_pinned_existing", + "create", + "pinned_create" + ]) + prompt: str = "Do you want to use an existing JEDI build or create a new build?" + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class jedi_forecast_model(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "jedi_forecast_model" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + depends: Dict = mutable_field({ + "window_type": "4D" + }) + prompt: str = "What forecast model should be used within JEDI for 4D window propagation?" + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_inflation_mult(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "local_ensemble_inflation_mult" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Specify the multiplicative prior inflation coefficient (0 inf]." + widget_type: WType = WType.FLOAT + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_inflation_rtpp(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "local_ensemble_inflation_rtpp" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Specify the Relaxation To Prior Perturbation (RTPP) coefficient (0 1]." + widget_type: WType = WType.FLOAT + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_inflation_rtps(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "local_ensemble_inflation_rtps" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Specify the Relaxation To Prior Spread (RTPS) coefficient (0 1]." + widget_type: WType = WType.FLOAT + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_save_posterior_ensemble(TaskQuestion): + default_value: bool = False + question_name: str = "local_ensemble_save_posterior_ensemble" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Save the posterior ensemble members?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_save_posterior_ensemble_increments(TaskQuestion): + default_value: bool = False + question_name: str = "local_ensemble_save_posterior_ensemble_increments" + ask_question: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Save the posterior ensemble member increments?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_save_posterior_mean(TaskQuestion): + default_value: bool = False + question_name: str = "local_ensemble_save_posterior_mean" + ask_question: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Save the posterior ensemble mean?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_save_posterior_mean_increment(TaskQuestion): + default_value: bool = True + question_name: str = "local_ensemble_save_posterior_mean_increment" + ask_question: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Save the posterior ensemble mean increment?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_solver(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "local_ensemble_solver" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which local ensemble solver type should be implemented?" + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_use_linear_observer(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "local_ensemble_use_linear_observer" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which local ensemble solver type should be implemented?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class minimizer(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "minimizer" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which data assimilation minimizer do you wish to use?" + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class mom6_iau(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "mom6_iau" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_marine", + ]) + prompt: str = "Do you wish to use IAU for MOM6?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class mom6_iau_nhours(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "mom6_iau_nhours" + options: List[str] = mutable_field([ + 'PT3H', + 'PT12H' + ]) + depends: dict = mutable_field({'mom6_iau': True}) + models: List[str] = mutable_field([ + "geos_marine", + ]) + prompt: str = "What is the IAU length (ODA_INCUPD_NHOURS) for MOM6?" + widget_type: WType = WType.ISO_DURATION + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ncdiag_experiments(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "ncdiag_experiments" + options: List[str] = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which previously run experiments do you wish to use for the NCdiag?" + widget_type: WType = WType.STRING_CHECK_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class npx(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "npx" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What is the number of grid points in the x-direction on each cube face?" + widget_type: WType = WType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class npx_proc(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "npx_proc" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere", + "geos_cf" + ]) + prompt: str = "What number of processors do you wish to use in the x-direction?" + widget_type: WType = WType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class npy(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "npy" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What is the number of grid points in the y-direction on each cube face?" + widget_type: WType = WType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class npy_proc(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "npy_proc" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere", + "geos_cf" + ]) + prompt: str = "What number of processors do you wish to use in the y-direction?" + widget_type: WType = WType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class number_of_iterations(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "number_of_iterations" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = ( + "What number of iterations do you wish to use for each outer loop?" + " Provide a list of integers the same length as the number of outer loops.") + widget_type: WType = WType.INTEGER_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class obs_experiment(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "obs_experiment" + ask_question: bool = True + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the database providing the observations?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class obs_thinning_rej_fraction(TaskQuestion): + default_value: float = 0.75 + question_name: str = "obs_thinning_rej_fraction" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the rejection fraction for obs thinning?" + widget_type: WType = WType.FLOAT + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class obs_to_ingest(TaskQuestion): + default_value: list = mutable_field([]) + question_name: str = "obs_to_ingest" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which observations do you want to ingest to R2D2?" + widget_type: WType = WType.STRING_CHECK_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class observations(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "observations" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which observations do you want to include?" + widget_type: WType = WType.STRING_CHECK_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class observing_system_records_mksi_path(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "observing_system_records_mksi_path" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to the GSI formatted observing system records?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class observing_system_records_mksi_path_tag(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "observing_system_records_mksi_path_tag" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the GSI formatted observing system records tag?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class observing_system_records_path(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "observing_system_records_path" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to the Swell formatted observing system records?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class path_to_ensemble(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "path_to_ensemble" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere", + "geos_marine" + ]) + prompt: str = "What is the path to where ensemble members are stored?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class path_to_geos_adas_background(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "path_to_geos_adas_background" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ( + "What is the path for the GEOSadas cubed sphere backgrounds?") + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class path_to_gsi_bc_coefficients(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "path_to_gsi_bc_coefficients" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the location where GSI bias correction files can be found?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class path_to_gsi_nc_diags(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "path_to_gsi_nc_diags" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to where the GSI ncdiags are stored?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class pause_on_tasks(TaskQuestion): + default_value: list = mutable_field([]) + question_name: str = "pause_on_tasks" + ask_question: bool = False + prompt: str = ("Specify any tasks that the workflow should pause on " + "(for development purposes).") + widget_type: WType = WType.STRING_CHECK_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class perhost(TaskQuestion): + default_value: str = None + question_name: str = "perhost" + ask_question: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the number of processors per host?" + widget_type: WType = WType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class produce_geovals(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "produce_geovals" + ask_question: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("When running the ncdiag to ioda converted do you " + "want to produce GeoVaLs files?") + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class r2d2_local_path(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "r2d2_local_path" + prompt: str = "What is the path to the R2D2 local directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class save_geovals(TaskQuestion): + default_value: bool = False + question_name: str = "save_geovals" + options: List[bool] = mutable_field([ + True, + False + ]) + prompt: str = "When running hofx do you want to output the GeoVaLs?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class set_obs_as_local(TaskQuestion): + default_value: bool = False + question_name: str = "set_obs_as_local" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + 'all_models' + ]) + prompt: str = "Treat observations as 'local' to the directory?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class single_observations(TaskQuestion): + default_value: bool = False + question_name: str = "single_observations" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Is it a single-observation test?" + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class swell_static_files(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "swell_static_files" + prompt: str = "What is the path to the Swell Static files directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class swell_static_files_user(TaskQuestion): + default_value: str = "None" + question_name: str = "swell_static_files_user" + prompt: str = "What is the path to the user provided Swell Static Files directory?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class task_email_parameters(TaskQuestion): + default_value: Union[Literal["auto"], dict] = "auto" + question_name: str = "task_email_parameters" + prompt: str = ("Provide a dictionary mapping tasks to cylc event statuses, or 'auto' to " + "automatically configure these based on the graph.") + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class total_processors(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "total_processors" + ask_question: bool = True + models: List[str] = mutable_field([ + "geos_marine", + ]) + prompt: str = "What is the number of processors for JEDI?" + widget_type: WType = WType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_localization_apply_log_transform(TaskQuestion): + default_value: bool = True + question_name: str = "vertical_localization_apply_log_transform" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("Should a log (base 10) transformation be applied " + "to vertical coordinate when " + "constructing vertical localization?") + widget_type: WType = WType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_localization_function(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_localization_function" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which localization scheme should be applied in the vertical?" + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_localization_ioda_vertical_coord(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_localization_ioda_vertical_coord" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which coordinate should be used in constructing vertical localization?" + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_localization_ioda_vertical_coord_group(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_localization_ioda_vertical_coord_group" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("Which vertical coordinate group should be used " + "in constructing vertical localization?") + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_localization_lengthscale(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_localization_lengthscale" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the length scale for vertical covariance localization?" + widget_type: WType = WType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_localization_method(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_localization_method" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("What localization scheme should be applied in " + "constructing a vertical localization?") + widget_type: WType = WType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_resolution(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "vertical_resolution" + ask_question: bool = True + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the vertical resolution for the forecast model and background?" + widget_type: WType = WType.STRING_DROP_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class window_length(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "window_length" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the duration for the data assimilation window?" + widget_type: WType = WType.ISO_DURATION # -------------------------------------------------------------------------------------------------- -class QuestionDefaults(): - - # -------------------------------------------------------------------------------------------------- - # Suite question defaults go here - # -------------------------------------------------------------------------------------------------- - - @dataclass - class comparison_experiment_paths(SuiteQuestion): - default_value: list = mutable_field([]) - question_name: str = "comparison_experiment_paths" - ask_question: bool = True - prompt: str = "Provide paths to two experiments to run comparison tests on." - widget_type: WType = WType.STRING_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class cycle_times(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "cycle_times" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the cycle times for this model." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class cycling_varbc(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "cycling_varbc" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Do you want to use cycling VarBC option?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class email_address(SuiteQuestion): - default_value: str = "defer_to_user" - question_name: str = "email_address" - prompt: str = "What email address should cylc messages be sent to?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_packets(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_packets" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the number of ensemble packets." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_strategy(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_strategy" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the ensemble hofx strategy." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class experiment_id(SuiteQuestion): - default_value: str = "defer_to_code" - question_name: str = "experiment_id" - ask_question: bool = True - prompt: str = "What is the experiment id?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class experiment_root(SuiteQuestion): - default_value: str = "defer_to_platform" - question_name: str = "experiment_root" - ask_question: bool = True - prompt: str = ("What is the experiment root (the directory where the " - "experiment will be stored)?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class final_cycle_point(SuiteQuestion): - default_value: str = "2023-10-10T06:00:00Z" - question_name: str = "final_cycle_point" - ask_question: bool = True - prompt: str = "What is the time of the final cycle (middle of the window)?" - widget_type: WType = WType.ISO_DATETIME - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class marine_models(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "marine_models" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_marine" - ]) - prompt: str = "Select the active SOCA models for this model." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class model_components(SuiteQuestion): - default_value: str = "defer_to_code" - question_name: str = "model_components" - ask_question: bool = True - options: str = "defer_to_code" - prompt: str = "Enter the model components for this model." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class parser_options(SuiteQuestion): - default_value: list = mutable_field(['fgrep_residual_norm']) - question_name: str = "parser_options" - ask_question: bool = True - options: list = mutable_field(['fgrep_residual_norm']) - prompt: str = "List the test types to run on the JEDI oops log." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class runahead_limit(SuiteQuestion): - default_value: str = "P4" - question_name: str = "runahead_limit" - ask_question: bool = True - prompt: str = ("Since this suite is non-cycling choose how " - "many hours the workflow can run ahead?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class skip_ensemble_hofx(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "skip_ensemble_hofx" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter if skip ensemble hofx." - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class start_cycle_point(SuiteQuestion): - default_value: str = "2023-10-10T00:00:00Z" - question_name: str = "start_cycle_point" - ask_question: bool = True - prompt: str = "What is the time of the first cycle (middle of the window)?" - widget_type: WType = WType.ISO_DATETIME - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class use_cycle_dir(SuiteQuestion): - default_value: bool = True - question_name: str = "use_cycle_dir" - ask_question: bool = False - prompt: str = ("For cycling tasks, send results to the experiment cycle directory?" - " If false, results will be stored in the current working directory.") - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class window_type(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "window_type" - options: List[str] = mutable_field([ - "3D", - "4D" - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the window type for this model." - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - # Task question defaults go here - # -------------------------------------------------------------------------------------------------- - - @dataclass - class analysis_variables(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "analysis_variables" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What are the analysis variables?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_error_model(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "background_error_model" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which background error model do you want to use?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_experiment(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "background_experiment" - ask_question: bool = True - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the name of the name of the experiment providing the backgrounds?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_frequency(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "background_frequency" - models: List[str] = mutable_field([ - "all_models" - ]) - depends: Dict = mutable_field({ - "window_type": "4D" - }) - prompt: str = "What is the frequency of the background files?" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_time_offset(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "background_time_offset" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = ("How long before the middle of the analysis window did" - " the background providing forecast begin?") - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - @dataclass - class bufr_obs_classes(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "bufr_obs_classes" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What BUFR observation classes will be used?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class bundles(TaskQuestion): - default_value: List[str] = mutable_field([ - "fv3-jedi", - "soca", - "iodaconv", - "ufo" - ]) - question_name: str = "bundles" - ask_question: bool = True - options: List[str] = mutable_field([ - "fv3-jedi", - "soca", - "iodaconv", - "ufo", - "ioda", - "oops", - "saber" - ]) - depends: Dict = mutable_field({ - "jedi_build_method": "create" - }) - prompt: str = "Which JEDI bundles do you wish to build?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class check_for_obs(TaskQuestion): - default_value: bool = True - question_name: str = "check_for_obs" - options: List[bool] = mutable_field([True, False]) - models: List[str] = mutable_field([ - 'all_models' - ]) - prompt: str = "Perform check for observations? Set to false for debugging purposes." - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class clean_patterns(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "clean_patterns" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Provide a list of patterns that you wish to remove from the cycle directory." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class comparison_log_type(TaskQuestion): - default_value: str = "variational" - question_name: str = "comparison_log_type" - options: List[str] = mutable_field([ - 'variational', - 'fgat', - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Provide the log naming convention (e.g. 'variational', 'fgat')." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class crtm_coeff_dir(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "crtm_coeff_dir" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to the CRTM coefficient files?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class dry_run(TaskQuestion): - default_value: bool = True - question_name: str = "dry_run" - ask_question: bool = False - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Dry-run mode: preview what would be ingested before storing to R2D2" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_packets(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_packets" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Enter number of packets in which ensemble observers should be computed." - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_strategy(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_strategy" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Enter hofx strategy." - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_num_members(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_num_members" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "How many members comprise the ensemble?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensmean_only(TaskQuestion): - default_value: bool = False - question_name: str = "ensmean_only" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Calculate ensemble mean only?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensmeanvariance_only(TaskQuestion): - default_value: bool = False - question_name: str = "ensmeanvariance_only" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Calculate ensemble mean and variance only?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_geos_gcm_build_path(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_geos_gcm_build_path" - ask_question: bool = True - depends: Dict = mutable_field({ - "geos_build_method": "use_existing" - }) - prompt: str = "What is the path to the existing GEOS build directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_geos_gcm_source_path(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_geos_gcm_source_path" - ask_question: bool = True - depends: Dict = mutable_field({ - "geos_build_method": "use_existing" - }) - prompt: str = "What is the path to the existing GEOS source code directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_jedi_build_directory(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_jedi_build_directory" - ask_question: bool = True - depends: Dict = mutable_field({ - "jedi_build_method": "use_existing" - }) - prompt: str = "What is the path to the existing JEDI build directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_jedi_build_directory_pinned(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_jedi_build_directory_pinned" - ask_question: bool = True - depends: Dict = mutable_field({ - "jedi_build_method": "use_pinned_existing" - }) - prompt: str = "What is the path to the existing pinned JEDI build directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_jedi_source_directory(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_jedi_source_directory" - ask_question: bool = True - depends: Dict = mutable_field({ - "jedi_build_method": "use_existing" - }) - prompt: str = "What is the path to the existing JEDI source code directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_jedi_source_directory_pinned(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_jedi_source_directory_pinned" - ask_question: bool = True - depends: Dict = mutable_field({ - "jedi_build_method": "use_pinned_existing" - }) - prompt: str = "What is the path to the existing pinned JEDI source code directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_perllib_path(TaskQuestion): - default_value: str = 'defer_to_platform' - question_name: str = 'existing_perllib_path' - prompt: str = "Provide a path to an existing location for GMAO_perllib." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class forecast_duration(TaskQuestion): - default_value: str = "PT12H" - question_name: str = "forecast_duration" - ask_question: bool = True - prompt: str = "GEOS forecast duration" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class generate_yaml_and_exit(TaskQuestion): - default_value: bool = False - question_name: str = "generate_yaml_and_exit" - prompt: str = "Generate JEDI executable YAML and exit?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_build_method(TaskQuestion): - default_value: str = "create" - question_name: str = "geos_build_method" - ask_question: bool = True - options: List[str] = mutable_field([ - "use_existing", - "create" - ]) - prompt: str = "Do you want to use an existing GEOS build or create a new build?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_experiment_directory(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "geos_experiment_directory" - ask_question: bool = True - prompt: str = "What is the path to the GEOS restarts directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_gcm_tag(TaskQuestion): - default_value: str = "v11.6.0" - question_name: str = "geos_gcm_tag" - ask_question: bool = True - prompt: str = "Which GEOS tag do you wish to clone?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_restarts_directory(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "geos_restarts_directory" - ask_question: bool = True - prompt: str = "What is the path to the GEOS restarts directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_x_background_directory(TaskQuestion): - default_value: str = "/dev/null/" - question_name: str = "geos_x_background_directory" - ask_question: bool = True - options: List[str] = mutable_field([ - "/dev/null/", - "/discover/nobackup/projects/gmao/dadev/rtodling/archive/Restarts/JEDI/541x" - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the path to the GEOS X-backgrounds directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_x_ensemble_directory(TaskQuestion): - default_value: str = "/dev/null/" - question_name: str = "geos_x_ensemble_directory" - ask_question: bool = True - options: List[str] = mutable_field([ - "/dev/null/", - "/gpfsm/dnb05/projects/p139/rtodling/archive/" - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to the GEOS X-backgrounds directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geovals_experiment(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "geovals_experiment" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the name of the R2D2 experiment providing the GeoVaLs?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geovals_provider(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "geovals_provider" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the name of the R2D2 database providing the GeoVaLs?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gmao_perllib_tag(TaskQuestion): - default_value: str = 'g1.0.1' - question_name: str = 'gmao_perllib_tag' - prompt: str = "Specify the tag at which GMAO_perllib should be cloned." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gradient_norm_reduction(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "gradient_norm_reduction" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What value of gradient norm reduction for convergence?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gsibec_configuration(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "gsibec_configuration" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which GSIBEC climatological or hybrid?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gsibec_nlats(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "gsibec_nlats" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "How many number of latutides in GSIBEC grid?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gsibec_nlons(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "gsibec_nlons" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "How many number of longitudes in GSIBEC grid?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class horizontal_localization_lengthscale(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "horizontal_localization_lengthscale" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the length scale for horizontal covariance localization?" - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class horizontal_localization_max_nobs(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "horizontal_localization_max_nobs" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("What is the maximum number of observations to consider" - " for horizontal covariance localization?") - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class horizontal_localization_method(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "horizontal_localization_method" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which localization scheme should be applied in the horizontal?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class horizontal_resolution(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "horizontal_resolution" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the horizontal resolution for the forecast model and backgrounds?" - widget_type: WType = WType.STRING_DROP_LIST - - # ------------------------------------------------------------------------------------------------ - - @dataclass - class ioda_locations_not_in_r2d2(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "ioda_locations_not_in_r2d2" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ( - "Provide a path that contains observation files not in r2d2.") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class jedi_build_method(TaskQuestion): - default_value: str = "create" - question_name: str = "jedi_build_method" - ask_question: bool = True - options: List[str] = mutable_field([ - "use_existing", - "use_pinned_existing", - "create", - "pinned_create" - ]) - prompt: str = "Do you want to use an existing JEDI build or create a new build?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class jedi_forecast_model(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "jedi_forecast_model" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - depends: Dict = mutable_field({ - "window_type": "4D" - }) - prompt: str = "What forecast model should be used within JEDI for 4D window propagation?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_inflation_mult(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_inflation_mult" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Specify the multiplicative prior inflation coefficient (0 inf]." - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_inflation_rtpp(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_inflation_rtpp" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Specify the Relaxation To Prior Perturbation (RTPP) coefficient (0 1]." - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_inflation_rtps(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_inflation_rtps" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Specify the Relaxation To Prior Spread (RTPS) coefficient (0 1]." - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_save_posterior_ensemble(TaskQuestion): - default_value: bool = False - question_name: str = "local_ensemble_save_posterior_ensemble" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Save the posterior ensemble members?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_save_posterior_ensemble_increments(TaskQuestion): - default_value: bool = False - question_name: str = "local_ensemble_save_posterior_ensemble_increments" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Save the posterior ensemble member increments?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_save_posterior_mean(TaskQuestion): - default_value: bool = False - question_name: str = "local_ensemble_save_posterior_mean" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Save the posterior ensemble mean?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_save_posterior_mean_increment(TaskQuestion): - default_value: bool = True - question_name: str = "local_ensemble_save_posterior_mean_increment" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Save the posterior ensemble mean increment?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_solver(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_solver" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which local ensemble solver type should be implemented?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_use_linear_observer(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_use_linear_observer" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which local ensemble solver type should be implemented?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class minimizer(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "minimizer" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which data assimilation minimizer do you wish to use?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class mom6_iau(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "mom6_iau" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_marine", - ]) - prompt: str = "Do you wish to use IAU for MOM6?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class mom6_iau_nhours(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "mom6_iau_nhours" - options: List[str] = mutable_field([ - 'PT3H', - 'PT12H' - ]) - depends: dict = mutable_field({'mom6_iau': True}) - models: List[str] = mutable_field([ - "geos_marine", - ]) - prompt: str = "What is the IAU length (ODA_INCUPD_NHOURS) for MOM6?" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ncdiag_experiments(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ncdiag_experiments" - options: List[str] = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which previously run experiments do you wish to use for the NCdiag?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class npx(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "npx" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What is the number of grid points in the x-direction on each cube face?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class npx_proc(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "npx_proc" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere", - "geos_cf" - ]) - prompt: str = "What number of processors do you wish to use in the x-direction?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class npy(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "npy" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What is the number of grid points in the y-direction on each cube face?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class npy_proc(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "npy_proc" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere", - "geos_cf" - ]) - prompt: str = "What number of processors do you wish to use in the y-direction?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class number_of_iterations(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "number_of_iterations" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = ( - "What number of iterations do you wish to use for each outer loop?" - " Provide a list of integers the same length as the number of outer loops.") - widget_type: WType = WType.INTEGER_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class obs_experiment(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "obs_experiment" - ask_question: bool = True - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the database providing the observations?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class obs_thinning_rej_fraction(TaskQuestion): - default_value: float = 0.75 - question_name: str = "obs_thinning_rej_fraction" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the rejection fraction for obs thinning?" - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class obs_to_ingest(TaskQuestion): - default_value: list = mutable_field([]) - question_name: str = "obs_to_ingest" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which observations do you want to ingest to R2D2?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observations(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "observations" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which observations do you want to include?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observing_system_records_mksi_path(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "observing_system_records_mksi_path" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to the GSI formatted observing system records?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observing_system_records_mksi_path_tag(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "observing_system_records_mksi_path_tag" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the GSI formatted observing system records tag?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observing_system_records_path(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "observing_system_records_path" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to the Swell formatted observing system records?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class path_to_ensemble(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "path_to_ensemble" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere", - "geos_marine" - ]) - prompt: str = "What is the path to where ensemble members are stored?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class path_to_geos_adas_background(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "path_to_geos_adas_background" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ( - "What is the path for the GEOSadas cubed sphere backgrounds?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class path_to_gsi_bc_coefficients(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "path_to_gsi_bc_coefficients" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the location where GSI bias correction files can be found?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class path_to_gsi_nc_diags(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "path_to_gsi_nc_diags" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to where the GSI ncdiags are stored?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class pause_on_tasks(TaskQuestion): - default_value: list = mutable_field([]) - question_name: str = "pause_on_tasks" - ask_question: bool = False - prompt: str = ("Specify any tasks that the workflow should pause on " - "(for development purposes).") - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class perhost(TaskQuestion): - default_value: str = None - question_name: str = "perhost" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the number of processors per host?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class produce_geovals(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "produce_geovals" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("When running the ncdiag to ioda converted do you " - "want to produce GeoVaLs files?") - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class r2d2_local_path(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "r2d2_local_path" - prompt: str = "What is the path to the R2D2 local directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class save_geovals(TaskQuestion): - default_value: bool = False - question_name: str = "save_geovals" - options: List[bool] = mutable_field([ - True, - False - ]) - prompt: str = "When running hofx do you want to output the GeoVaLs?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class set_obs_as_local(TaskQuestion): - default_value: bool = False - question_name: str = "set_obs_as_local" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - 'all_models' - ]) - prompt: str = "Treat observations as 'local' to the directory?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class single_observations(TaskQuestion): - default_value: bool = False - question_name: str = "single_observations" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Is it a single-observation test?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class swell_static_files(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "swell_static_files" - prompt: str = "What is the path to the Swell Static files directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class swell_static_files_user(TaskQuestion): - default_value: str = "None" - question_name: str = "swell_static_files_user" - prompt: str = "What is the path to the user provided Swell Static Files directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class task_email_parameters(TaskQuestion): - default_value: Union[Literal["auto"], dict] = "auto" - question_name: str = "task_email_parameters" - prompt: str = ("Provide a dictionary mapping tasks to cylc event statuses, or 'auto' to " - "automatically configure these based on the graph.") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class total_processors(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "total_processors" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_marine", - ]) - prompt: str = "What is the number of processors for JEDI?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_apply_log_transform(TaskQuestion): - default_value: bool = True - question_name: str = "vertical_localization_apply_log_transform" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("Should a log (base 10) transformation be applied " - "to vertical coordinate when " - "constructing vertical localization?") - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_function(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_function" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which localization scheme should be applied in the vertical?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_ioda_vertical_coord(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_ioda_vertical_coord" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which coordinate should be used in constructing vertical localization?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_ioda_vertical_coord_group(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_ioda_vertical_coord_group" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("Which vertical coordinate group should be used " - "in constructing vertical localization?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_lengthscale(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_lengthscale" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the length scale for vertical covariance localization?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_method(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_method" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("What localization scheme should be applied in " - "constructing a vertical localization?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_resolution(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_resolution" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the vertical resolution for the forecast model and background?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class window_length(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "window_length" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the duration for the data assimilation window?" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class window_type(TaskQuestion): - question_name: str = "window_type" - default_value: str = "defer_to_model" - ask_question: bool = True - options: List[str] = mutable_field([ - "3D", - "4D" - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Do you want to use a 3D or 4D (including FGAT) window?" - widget_type: WType = WType.STRING_DROP_LIST +@dataclass +class window_type(TaskQuestion): + question_name: str = "window_type" + default_value: str = "defer_to_model" + ask_question: bool = True + options: List[str] = mutable_field([ + "3D", + "4D" + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Do you want to use a 3D or 4D (including FGAT) window?" + widget_type: WType = WType.STRING_DROP_LIST # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_atmos/suite_config.py b/src/swell/suites/3dfgat_atmos/suite_config.py index c2f875432..0829a27b2 100644 --- a/src/swell/suites/3dfgat_atmos/suite_config.py +++ b/src/swell/suites/3dfgat_atmos/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import common from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/3dfgat_cycle/suite_config.py b/src/swell/suites/3dfgat_cycle/suite_config.py index 05ff9a449..20e1eb3bf 100644 --- a/src/swell/suites/3dfgat_cycle/suite_config.py +++ b/src/swell/suites/3dfgat_cycle/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import marine from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/3dvar/suite_config.py b/src/swell/suites/3dvar/suite_config.py index a6f396a02..d832e46bb 100644 --- a/src/swell/suites/3dvar/suite_config.py +++ b/src/swell/suites/3dvar/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.all_suites import suite_configs from swell.suites.base.suite_questions import marine diff --git a/src/swell/suites/3dvar_atmos/suite_config.py b/src/swell/suites/3dvar_atmos/suite_config.py index ff00118e2..d6b6d1e1a 100644 --- a/src/swell/suites/3dvar_atmos/suite_config.py +++ b/src/swell/suites/3dvar_atmos/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.all_suites import suite_configs from swell.suites.base.suite_questions import common diff --git a/src/swell/suites/3dvar_cycle/suite_config.py b/src/swell/suites/3dvar_cycle/suite_config.py index 60329fbb3..2447fa6b7 100644 --- a/src/swell/suites/3dvar_cycle/suite_config.py +++ b/src/swell/suites/3dvar_cycle/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import marine from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/base/suite_questions.py b/src/swell/suites/base/suite_questions.py index 504a48e73..dfe4f475b 100644 --- a/src/swell/suites/base/suite_questions.py +++ b/src/swell/suites/base/suite_questions.py @@ -9,7 +9,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.all_suites import suite_configs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare/suite_config.py b/src/swell/suites/compare/suite_config.py index d3b43da02..0ba09739e 100644 --- a/src/swell/suites/compare/suite_config.py +++ b/src/swell/suites/compare/suite_config.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.swell_questions import QuestionList, WidgetType -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.all_suites import suite_configs from swell.suites.base.suite_questions import all_suites diff --git a/src/swell/suites/convert_bufr/suite_config.py b/src/swell/suites/convert_bufr/suite_config.py index 6df7a15ad..b4d61b758 100644 --- a/src/swell/suites/convert_bufr/suite_config.py +++ b/src/swell/suites/convert_bufr/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import common from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/convert_ncdiags/suite_config.py b/src/swell/suites/convert_ncdiags/suite_config.py index 739c7ddf3..63d288a99 100644 --- a/src/swell/suites/convert_ncdiags/suite_config.py +++ b/src/swell/suites/convert_ncdiags/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import common from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/eva_capabilities/suite_config.py b/src/swell/suites/eva_capabilities/suite_config.py index 941f440df..86778c06d 100644 --- a/src/swell/suites/eva_capabilities/suite_config.py +++ b/src/swell/suites/eva_capabilities/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import marine from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/forecast_geos/suite_config.py b/src/swell/suites/forecast_geos/suite_config.py index 89e33ae26..08df668f0 100644 --- a/src/swell/suites/forecast_geos/suite_config.py +++ b/src/swell/suites/forecast_geos/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import all_suites from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/geosadas/suite_config.py b/src/swell/suites/geosadas/suite_config.py index 1405b31c8..08e2db3a6 100644 --- a/src/swell/suites/geosadas/suite_config.py +++ b/src/swell/suites/geosadas/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import all_suites from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/hofx/suite_config.py b/src/swell/suites/hofx/suite_config.py index eefc0c107..4172e20df 100644 --- a/src/swell/suites/hofx/suite_config.py +++ b/src/swell/suites/hofx/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import marine from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/hofx_cf/suite_config.py b/src/swell/suites/hofx_cf/suite_config.py index 37bab4e15..e3ddb8a93 100644 --- a/src/swell/suites/hofx_cf/suite_config.py +++ b/src/swell/suites/hofx_cf/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import common from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/ingest_obs/suite_config.py b/src/swell/suites/ingest_obs/suite_config.py index 642a1db83..9922d38a5 100644 --- a/src/swell/suites/ingest_obs/suite_config.py +++ b/src/swell/suites/ingest_obs/suite_config.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import common, marine from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index 8b6fe7313..8959ef3f4 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import marine from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/suites/ufo_testing/suite_config.py b/src/swell/suites/ufo_testing/suite_config.py index d60f74918..18a7901a1 100644 --- a/src/swell/suites/ufo_testing/suite_config.py +++ b/src/swell/suites/ufo_testing/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import common from swell.suites.base.all_suites import suite_configs diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index 5465ad65b..04634d1fe 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.shell_commands import run_subprocess, create_executable_file diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index f5e91a5f6..8ffdf19bf 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index b5e71c183..b2e3d9a61 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index cc870100c..fa8fb0156 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.tasks.base.task_attributes import task_attributes # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index 8f250d822..bcbaba933 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -11,7 +11,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.datetime_util import datetime_formats from datetime import datetime as dt import glob diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 3820ab617..b36512a1a 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.git_utils import git_clone from swell.utilities.shell_commands import run_subprocess diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index edafe2558..a4eda2ead 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -12,7 +12,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/clone_gmao_perllib.py b/src/swell/tasks/clone_gmao_perllib.py index 1e80ed483..5123ca75e 100644 --- a/src/swell/tasks/clone_gmao_perllib.py +++ b/src/swell/tasks/clone_gmao_perllib.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index a73a49804..8b26b6430 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.pinned_versions.check_hashes import check_hashes from swell.tasks.base.task_attributes import task_attributes diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index 396ee989d..d791338e8 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.data_assimilation_window_params import DataAssimilationWindowParams from swell.utilities.comparisons import comparison_tags, experiment_ids diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index e4d69a6e1..6f8688633 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.comparisons import comparison_tags diff --git a/src/swell/tasks/eva_comparison_observations.py b/src/swell/tasks/eva_comparison_observations.py index f0383d12c..9fac4763f 100644 --- a/src/swell/tasks/eva_comparison_observations.py +++ b/src/swell/tasks/eva_comparison_observations.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.observations import ioda_name_to_long_name diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index f45ded4c6..88dc73aa1 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 24ce02a10..8c72a09c9 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.observations import ioda_name_to_long_name diff --git a/src/swell/tasks/eva_timeseries.py b/src/swell/tasks/eva_timeseries.py index 8d013aa40..cb78d53ce 100644 --- a/src/swell/tasks/eva_timeseries.py +++ b/src/swell/tasks/eva_timeseries.py @@ -18,7 +18,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index 54160aa11..7113e23b9 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -11,7 +11,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.shell_commands import run_track_log_subprocess from swell.utilities.file_system_operations import check_if_files_exist_in_path diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 0afd240a4..bc5ab9ee9 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -10,7 +10,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.file_system_operations import link_all_files_from_first_in_hierarchy_of_sources diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index ba876a0c0..1f89a67b6 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.observing_system_records import ObservingSystemRecords # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 5701172fe..068c3e817 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -11,7 +11,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd import isodate import os diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index f885dcf00..7a5ff4b4f 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.datetime_util import datetime_formats # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_bufr.py b/src/swell/tasks/get_bufr.py index f2ef5b1be..ef1b6a716 100644 --- a/src/swell/tasks/get_bufr.py +++ b/src/swell/tasks/get_bufr.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_ensemble.py b/src/swell/tasks/get_ensemble.py index abb09d615..b3c9c7f29 100644 --- a/src/swell/tasks/get_ensemble.py +++ b/src/swell/tasks/get_ensemble.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_ensemble_geos_experiment.py b/src/swell/tasks/get_ensemble_geos_experiment.py index ca02d1974..166a96fc7 100644 --- a/src/swell/tasks/get_ensemble_geos_experiment.py +++ b/src/swell/tasks/get_ensemble_geos_experiment.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.datetime_util import datetime_formats # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py index 13cd57691..c6ea42689 100644 --- a/src/swell/tasks/get_geos_adas_background.py +++ b/src/swell/tasks/get_geos_adas_background.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_geos_restart.py b/src/swell/tasks/get_geos_restart.py index 3c68cf357..416f6783d 100644 --- a/src/swell/tasks/get_geos_restart.py +++ b/src/swell/tasks/get_geos_restart.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.file_system_operations import copy_to_dst_dir, check_if_files_exist_in_path # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index ca4da51c8..8d110c1e4 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.r2d2 import create_r2d2_config diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index ff2547137..ca02b804a 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index 8465b7c1f..45e2c39fd 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index b1682c60e..7d12c3b78 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -11,7 +11,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py index 33046392e..418700784 100644 --- a/src/swell/tasks/get_obs_not_in_r2d2.py +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index a67d77d52..e26b309fb 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.r2d2 import create_r2d2_config from swell.utilities.datetime_util import datetime_formats from swell.utilities.observations import get_ioda_names_list, get_provider_for_observation diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 9738ce29f..cd2811fec 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.dictionary import write_dict_to_yaml from swell.utilities.shell_commands import run_track_log_subprocess diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 851d0b5de..cb558cfaf 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.shell_commands import run_subprocess, create_executable_file diff --git a/src/swell/tasks/ingest_obs.py b/src/swell/tasks/ingest_obs.py index be8912f50..07930c39b 100644 --- a/src/swell/tasks/ingest_obs.py +++ b/src/swell/tasks/ingest_obs.py @@ -20,7 +20,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.r2d2 import create_r2d2_config from swell.utilities.observations import get_ioda_names_list, get_provider_for_observation import r2d2 diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py index ec6a93aee..b7eb81bb0 100644 --- a/src/swell/tasks/jedi_log_comparison.py +++ b/src/swell/tasks/jedi_log_comparison.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.comparisons import comparison_tags # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/jedi_oops_log_parser.py b/src/swell/tasks/jedi_oops_log_parser.py index 2a90145ca..01781072e 100644 --- a/src/swell/tasks/jedi_oops_log_parser.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index 0febac9b1..6f0cb3d28 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -18,7 +18,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/move_da_restart.py b/src/swell/tasks/move_da_restart.py index d8b986177..953b907c5 100644 --- a/src/swell/tasks/move_da_restart.py +++ b/src/swell/tasks/move_da_restart.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.file_system_operations import move_files # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/move_forecast_restart.py b/src/swell/tasks/move_forecast_restart.py index 494ee9dcc..e42a09675 100644 --- a/src/swell/tasks/move_forecast_restart.py +++ b/src/swell/tasks/move_forecast_restart.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.file_system_operations import move_files # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/prep_geos_run_dir.py b/src/swell/tasks/prep_geos_run_dir.py index c93b6a892..d845088d4 100644 --- a/src/swell/tasks/prep_geos_run_dir.py +++ b/src/swell/tasks/prep_geos_run_dir.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.file_system_operations import copy_to_dst_dir, check_if_files_exist_in_path # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index 7ef1da8ee..520c30ba5 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -17,7 +17,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index 90eca8c47..20763799d 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.run_jedi_executables import check_obs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py index fcf2be0d2..62338d55d 100644 --- a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py +++ b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.run_jedi_executables import run_executable diff --git a/src/swell/tasks/run_jedi_ensemble_mean_variance.py b/src/swell/tasks/run_jedi_ensemble_mean_variance.py index 7a62678c0..7bc8f59de 100644 --- a/src/swell/tasks/run_jedi_ensemble_mean_variance.py +++ b/src/swell/tasks/run_jedi_ensemble_mean_variance.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.run_jedi_executables import run_executable diff --git a/src/swell/tasks/run_jedi_fgat_executable.py b/src/swell/tasks/run_jedi_fgat_executable.py index 22595e44d..d2bee09cc 100644 --- a/src/swell/tasks/run_jedi_fgat_executable.py +++ b/src/swell/tasks/run_jedi_fgat_executable.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.run_jedi_executables import run_executable diff --git a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py index db7a801cd..76ac4208c 100644 --- a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py +++ b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.run_jedi_executables import run_executable from swell.tasks.run_jedi_hofx_executable import RunJediHofxExecutable diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index a13aa57ad..dcc52ec6e 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.run_jedi_executables import run_executable diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 2c71ef2f0..d3ab510f2 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index c85202727..994f2be08 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.run_jedi_executables import run_executable # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 439878b65..2882ae7fd 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -15,7 +15,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.dictionary import update_dict from swell.utilities.run_jedi_executables import run_executable diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index e390da069..c03e6f263 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -13,7 +13,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.run_jedi_executables import run_executable diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 1bf83a640..9b80641cd 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -10,7 +10,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.r2d2 import create_r2d2_config from swell.utilities.run_jedi_executables import check_obs diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index bf104bceb..4abd0ea50 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.file_system_operations import copy_to_dst_dir diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 62b99a147..265ec80e4 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -14,7 +14,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.filehandler import get_file_handler from swell.utilities.exceptions import SwellError from swell.utilities.file_system_operations import check_if_files_exist_in_path diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index 8b14fe446..34cad4abb 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes -from swell.configuration.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.utilities.datetime_util import datetime_formats From 6ca17912ea7719204e60ff6ca7f8554c30892775 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 12:29:51 -0500 Subject: [PATCH 223/299] Update docs for question defaults --- docs/adding_a_suite.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/adding_a_suite.md b/docs/adding_a_suite.md index d2df71b9e..8a15b3bcc 100644 --- a/docs/adding_a_suite.md +++ b/docs/adding_a_suite.md @@ -346,4 +346,4 @@ suite_configs.register('3dvar', '3dvar_tier1', _3dvar_tier1) ``` The class `SuiteQuestions` contains lists of questions which are common to many suites. This avoids the need for redundantly setting the same questions for every suite. -`_3dvar_base` is responsible for establishing the baseline for questions used by the suite. The 'base' list should be used to associate all questions used by the suite. This list will be populated with the questions that match the defaults in `QuestionDefaults` (`src/swell/configuration/question_defaults.py`). However, in many cases, those defaults will not be ideal defaults for the individual suite. Thus, `_3dvar_tier1` sets different default values which override the question defaults. If desired, other configurations can then inherit question defaults from `_3dvar_tier1`, and set their own defaults on top of the existing ones. +`_3dvar_base` is responsible for establishing the baseline for questions used by the suite. The 'base' list should be used to associate all questions used by the suite. This list will be populated with the questions that match the defaults in `question_defaults.py` (`src/swell/configuration/question_defaults.py`). However, in many cases, those defaults will not be ideal defaults for the individual suite. Thus, `_3dvar_tier1` sets different default values which override the question defaults. If desired, other configurations can then inherit question defaults from `_3dvar_tier1`, and set their own defaults on top of the existing ones. From 898ed8ed5f7de6e06dea81d2cafb00777ca5a5e8 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 12:31:44 -0500 Subject: [PATCH 224/299] Refactor suite_attributes.py --- src/swell/suites/base/suite_attributes.py | 106 ++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 src/swell/suites/base/suite_attributes.py diff --git a/src/swell/suites/base/suite_attributes.py b/src/swell/suites/base/suite_attributes.py new file mode 100644 index 000000000..24a21b27d --- /dev/null +++ b/src/swell/suites/base/suite_attributes.py @@ -0,0 +1,106 @@ +# -------------------------------------------------------------------------------------------------- +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.suites.base.cylc_workflow import CylcWorkflow +from swell.utilities.swell_questions import QuestionList +import swell.suites +from swell.utilities.plugins import discover_plugins + +# -------------------------------------------------------------------------------------------------- + + +def format_suite_name(suite_name): + # Format suite names starting with a digit + return suite_name[1:] if suite_name[0] == '_' else suite_name + +# -------------------------------------------------------------------------------------------------- + + +class Workflows(): + + def __init__(self) -> None: + self.__workflow_names__ = [] + + def register(self, name: str) -> None: + self.__workflow_names__.append(name) + + def wrapper(cls): + setattr(self, name, cls) + return cls + return wrapper + + def get(self, name: str) -> type[CylcWorkflow]: + return getattr(self, name) + + def all(self) -> list: + return self.__workflow_names__ + +# -------------------------------------------------------------------------------------------------- + + +class SuiteConfigs(): + + def __init__(self) -> None: + + # Dictionary tracking the suite for each config + self.__config_map__ = {} + + # -------------------------------------------------------------------------------------------------- + + def register(self, + base_suite: str, + config_name: str, + question_list: QuestionList) -> None: + + self.__config_map__[config_name] = sub_dict = {} + + sub_dict['suite'] = base_suite + sub_dict['list'] = question_list + + # -------------------------------------------------------------------------------------------------- + + def get_config(self, config_name: str) -> QuestionList: + return self.__config_map__[config_name]['list'] + + # -------------------------------------------------------------------------------------------------- + + def base_suite(self, config_name: str) -> str: + return self.__config_map__[config_name]['suite'] + + # -------------------------------------------------------------------------------------------------- + + def all_configs(self) -> str: + return list(self.__config_map__.keys()) + + # -------------------------------------------------------------------------------------------------- + + def configs_under_suites(self) -> dict: + suite_map = {} + + for config_name, config_dict in self.__config_map__.items(): + suite_name = config_dict['suite'] + + if suite_name not in suite_map: + suite_map[suite_name] = [] + + suite_map[suite_name].append(config_name) + + return suite_map + +# -------------------------------------------------------------------------------------------------- + + +# Objects to reference in imports +suite_configs = SuiteConfigs() +workflows = Workflows() + +discover_plugins(swell.suites) + +# -------------------------------------------------------------------------------------------------- From 13411e2c549e7e6f0b3540a09666df9ce240e615 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 12:43:24 -0500 Subject: [PATCH 225/299] code test fixes --- src/swell/configuration/question_defaults.py | 137 +++++++++++++++++-- 1 file changed, 125 insertions(+), 12 deletions(-) diff --git a/src/swell/configuration/question_defaults.py b/src/swell/configuration/question_defaults.py index a46720f2c..5eef291e9 100644 --- a/src/swell/configuration/question_defaults.py +++ b/src/swell/configuration/question_defaults.py @@ -19,6 +19,7 @@ # Suite question defaults go here # -------------------------------------------------------------------------------------------------- + @dataclass class comparison_experiment_paths(SuiteQuestion): default_value: list = mutable_field([]) @@ -29,6 +30,7 @@ class comparison_experiment_paths(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class cycle_times(SuiteQuestion): default_value: str = "defer_to_model" @@ -43,6 +45,7 @@ class cycle_times(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class cycling_varbc(SuiteQuestion): default_value: str = "defer_to_model" @@ -56,6 +59,7 @@ class cycling_varbc(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class email_address(SuiteQuestion): default_value: str = "defer_to_user" @@ -65,6 +69,7 @@ class email_address(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_hofx_packets(SuiteQuestion): default_value: str = "defer_to_model" @@ -77,6 +82,7 @@ class ensemble_hofx_packets(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_hofx_strategy(SuiteQuestion): default_value: str = "defer_to_model" @@ -89,6 +95,7 @@ class ensemble_hofx_strategy(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class experiment_id(SuiteQuestion): default_value: str = "defer_to_code" @@ -99,17 +106,19 @@ class experiment_id(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class experiment_root(SuiteQuestion): default_value: str = "defer_to_platform" question_name: str = "experiment_root" ask_question: bool = True prompt: str = ("What is the experiment root (the directory where the " - "experiment will be stored)?") + "experiment will be stored)?") widget_type: WType = WType.STRING # -------------------------------------------------------------------------------------------------- + @dataclass class final_cycle_point(SuiteQuestion): default_value: str = "2023-10-10T06:00:00Z" @@ -120,6 +129,7 @@ class final_cycle_point(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class marine_models(SuiteQuestion): default_value: str = "defer_to_model" @@ -134,6 +144,7 @@ class marine_models(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class model_components(SuiteQuestion): default_value: str = "defer_to_code" @@ -145,6 +156,7 @@ class model_components(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class parser_options(SuiteQuestion): default_value: list = mutable_field(['fgrep_residual_norm']) @@ -156,17 +168,19 @@ class parser_options(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class runahead_limit(SuiteQuestion): default_value: str = "P4" question_name: str = "runahead_limit" ask_question: bool = True prompt: str = ("Since this suite is non-cycling choose how " - "many hours the workflow can run ahead?") + "many hours the workflow can run ahead?") widget_type: WType = WType.STRING # -------------------------------------------------------------------------------------------------- + @dataclass class skip_ensemble_hofx(SuiteQuestion): default_value: str = "defer_to_model" @@ -179,6 +193,7 @@ class skip_ensemble_hofx(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class start_cycle_point(SuiteQuestion): default_value: str = "2023-10-10T00:00:00Z" @@ -189,17 +204,19 @@ class start_cycle_point(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class use_cycle_dir(SuiteQuestion): default_value: bool = True question_name: str = "use_cycle_dir" ask_question: bool = False prompt: str = ("For cycling tasks, send results to the experiment cycle directory?" - " If false, results will be stored in the current working directory.") + " If false, results will be stored in the current working directory.") widget_type: WType = WType.BOOLEAN # -------------------------------------------------------------------------------------------------- + @dataclass class window_type(SuiteQuestion): default_value: str = "defer_to_model" @@ -218,6 +235,7 @@ class window_type(SuiteQuestion): # Task question defaults go here # -------------------------------------------------------------------------------------------------- + @dataclass class analysis_variables(TaskQuestion): default_value: str = "defer_to_model" @@ -231,6 +249,7 @@ class analysis_variables(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class background_error_model(TaskQuestion): default_value: str = "defer_to_model" @@ -244,6 +263,7 @@ class background_error_model(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class background_experiment(TaskQuestion): default_value: str = "defer_to_model" @@ -257,6 +277,7 @@ class background_experiment(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class background_frequency(TaskQuestion): default_value: str = "defer_to_model" @@ -272,6 +293,7 @@ class background_frequency(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class background_time_offset(TaskQuestion): default_value: str = "defer_to_model" @@ -280,10 +302,12 @@ class background_time_offset(TaskQuestion): "all_models" ]) prompt: str = ("How long before the middle of the analysis window did" - " the background providing forecast begin?") + " the background providing forecast begin?") widget_type: WType = WType.ISO_DURATION # -------------------------------------------------------------------------------------------------- + + @dataclass class bufr_obs_classes(TaskQuestion): default_value: str = "defer_to_model" @@ -298,6 +322,7 @@ class bufr_obs_classes(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class bundles(TaskQuestion): default_value: List[str] = mutable_field([ @@ -325,6 +350,7 @@ class bundles(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class check_for_obs(TaskQuestion): default_value: bool = True @@ -338,6 +364,7 @@ class check_for_obs(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class clean_patterns(TaskQuestion): default_value: str = "defer_to_model" @@ -351,6 +378,7 @@ class clean_patterns(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class comparison_log_type(TaskQuestion): default_value: str = "variational" @@ -367,6 +395,7 @@ class comparison_log_type(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class crtm_coeff_dir(TaskQuestion): default_value: str = "defer_to_platform" @@ -379,6 +408,7 @@ class crtm_coeff_dir(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class dry_run(TaskQuestion): default_value: bool = True @@ -392,6 +422,7 @@ class dry_run(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_hofx_packets(TaskQuestion): default_value: str = "defer_to_model" @@ -406,6 +437,7 @@ class ensemble_hofx_packets(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_hofx_strategy(TaskQuestion): default_value: str = "defer_to_model" @@ -420,6 +452,7 @@ class ensemble_hofx_strategy(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_num_members(TaskQuestion): default_value: str = "defer_to_model" @@ -433,6 +466,7 @@ class ensemble_num_members(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensmean_only(TaskQuestion): default_value: bool = False @@ -449,6 +483,7 @@ class ensmean_only(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensmeanvariance_only(TaskQuestion): default_value: bool = False @@ -465,6 +500,7 @@ class ensmeanvariance_only(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_geos_gcm_build_path(TaskQuestion): default_value: str = "defer_to_platform" @@ -478,6 +514,7 @@ class existing_geos_gcm_build_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_geos_gcm_source_path(TaskQuestion): default_value: str = "defer_to_platform" @@ -491,6 +528,7 @@ class existing_geos_gcm_source_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_jedi_build_directory(TaskQuestion): default_value: str = "defer_to_platform" @@ -504,6 +542,7 @@ class existing_jedi_build_directory(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_jedi_build_directory_pinned(TaskQuestion): default_value: str = "defer_to_platform" @@ -517,6 +556,7 @@ class existing_jedi_build_directory_pinned(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_jedi_source_directory(TaskQuestion): default_value: str = "defer_to_platform" @@ -530,6 +570,7 @@ class existing_jedi_source_directory(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_jedi_source_directory_pinned(TaskQuestion): default_value: str = "defer_to_platform" @@ -543,6 +584,7 @@ class existing_jedi_source_directory_pinned(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_perllib_path(TaskQuestion): default_value: str = 'defer_to_platform' @@ -552,6 +594,7 @@ class existing_perllib_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class forecast_duration(TaskQuestion): default_value: str = "PT12H" @@ -562,6 +605,7 @@ class forecast_duration(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class generate_yaml_and_exit(TaskQuestion): default_value: bool = False @@ -571,6 +615,7 @@ class generate_yaml_and_exit(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_build_method(TaskQuestion): default_value: str = "create" @@ -585,6 +630,7 @@ class geos_build_method(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_experiment_directory(TaskQuestion): default_value: str = "defer_to_platform" @@ -595,6 +641,7 @@ class geos_experiment_directory(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_gcm_tag(TaskQuestion): default_value: str = "v11.6.0" @@ -605,6 +652,7 @@ class geos_gcm_tag(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_restarts_directory(TaskQuestion): default_value: str = "defer_to_platform" @@ -615,6 +663,7 @@ class geos_restarts_directory(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_x_background_directory(TaskQuestion): default_value: str = "/dev/null/" @@ -632,6 +681,7 @@ class geos_x_background_directory(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_x_ensemble_directory(TaskQuestion): default_value: str = "/dev/null/" @@ -649,6 +699,7 @@ class geos_x_ensemble_directory(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geovals_experiment(TaskQuestion): default_value: str = "defer_to_model" @@ -662,6 +713,7 @@ class geovals_experiment(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geovals_provider(TaskQuestion): default_value: str = "defer_to_model" @@ -674,6 +726,7 @@ class geovals_provider(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class gmao_perllib_tag(TaskQuestion): default_value: str = 'g1.0.1' @@ -683,6 +736,7 @@ class gmao_perllib_tag(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class gradient_norm_reduction(TaskQuestion): default_value: str = "defer_to_model" @@ -695,6 +749,7 @@ class gradient_norm_reduction(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class gsibec_configuration(TaskQuestion): default_value: str = "defer_to_model" @@ -707,6 +762,7 @@ class gsibec_configuration(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class gsibec_nlats(TaskQuestion): default_value: str = "defer_to_model" @@ -719,6 +775,7 @@ class gsibec_nlats(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class gsibec_nlons(TaskQuestion): default_value: str = "defer_to_model" @@ -731,6 +788,7 @@ class gsibec_nlons(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class horizontal_localization_lengthscale(TaskQuestion): default_value: str = "defer_to_model" @@ -743,6 +801,7 @@ class horizontal_localization_lengthscale(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class horizontal_localization_max_nobs(TaskQuestion): default_value: str = "defer_to_model" @@ -751,11 +810,12 @@ class horizontal_localization_max_nobs(TaskQuestion): "geos_atmosphere" ]) prompt: str = ("What is the maximum number of observations to consider" - " for horizontal covariance localization?") + " for horizontal covariance localization?") widget_type: WType = WType.INTEGER # -------------------------------------------------------------------------------------------------- + @dataclass class horizontal_localization_method(TaskQuestion): default_value: str = "defer_to_model" @@ -769,6 +829,7 @@ class horizontal_localization_method(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class horizontal_resolution(TaskQuestion): default_value: str = "defer_to_model" @@ -783,6 +844,7 @@ class horizontal_resolution(TaskQuestion): # ------------------------------------------------------------------------------------------------ + @dataclass class ioda_locations_not_in_r2d2(TaskQuestion): default_value: str = "defer_to_platform" @@ -797,6 +859,7 @@ class ioda_locations_not_in_r2d2(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class jedi_build_method(TaskQuestion): default_value: str = "create" @@ -813,6 +876,7 @@ class jedi_build_method(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class jedi_forecast_model(TaskQuestion): default_value: str = "defer_to_model" @@ -830,6 +894,7 @@ class jedi_forecast_model(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_inflation_mult(TaskQuestion): default_value: str = "defer_to_model" @@ -842,6 +907,7 @@ class local_ensemble_inflation_mult(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_inflation_rtpp(TaskQuestion): default_value: str = "defer_to_model" @@ -854,6 +920,7 @@ class local_ensemble_inflation_rtpp(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_inflation_rtps(TaskQuestion): default_value: str = "defer_to_model" @@ -866,6 +933,7 @@ class local_ensemble_inflation_rtps(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_save_posterior_ensemble(TaskQuestion): default_value: bool = False @@ -882,6 +950,7 @@ class local_ensemble_save_posterior_ensemble(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_save_posterior_ensemble_increments(TaskQuestion): default_value: bool = False @@ -899,6 +968,7 @@ class local_ensemble_save_posterior_ensemble_increments(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_save_posterior_mean(TaskQuestion): default_value: bool = False @@ -916,6 +986,7 @@ class local_ensemble_save_posterior_mean(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_save_posterior_mean_increment(TaskQuestion): default_value: bool = True @@ -933,6 +1004,7 @@ class local_ensemble_save_posterior_mean_increment(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_solver(TaskQuestion): default_value: str = "defer_to_model" @@ -947,6 +1019,7 @@ class local_ensemble_solver(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_use_linear_observer(TaskQuestion): default_value: str = "defer_to_model" @@ -961,6 +1034,7 @@ class local_ensemble_use_linear_observer(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class minimizer(TaskQuestion): default_value: str = "defer_to_model" @@ -974,6 +1048,7 @@ class minimizer(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class mom6_iau(TaskQuestion): default_value: str = "defer_to_model" @@ -990,6 +1065,7 @@ class mom6_iau(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class mom6_iau_nhours(TaskQuestion): default_value: str = "defer_to_model" @@ -1007,6 +1083,7 @@ class mom6_iau_nhours(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ncdiag_experiments(TaskQuestion): default_value: str = "defer_to_model" @@ -1020,6 +1097,7 @@ class ncdiag_experiments(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class npx(TaskQuestion): default_value: str = "defer_to_model" @@ -1033,6 +1111,7 @@ class npx(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class npx_proc(TaskQuestion): default_value: str = "defer_to_model" @@ -1047,6 +1126,7 @@ class npx_proc(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class npy(TaskQuestion): default_value: str = "defer_to_model" @@ -1060,6 +1140,7 @@ class npy(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class npy_proc(TaskQuestion): default_value: str = "defer_to_model" @@ -1074,6 +1155,7 @@ class npy_proc(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class number_of_iterations(TaskQuestion): default_value: str = "defer_to_model" @@ -1088,6 +1170,7 @@ class number_of_iterations(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class obs_experiment(TaskQuestion): default_value: str = "defer_to_model" @@ -1101,6 +1184,7 @@ class obs_experiment(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class obs_thinning_rej_fraction(TaskQuestion): default_value: float = 0.75 @@ -1113,6 +1197,7 @@ class obs_thinning_rej_fraction(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class obs_to_ingest(TaskQuestion): default_value: list = mutable_field([]) @@ -1127,6 +1212,7 @@ class obs_to_ingest(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class observations(TaskQuestion): default_value: str = "defer_to_model" @@ -1141,6 +1227,7 @@ class observations(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class observing_system_records_mksi_path(TaskQuestion): default_value: str = "defer_to_model" @@ -1153,6 +1240,7 @@ class observing_system_records_mksi_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class observing_system_records_mksi_path_tag(TaskQuestion): default_value: str = "defer_to_model" @@ -1165,6 +1253,7 @@ class observing_system_records_mksi_path_tag(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class observing_system_records_path(TaskQuestion): default_value: str = "defer_to_model" @@ -1177,6 +1266,7 @@ class observing_system_records_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class path_to_ensemble(TaskQuestion): default_value: str = "defer_to_model" @@ -1191,6 +1281,7 @@ class path_to_ensemble(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class path_to_geos_adas_background(TaskQuestion): default_value: str = "defer_to_model" @@ -1205,6 +1296,7 @@ class path_to_geos_adas_background(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class path_to_gsi_bc_coefficients(TaskQuestion): default_value: str = "defer_to_model" @@ -1218,6 +1310,7 @@ class path_to_gsi_bc_coefficients(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class path_to_gsi_nc_diags(TaskQuestion): default_value: str = "defer_to_model" @@ -1231,17 +1324,19 @@ class path_to_gsi_nc_diags(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class pause_on_tasks(TaskQuestion): default_value: list = mutable_field([]) question_name: str = "pause_on_tasks" ask_question: bool = False prompt: str = ("Specify any tasks that the workflow should pause on " - "(for development purposes).") + "(for development purposes).") widget_type: WType = WType.STRING_CHECK_LIST # -------------------------------------------------------------------------------------------------- + @dataclass class perhost(TaskQuestion): default_value: str = None @@ -1259,6 +1354,7 @@ class perhost(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class produce_geovals(TaskQuestion): default_value: str = "defer_to_model" @@ -1272,11 +1368,12 @@ class produce_geovals(TaskQuestion): "geos_atmosphere" ]) prompt: str = ("When running the ncdiag to ioda converted do you " - "want to produce GeoVaLs files?") + "want to produce GeoVaLs files?") widget_type: WType = WType.BOOLEAN # -------------------------------------------------------------------------------------------------- + @dataclass class r2d2_local_path(TaskQuestion): default_value: str = "defer_to_platform" @@ -1286,6 +1383,7 @@ class r2d2_local_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class save_geovals(TaskQuestion): default_value: bool = False @@ -1299,6 +1397,7 @@ class save_geovals(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class set_obs_as_local(TaskQuestion): default_value: bool = False @@ -1315,6 +1414,7 @@ class set_obs_as_local(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class single_observations(TaskQuestion): default_value: bool = False @@ -1331,6 +1431,7 @@ class single_observations(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class swell_static_files(TaskQuestion): default_value: str = "defer_to_platform" @@ -1340,6 +1441,7 @@ class swell_static_files(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class swell_static_files_user(TaskQuestion): default_value: str = "None" @@ -1349,16 +1451,18 @@ class swell_static_files_user(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class task_email_parameters(TaskQuestion): default_value: Union[Literal["auto"], dict] = "auto" question_name: str = "task_email_parameters" prompt: str = ("Provide a dictionary mapping tasks to cylc event statuses, or 'auto' to " - "automatically configure these based on the graph.") + "automatically configure these based on the graph.") widget_type: WType = WType.STRING # -------------------------------------------------------------------------------------------------- + @dataclass class total_processors(TaskQuestion): default_value: str = "defer_to_model" @@ -1372,6 +1476,7 @@ class total_processors(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_localization_apply_log_transform(TaskQuestion): default_value: bool = True @@ -1384,12 +1489,13 @@ class vertical_localization_apply_log_transform(TaskQuestion): "geos_atmosphere" ]) prompt: str = ("Should a log (base 10) transformation be applied " - "to vertical coordinate when " - "constructing vertical localization?") + "to vertical coordinate when " + "constructing vertical localization?") widget_type: WType = WType.BOOLEAN # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_localization_function(TaskQuestion): default_value: str = "defer_to_model" @@ -1403,6 +1509,7 @@ class vertical_localization_function(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_localization_ioda_vertical_coord(TaskQuestion): default_value: str = "defer_to_model" @@ -1416,6 +1523,7 @@ class vertical_localization_ioda_vertical_coord(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_localization_ioda_vertical_coord_group(TaskQuestion): default_value: str = "defer_to_model" @@ -1425,11 +1533,12 @@ class vertical_localization_ioda_vertical_coord_group(TaskQuestion): "geos_atmosphere" ]) prompt: str = ("Which vertical coordinate group should be used " - "in constructing vertical localization?") + "in constructing vertical localization?") widget_type: WType = WType.STRING # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_localization_lengthscale(TaskQuestion): default_value: str = "defer_to_model" @@ -1442,6 +1551,7 @@ class vertical_localization_lengthscale(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_localization_method(TaskQuestion): default_value: str = "defer_to_model" @@ -1451,11 +1561,12 @@ class vertical_localization_method(TaskQuestion): "geos_atmosphere" ]) prompt: str = ("What localization scheme should be applied in " - "constructing a vertical localization?") + "constructing a vertical localization?") widget_type: WType = WType.STRING # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_resolution(TaskQuestion): default_value: str = "defer_to_model" @@ -1470,6 +1581,7 @@ class vertical_resolution(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class window_length(TaskQuestion): default_value: str = "defer_to_model" @@ -1482,6 +1594,7 @@ class window_length(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class window_type(TaskQuestion): question_name: str = "window_type" From f2a8d1677acad5be79f440a6cae4bd9bf7313362 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 12:49:04 -0500 Subject: [PATCH 226/299] Remove unused list_name --- src/swell/suites/3dfgat_atmos/suite_config.py | 3 --- src/swell/suites/3dfgat_cycle/suite_config.py | 2 -- src/swell/suites/3dvar/suite_config.py | 2 -- src/swell/suites/3dvar_atmos/suite_config.py | 2 -- src/swell/suites/3dvar_cycle/suite_config.py | 2 -- src/swell/suites/base/suite_questions.py | 5 ----- src/swell/suites/build_geos/suite_config.py | 1 - src/swell/suites/build_jedi/suite_config.py | 1 - src/swell/suites/compare/suite_config.py | 4 ---- src/swell/suites/convert_bufr/suite_config.py | 1 - src/swell/suites/convert_ncdiags/suite_config.py | 2 -- src/swell/suites/eva_capabilities/suite_config.py | 2 -- src/swell/suites/forecast_geos/suite_config.py | 2 -- src/swell/suites/geosadas/suite_config.py | 2 -- src/swell/suites/hofx/suite_config.py | 2 -- src/swell/suites/hofx_cf/suite_config.py | 2 -- src/swell/suites/ingest_obs/suite_config.py | 2 -- src/swell/suites/localensembleda/suite_config.py | 3 --- src/swell/suites/ufo_testing/suite_config.py | 2 -- src/swell/utilities/swell_questions.py | 1 - 20 files changed, 43 deletions(-) diff --git a/src/swell/suites/3dfgat_atmos/suite_config.py b/src/swell/suites/3dfgat_atmos/suite_config.py index 0829a27b2..b06306368 100644 --- a/src/swell/suites/3dfgat_atmos/suite_config.py +++ b/src/swell/suites/3dfgat_atmos/suite_config.py @@ -18,7 +18,6 @@ # -------------------------------------------------------------------------------------------------- _3dfgat_atmos_tier1 = QuestionList( - list_name="3dfgat_atmos", questions=[ common, qd.start_cycle_point("2023-10-10T00:00:00Z"), @@ -86,7 +85,6 @@ # -------------------------------------------------------------------------------------------------- _3dfgat_atmos = QuestionList( - list_name="3dfgat_atmos", questions=[ _3dfgat_atmos_tier1 ] @@ -97,7 +95,6 @@ # -------------------------------------------------------------------------------------------------- _3dfgat_atmos_tier2 = QuestionList( - list_name="3dfgat_atmos_tier2", questions=[ _3dfgat_atmos_tier1, ], diff --git a/src/swell/suites/3dfgat_cycle/suite_config.py b/src/swell/suites/3dfgat_cycle/suite_config.py index 20e1eb3bf..284a7c34c 100644 --- a/src/swell/suites/3dfgat_cycle/suite_config.py +++ b/src/swell/suites/3dfgat_cycle/suite_config.py @@ -18,7 +18,6 @@ # -------------------------------------------------------------------------------------------------- _3dfgat_cycle_tier1 = QuestionList( - list_name="3dfgat_cycle", questions=[ marine, qd.cycling_varbc(), @@ -86,7 +85,6 @@ # -------------------------------------------------------------------------------------------------- _3dfgat_cycle = QuestionList( - list_name="3dfgat_cycle", questions=[ _3dfgat_cycle_tier1 ] diff --git a/src/swell/suites/3dvar/suite_config.py b/src/swell/suites/3dvar/suite_config.py index d832e46bb..4e7ac6b06 100644 --- a/src/swell/suites/3dvar/suite_config.py +++ b/src/swell/suites/3dvar/suite_config.py @@ -18,7 +18,6 @@ suite_name = '3dvar' _3dvar_tier1 = QuestionList( - list_name="3dvar", questions=[ marine, qd.cycling_varbc(), @@ -61,7 +60,6 @@ # -------------------------------------------------------------------------------------------------- _3dvar = QuestionList( - list_name="3dvar", questions=[ _3dvar_tier1 ] diff --git a/src/swell/suites/3dvar_atmos/suite_config.py b/src/swell/suites/3dvar_atmos/suite_config.py index d6b6d1e1a..500aac557 100644 --- a/src/swell/suites/3dvar_atmos/suite_config.py +++ b/src/swell/suites/3dvar_atmos/suite_config.py @@ -18,7 +18,6 @@ # -------------------------------------------------------------------------------------------------- _3dvar_atmos_tier1 = QuestionList( - list_name="3dvar_atmos", questions=[ common, qd.start_cycle_point("2023-10-10T00:00:00Z"), @@ -88,7 +87,6 @@ # -------------------------------------------------------------------------------------------------- _3dvar_atmos = QuestionList( - list_name="3dvar_atmos", questions=[ _3dvar_atmos_tier1 ] diff --git a/src/swell/suites/3dvar_cycle/suite_config.py b/src/swell/suites/3dvar_cycle/suite_config.py index 2447fa6b7..dc39223fc 100644 --- a/src/swell/suites/3dvar_cycle/suite_config.py +++ b/src/swell/suites/3dvar_cycle/suite_config.py @@ -18,7 +18,6 @@ # -------------------------------------------------------------------------------------------------- _3dvar_cycle_tier1 = QuestionList( - list_name="3dvar_cycle", questions=[ marine, qd.cycling_varbc(), @@ -73,7 +72,6 @@ # -------------------------------------------------------------------------------------------------- _3dvar_cycle = QuestionList( - list_name="3dvar_cycle", questions=[ _3dvar_cycle_tier1 ] diff --git a/src/swell/suites/base/suite_questions.py b/src/swell/suites/base/suite_questions.py index dfe4f475b..9f2955dfb 100644 --- a/src/swell/suites/base/suite_questions.py +++ b/src/swell/suites/base/suite_questions.py @@ -17,7 +17,6 @@ # -------------------------------------------------------------------------------------------------- all_suites = QuestionList( - list_name="all_suites", questions=[ qd.experiment_id(), qd.experiment_root(), @@ -32,7 +31,6 @@ # -------------------------------------------------------------------------------------------------- common = QuestionList( - list_name="common", questions=[ all_suites, qd.cycle_times(), @@ -46,7 +44,6 @@ # -------------------------------------------------------------------------------------------------- marine = QuestionList( - list_name="marine", questions=[ common, qd.marine_models() @@ -56,7 +53,6 @@ # -------------------------------------------------------------------------------------------------- compare = QuestionList( - list_name="compare", questions=[ all_suites, qd.comparison_experiment_paths() @@ -66,7 +62,6 @@ # -------------------------------------------------------------------------------------------------- task_minimum = QuestionList( - list_name="task_minimum", questions=[ qd.experiment_id(), qd.experiment_root(), diff --git a/src/swell/suites/build_geos/suite_config.py b/src/swell/suites/build_geos/suite_config.py index db0fe567b..12eb3114c 100644 --- a/src/swell/suites/build_geos/suite_config.py +++ b/src/swell/suites/build_geos/suite_config.py @@ -17,7 +17,6 @@ # -------------------------------------------------------------------------------------------------- build_geos = QuestionList( - list_name="build_geos", questions=[ all_suites ] diff --git a/src/swell/suites/build_jedi/suite_config.py b/src/swell/suites/build_jedi/suite_config.py index c37cd4ebf..0cad7c32d 100644 --- a/src/swell/suites/build_jedi/suite_config.py +++ b/src/swell/suites/build_jedi/suite_config.py @@ -17,7 +17,6 @@ # -------------------------------------------------------------------------------------------------- build_jedi = QuestionList( - list_name="build_jedi", questions=[ all_suites ] diff --git a/src/swell/suites/compare/suite_config.py b/src/swell/suites/compare/suite_config.py index 0ba09739e..a86507211 100644 --- a/src/swell/suites/compare/suite_config.py +++ b/src/swell/suites/compare/suite_config.py @@ -17,7 +17,6 @@ suite_name = 'compare' compare = QuestionList( - list_name="compare", questions=[ all_suites, qd.comparison_experiment_paths(), @@ -34,7 +33,6 @@ # -------------------------------------------------------------------------------------------------- compare_variational_marine = QuestionList( - list_name="compare_variational_marine", questions=[ compare, qd.comparison_log_type('variational'), @@ -47,7 +45,6 @@ # -------------------------------------------------------------------------------------------------- compare_variational_atmosphere = QuestionList( - list_name="compare_variational_atmosphere", questions=[ compare, qd.comparison_log_type('variational'), @@ -60,7 +57,6 @@ # -------------------------------------------------------------------------------------------------- compare_fgat_marine = QuestionList( - list_name="compare_fgat_marine", questions=[ compare, qd.comparison_log_type('fgat'), diff --git a/src/swell/suites/convert_bufr/suite_config.py b/src/swell/suites/convert_bufr/suite_config.py index b4d61b758..11160b055 100644 --- a/src/swell/suites/convert_bufr/suite_config.py +++ b/src/swell/suites/convert_bufr/suite_config.py @@ -19,7 +19,6 @@ suite_name = 'convert_bufr' convert_bufr = QuestionList( - list_name="convert_bufr", questions=[ common, qd.start_cycle_point("2023-10-10T00:00:00Z"), diff --git a/src/swell/suites/convert_ncdiags/suite_config.py b/src/swell/suites/convert_ncdiags/suite_config.py index 63d288a99..ad902dda9 100644 --- a/src/swell/suites/convert_ncdiags/suite_config.py +++ b/src/swell/suites/convert_ncdiags/suite_config.py @@ -19,7 +19,6 @@ suite_name = 'convert_ncdiags' convert_ncdiags_tier1 = QuestionList( - list_name="convert_ncdiags", questions=[ common, qd.start_cycle_point("2021-12-12T00:00:00Z"), @@ -85,7 +84,6 @@ # -------------------------------------------------------------------------------------------------- convert_ncdiags = QuestionList( - list_name="convert_ncdiags", questions=[ convert_ncdiags_tier1 ] diff --git a/src/swell/suites/eva_capabilities/suite_config.py b/src/swell/suites/eva_capabilities/suite_config.py index 86778c06d..282ebdccd 100644 --- a/src/swell/suites/eva_capabilities/suite_config.py +++ b/src/swell/suites/eva_capabilities/suite_config.py @@ -18,7 +18,6 @@ suite_name = 'eva_capabilities' eva_capabilities = QuestionList( - list_name="eva_capabilities", questions=[ marine, qd.start_cycle_point("2021-07-02T06:00:00Z"), @@ -52,7 +51,6 @@ # -------------------------------------------------------------------------------------------------- eva_capabilities_atmosphere = QuestionList( - list_name="eva_capabilities_atmosphere", questions=[ eva_capabilities, qd.start_cycle_point("2023-10-10T00:00:00Z"), diff --git a/src/swell/suites/forecast_geos/suite_config.py b/src/swell/suites/forecast_geos/suite_config.py index 08df668f0..4097a59d5 100644 --- a/src/swell/suites/forecast_geos/suite_config.py +++ b/src/swell/suites/forecast_geos/suite_config.py @@ -18,7 +18,6 @@ suite_name = 'forecast_geos' forecast_geos_tier1 = QuestionList( - list_name="forecast_geos", questions=[ all_suites, qd.cycle_times(), @@ -42,7 +41,6 @@ # -------------------------------------------------------------------------------------------------- forecast_geos = QuestionList( - list_name="forecast_geos", questions=[ forecast_geos_tier1 ] diff --git a/src/swell/suites/geosadas/suite_config.py b/src/swell/suites/geosadas/suite_config.py index 08e2db3a6..ce6981cb7 100644 --- a/src/swell/suites/geosadas/suite_config.py +++ b/src/swell/suites/geosadas/suite_config.py @@ -19,7 +19,6 @@ suite_name = 'geosadas' geosadas_tier1 = QuestionList( - list_name="geosadas", questions=[ all_suites, qd.jedi_build_method("use_existing"), @@ -71,7 +70,6 @@ # -------------------------------------------------------------------------------------------------- geosadas = QuestionList( - list_name="geosadas", questions=[ geosadas_tier1 ] diff --git a/src/swell/suites/hofx/suite_config.py b/src/swell/suites/hofx/suite_config.py index 4172e20df..fe027836c 100644 --- a/src/swell/suites/hofx/suite_config.py +++ b/src/swell/suites/hofx/suite_config.py @@ -19,7 +19,6 @@ suite_name = 'hofx' hofx_tier1 = QuestionList( - list_name="hofx", questions=[ marine, qd.cycling_varbc(), @@ -79,7 +78,6 @@ # -------------------------------------------------------------------------------------------------- hofx = QuestionList( - list_name="hofx", questions=[ hofx_tier1 ] diff --git a/src/swell/suites/hofx_cf/suite_config.py b/src/swell/suites/hofx_cf/suite_config.py index e3ddb8a93..1633b530f 100644 --- a/src/swell/suites/hofx_cf/suite_config.py +++ b/src/swell/suites/hofx_cf/suite_config.py @@ -19,7 +19,6 @@ suite_name = 'hofx_cf' hofx_cf = QuestionList( - list_name="hofx_cf", questions=[ common, qd.start_cycle_point("2023-08-05T18:00:00Z"), @@ -38,7 +37,6 @@ # -------------------------------------------------------------------------------------------------- hofx_cf_tier1 = QuestionList( - list_name="hofx_cf_tier1", questions=[ hofx_cf ] diff --git a/src/swell/suites/ingest_obs/suite_config.py b/src/swell/suites/ingest_obs/suite_config.py index 9922d38a5..e456d58e7 100644 --- a/src/swell/suites/ingest_obs/suite_config.py +++ b/src/swell/suites/ingest_obs/suite_config.py @@ -18,7 +18,6 @@ suite_name = 'ingest_obs' ingest_obs = QuestionList( - list_name="ingest_obs", questions=[ common, ], @@ -31,7 +30,6 @@ # This name should be unique and not conflict with other suites # (otherwise it might get overwritten) ingest_obs_marine = QuestionList( - list_name="ingest_obs_marine", questions=[ ingest_obs, marine, diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index 8959ef3f4..ef5aacde5 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -18,7 +18,6 @@ suite_name = 'localensembleda' localensembleda_tier1 = QuestionList( - list_name="localensembleda", questions=[ marine, qd.cycling_varbc(), @@ -62,7 +61,6 @@ # -------------------------------------------------------------------------------------------------- localensembleda_tier2 = QuestionList( - list_name="localensembleda", questions=[ marine, qd.ensemble_hofx_packets(), @@ -132,7 +130,6 @@ # -------------------------------------------------------------------------------------------------- localensembleda = QuestionList( - list_name="localensembleda", questions=[ localensembleda_tier2 ] diff --git a/src/swell/suites/ufo_testing/suite_config.py b/src/swell/suites/ufo_testing/suite_config.py index 18a7901a1..2e9a11303 100644 --- a/src/swell/suites/ufo_testing/suite_config.py +++ b/src/swell/suites/ufo_testing/suite_config.py @@ -18,7 +18,6 @@ suite_name = 'ufo_testing' ufo_testing_tier1 = QuestionList( - list_name="ufo_testing", questions=[ common, qd.final_cycle_point("2023-10-10T00:00:00Z"), @@ -87,7 +86,6 @@ # -------------------------------------------------------------------------------------------------- ufo_testing = QuestionList( - list_name="ufo_testing", questions=[ ufo_testing_tier1 ] diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index 066453b4a..049f9a639 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -121,7 +121,6 @@ class SwellQuestion: class QuestionList: """Basic dataclass containing a list of questions for each model, suite, task""" questions: List[Union[SwellQuestion, Self]] = mutable_field([]) - list_name: str = '' geos_atmosphere: list = mutable_field([]) geos_cf: list = mutable_field([]) From aa468bbaf5253888866923492e0e0dc484961201 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 12:50:08 -0500 Subject: [PATCH 227/299] Remove get_suites --- src/swell/utilities/suite_utils.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/swell/utilities/suite_utils.py b/src/swell/utilities/suite_utils.py index 0d63ce745..5520a4abb 100644 --- a/src/swell/utilities/suite_utils.py +++ b/src/swell/utilities/suite_utils.py @@ -25,19 +25,3 @@ def get_model_components() -> list: return os.listdir(interface_directory) # -------------------------------------------------------------------------------------------------- - - -def get_suites() -> list: - - # Path to platforms - suites_directory = os.path.join(get_swell_path(), 'suites') - - # List of base suites - suites = sorted([sdir for sdir in os.listdir(suites_directory) - if (os.path.isdir(os.path.join(suites_directory, sdir)) - and os.path.exists(os.path.join(suites_directory, sdir, - 'suite_config.py')))]) - - return suites - -# -------------------------------------------------------------------------------------------------- From 433ba0dba5eb434ea9c5d6dacb1b7196dc72791a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 13:32:53 -0500 Subject: [PATCH 228/299] refactor suite attributes --- src/swell/deployment/create_experiment.py | 2 +- .../prepare_config_and_suite.py | 2 +- src/swell/suites/3dfgat_atmos/suite_config.py | 2 +- src/swell/suites/3dfgat_atmos/workflow.py | 2 +- src/swell/suites/3dfgat_cycle/suite_config.py | 2 +- src/swell/suites/3dfgat_cycle/workflow.py | 2 +- src/swell/suites/3dvar/suite_config.py | 2 +- src/swell/suites/3dvar/workflow.py | 2 +- src/swell/suites/3dvar_atmos/suite_config.py | 2 +- src/swell/suites/3dvar_atmos/workflow.py | 2 +- src/swell/suites/3dvar_cycle/suite_config.py | 2 +- src/swell/suites/3dvar_cycle/workflow.py | 2 +- src/swell/suites/base/all_suites.py | 106 ------------------ src/swell/suites/base/suite_questions.py | 2 +- src/swell/suites/build_geos/suite_config.py | 2 +- src/swell/suites/build_geos/workflow.py | 2 +- src/swell/suites/build_jedi/suite_config.py | 2 +- src/swell/suites/build_jedi/workflow.py | 2 +- src/swell/suites/compare/suite_config.py | 2 +- src/swell/suites/compare/workflow.py | 2 +- src/swell/suites/convert_bufr/suite_config.py | 2 +- src/swell/suites/convert_bufr/workflow.py | 2 +- .../suites/convert_ncdiags/suite_config.py | 2 +- src/swell/suites/convert_ncdiags/workflow.py | 2 +- .../suites/eva_capabilities/suite_config.py | 2 +- src/swell/suites/eva_capabilities/workflow.py | 2 +- .../suites/forecast_geos/suite_config.py | 2 +- src/swell/suites/forecast_geos/workflow.py | 2 +- src/swell/suites/geosadas/suite_config.py | 2 +- src/swell/suites/geosadas/workflow.py | 2 +- src/swell/suites/hofx/suite_config.py | 2 +- src/swell/suites/hofx/workflow.py | 2 +- src/swell/suites/hofx_cf/suite_config.py | 2 +- src/swell/suites/hofx_cf/workflow.py | 2 +- src/swell/suites/ingest_obs/suite_config.py | 2 +- src/swell/suites/ingest_obs/workflow.py | 2 +- .../suites/localensembleda/suite_config.py | 2 +- src/swell/suites/localensembleda/workflow.py | 2 +- src/swell/suites/ufo_testing/suite_config.py | 2 +- src/swell/suites/ufo_testing/workflow.py | 2 +- src/swell/swell.py | 2 +- src/swell/utilities/config.py | 2 +- 42 files changed, 41 insertions(+), 147 deletions(-) delete mode 100644 src/swell/suites/base/all_suites.py diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 48bd77cc5..c69f8a843 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -23,7 +23,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.logger import Logger, get_logger from swell.utilities.slurm import prepare_slurm_defaults_and_overrides -from swell.suites.base.all_suites import suite_configs, workflows +from swell.suites.base.suite_attributes import suite_configs, workflows from swell.utilities.check_da_params import check_da_params diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 6c90a9c3b..0837e7492 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -21,7 +21,7 @@ from swell.utilities.dictionary import dict_get from swell.utilities.logger import Logger from swell.utilities.dictionary import update_dict, add_dict -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs from swell.utilities.swell_questions import QuestionType diff --git a/src/swell/suites/3dfgat_atmos/suite_config.py b/src/swell/suites/3dfgat_atmos/suite_config.py index b06306368..9245cb06f 100644 --- a/src/swell/suites/3dfgat_atmos/suite_config.py +++ b/src/swell/suites/3dfgat_atmos/suite_config.py @@ -11,7 +11,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import common -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs suite_name = '3dfgat_atmos' diff --git a/src/swell/suites/3dfgat_atmos/workflow.py b/src/swell/suites/3dfgat_atmos/workflow.py index 28621aa48..a88e07ffb 100644 --- a/src/swell/suites/3dfgat_atmos/workflow.py +++ b/src/swell/suites/3dfgat_atmos/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_cycle/suite_config.py b/src/swell/suites/3dfgat_cycle/suite_config.py index 284a7c34c..33269dedf 100644 --- a/src/swell/suites/3dfgat_cycle/suite_config.py +++ b/src/swell/suites/3dfgat_cycle/suite_config.py @@ -11,7 +11,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import marine -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs suite_name = '3dfgat_cycle' diff --git a/src/swell/suites/3dfgat_cycle/workflow.py b/src/swell/suites/3dfgat_cycle/workflow.py index 1ac12059e..2b601e50d 100644 --- a/src/swell/suites/3dfgat_cycle/workflow.py +++ b/src/swell/suites/3dfgat_cycle/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/suite_config.py b/src/swell/suites/3dvar/suite_config.py index 4e7ac6b06..c607add1c 100644 --- a/src/swell/suites/3dvar/suite_config.py +++ b/src/swell/suites/3dvar/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs from swell.suites.base.suite_questions import marine # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar/workflow.py b/src/swell/suites/3dvar/workflow.py index 5518c1ad9..4fde57879 100644 --- a/src/swell/suites/3dvar/workflow.py +++ b/src/swell/suites/3dvar/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_atmos/suite_config.py b/src/swell/suites/3dvar_atmos/suite_config.py index 500aac557..9cbb3df91 100644 --- a/src/swell/suites/3dvar_atmos/suite_config.py +++ b/src/swell/suites/3dvar_atmos/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs from swell.suites.base.suite_questions import common suite_name = '3dvar_atmos' diff --git a/src/swell/suites/3dvar_atmos/workflow.py b/src/swell/suites/3dvar_atmos/workflow.py index 9536aac67..9699bf9f4 100644 --- a/src/swell/suites/3dvar_atmos/workflow.py +++ b/src/swell/suites/3dvar_atmos/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cycle/suite_config.py b/src/swell/suites/3dvar_cycle/suite_config.py index dc39223fc..89e6bfaa6 100644 --- a/src/swell/suites/3dvar_cycle/suite_config.py +++ b/src/swell/suites/3dvar_cycle/suite_config.py @@ -11,7 +11,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import marine -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs suite_name = '3dvar_cycle' diff --git a/src/swell/suites/3dvar_cycle/workflow.py b/src/swell/suites/3dvar_cycle/workflow.py index 74d759a23..0e7429a45 100644 --- a/src/swell/suites/3dvar_cycle/workflow.py +++ b/src/swell/suites/3dvar_cycle/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/base/all_suites.py b/src/swell/suites/base/all_suites.py deleted file mode 100644 index 24a21b27d..000000000 --- a/src/swell/suites/base/all_suites.py +++ /dev/null @@ -1,106 +0,0 @@ -# -------------------------------------------------------------------------------------------------- -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - -from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.utilities.swell_questions import QuestionList -import swell.suites -from swell.utilities.plugins import discover_plugins - -# -------------------------------------------------------------------------------------------------- - - -def format_suite_name(suite_name): - # Format suite names starting with a digit - return suite_name[1:] if suite_name[0] == '_' else suite_name - -# -------------------------------------------------------------------------------------------------- - - -class Workflows(): - - def __init__(self) -> None: - self.__workflow_names__ = [] - - def register(self, name: str) -> None: - self.__workflow_names__.append(name) - - def wrapper(cls): - setattr(self, name, cls) - return cls - return wrapper - - def get(self, name: str) -> type[CylcWorkflow]: - return getattr(self, name) - - def all(self) -> list: - return self.__workflow_names__ - -# -------------------------------------------------------------------------------------------------- - - -class SuiteConfigs(): - - def __init__(self) -> None: - - # Dictionary tracking the suite for each config - self.__config_map__ = {} - - # -------------------------------------------------------------------------------------------------- - - def register(self, - base_suite: str, - config_name: str, - question_list: QuestionList) -> None: - - self.__config_map__[config_name] = sub_dict = {} - - sub_dict['suite'] = base_suite - sub_dict['list'] = question_list - - # -------------------------------------------------------------------------------------------------- - - def get_config(self, config_name: str) -> QuestionList: - return self.__config_map__[config_name]['list'] - - # -------------------------------------------------------------------------------------------------- - - def base_suite(self, config_name: str) -> str: - return self.__config_map__[config_name]['suite'] - - # -------------------------------------------------------------------------------------------------- - - def all_configs(self) -> str: - return list(self.__config_map__.keys()) - - # -------------------------------------------------------------------------------------------------- - - def configs_under_suites(self) -> dict: - suite_map = {} - - for config_name, config_dict in self.__config_map__.items(): - suite_name = config_dict['suite'] - - if suite_name not in suite_map: - suite_map[suite_name] = [] - - suite_map[suite_name].append(config_name) - - return suite_map - -# -------------------------------------------------------------------------------------------------- - - -# Objects to reference in imports -suite_configs = SuiteConfigs() -workflows = Workflows() - -discover_plugins(swell.suites) - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/base/suite_questions.py b/src/swell/suites/base/suite_questions.py index 9f2955dfb..bfd9647a8 100644 --- a/src/swell/suites/base/suite_questions.py +++ b/src/swell/suites/base/suite_questions.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- # Shared groups of questions across suites diff --git a/src/swell/suites/build_geos/suite_config.py b/src/swell/suites/build_geos/suite_config.py index 12eb3114c..d3d3d0382 100644 --- a/src/swell/suites/build_geos/suite_config.py +++ b/src/swell/suites/build_geos/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionList from swell.suites.base.suite_questions import all_suites -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs suite_name = 'build_geos' diff --git a/src/swell/suites/build_geos/workflow.py b/src/swell/suites/build_geos/workflow.py index e444b6a16..a97f4c544 100644 --- a/src/swell/suites/build_geos/workflow.py +++ b/src/swell/suites/build_geos/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/build_jedi/suite_config.py b/src/swell/suites/build_jedi/suite_config.py index 0cad7c32d..12897233c 100644 --- a/src/swell/suites/build_jedi/suite_config.py +++ b/src/swell/suites/build_jedi/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionList from swell.suites.base.suite_questions import all_suites -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs suite_name = 'build_jedi' diff --git a/src/swell/suites/build_jedi/workflow.py b/src/swell/suites/build_jedi/workflow.py index 1e7b62406..6850c9715 100644 --- a/src/swell/suites/build_jedi/workflow.py +++ b/src/swell/suites/build_jedi/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare/suite_config.py b/src/swell/suites/compare/suite_config.py index a86507211..e445768d8 100644 --- a/src/swell/suites/compare/suite_config.py +++ b/src/swell/suites/compare/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionList, WidgetType import swell.configuration.question_defaults as qd -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs from swell.suites.base.suite_questions import all_suites # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare/workflow.py b/src/swell/suites/compare/workflow.py index ae1895c81..fb4348bb8 100644 --- a/src/swell/suites/compare/workflow.py +++ b/src/swell/suites/compare/workflow.py @@ -12,7 +12,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_bufr/suite_config.py b/src/swell/suites/convert_bufr/suite_config.py index 11160b055..9a3f55a50 100644 --- a/src/swell/suites/convert_bufr/suite_config.py +++ b/src/swell/suites/convert_bufr/suite_config.py @@ -11,7 +11,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import common -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_bufr/workflow.py b/src/swell/suites/convert_bufr/workflow.py index d17e9e563..e999d13c1 100644 --- a/src/swell/suites/convert_bufr/workflow.py +++ b/src/swell/suites/convert_bufr/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/suite_config.py b/src/swell/suites/convert_ncdiags/suite_config.py index ad902dda9..15a7f8971 100644 --- a/src/swell/suites/convert_ncdiags/suite_config.py +++ b/src/swell/suites/convert_ncdiags/suite_config.py @@ -11,7 +11,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import common -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/workflow.py b/src/swell/suites/convert_ncdiags/workflow.py index b86704da6..9118f5e8b 100644 --- a/src/swell/suites/convert_ncdiags/workflow.py +++ b/src/swell/suites/convert_ncdiags/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eva_capabilities/suite_config.py b/src/swell/suites/eva_capabilities/suite_config.py index 282ebdccd..a2aa04b65 100644 --- a/src/swell/suites/eva_capabilities/suite_config.py +++ b/src/swell/suites/eva_capabilities/suite_config.py @@ -11,7 +11,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import marine -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eva_capabilities/workflow.py b/src/swell/suites/eva_capabilities/workflow.py index 0447a7b44..4beae3288 100644 --- a/src/swell/suites/eva_capabilities/workflow.py +++ b/src/swell/suites/eva_capabilities/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/suite_config.py b/src/swell/suites/forecast_geos/suite_config.py index 4097a59d5..635fc974a 100644 --- a/src/swell/suites/forecast_geos/suite_config.py +++ b/src/swell/suites/forecast_geos/suite_config.py @@ -11,7 +11,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import all_suites -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_geos/workflow.py b/src/swell/suites/forecast_geos/workflow.py index cf256d77e..566d5c097 100644 --- a/src/swell/suites/forecast_geos/workflow.py +++ b/src/swell/suites/forecast_geos/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/suite_config.py b/src/swell/suites/geosadas/suite_config.py index ce6981cb7..e84769bb0 100644 --- a/src/swell/suites/geosadas/suite_config.py +++ b/src/swell/suites/geosadas/suite_config.py @@ -11,7 +11,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import all_suites -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/workflow.py b/src/swell/suites/geosadas/workflow.py index 7282b00d1..ae1af9d42 100644 --- a/src/swell/suites/geosadas/workflow.py +++ b/src/swell/suites/geosadas/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/suite_config.py b/src/swell/suites/hofx/suite_config.py index fe027836c..52c193943 100644 --- a/src/swell/suites/hofx/suite_config.py +++ b/src/swell/suites/hofx/suite_config.py @@ -11,7 +11,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import marine -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx/workflow.py b/src/swell/suites/hofx/workflow.py index 397da336b..e363e1b91 100644 --- a/src/swell/suites/hofx/workflow.py +++ b/src/swell/suites/hofx/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx_cf/suite_config.py b/src/swell/suites/hofx_cf/suite_config.py index 1633b530f..c1748c95f 100644 --- a/src/swell/suites/hofx_cf/suite_config.py +++ b/src/swell/suites/hofx_cf/suite_config.py @@ -11,7 +11,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import common -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/hofx_cf/workflow.py b/src/swell/suites/hofx_cf/workflow.py index 67c6e68ea..e687330ae 100644 --- a/src/swell/suites/hofx_cf/workflow.py +++ b/src/swell/suites/hofx_cf/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ingest_obs/suite_config.py b/src/swell/suites/ingest_obs/suite_config.py index e456d58e7..fc0bd76d7 100644 --- a/src/swell/suites/ingest_obs/suite_config.py +++ b/src/swell/suites/ingest_obs/suite_config.py @@ -10,7 +10,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import common, marine -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ingest_obs/workflow.py b/src/swell/suites/ingest_obs/workflow.py index f7444f569..1ba080a8c 100644 --- a/src/swell/suites/ingest_obs/workflow.py +++ b/src/swell/suites/ingest_obs/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index ef5aacde5..5f2536586 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -11,7 +11,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import marine -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/localensembleda/workflow.py b/src/swell/suites/localensembleda/workflow.py index 347557805..b70b39444 100644 --- a/src/swell/suites/localensembleda/workflow.py +++ b/src/swell/suites/localensembleda/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/suite_config.py b/src/swell/suites/ufo_testing/suite_config.py index 2e9a11303..f2a62ad1a 100644 --- a/src/swell/suites/ufo_testing/suite_config.py +++ b/src/swell/suites/ufo_testing/suite_config.py @@ -11,7 +11,7 @@ from swell.utilities.swell_questions import QuestionList import swell.configuration.question_defaults as qd from swell.suites.base.suite_questions import common -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/ufo_testing/workflow.py b/src/swell/suites/ufo_testing/workflow.py index de623a0fe..672d25f24 100644 --- a/src/swell/suites/ufo_testing/workflow.py +++ b/src/swell/suites/ufo_testing/workflow.py @@ -10,7 +10,7 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.all_suites import workflows +from swell.suites.base.suite_attributes import workflows # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index 8b781ed6b..78d5af12e 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -18,7 +18,7 @@ from swell.tasks.base.task_base import task_wrapper, get_tasks from swell.test.test_driver import test_wrapper, valid_tests from swell.test.suite_tests.suite_tests import run_suite, TestSuite -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs from swell.utilities.welcome_message import write_welcome_message from swell.utilities.scripts.utility_driver import get_utilities, utility_wrapper from swell.deployment.create_task_config import task_config_wrapper diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 5f0fc2e8c..456f66342 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -12,7 +12,7 @@ from swell.tasks.base.task_attributes import task_attributes from swell.utilities.logger import Logger -from swell.suites.base.all_suites import suite_configs +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- From 904e679f30c015f6e31796600936658c26c74edb Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 13:36:24 -0500 Subject: [PATCH 229/299] Fix task minimum --- src/swell/suites/base/suite_questions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/suites/base/suite_questions.py b/src/swell/suites/base/suite_questions.py index bfd9647a8..0d4c99e86 100644 --- a/src/swell/suites/base/suite_questions.py +++ b/src/swell/suites/base/suite_questions.py @@ -72,6 +72,6 @@ ] ) -suite_configs.register('TaskMinimum', 'TaskMinimum', task_minimum) +suite_configs.register('task_minimum', 'task_minimum', task_minimum) # -------------------------------------------------------------------------------------------------- From 2014fd1dfb39b40a34011e15ec2aed13e23f5fd4 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 3 Mar 2026 15:46:56 -0500 Subject: [PATCH 230/299] Use subprocess and check exit status for mksi --- src/swell/tasks/clone_geos_mksi.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index a4eda2ead..3dd5c2501 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -9,6 +9,7 @@ import os +import subprocess from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes @@ -60,9 +61,17 @@ def execute(self) -> None: branch = 'develop' else: branch = tag - # Clone GEOS_mksi develop repo to experiment directory - os.system(f'git clone -b {branch} https://github.com/GEOS-ESM/GEOS_mksi.git ' - + os.path.join(self.experiment_path(), 'GEOS_mksi')) + + mksi_path = os.path.join(self.experiment_path(), 'GEOS_mksi') + + if os.path.exists(mksi_path): + # Checkout the branch + subprocess.run(['git', 'checkout', branch], cwd=mksi_path, check=True) + else: + # Clone GEOS_mksi develop repo to experiment directory + subprocess.run(['git', 'clone', '-b', branch, + 'https://github.com/GEOS-ESM/GEOS_mksi.git', + mksi_path], check=True) else: # Link the source code directory link_path(self.config.observing_system_records_mksi_path(), From 08e8674314c39e9bfb94f4a6c0f1d4708a0e8b9b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 10 Mar 2026 13:54:55 -0400 Subject: [PATCH 231/299] Fixes --- src/swell/deployment/create_experiment.py | 5 +++-- .../prepare_config_and_suite/prepare_config_and_suite.py | 2 +- src/swell/suites/3dvar_marine/flow.cylc | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index e60515505..29050c7e6 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -15,7 +15,7 @@ import shutil import sys from ruamel.yaml import YAML -from typing import Union, Optional +from typing import Optional from swell.suites.all_suites import AllSuites from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import \ @@ -178,7 +178,8 @@ def prepare_config( # Register the experiment in R2D2 # ------------------------------- - if 'r2d2_experiment_id' in experiment_dict: + if 'r2d2_experiment_id' in experiment_dict and 'skip_r2d2' in experiment_dict \ + and not experiment_dict['skip_r2d2']: from swell.utilities.r2d2 import load_r2d2_credentials, load_r2d2_module, unique_r2d2_id diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index c260c0bde..69987bf5f 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -12,7 +12,7 @@ import os from ruamel.yaml import YAML from collections.abc import Mapping -from typing import Union, Tuple, Optional +from typing import Tuple, Optional from swell.swell_path import get_swell_path from swell.deployment.prepare_config_and_suite.question_and_answer_cli import GetAnswerCli diff --git a/src/swell/suites/3dvar_marine/flow.cylc b/src/swell/suites/3dvar_marine/flow.cylc index ef036e0be..7afdb11af 100644 --- a/src/swell/suites/3dvar_marine/flow.cylc +++ b/src/swell/suites/3dvar_marine/flow.cylc @@ -71,7 +71,7 @@ # EvaIncrement RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - + {% if not skip_r2d2 %} # Save observations RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} From e42e3b9297d225419f022bbb137d99cbde01fc4f Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 10 Mar 2026 14:37:35 -0400 Subject: [PATCH 232/299] remove comma --- src/swell/deployment/create_experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 29050c7e6..0b7fa81e3 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -89,7 +89,7 @@ def prepare_config( platform: str, override: dict, advanced: bool, - slurm: str, + slurm: str ) -> str: # Create a logger From a97c229b7cca9fe5d7b719a371330485b9ac9816 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 10 Mar 2026 14:47:05 -0400 Subject: [PATCH 233/299] code_test fix --- src/swell/utilities/r2d2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/utilities/r2d2.py b/src/swell/utilities/r2d2.py index b735dbc0c..2d96a82de 100644 --- a/src/swell/utilities/r2d2.py +++ b/src/swell/utilities/r2d2.py @@ -14,7 +14,7 @@ from swell.swell_path import get_swell_path from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.logger import get_logger, Logger +from swell.utilities.logger import Logger # -------------------------------------------------------------------------------------------------- From 5f332cd8a8978a111363c1c7ebc240f9d2d478fe Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 19 Mar 2026 10:18:37 -0400 Subject: [PATCH 234/299] fix templating --- src/swell/suites/3dfgat_marine_cycle/flow.cylc | 2 +- src/swell/suites/3dvar_atmos/flow.cylc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/suites/3dfgat_marine_cycle/flow.cylc b/src/swell/suites/3dfgat_marine_cycle/flow.cylc index 41dac74e9..438fb8186 100644 --- a/src/swell/suites/3dfgat_marine_cycle/flow.cylc +++ b/src/swell/suites/3dfgat_marine_cycle/flow.cylc @@ -102,7 +102,7 @@ # Move restart to next cycle and then erase current forecast folder SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} - {% if not save_r2d2 %} + {% if not skip_r2d2 %} # Save analysis output # RunJediFgatExecutable-{{model_component}} => SaveAnalysis-{{model_component}} RunJediFgatExecutable-{{model_component}} => SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc index 30bf4d0a0..5e400f6a9 100644 --- a/src/swell/suites/3dvar_atmos/flow.cylc +++ b/src/swell/suites/3dvar_atmos/flow.cylc @@ -86,7 +86,7 @@ # EvaIncrement RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - {% if not save_r2d2 %} + {% if not skip_r2d2 %} # Save observations RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} {% endif %} From 2a5a0f85189b2c765fc91a73ce375883af52a5af Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 19 Mar 2026 11:54:14 -0400 Subject: [PATCH 235/299] Clean cycle after move da restart --- src/swell/suites/3dfgat_marine_cycle/flow.cylc | 2 +- src/swell/suites/3dvar_marine_cycle/flow.cylc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/suites/3dfgat_marine_cycle/flow.cylc b/src/swell/suites/3dfgat_marine_cycle/flow.cylc index 438fb8186..854ba71bb 100644 --- a/src/swell/suites/3dfgat_marine_cycle/flow.cylc +++ b/src/swell/suites/3dfgat_marine_cycle/flow.cylc @@ -100,7 +100,7 @@ {% endif %} # Move restart to next cycle and then erase current forecast folder - SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} + SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} => CleanCycle-{{model_component}} {% if not skip_r2d2 %} # Save analysis output diff --git a/src/swell/suites/3dvar_marine_cycle/flow.cylc b/src/swell/suites/3dvar_marine_cycle/flow.cylc index 8f9a9e8be..5792460d8 100644 --- a/src/swell/suites/3dvar_marine_cycle/flow.cylc +++ b/src/swell/suites/3dvar_marine_cycle/flow.cylc @@ -99,7 +99,7 @@ {% endif %} # Move restart to next cycle and then erase current forecast folder - SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} + SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} => CleanCycle-{{model_component}} {% if skip_r2d2 %} # Save analysis output From 987bbafb3f00272060f2d7d739452020b08b0b4a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 19 Mar 2026 15:22:57 -0400 Subject: [PATCH 236/299] Fixes to workflows --- src/swell/suites/3dfgat_atmos/flow.cylc | 2 +- src/swell/suites/3dvar_atmos/flow.cylc | 2 +- src/swell/suites/3dvar_marine_cycle/flow.cylc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/swell/suites/3dfgat_atmos/flow.cylc b/src/swell/suites/3dfgat_atmos/flow.cylc index 44cdcb696..878637314 100644 --- a/src/swell/suites/3dfgat_atmos/flow.cylc +++ b/src/swell/suites/3dfgat_atmos/flow.cylc @@ -97,7 +97,7 @@ # Clean up large files EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + EvaObservations-{{model_component}} => CleanCycle-{{model_component}} {% endif %} diff --git a/src/swell/suites/3dvar_atmos/flow.cylc b/src/swell/suites/3dvar_atmos/flow.cylc index 5e400f6a9..b71bba42a 100644 --- a/src/swell/suites/3dvar_atmos/flow.cylc +++ b/src/swell/suites/3dvar_atmos/flow.cylc @@ -93,7 +93,7 @@ # Clean up large files EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + EvaObservations-{{model_component}} => CleanCycle-{{model_component}} {% endif %} diff --git a/src/swell/suites/3dvar_marine_cycle/flow.cylc b/src/swell/suites/3dvar_marine_cycle/flow.cylc index 5792460d8..8b76edf85 100644 --- a/src/swell/suites/3dvar_marine_cycle/flow.cylc +++ b/src/swell/suites/3dvar_marine_cycle/flow.cylc @@ -101,7 +101,7 @@ # Move restart to next cycle and then erase current forecast folder SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} => CleanCycle-{{model_component}} - {% if skip_r2d2 %} + {% if not skip_r2d2 %} # Save analysis output # RunJediVariationalExecutable-{{model_component}} => SaveAnalysis-{{model_component}} RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} @@ -111,7 +111,7 @@ # MoveBackground-{{model_component}} => StoreBackground-{{model_component}} # Clean up large files - EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & SaveObsDiags-{{model_component}} => + EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} {% endfor %} """ From 9119709313a9bae2a9c9f56647834a6686472448 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 20 Mar 2026 16:24:53 -0400 Subject: [PATCH 237/299] Add suite creation test --- .../test/code_tests/suite_creation_test.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/swell/test/code_tests/suite_creation_test.py diff --git a/src/swell/test/code_tests/suite_creation_test.py b/src/swell/test/code_tests/suite_creation_test.py new file mode 100644 index 000000000..bceb07650 --- /dev/null +++ b/src/swell/test/code_tests/suite_creation_test.py @@ -0,0 +1,42 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import tempfile +import os +from ruamel.yaml import YAML + +from swell.deployment.create_experiment import create_experiment_directory + +# -------------------------------------------------------------------------------------------------- + +def suite_creation_test(suite: str) -> None: + + tempdir = tempfile.mkdtemp() + + override_dict = {} + + override_dict['experiment_root'] = tempdir + + create_experiment_directory(suite, 'defaults', 'nccs_discover_sles15', + override_dict, False, None) + + experiment_yaml = os.path.join(tempdir, f'swell-{suite}', f'swell-{suite}-suite', 'experiment.yaml') + + yaml = YAML(typ='safe') + + with open(experiment_yaml, 'r') as f: + experiment_yaml_str = yaml(f) + + assert 'defer_to_model' not in experiment_yaml_str + assert 'defer_to_platform' not in experiment_yaml_str + +# -------------------------------------------------------------------------------------------------- + +if __name__ == '__main__': + suite_creation_test('3dvar_marine') \ No newline at end of file From 4b1a98cc24d232b75aed547b08d5d72a5dbf4926 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 20 Mar 2026 16:36:58 -0400 Subject: [PATCH 238/299] Add to code tests --- src/swell/test/code_tests/code_tests.py | 4 ++ .../test/code_tests/suite_creation_test.py | 41 +++++++++++-------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/swell/test/code_tests/code_tests.py b/src/swell/test/code_tests/code_tests.py index ff3e4e4c2..2832a937d 100644 --- a/src/swell/test/code_tests/code_tests.py +++ b/src/swell/test/code_tests/code_tests.py @@ -17,6 +17,7 @@ from swell.test.code_tests.unused_variables_test import UnusedVariablesTest from swell.test.code_tests.question_dictionary_comparison_test import QuestionDictionaryTest from swell.test.code_tests.test_generate_observing_system import GenerateObservingSystemTest +from swell.test.code_tests.suite_creation_test import SuiteCreationTest # -------------------------------------------------------------------------------------------------- @@ -52,6 +53,9 @@ def code_tests() -> None: # Load Pinned Versions Test test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(PinnedVersionsTest)) + # Load Suite Creation Test + test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(SuiteCreationTest)) + # Create a test runner test_runner = unittest.TextTestRunner() diff --git a/src/swell/test/code_tests/suite_creation_test.py b/src/swell/test/code_tests/suite_creation_test.py index bceb07650..adf5bae35 100644 --- a/src/swell/test/code_tests/suite_creation_test.py +++ b/src/swell/test/code_tests/suite_creation_test.py @@ -10,33 +10,42 @@ import tempfile import os from ruamel.yaml import YAML +import unittest +from swell.suites.all_suites import get_suites from swell.deployment.create_experiment import create_experiment_directory # -------------------------------------------------------------------------------------------------- -def suite_creation_test(suite: str) -> None: +class SuiteCreationTest(unittest.TestCase): - tempdir = tempfile.mkdtemp() + def run_suite_creation_test(self) -> None: - override_dict = {} + suites = get_suites() - override_dict['experiment_root'] = tempdir + for suite in suites: + self.suite_creation_test(suite) - create_experiment_directory(suite, 'defaults', 'nccs_discover_sles15', - override_dict, False, None) - - experiment_yaml = os.path.join(tempdir, f'swell-{suite}', f'swell-{suite}-suite', 'experiment.yaml') + def suite_creation_test(self, suite: str) -> None: - yaml = YAML(typ='safe') + tempdir = tempfile.mkdtemp() - with open(experiment_yaml, 'r') as f: - experiment_yaml_str = yaml(f) + override_dict = {} - assert 'defer_to_model' not in experiment_yaml_str - assert 'defer_to_platform' not in experiment_yaml_str + override_dict['experiment_root'] = tempdir + override_dict['skip_r2d2'] = True -# -------------------------------------------------------------------------------------------------- + create_experiment_directory(suite, 'defaults', 'nccs_discover_sles15', + override_dict, False, None) + + experiment_yaml = os.path.join(tempdir, f'swell-{suite}', f'swell-{suite}-suite', 'experiment.yaml') + + yaml = YAML(typ='safe') + + with open(experiment_yaml, 'r') as f: + experiment_yaml_str = yaml.load(f) + + assert 'defer_to_model' not in experiment_yaml_str + assert 'defer_to_platform' not in experiment_yaml_str -if __name__ == '__main__': - suite_creation_test('3dvar_marine') \ No newline at end of file +# -------------------------------------------------------------------------------------------------- \ No newline at end of file From eb58146a59fc53e50826b2e5f4ea44c01c769231 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 23 Mar 2026 10:57:12 -0400 Subject: [PATCH 239/299] Add jedi config comparison tests --- src/swell/test/code_tests/jedi_config_test.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/swell/test/code_tests/jedi_config_test.py diff --git a/src/swell/test/code_tests/jedi_config_test.py b/src/swell/test/code_tests/jedi_config_test.py new file mode 100644 index 000000000..ed7928076 --- /dev/null +++ b/src/swell/test/code_tests/jedi_config_test.py @@ -0,0 +1,66 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +import tempfile +from ruamel.yaml import YAML + +from swell.swell_path import get_swell_path +from swell.deployment.create_experiment import create_experiment_directory +from swell.tasks.base.task_base import task_wrapper + +# -------------------------------------------------------------------------------------------------- + +def test_jedi_config(suite: str, + model: str, + datetime: str, + executable_type: str) -> None: + + tempdir = tempfile.mkdtemp() + + override_dict = {'models': {}} + override_dict['experiment_root'] = tempdir + override_dict['models'][model] = {'check_for_obs': False, + 'generate_yaml_and_exit': True} + + create_experiment_directory(suite, 'defaults', 'nccs_discover_sles15', + override_dict, 'defaults', None) + + experiment_yaml = os.path.join(tempdir, f'swell-{suite}', + f'swell-{suite}-suite', 'experiment.yaml') + + task_wrapper('RenderJediObservations', experiment_yaml, datetime, + model, ensemblePacket=None) + + task_wrapper(f'RunJedi{executable_type.upper()}Executable', experiment_yaml, datetime, + model, ensemblePacket=None) + + cycle_dir = os.path.join(tempdir, f'swell-{suite}', 'run', datetime, model) + + config_file = os.path.join(cycle_dir, f'jedi_{executable_type}_config.yaml') + + yaml = YAML(typ='safe') + + with open(config_file, 'r') as f: + config_dict = yaml.load(f) + + comparison_config = os.path.join(get_swell_path(), 'test', f'jedi_config_{suite}.yaml') + + with open(comparison_config, 'r') as f: + comparison_dict = yaml.load(f) + + assert config_dict == comparison_dict + +# -------------------------------------------------------------------------------------------------- + + +if __name__ == '__main__': + test_jedi_config('3dvar_marine') + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file From 912b3f07a2b03935d10183b81338b6a08bfdd417 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 23 Mar 2026 10:58:47 -0400 Subject: [PATCH 240/299] Change name of runTest --- src/swell/test/code_tests/suite_creation_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/test/code_tests/suite_creation_test.py b/src/swell/test/code_tests/suite_creation_test.py index adf5bae35..dd9a22a3b 100644 --- a/src/swell/test/code_tests/suite_creation_test.py +++ b/src/swell/test/code_tests/suite_creation_test.py @@ -19,7 +19,7 @@ class SuiteCreationTest(unittest.TestCase): - def run_suite_creation_test(self) -> None: + def runTest(self) -> None: suites = get_suites() @@ -48,4 +48,4 @@ def suite_creation_test(self, suite: str) -> None: assert 'defer_to_model' not in experiment_yaml_str assert 'defer_to_platform' not in experiment_yaml_str -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- From 9c342ca85b3f85560fe0f01c5d00a2de4dee941e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 23 Mar 2026 14:07:23 -0400 Subject: [PATCH 241/299] Separate into utility --- src/swell/test/code_tests/jedi_config_test.py | 34 +++-------- src/swell/utilities/render_jedi_config.py | 56 +++++++++++++++++++ 2 files changed, 63 insertions(+), 27 deletions(-) create mode 100644 src/swell/utilities/render_jedi_config.py diff --git a/src/swell/test/code_tests/jedi_config_test.py b/src/swell/test/code_tests/jedi_config_test.py index ed7928076..e8667b386 100644 --- a/src/swell/test/code_tests/jedi_config_test.py +++ b/src/swell/test/code_tests/jedi_config_test.py @@ -12,36 +12,16 @@ from ruamel.yaml import YAML from swell.swell_path import get_swell_path -from swell.deployment.create_experiment import create_experiment_directory -from swell.tasks.base.task_base import task_wrapper +from swell.utilities.render_jedi_config import render_jedi_config # -------------------------------------------------------------------------------------------------- -def test_jedi_config(suite: str, - model: str, - datetime: str, - executable_type: str) -> None: +def run_test(suite: str, + model: str, + datetime: str, + executable_type: str) -> None: - tempdir = tempfile.mkdtemp() - - override_dict = {'models': {}} - override_dict['experiment_root'] = tempdir - override_dict['models'][model] = {'check_for_obs': False, - 'generate_yaml_and_exit': True} - - create_experiment_directory(suite, 'defaults', 'nccs_discover_sles15', - override_dict, 'defaults', None) - - experiment_yaml = os.path.join(tempdir, f'swell-{suite}', - f'swell-{suite}-suite', 'experiment.yaml') - - task_wrapper('RenderJediObservations', experiment_yaml, datetime, - model, ensemblePacket=None) - - task_wrapper(f'RunJedi{executable_type.upper()}Executable', experiment_yaml, datetime, - model, ensemblePacket=None) - - cycle_dir = os.path.join(tempdir, f'swell-{suite}', 'run', datetime, model) + render_jedi_config(suite, model, datetime, executable_type) config_file = os.path.join(cycle_dir, f'jedi_{executable_type}_config.yaml') @@ -61,6 +41,6 @@ def test_jedi_config(suite: str, if __name__ == '__main__': - test_jedi_config('3dvar_marine') + run_test('3dvar_marine', 'geos_marine', '20210701T120000Z', 'variational') # -------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/swell/utilities/render_jedi_config.py b/src/swell/utilities/render_jedi_config.py new file mode 100644 index 000000000..dc673faa8 --- /dev/null +++ b/src/swell/utilities/render_jedi_config.py @@ -0,0 +1,56 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +import shutil +import tempfile + +from swell.tasks.base.task_base import task_wrapper +from swell.deployment.create_experiment import create_experiment_directory + +# -------------------------------------------------------------------------------------------------- + +def render_jedi_config(suite: str, + model: str, + datetime: str, + executable_type: str, + copy_to_wd: bool = False) -> str: + + tempdir = tempfile.mkdtemp() + + override_dict = {'models': {}} + override_dict['experiment_root'] = tempdir + override_dict['models'][model] = {'check_for_obs': False, + 'generate_yaml_and_exit': True} + + create_experiment_directory(suite, method='defaults', platform='nccs_discover_sles15', + override=override_dict, advanced=False, slurm=None) + + experiment_yaml = os.path.join(tempdir, f'swell-{suite}', + f'swell-{suite}-suite', 'experiment.yaml') + + task_wrapper('RenderJediObservations', experiment_yaml, datetime, + model, ensemblePacket=None) + + task_wrapper(f'RunJedi{executable_type.upper()}Executable', experiment_yaml, datetime, + model, ensemblePacket=None) + + cycle_dir = os.path.join(tempdir, f'swell-{suite}', 'run', datetime, model) + + filename = f'jedi_{executable_type}_config.yaml' + config_file = os.path.join(cycle_dir, filename) + + if copy_to_wd: + new_path = os.path.join(os.getcwd(), f'jedi_{suite}_config.yaml') + shutil.copy(config_file, new_path) + config_file = new_path + + return config_file + +# -------------------------------------------------------------------------------------------------- \ No newline at end of file From 13247a2ad88805d5c0996359c0e36eba5bfc1e7a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 23 Mar 2026 14:54:56 -0400 Subject: [PATCH 242/299] Add 3dvar_cf --- src/swell/suites/3dvar_cf/flow.cylc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/swell/suites/3dvar_cf/flow.cylc b/src/swell/suites/3dvar_cf/flow.cylc index 8c618b5f6..22c014bfe 100644 --- a/src/swell/suites/3dvar_cf/flow.cylc +++ b/src/swell/suites/3dvar_cf/flow.cylc @@ -66,11 +66,14 @@ # EvaIncrement RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} + {% if not skip_r2d2 %} # Save observations RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} + {% endif %} # Clean up large files - EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & SaveObsDiags-{{model_component}} => + EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} {% endif %} From a0a9d2867bc9edc76b7134b3bb554deaa16aead1 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 23 Mar 2026 14:55:49 -0400 Subject: [PATCH 243/299] Fix update_dict --- .../prepare_config_and_suite/prepare_config_and_suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 7cafb1033..c12fe1d47 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -360,7 +360,7 @@ def override_with_external(self) -> None: override_dict = {} if isinstance(self.override, Mapping): - override_dict.update_dict(override_dict, self.override) + override_dict = update_dict(override_dict, self.override) elif isinstance(self.override, str): yaml = YAML(typ='safe') From 05f8a7d828a11fdde2e585bacf45889563c8ca6a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 23 Mar 2026 16:55:10 -0400 Subject: [PATCH 244/299] Add option to mock cycle dir --- src/swell/tasks/render_jedi_observations.py | 3 +++ src/swell/tasks/task_questions.py | 3 ++- src/swell/utilities/question_defaults.py | 13 +++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index 834962cfe..471c3806b 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -51,6 +51,9 @@ def execute(self) -> None: cwd = os.getcwd() + if self.config.mock_cycle_dir(False): + self.jedi_rendering.add_key('cycle_dir', './') + observations = [] # Iterate through list diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 8347850a7..0d8c82e0b 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -572,7 +572,8 @@ class TaskQuestions(QuestionContainer, Enum): qd.background_time_offset(), qd.observing_system_records_path(), qd.observations(), - qd.window_length() + qd.window_length(), + qd.mock_cycle_dir() ] ) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index fe59001b9..da37ec096 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -809,6 +809,19 @@ class dry_run(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class mock_cycle_dir(TaskQuestion): + default_value: bool = False + question_name: str = "mock_cycle_dir" + ask_question: bool = False + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Dry-run option for comparing configs." + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + @dataclass class obs_to_ingest(TaskQuestion): default_value: list = mutable_field([]) From 1ff3b2433fa896827525290b7e897e29ebf25c23 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 23 Mar 2026 17:32:53 -0400 Subject: [PATCH 245/299] Fix dictionary --- src/swell/deployment/create_experiment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index eb9a54682..c50e3451e 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -251,6 +251,8 @@ def create_experiment_directory( # ------------------ if isinstance(override, str): override_dict = read_override_file(override) + elif override is None: + override_dict = {} else: override_dict = override From d5602da4a5143246831cf93711e8c7cf370b6f54 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 24 Mar 2026 10:55:13 -0400 Subject: [PATCH 246/299] Working --- src/swell/tasks/render_jedi_observations.py | 6 ++++-- .../tasks/run_jedi_variational_executable.py | 5 +++++ src/swell/tasks/task_questions.py | 17 +++++++++++++---- src/swell/utilities/question_defaults.py | 4 ++-- src/swell/utilities/render_jedi_config.py | 3 ++- .../utilities/render_jedi_interface_files.py | 2 ++ 6 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index 471c3806b..aa2f639e6 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -51,8 +51,10 @@ def execute(self) -> None: cwd = os.getcwd() - if self.config.mock_cycle_dir(False): - self.jedi_rendering.add_key('cycle_dir', './') + if self.config.mock_experiment_directory(False): + self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') + self.jedi_rendering.add_key('experiment_id', 'experiment_id') + self.jedi_rendering.add_key('experiment_root', 'experiment_root') observations = [] diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 3c9e0b5a8..25b9dde76 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -96,6 +96,11 @@ def execute(self) -> None: self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) self.jedi_rendering.add_key('window_begin', window_begin) + if self.config.mock_experiment_directory(False): + self.jedi_rendering.add_key('experiment_root', 'experiment_root') + self.jedi_rendering.add_key('experiment_id', 'experiment_id') + self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') + # Atmosphere background error model # --------------------------------- if gsibec_configuration is not None: diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 0d8c82e0b..09278f326 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -573,7 +573,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.observing_system_records_path(), qd.observations(), qd.window_length(), - qd.mock_cycle_dir() + qd.mock_experiment_directory() ] ) @@ -591,6 +591,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.window_length(), qd.window_type(), qd.comparison_log_type('convert_state_soca2cice'), + qd.mock_experiment_directory() ] ) @@ -608,6 +609,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.observations(), qd.observing_system_records_path(), qd.comparison_log_type('ensmeanvariance'), + qd.mock_experiment_directory() ] ) @@ -618,7 +620,8 @@ class TaskQuestions(QuestionContainer, Enum): questions=[ run_jedi_executable, qd.marine_models(), - qd.comparison_log_type('fgat') + qd.comparison_log_type('fgat'), + qd.mock_experiment_directory() ] ) @@ -637,7 +640,8 @@ class TaskQuestions(QuestionContainer, Enum): qd.generate_yaml_and_exit(), qd.jedi_forecast_model(), qd.total_processors(), - qd.comparison_log_type('hofx') + qd.comparison_log_type('hofx'), + qd.mock_experiment_directory() ] ) @@ -655,6 +659,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.save_geovals(), qd.total_processors(), qd.comparison_log_type('ensemblehofx'), + qd.mock_experiment_directory() ] ) @@ -695,6 +700,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.vertical_localization_method(), qd.perhost(), qd.comparison_log_type('localensembleda'), + qd.mock_experiment_directory() ] ) @@ -712,7 +718,8 @@ class TaskQuestions(QuestionContainer, Enum): qd.observing_system_records_path(), qd.total_processors(), qd.obs_thinning_rej_fraction(), - qd.comparison_log_type('obsfilters') + qd.comparison_log_type('obsfilters'), + qd.mock_experiment_directory() ] ) @@ -726,6 +733,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.single_observations(), qd.window_length(), qd.comparison_log_type('ufo_tests'), + qd.mock_experiment_directory() ] ) @@ -737,6 +745,7 @@ class TaskQuestions(QuestionContainer, Enum): run_jedi_executable, qd.perhost(), qd.comparison_log_type('variational'), + qd.mock_experiment_directory() ] ) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index da37ec096..5b0257a4c 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -810,9 +810,9 @@ class dry_run(TaskQuestion): # -------------------------------------------------------------------------------------------------- @dataclass - class mock_cycle_dir(TaskQuestion): + class mock_experiment_directory(TaskQuestion): default_value: bool = False - question_name: str = "mock_cycle_dir" + question_name: str = "mock_experiment_directory" ask_question: bool = False models: List[str] = mutable_field([ "all_models" diff --git a/src/swell/utilities/render_jedi_config.py b/src/swell/utilities/render_jedi_config.py index 2aa748c49..ddfaeb9d9 100644 --- a/src/swell/utilities/render_jedi_config.py +++ b/src/swell/utilities/render_jedi_config.py @@ -27,7 +27,8 @@ def render_jedi_config(suite: str, override_dict = {'models': {}} override_dict['experiment_root'] = tempdir override_dict['generate_yaml_and_exit'] = True - override_dict['models'][model] = {'check_for_obs': False} + override_dict['models'][model] = {'check_for_obs': False, + 'mock_experiment_directory': True} create_experiment_directory(suite, method='defaults', platform='nccs_discover_sles15', override=override_dict, advanced=False, slurm=None, skip_r2d2=True) diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 779a73fb1..491ceff14 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -83,6 +83,8 @@ def __init__( 'ensemble_num_members', 'ensmean_only', 'ensmeanvariance_only', + 'experiment_id', + 'experiment_root', 'final_cycle_point', 'gradient_norm_reduction', 'gsibec_configuration', From 8cc96f8b845565c3e58650f4730b3b61ec603246 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 24 Mar 2026 11:24:23 -0400 Subject: [PATCH 247/299] Refactor and add to other suites --- ...jedi_convert_state_soca2cice_executable.py | 7 +++++ .../tasks/run_jedi_ensemble_mean_variance.py | 7 +++++ src/swell/tasks/run_jedi_fgat_executable.py | 7 +++++ .../run_jedi_hofx_ensemble_executable.py | 7 +++++ src/swell/tasks/run_jedi_hofx_executable.py | 7 +++++ .../run_jedi_local_ensemble_da_executable.py | 7 +++++ .../tasks/run_jedi_obsfilters_executable.py | 7 +++++ .../tasks/run_jedi_ufo_tests_executable.py | 7 +++++ .../tasks/run_jedi_variational_executable.py | 4 ++- src/swell/tasks/task_questions.py | 18 ++++++------- src/swell/test/code_tests/jedi_config_test.py | 4 +-- ...der_jedi_config.py => mock_jedi_config.py} | 10 +++---- src/swell/utilities/question_defaults.py | 26 +++++++++---------- 13 files changed, 88 insertions(+), 30 deletions(-) rename src/swell/utilities/{render_jedi_config.py => mock_jedi_config.py} (90%) diff --git a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py index 104a9a5dc..297f0bfb3 100644 --- a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py +++ b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py @@ -54,6 +54,13 @@ def execute(self) -> None: self.jedi_rendering.add_key('analysis_time', analysis_time) self.jedi_rendering.add_key('analysis_time_iso', analysis_time_iso) + # Add placeholder names if mock experiment + # ---------------------------------------- + if self.config.mock_experiment(False): + self.jedi_rendering.add_key('experiment_root', 'experiment_root') + self.jedi_rendering.add_key('experiment_id', 'experiment_id') + self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') + # Geometry # -------- self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) diff --git a/src/swell/tasks/run_jedi_ensemble_mean_variance.py b/src/swell/tasks/run_jedi_ensemble_mean_variance.py index 3a398b454..7d98c327c 100644 --- a/src/swell/tasks/run_jedi_ensemble_mean_variance.py +++ b/src/swell/tasks/run_jedi_ensemble_mean_variance.py @@ -71,6 +71,13 @@ def execute(self) -> None: # Ensemble self.jedi_rendering.add_key('ensemble_num_members', self.config.ensemble_num_members(None)) + # Add placeholder names if mock experiment + # ---------------------------------------- + if self.config.mock_experiment(False): + self.jedi_rendering.add_key('experiment_root', 'experiment_root') + self.jedi_rendering.add_key('experiment_id', 'experiment_id') + self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') + # Jedi configuration file # ----------------------- jedi_config_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_config.yaml') diff --git a/src/swell/tasks/run_jedi_fgat_executable.py b/src/swell/tasks/run_jedi_fgat_executable.py index f65297508..0c2eaed80 100644 --- a/src/swell/tasks/run_jedi_fgat_executable.py +++ b/src/swell/tasks/run_jedi_fgat_executable.py @@ -105,6 +105,13 @@ def execute(self) -> None: background_frequency = self.config.background_frequency() self.jedi_rendering.add_key('background_frequency', background_frequency) + # Add placeholder names if mock experiment + # ---------------------------------------- + if self.config.mock_experiment(False): + self.jedi_rendering.add_key('experiment_root', 'experiment_root') + self.jedi_rendering.add_key('experiment_id', 'experiment_id') + self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') + # Use GEOS utility to generate states # ----------------------------------- states = self.geos.states_generator(background_frequency, window_length, diff --git a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py index 8e9aee39e..2a5be2c7a 100644 --- a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py +++ b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py @@ -107,6 +107,13 @@ def execute(self) -> None: self.jedi_rendering.add_key('ensemble_hofx_packets', ensemble_hofx_packets) self.jedi_rendering.add_key('packet_ensemble_members', packet_ensemble_members) + # Add placeholder names if mock experiment + # ---------------------------------------- + if self.config.mock_experiment(False): + self.jedi_rendering.add_key('experiment_root', 'experiment_root') + self.jedi_rendering.add_key('experiment_id', 'experiment_id') + self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') + # Jedi configuration file # ----------------------- jedi_config_file = os.path.join(self.cycle_dir(), diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 6d63cf5fc..9fdee90e6 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -84,6 +84,13 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) self.jedi_rendering.add_key('window_begin', window_begin) + # Add placeholder names if mock experiment + # ---------------------------------------- + if self.config.mock_experiment(False): + self.jedi_rendering.add_key('experiment_root', 'experiment_root') + self.jedi_rendering.add_key('experiment_id', 'experiment_id') + self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') + # Model # ----- if window_type == '4D': diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index b0d67a5b8..30bc0db45 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -140,6 +140,13 @@ def execute(self) -> None: self.config.local_ensemble_use_linear_observer()) self.jedi_rendering.add_key('skip_ensemble_hofx', self.config.skip_ensemble_hofx()) + # Add placeholder names if mock experiment + # ---------------------------------------- + if self.config.mock_experiment(False): + self.jedi_rendering.add_key('experiment_root', 'experiment_root') + self.jedi_rendering.add_key('experiment_id', 'experiment_id') + self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') + # Prevent both 'local_ensemble_save_posterior_mean' and # 'local_ensemble_save_posterior_ensemble' from being true # -------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index 7af6e4867..a76f934ce 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -76,6 +76,13 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) self.jedi_rendering.add_key('window_begin', window_begin) + # Add placeholder names if mock experiment + # ---------------------------------------- + if self.config.mock_experiment(False): + self.jedi_rendering.add_key('experiment_root', 'experiment_root') + self.jedi_rendering.add_key('experiment_id', 'experiment_id') + self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') + # Model # ----- if window_type == '4D': diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 0613c5990..308d5c8ee 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -57,6 +57,13 @@ def execute(self) -> None: self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) self.jedi_rendering.add_key('window_begin', window_begin) + # Add placeholder names if mock experiment + # ---------------------------------------- + if self.config.mock_experiment(False): + self.jedi_rendering.add_key('experiment_root', 'experiment_root') + self.jedi_rendering.add_key('experiment_id', 'experiment_id') + self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') + # Open the JEDI config file and fill templates # -------------------------------------------- jedi_config_dict = self.jedi_rendering.render_oops_file(f'{jedi_application}', '3D') diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 25b9dde76..862cf5af5 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -96,7 +96,9 @@ def execute(self) -> None: self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) self.jedi_rendering.add_key('window_begin', window_begin) - if self.config.mock_experiment_directory(False): + # Add placeholder names if mock experiment + # ---------------------------------------- + if self.config.mock_experiment(False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 09278f326..8575c1f16 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -573,7 +573,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.observing_system_records_path(), qd.observations(), qd.window_length(), - qd.mock_experiment_directory() + qd.mock_experiment() ] ) @@ -591,7 +591,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.window_length(), qd.window_type(), qd.comparison_log_type('convert_state_soca2cice'), - qd.mock_experiment_directory() + qd.mock_experiment() ] ) @@ -609,7 +609,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.observations(), qd.observing_system_records_path(), qd.comparison_log_type('ensmeanvariance'), - qd.mock_experiment_directory() + qd.mock_experiment() ] ) @@ -621,7 +621,7 @@ class TaskQuestions(QuestionContainer, Enum): run_jedi_executable, qd.marine_models(), qd.comparison_log_type('fgat'), - qd.mock_experiment_directory() + qd.mock_experiment() ] ) @@ -641,7 +641,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.jedi_forecast_model(), qd.total_processors(), qd.comparison_log_type('hofx'), - qd.mock_experiment_directory() + qd.mock_experiment() ] ) @@ -659,7 +659,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.save_geovals(), qd.total_processors(), qd.comparison_log_type('ensemblehofx'), - qd.mock_experiment_directory() + qd.mock_experiment() ] ) @@ -719,7 +719,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.total_processors(), qd.obs_thinning_rej_fraction(), qd.comparison_log_type('obsfilters'), - qd.mock_experiment_directory() + qd.mock_experiment() ] ) @@ -733,7 +733,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.single_observations(), qd.window_length(), qd.comparison_log_type('ufo_tests'), - qd.mock_experiment_directory() + qd.mock_experiment() ] ) @@ -745,7 +745,7 @@ class TaskQuestions(QuestionContainer, Enum): run_jedi_executable, qd.perhost(), qd.comparison_log_type('variational'), - qd.mock_experiment_directory() + qd.mock_experiment() ] ) diff --git a/src/swell/test/code_tests/jedi_config_test.py b/src/swell/test/code_tests/jedi_config_test.py index 7dae4ecd1..f527465da 100644 --- a/src/swell/test/code_tests/jedi_config_test.py +++ b/src/swell/test/code_tests/jedi_config_test.py @@ -12,7 +12,7 @@ from ruamel.yaml import YAML from swell.swell_path import get_swell_path -from swell.utilities.render_jedi_config import render_jedi_config +from swell.utilities.mock_jedi_config import mock_jedi_config # -------------------------------------------------------------------------------------------------- @@ -21,7 +21,7 @@ def run_test(suite: str, datetime: str, executable_type: str) -> None: - config_file = render_jedi_config(suite, model, datetime, executable_type) + config_file = mock_jedi_config(suite, model, datetime, executable_type) yaml = YAML(typ='safe') diff --git a/src/swell/utilities/render_jedi_config.py b/src/swell/utilities/mock_jedi_config.py similarity index 90% rename from src/swell/utilities/render_jedi_config.py rename to src/swell/utilities/mock_jedi_config.py index ddfaeb9d9..57ea8c409 100644 --- a/src/swell/utilities/render_jedi_config.py +++ b/src/swell/utilities/mock_jedi_config.py @@ -16,11 +16,11 @@ # -------------------------------------------------------------------------------------------------- -def render_jedi_config(suite: str, - model: str, - datetime: str, - executable_type: str, - copy_to_wd: bool = False) -> str: +def mock_jedi_config(suite: str, + model: str, + datetime: str, + executable_type: str, + copy_to_wd: bool = False) -> str: tempdir = tempfile.mkdtemp() diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 5b0257a4c..46419a783 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -130,6 +130,19 @@ class marine_models(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class mock_experiment(SuiteQuestion): + default_value: bool = False + question_name: str = "mock_experiment" + ask_question: bool = False + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Dry-run option for comparing configs." + widget_type: WType = WType.BOOLEAN + + # -------------------------------------------------------------------------------------------------- + @dataclass class model_components(SuiteQuestion): default_value: str = "defer_to_code" @@ -809,19 +822,6 @@ class dry_run(TaskQuestion): # -------------------------------------------------------------------------------------------------- - @dataclass - class mock_experiment_directory(TaskQuestion): - default_value: bool = False - question_name: str = "mock_experiment_directory" - ask_question: bool = False - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Dry-run option for comparing configs." - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - @dataclass class obs_to_ingest(TaskQuestion): default_value: list = mutable_field([]) From df0d9df55bdcba520ad5fba8c7213dcbd1c80e83 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 24 Mar 2026 11:25:11 -0400 Subject: [PATCH 248/299] add test --- src/swell/test/code_tests/jedi_config_test.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/swell/test/code_tests/jedi_config_test.py b/src/swell/test/code_tests/jedi_config_test.py index f527465da..42c8a758d 100644 --- a/src/swell/test/code_tests/jedi_config_test.py +++ b/src/swell/test/code_tests/jedi_config_test.py @@ -21,20 +21,25 @@ def run_test(suite: str, datetime: str, executable_type: str) -> None: + # Create the mock jedi config config_file = mock_jedi_config(suite, model, datetime, executable_type) + # Read the file yaml = YAML(typ='safe') - with open(config_file, 'r') as f: config_dict = yaml.load(f) + # Open the comparison yaml for the suite comparison_config = os.path.join(get_swell_path(), 'test', 'jedi_yamls', f'jedi_config_{suite}.yaml') - with open(comparison_config, 'r') as f: comparison_dict = yaml.load(f) - assert config_dict == comparison_dict + if config_dict != comparison_dict: + raise AssertionError(f'Rendered JEDI config for suite {suite}, {config_file} ' + f'did not match comparison version {comparison_config}. ' + 'Please check the file diff. If these differences are intentional, ' + 'create a mock file using `swell utility mock_jedi_config ') # -------------------------------------------------------------------------------------------------- From 3e9a44ffe303961c9602314e6643ec9dc8f02393 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 24 Mar 2026 17:34:31 -0400 Subject: [PATCH 249/299] staging --- src/swell/tasks/task_questions.py | 2 +- src/swell/utilities/mock_jedi_config.py | 23 +++++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 8575c1f16..e72de4f65 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -700,7 +700,7 @@ class TaskQuestions(QuestionContainer, Enum): qd.vertical_localization_method(), qd.perhost(), qd.comparison_log_type('localensembleda'), - qd.mock_experiment_directory() + qd.mock_experiment() ] ) diff --git a/src/swell/utilities/mock_jedi_config.py b/src/swell/utilities/mock_jedi_config.py index 57ea8c409..a9def2e31 100644 --- a/src/swell/utilities/mock_jedi_config.py +++ b/src/swell/utilities/mock_jedi_config.py @@ -16,6 +16,25 @@ # -------------------------------------------------------------------------------------------------- +defaults_dict = {} + +marine_default_datetime = '20210701T120000Z' +atmosphere_default_datetime = '20231010T000000Z' + +defaults_dict['3dvar_marine'] = {'datetime': marine_default_datetime, + 'model': 'geos_marine', + 'executable_type': 'variational'} + +defaults_dict['3dvar_marine_cycle'] = defaults_dict['3dvar_marine'].copy() + +defaults_dict['3dfgat_marine_cycle'] = {'datetime': marine_default_datetime, + 'model': 'geos_marine', + 'executable_type': 'fgat'} + +defaults_dict['3dvar_atmos'] = {'datetime' + +# -------------------------------------------------------------------------------------------------- + def mock_jedi_config(suite: str, model: str, datetime: str, @@ -27,8 +46,8 @@ def mock_jedi_config(suite: str, override_dict = {'models': {}} override_dict['experiment_root'] = tempdir override_dict['generate_yaml_and_exit'] = True - override_dict['models'][model] = {'check_for_obs': False, - 'mock_experiment_directory': True} + override_dict['mock_experiment_directory'] = True + override_dict['models'][model] = {'check_for_obs': False} create_experiment_directory(suite, method='defaults', platform='nccs_discover_sles15', override=override_dict, advanced=False, slurm=None, skip_r2d2=True) From 3d9a7f2276e5290280c3eb3b1bd740034008a934 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 24 Mar 2026 18:04:26 -0400 Subject: [PATCH 250/299] Add to utility scripts --- src/swell/utilities/mock_jedi_config.py | 19 ------ .../utilities/scripts/create_mock_files.py | 60 +++++++++++++++++++ 2 files changed, 60 insertions(+), 19 deletions(-) create mode 100644 src/swell/utilities/scripts/create_mock_files.py diff --git a/src/swell/utilities/mock_jedi_config.py b/src/swell/utilities/mock_jedi_config.py index a9def2e31..480514a72 100644 --- a/src/swell/utilities/mock_jedi_config.py +++ b/src/swell/utilities/mock_jedi_config.py @@ -16,25 +16,6 @@ # -------------------------------------------------------------------------------------------------- -defaults_dict = {} - -marine_default_datetime = '20210701T120000Z' -atmosphere_default_datetime = '20231010T000000Z' - -defaults_dict['3dvar_marine'] = {'datetime': marine_default_datetime, - 'model': 'geos_marine', - 'executable_type': 'variational'} - -defaults_dict['3dvar_marine_cycle'] = defaults_dict['3dvar_marine'].copy() - -defaults_dict['3dfgat_marine_cycle'] = {'datetime': marine_default_datetime, - 'model': 'geos_marine', - 'executable_type': 'fgat'} - -defaults_dict['3dvar_atmos'] = {'datetime' - -# -------------------------------------------------------------------------------------------------- - def mock_jedi_config(suite: str, model: str, datetime: str, diff --git a/src/swell/utilities/scripts/create_mock_files.py b/src/swell/utilities/scripts/create_mock_files.py new file mode 100644 index 000000000..f19342ac2 --- /dev/null +++ b/src/swell/utilities/scripts/create_mock_files.py @@ -0,0 +1,60 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.mock_jedi_config import mock_jedi_config + +# -------------------------------------------------------------------------------------------------- + +defaults_dict = {} + +marine_default_datetime = '20210701T120000Z' +atmosphere_default_datetime = '20231010T000000Z' +compo_default_datetime = '20230805T1800Z' + +defaults_dict['3dvar_marine'] = {'datetime': marine_default_datetime, + 'model': 'geos_marine', + 'executable_type': 'variational'} + +defaults_dict['3dvar_marine_cycle'] = defaults_dict['3dvar_marine'].copy() + +defaults_dict['3dfgat_marine_cycle'] = {'datetime': marine_default_datetime, + 'model': 'geos_marine', + 'executable_type': 'fgat'} + +defaults_dict['3dvar_atmos'] = {'datetime': atmosphere_default_datetime, + 'model': 'geos_atmosphere', + 'executable_type': 'variational'} + +defaults_dict['3dfgat_atmos'] = {'datetime': atmosphere_default_datetime, + 'model': 'geos_atmosphere', + 'executable_type': 'variational'} + +defaults_dict['hofx'] = {'datetime': atmosphere_default_datetime, + 'model': 'geos_atmosphere', + 'executable_type': 'hofx'} + +defaults_dict['hofx_cf'] = {'datetime': compo_default_datetime, + 'model': 'geos_cf', + 'executable_type': 'hofx'} + +defaults_dict['localensembleda'] = {'datetime': atmosphere_default_datetime, + 'model': 'geos_atmosphere', + 'executable_type': 'localensembleda'} +# -------------------------------------------------------------------------------------------------- + +def create_mock_files() -> None: + for suite, defaults in defaults_dict.items(): + model = defaults['model'] + datetime = defaults['datetime'] + executable_type = defaults['executable_type'] + + mock_jedi_config(suite, model, datetime, executable_type, True) + + +# -------------------------------------------------------------------------------------------------- From b25a6dd1305bdffff6f7ae25b862733ad2d99c0e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 25 Mar 2026 17:15:25 -0400 Subject: [PATCH 251/299] Functional --- docs/code_tests/code_tests.md | 8 + .../interfaces/geos_cf/task_questions.yaml | 3 + .../nccs_discover_sles15/task_questions.yaml | 6 +- src/swell/suites/geosadas/flow.cylc | 2 +- src/swell/tasks/render_jedi_observations.py | 2 +- src/swell/tasks/run_jedi_hofx_executable.py | 8 +- .../tasks/run_jedi_variational_executable.py | 1 + src/swell/test/code_tests/code_tests.py | 4 + src/swell/test/code_tests/jedi_config_test.py | 63 +- .../test/code_tests/suite_creation_test.py | 34 +- .../jedi_3dfgat_atmos_config.yaml | 19226 +++++++++++++++ .../jedi_3dfgat_marine_cycle_config.yaml | 959 + .../jedi_configs/jedi_3dvar_atmos_config.yaml | 19206 +++++++++++++++ .../jedi_configs/jedi_3dvar_cf_config.yaml | 179 + .../jedi_3dvar_marine_config.yaml | 694 + .../jedi_3dvar_marine_cycle_config.yaml | 694 + .../jedi_configs/jedi_hofx_cf_config.yaml | 111 + .../test/jedi_configs/jedi_hofx_config.yaml | 19233 ++++++++++++++++ .../jedi_localensembleda_config.yaml | 9498 ++++++++ src/swell/utilities/mock_jedi_config.py | 34 +- src/swell/utilities/question_defaults.py | 3 - .../utilities/scripts/create_mock_files.py | 60 - 22 files changed, 69919 insertions(+), 109 deletions(-) create mode 100644 src/swell/test/jedi_configs/jedi_3dfgat_atmos_config.yaml create mode 100644 src/swell/test/jedi_configs/jedi_3dfgat_marine_cycle_config.yaml create mode 100644 src/swell/test/jedi_configs/jedi_3dvar_atmos_config.yaml create mode 100644 src/swell/test/jedi_configs/jedi_3dvar_cf_config.yaml create mode 100644 src/swell/test/jedi_configs/jedi_3dvar_marine_config.yaml create mode 100644 src/swell/test/jedi_configs/jedi_3dvar_marine_cycle_config.yaml create mode 100644 src/swell/test/jedi_configs/jedi_hofx_cf_config.yaml create mode 100644 src/swell/test/jedi_configs/jedi_hofx_config.yaml create mode 100644 src/swell/test/jedi_configs/jedi_localensembleda_config.yaml delete mode 100644 src/swell/utilities/scripts/create_mock_files.py diff --git a/docs/code_tests/code_tests.md b/docs/code_tests/code_tests.md index 7d369d3da..2dbe279a4 100644 --- a/docs/code_tests/code_tests.md +++ b/docs/code_tests/code_tests.md @@ -17,3 +17,11 @@ By default, swell will create several directories in the working directory that ```yaml test_cache_location: /discover/nobackup//swell-test-cache ``` + +## Code tests + +### Suite creation test +The suite creation test attempts to construct experiments for all suites within swell in a temporary directory. If one fails, try creating the suite on its own to make sure it is configured properly. Ensure all values are valid and are not filled by the templates `defer_to_model` or `defer_to_platform`. + +### JEDI Config test +The JEDI config test generates mock configs for jedi executables in a dry-run mode, where obs will not be checked and placeholders will be used for experiment filepaths. These configs are compared against reference files located in `src/swell/test/jedi_configs/`, and named `jedi__config.yaml`. Any difference in values in these yamls will cause this test to fail, so ensure any differences created are intentional, then run `swell utility CreateMockConfigs` to automatically generate new reference files for all suites. These new files are placed in the `jedi_config` location in the source code. diff --git a/src/swell/configuration/jedi/interfaces/geos_cf/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_cf/task_questions.yaml index 8db8f8996..43d6ef951 100644 --- a/src/swell/configuration/jedi/interfaces/geos_cf/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_cf/task_questions.yaml @@ -80,6 +80,9 @@ observations: - tropomi_s5p_no2_tropo - tempo_no2_total +obs_experiment: + default_value: None + observing_system_records_path: default_value: None diff --git a/src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml b/src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml index 28a448ff4..be2b5f33f 100644 --- a/src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml +++ b/src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml @@ -19,15 +19,15 @@ existing_jedi_source_directory: existing_jedi_source_directory_pinned: default_value: /discover/nobackup/projects/gmao/advda/jedi_bundles_sles15/current_pinned_jedi_bundle/source/ +existing_perllib_path: + default_value: /discover/nobackup/projects/gmao/advda/perllib_opt/GMAO_perllib/g1.0.1/ + geos_homdir: default_value: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/geos/homdirs/coupled_5deg initial_restarts_method: default_value: geos_expdir -gmao_perllib_path: - default_value: /discover/nobackup/projects/gmao/advda/perllib_opt/GMAO_perllib/g1.0.1/ - path_to_bufr: default_value: None diff --git a/src/swell/suites/geosadas/flow.cylc b/src/swell/suites/geosadas/flow.cylc index 9bd6b7936..1e514a2bb 100644 --- a/src/swell/suites/geosadas/flow.cylc +++ b/src/swell/suites/geosadas/flow.cylc @@ -107,7 +107,7 @@ [[ GetGeosAdasBackground ]] script = "swell task GetGeosAdasBackground $config -d $datetime -m geos_atmosphere" - [[RenderJediObservations-{{model_component}}]] + [[RenderJediObservations-geos_atmosphere]] script = "swell task RenderJediObservations $config -d $datetime -m geos_atmosphere" [[RunJediVariationalExecutable]] diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index aa2f639e6..a2c43f26a 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -51,7 +51,7 @@ def execute(self) -> None: cwd = os.getcwd() - if self.config.mock_experiment_directory(False): + if self.config.mock_experiment(False): self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('experiment_root', 'experiment_root') diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 9fdee90e6..46e039811 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -164,6 +164,7 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: jedi_config_file, output_log_file) else: self.logger.info('YAML generated, now exiting.') + return # If saving the geovals they need to be combined # ---------------------------------------------- @@ -269,6 +270,11 @@ def append_gomsaver( # Add mem to the filename if it is not None mem_str = f'_mem{mem}' if mem is not None else '' + if not self.config.mock_experiment(False): + cycle_dir = self.cycle_dir() + else: + cycle_dir = 'cycle_dir' + for observer in jedi_config_dict['observations']['observers']: observation = observer['observation_name'] @@ -276,7 +282,7 @@ def append_gomsaver( # Define the GeoVaLs saver dictionary gom_saver_dict = { 'filter': 'GOMsaver', - 'filename': os.path.join(self.cycle_dir(), + 'filename': os.path.join(cycle_dir, f'{observation}-geovals.{window_begin}{mem_str}.nc4') } diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 862cf5af5..ba0c1c42d 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -99,6 +99,7 @@ def execute(self) -> None: # Add placeholder names if mock experiment # ---------------------------------------- if self.config.mock_experiment(False): + print('MOCK EXPERIMENT') self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') diff --git a/src/swell/test/code_tests/code_tests.py b/src/swell/test/code_tests/code_tests.py index 2832a937d..1b7b790f1 100644 --- a/src/swell/test/code_tests/code_tests.py +++ b/src/swell/test/code_tests/code_tests.py @@ -18,6 +18,7 @@ from swell.test.code_tests.question_dictionary_comparison_test import QuestionDictionaryTest from swell.test.code_tests.test_generate_observing_system import GenerateObservingSystemTest from swell.test.code_tests.suite_creation_test import SuiteCreationTest +from swell.test.code_tests.jedi_config_test import JEDIConfigTest # -------------------------------------------------------------------------------------------------- @@ -56,6 +57,9 @@ def code_tests() -> None: # Load Suite Creation Test test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(SuiteCreationTest)) + # Load Suite Creation Test + test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(JEDIConfigTest)) + # Create a test runner test_runner = unittest.TextTestRunner() diff --git a/src/swell/test/code_tests/jedi_config_test.py b/src/swell/test/code_tests/jedi_config_test.py index 42c8a758d..adb8f5dba 100644 --- a/src/swell/test/code_tests/jedi_config_test.py +++ b/src/swell/test/code_tests/jedi_config_test.py @@ -8,43 +8,58 @@ # -------------------------------------------------------------------------------------------------- import os -import tempfile from ruamel.yaml import YAML +import unittest from swell.swell_path import get_swell_path from swell.utilities.mock_jedi_config import mock_jedi_config +from swell.utilities.scripts.create_mock_configs import defaults_dict # -------------------------------------------------------------------------------------------------- -def run_test(suite: str, - model: str, - datetime: str, - executable_type: str) -> None: - # Create the mock jedi config - config_file = mock_jedi_config(suite, model, datetime, executable_type) +class JEDIConfigTest(unittest.TestCase): - # Read the file - yaml = YAML(typ='safe') - with open(config_file, 'r') as f: - config_dict = yaml.load(f) + def runTest(self) -> None: - # Open the comparison yaml for the suite - comparison_config = os.path.join(get_swell_path(), 'test', 'jedi_yamls', - f'jedi_config_{suite}.yaml') - with open(comparison_config, 'r') as f: - comparison_dict = yaml.load(f) + '''Generates new jedi config mocks for all suites and compares them to previous versions. - if config_dict != comparison_dict: - raise AssertionError(f'Rendered JEDI config for suite {suite}, {config_file} ' - f'did not match comparison version {comparison_config}. ' - 'Please check the file diff. If these differences are intentional, ' - 'create a mock file using `swell utility mock_jedi_config ') + Comparison mocks are located under `src/swell/test/jedi_configs`. The intention of this + test is to ensure changes to the jedi configs generated by swell are intentional. These + mock files are run in a 'dry' mode, in that they do not fetch or check observations, + and all filepaths are filled by placeholders. The JEDI configs can then be neatly checked + for equivalence. -# -------------------------------------------------------------------------------------------------- + It is expected that this test will fail when any change is made to JEDI, thus new mock + files will need to be generated when changes are made. New mock files can be generated + automatically by running `swell utility CreateMockConfigs`. + ''' + + for suite, defaults in defaults_dict.items(): + model = defaults['model'] + datetime = defaults['datetime'] + executable_type = defaults['executable_type'] + + # Create the mock jedi config + config_file = mock_jedi_config(suite, model, datetime, executable_type) + + # Read the file + yaml = YAML(typ='safe') + with open(config_file, 'r') as f: + config_dict = yaml.load(f) + # Open the comparison yaml for the suite + comparison_path = os.path.join(get_swell_path(), 'test', 'jedi_configs') + comparison_config = os.path.join(comparison_path, + f'jedi_{suite}_config.yaml') + with open(comparison_config, 'r') as f: + comparison_dict = yaml.load(f) -if __name__ == '__main__': - run_test('3dvar_marine', 'geos_marine', '20210701T120000Z', 'variational') + if config_dict != comparison_dict: + raise AssertionError(f'Rendered JEDI config for suite {suite}, {config_file} ' + f'did not match comparison version {comparison_config}. ' + 'Please check the file diffs. If these differences ' + 'are intentional, new mock test files can be generated using ' + '`swell utility MockJediConfigs`') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/test/code_tests/suite_creation_test.py b/src/swell/test/code_tests/suite_creation_test.py index dd9a22a3b..846e4b80c 100644 --- a/src/swell/test/code_tests/suite_creation_test.py +++ b/src/swell/test/code_tests/suite_creation_test.py @@ -9,22 +9,34 @@ import tempfile import os -from ruamel.yaml import YAML import unittest from swell.suites.all_suites import get_suites from swell.deployment.create_experiment import create_experiment_directory +from swell.utilities.logger import get_logger # -------------------------------------------------------------------------------------------------- + class SuiteCreationTest(unittest.TestCase): def runTest(self) -> None: + '''Test generating all suites to make sure they are configured correctly. + + Checks that templates `defer_to_model` and `defer_to_platform` are filled. + + ''' + suites = get_suites() + self.logger = get_logger('SuiteCreationTest') + for suite in suites: - self.suite_creation_test(suite) + try: + self.suite_creation_test(suite) + except Exception as e: + raise Exception(f'Error generating {suite} experiment: {e}') def suite_creation_test(self, suite: str) -> None: @@ -36,16 +48,20 @@ def suite_creation_test(self, suite: str) -> None: override_dict['skip_r2d2'] = True create_experiment_directory(suite, 'defaults', 'nccs_discover_sles15', - override_dict, False, None) - - experiment_yaml = os.path.join(tempdir, f'swell-{suite}', f'swell-{suite}-suite', 'experiment.yaml') + override_dict, advanced=False, slurm=None, skip_r2d2=True) - yaml = YAML(typ='safe') + experiment_yaml = os.path.join(tempdir, f'swell-{suite}', f'swell-{suite}-suite', + 'experiment.yaml') with open(experiment_yaml, 'r') as f: - experiment_yaml_str = yaml.load(f) + experiment_yaml_str = f.read() + + if 'defer_to_model' in experiment_yaml_str: + raise AssertionError(f'Improperly filled template, `defer_to_model`' + 'present in experiment yaml') - assert 'defer_to_model' not in experiment_yaml_str - assert 'defer_to_platform' not in experiment_yaml_str + if 'defer_to_platform' in experiment_yaml_str: + raise AssertionError(f'Improperly filled template, `defer_to_platform`' + 'present in experiment.yaml') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/test/jedi_configs/jedi_3dfgat_atmos_config.yaml b/src/swell/test/jedi_configs/jedi_3dfgat_atmos_config.yaml new file mode 100644 index 000000000..c6aa44731 --- /dev/null +++ b/src/swell/test/jedi_configs/jedi_3dfgat_atmos_config.yaml @@ -0,0 +1,19226 @@ +cost function: + cost type: 4D-Var + jb evaluation: false + time window: + begin: '2023-10-09T21:00:00Z' + length: PT6H + bound to include: begin + geometry: + fms initialization: + namelist filename: ./fv3-jedi/fv3files/fmsmpp.nml + field table filename: ./fv3-jedi/fv3files/field_table_gmao + akbk: ./fv3-jedi/fv3files/akbk72.nc4 + layout: + - 4 + - 4 + npx: '91' + npy: '91' + npz: '72' + model: + name: PSEUDO + tstep: PT1H + filetype: cube sphere history + provider: geos + compute edge pressure from surface pressure: true + max allowable geometry difference: 0.001 + datapath: cycle_dir + filenames: + - bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4 + - fv3-jedi/bkg/geos.crtmsrf.91.nc4 + field io names: &id001 + eastward_wind: ua + northward_wind: va + air_temperature: t + air_pressure_at_surface: ps + air_pressure_levels: pe + water_vapor_mixing_ratio_wrt_moist_air: q + cloud_liquid_ice: qi + cloud_liquid_water: ql + rain_water: qr + snow_water: qs + mole_fraction_of_ozone_in_air: o3ppmv + geopotential_height_times_gravity_at_surface: phis + initial_mass_fraction_of_large_scale_cloud_condensate: qls + initial_mass_fraction_of_convective_cloud_condensate: qcn + convective_cloud_area_fraction: cfcn + fraction_of_ocean: frocean + fraction_of_land: frland + isotropic_variance_of_filtered_topography: varflt + surface_velocity_scale: ustar + surface_buoyancy_scale: bstar + planetary_boundary_layer_height: zpbl + surface_exchange_coefficient_for_momentum: cm + surface_exchange_coefficient_for_heat: ct + surface_exchange_coefficient_for_moisture: cq + KCBL_before_moist: kcbl + surface_temp_before_moist: tsm + lower_index_where_Kh_greater_than_2: khl + upper_index_where_Kh_greater_than_2: khu + fraction_of_lake: frlake + fraction_of_ice: frseaice + skin_temperature_at_surface: ts + eastward_wind_at_surface: u10m + northward_wind_at_surface: v10m + variable change: + variable change name: Analysis2Model + forecast length: PT6H + analysis variables: &id003 + - eastward_wind + - northward_wind + - air_temperature + - water_vapor_mixing_ratio_wrt_moist_air + - air_pressure_at_surface + - air_pressure_levels + - cloud_liquid_ice + - cloud_liquid_water + - rain_water + - snow_water + - mole_fraction_of_ozone_in_air + - geopotential_height_times_gravity_at_surface + - fraction_of_ocean + - fraction_of_lake + - fraction_of_ice + - skin_temperature_at_surface + background: + datetime: '2023-10-09T21:00:00Z' + filetype: cube sphere history + provider: geos + compute edge pressure from surface pressure: true + max allowable geometry difference: 0.001 + datapath: cycle_dir + filenames: + - bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4 + - fv3-jedi/bkg/geos.crtmsrf.91.nc4 + state variables: + - eastward_wind + - northward_wind + - air_temperature + - air_pressure_at_surface + - air_pressure_levels + - water_vapor_mixing_ratio_wrt_moist_air + - cloud_liquid_ice + - cloud_liquid_water + - rain_water + - snow_water + - mole_fraction_of_ozone_in_air + - geopotential_height_times_gravity_at_surface + - initial_mass_fraction_of_large_scale_cloud_condensate + - initial_mass_fraction_of_convective_cloud_condensate + - convective_cloud_area_fraction + - fraction_of_ocean + - fraction_of_land + - isotropic_variance_of_filtered_topography + - surface_velocity_scale + - surface_buoyancy_scale + - planetary_boundary_layer_height + - surface_exchange_coefficient_for_momentum + - surface_exchange_coefficient_for_heat + - surface_exchange_coefficient_for_moisture + - KCBL_before_moist + - surface_temp_before_moist + - lower_index_where_Kh_greater_than_2 + - upper_index_where_Kh_greater_than_2 + - fraction_of_lake + - fraction_of_ice + - vtype + - stype + - vfrac + - sheleg + - skin_temperature_at_surface + - soilt + - soilm + - eastward_wind_at_surface + - northward_wind_at_surface + field io names: *id001 + background error: + covariance model: SABER + covariance type: gsi hybrid covariance + saber central block: + saber block name: gsi hybrid covariance + read: + gsi akbk: ./fv3-jedi/fv3files/akbk72.nc4 + gsi error covariance file: ./fv3-jedi/gsibec/gsi-coeffs-gmao-global-l72x144y91.nc4 + gsi berror namelist file: ./fv3-jedi/gsibec/cli_gsibec_configuration_l72x144y91.nml + processor layout x direction: 4 + processor layout y direction: 24 + debugging mode: false + saber outer blocks: + - saber block name: interpolation + inner geometry: + function space: StructuredColumns + custom grid matching gsi: + type: latlon + lats: 91 + lons: 144 + custom partitioner matching gsi: + bands: 24 + halo: 1 + forward interpolator: + local interpolator type: oops unstructured grid interpolator + inverse interpolator: + local interpolator type: oops unstructured grid interpolator + state variables to inverse: &id002 + - eastward_wind + - northward_wind + - air_temperature + - air_pressure_at_surface + - air_pressure_levels + - water_vapor_mixing_ratio_wrt_moist_air + - cloud_liquid_ice + - cloud_liquid_water + - rain_water + - snow_water + - mole_fraction_of_ozone_in_air + - fraction_of_ocean + - fraction_of_lake + - fraction_of_ice + - geopotential_height_times_gravity_at_surface + - skin_temperature_at_surface + linear variable change: + linear variable change name: Control2Analysis + input variables: *id002 + output variables: *id003 + observations: + get values: + variable change: + variable change name: Model2GeoVaLs + hydrometeor effective radii method: gsi + tropopause pressure method: gsi + observers: + - obs space: + name: Aircraft Temperature + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/aircraft_temperature.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.aircraft_temperature.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - airTemperature + obs bias: + input file: cycle_dir/aircraft_temperature.20231009T150000Z.acftbias + output file: cycle_dir/aircraft_temperature.20231009T210000Z.acftbias + bc by record: true + variational bc: + predictors: + - name: constant + - name: obsMetadataPredictor + variable: instantaneousAltitudeRate + - name: obsMetadataPredictor + variable: instantaneousAltitudeRate + order: 2 + covariance: + minimal required obs number: 3 + variance range: + - 1e-06 + - 1.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/aircraft_temperature.20231009T150000Z.acftbias_cov + inflation: + ratio: 1.005 + ratio for small dataset: 2.0 + obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - airTemperature + linear obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - airTemperature + obs filters: + - filter: Variable Assignment + where: + - variable: + name: ObsType/airTemperature + is_in: 130 + assignments: + - name: MetaData/stationIdentification + value: 'KX130 ' + - filter: Variable Assignment + where: + - variable: + name: MetaData/instantaneousAltitudeRate + minvalue: 50.0 + assignments: + - name: MetaData/instantaneousAltitudeRate + value: 0.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 2.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + errors: + - 2.5 + - 2.3 + - 2.1 + - 1.9 + - 1.7 + where: + - variable: + name: ObsType/airTemperature + is_in: 130 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + errors: + - 1.4706 + - 1.3529 + - 1.2353 + - 1.1176 + - 1.0 + where: + - variable: + name: ObsType/airTemperature + is_in: 131,133 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 70000 + - 65000 + - 60000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 5000 + errors: + - 1.5 + - 1.3 + - 1.1 + - 0.9 + - 0.8 + - 0.8 + - 0.75 + - 0.7 + - 0.7 + - 0.75 + - 0.85 + - 1.3 + - 1.5 + - 1.5 + where: + - variable: + name: ObsType/airTemperature + is_in: 132 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 60000 + - 40000 + errors: + - 1.5 + - 1.35 + - 1.25 + - 1.1 + - 1.0 + - 1.3 + - 1.7 + where: + - variable: + name: ObsType/airTemperature + is_in: 134 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + errors: + - 1.4706 + - 1.3529 + - 1.2353 + - 1.1176 + - 1000000000.0 + where: + - variable: + name: ObsType/airTemperature + is_in: 135 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: + - airTemperature + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreQC/airTemperature + is_in: 4-15 + action: + name: passivate + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: + name: ObsType/airTemperature + is_in: 134, 135 + action: + name: passivate + defer to post: true + - filter: Perform Action + action: + name: inflate error + inflation factor: 10.0 + filter variables: + - name: airTemperature + where: + - variable: + name: MetaData/pressure + maxvalue: 110000 + minvalue: 50000 + - variable: + name: ObsType/airTemperature + is_in: 130 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + observation_name: aircraft_temperature + - obs space: + name: Aircraft Wind + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/aircraft_wind.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.aircraft_wind.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + obs prior filters: + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + obs post filters: + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: PreQC/windEastward + is_in: 4-15 + action: + name: passivate + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: ObsType/windEastward + is_in: 234, 235 + action: + name: passivate + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 80000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 1.4 + - 1.5 + - 1.6 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.3 + - 2.6 + - 2.8 + - 3.0 + - 3.2 + - 2.7 + - 2.4 + - 2.1 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.6 + where: + - variable: + name: ObsType/windEastward + is_in: 230 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.0 + where: + - variable: + name: ObsType/windEastward + is_in: 231 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 2.5 + where: + - variable: + name: ObsType/windEastward + is_in: 233 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.0 + where: + - variable: + name: ObsType/windEastward + is_in: 234, 235 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + errors: + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.2 + - 2.2 + - 2.3 + - 2.3 + - 2.4 + - 2.4 + - 2.5 + - 2.7 + - 2.9 + - 3.1 + where: + - variable: + name: ObsType/windEastward + is_in: 232 + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: + - windEastward + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: + - windNorthward + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 230 + - 231 + - 232 + - 233 + - 234 + - 235 + cgross: + - 6.0 + - 6.5 + - 7.0 + - 7.5 + - 7.5 + - 7.5 + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + variable: windEastward + action: + name: reject + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 230 + - 231 + - 232 + - 233 + - 234 + - 235 + cgross: + - 6.0 + - 6.5 + - 7.0 + - 7.5 + - 7.5 + - 7.5 + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + variable: windNorthward + action: + name: reject + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + defer to post: true + observation_name: aircraft_wind + - obs space: + name: AIRS AQUA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/airs_aqua.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.airs_aqua.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: airs_aqua + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/airs_aqua.20231009T150000Z.satbias.nc4 + output file: cycle_dir/airs_aqua.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/airs_aqua.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/airs_aqua.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/airs_aqua.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/airs_aqua.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/airs_aqua_119_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.2 + - 1.2 + - 1.3136 + - 1.4 + - 1.4 + - 1.2639 + - 1.4 + - 1.4 + - 1.1802 + - 1.2517 + - 1.1719 + - 1.2 + - 1.1728 + - 1.1442 + - 1.2 + - 1.2 + - 1.15 + - 1.0801 + - 1.15 + - 1.15 + - 1.0396 + - 1.15 + - 1.15 + - 1.15 + - 1.15 + - 1.15 + - 1.15 + - 1.15 + - 0.9946 + - 1.05 + - 0.9217 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.9591 + - 0.9465 + - 0.9593 + - 0.9337 + - 1.0 + - 0.9861 + - 1.0017 + - 1.1 + - 1.0083 + - 1.0024 + - 1.1 + - 0.9967 + - 1.0094 + - 0.9412 + - 1.1 + - 0.998 + - 0.9807 + - 0.857 + - 0.8727 + - 0.8114 + - 0.879 + - 0.871 + - 0.8853 + - 0.7937 + - 0.8243 + - 0.8 + - 0.8016 + - 0.8 + - 0.7781 + - 0.7475 + - 0.85 + - 0.7405 + - 0.715 + - 0.7416 + - 0.7465 + - 0.9 + - 0.7198 + - 0.7157 + - 0.9 + - 0.727 + - 0.7246 + - 0.704 + - 0.7039 + - 0.66 + - 0.6694 + - 0.6669 + - 0.7031 + - 0.6977 + - 0.6488 + - 0.6653 + - 0.9 + - 0.6265 + - 0.622 + - 0.6308 + - 0.6297 + - 0.621 + - 0.6225 + - 0.6229 + - 0.6234 + - 0.6238 + - 0.6332 + - 0.6425 + - 0.7028 + - 0.6152 + - 0.9 + - 0.7257 + - 0.7288 + - 1.15 + - 0.9 + - 0.6673 + - 0.7473 + - 0.6767 + - 0.7056 + - 0.9 + - 0.95 + - 0.7271 + - 0.95 + - 0.725 + - 0.7601 + - 0.6973 + - 0.7573 + - 0.6011 + - 0.606 + - 0.9 + - 0.6635 + - 0.586 + - 0.5766 + - 0.75 + - 2.0386 + - 0.75 + - 1.0 + - 0.9 + - 0.9 + - 0.9 + - 0.9 + - 0.9 + - 0.9 + - 1.0 + - 1.3386 + - 1.0 + - 1.0 + - 0.85 + - 0.95 + - 1.7386 + - 0.95 + - 0.9 + - 0.8 + - 1.7386 + - 0.75 + - 0.75 + - 0.75 + - 0.8 + - 0.75 + - 0.8 + - 0.9 + - 0.75 + - 0.8 + - 0.8 + - 1.1 + - 0.75 + - 1.1 + - 0.75 + - 0.5991 + - 0.5348 + - 0.6541 + - 0.7421 + - 0.6192 + - 0.8186 + - 1.0616 + - 0.8848 + - 1.024 + - 2.5 + - 1.0249 + - 1.0795 + - 1.2199 + - 2.5 + - 2.5 + - 1.3103 + - 1.3603 + - 2.5 + - 2.5 + - 2.5 + - 1.323 + - 2.5 + - 2.5 + - 2.5 + - 1.4406 + - 2.5 + - 2.5 + - 1.3965 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 1.6997 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 1.6264 + - 2.5 + - 2.5 + - 2.5 + - 1.3436 + - 2.5 + - 2.5 + - 0.5727 + - 0.6838 + - 0.5994 + - 0.5178 + - 0.5145 + - 0.547 + - 0.5572 + - 0.5002 + - 0.4974 + - 0.55 + - 0.4953 + - 0.4883 + - 0.4948 + - 0.5446 + - 0.5777 + - 1.5 + - 1.5 + - 3.0 + - 3.0 + - 2.5 + - 2.5 + - 2.0 + - 1.0 + - 1.5 + - 1.5 + - 1.8 + - 0.6 + - 0.7 + - 0.65 + - 0.675 + - 0.7 + - 0.75 + - 0.775 + - 0.8 + - 0.8 + - 0.85 + - 0.85 + - 0.85 + - 0.7 + - 0.7 + - 0.7 + - 0.7 + - 0.7 + - 0.7 + - 0.7 + - 0.725 + - 0.75 + - 0.775 + - 0.8 + - 0.825 + - 0.8 + - 0.8 + - 0.8 + - 0.75 + - 0.8 + - 0.8 + - 0.8 + - 0.8 + - 0.8 + - 0.85 + - 0.8 + - 0.8 + - 2.5 + - 0.75 + - 0.75 + - 0.75 + - 0.75 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 2122, 2123, 2128, 2134, 2141, 2145, 2149, 2153, 2164, 2189, 2197, + 2209, 2226, 2234, 2280, 2318, 2321, 2325, 2328, 2333, 2339, 2348, 2353, + 2355, 2357, 2363, 2370, 2371, 2377 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: airs_aqua + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: &id004 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: airs_aqua + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 1.0 + - 1.0 + - 3.0 + - 1.0 + - 3.0 + - 1.0 + - 1.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 3.0 + - 1.0 + - 1.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 3.0 + - 3.0 + - 3.5 + - 3.0 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 1.7 + - 3.0 + - 1.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.5 + - 1.25 + - 3.5 + - 3.5 + - 3.0 + - 3.0 + - 1.5 + - 3.0 + - 3.0 + - 3.0 + - 1.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.5 + - 3.0 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.5 + - 3.0 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id004 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: airs_aqua + - obs space: + name: AMSR2 GCOM-W1 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsr2_gcom-w1.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsr2_gcom-w1.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: amsr2_gcom-w1 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsr2_gcom-w1.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsr2_gcom-w1.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsr2_gcom-w1.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsr2_gcom-w1.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsr2_gcom-w1.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsr2_gcom-w1.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 340.0 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.999 + - variable: + name: GeoVaLs/skin_temperature_at_surface_where_sea + minvalue: 275 + - variable: + name: GeoVaLs/wind_speed_at_surface + maxvalue: 12 + - variable: + name: MetaData/latitude + minvalue: -60.0 + maxvalue: 60.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/TotalColumnVaporGuess + minvalue: 10.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SunGlintAngle + minvalue: 20.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: &id005 + - 0.48 + - 3.0737 + - 0.7433 + - 3.643 + - 3.5304 + - 4.427 + - 5.1448 + - 5.0785 + - 4.9763 + - 9.3215 + - 2.5789 + - 5.5274 + - 0.6641 + - 1.3674 + clwret_types: + - ObsValue + maxvalue: 1.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id005 + clwret_types: + - HofX + maxvalue: 1.0 + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: None + value: + name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id005 + clwret_types: + - ObsValue + reference: + name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id005 + clwret_types: + - HofX + minvalue: -0.5 + maxvalue: 0.5 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id005 + clwret_types: + - ObsValue + - HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.1 + - 0.1 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 0.6 + - 0.6 + - 0.6 + - 0.6 + - 0.6 + - 0.5 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + err0: + - 0.8 + - 0.9 + - 0.8 + - 0.9 + - 1.0 + - 1.1 + - 2.0 + - 3.5 + - 3.0 + - 4.8 + - 5.0 + - 6.0 + - 4.5 + - 6.3 + err1: + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 18.5 + - 20.0 + - 40.0 + - 20.0 + - 25.0 + - 30.0 + - 30.0 + - 30.0 + - 20.0 + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: None + threshold: 2.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 7-10 + absolute threshold: 30 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 11-14 + absolute threshold: 50 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: amsr2_gcom-w1 + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsr2_gcom-w1 + - obs space: + name: AMSU-A AQUA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_aqua.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_aqua.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_aqua + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_aqua.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_aqua.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_aqua.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_aqua.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_aqua.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_aqua.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id006 + - 2.5 + - 2.0 + - 2.0 + - 0.5 + - 0.4 + - 0.4 + - 0.5 + - 0.3 + - 0.35 + - 0.35 + - 0.45 + - 1.0 + - 1.5 + - 3.75 + - 6.3 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_aqua + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_aqua + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_aqua + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_aqua + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_aqua + error parameter vector: *id006 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 3.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 3.0 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_aqua + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_aqua + - obs space: + name: AMSU-A METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_metop-b.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_metop-b.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_metop-b.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_metop-b.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_metop-b.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_metop-b.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id007 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_metop-b + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_metop-b + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_metop-b + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_metop-b + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_metop-b + error parameter vector: *id007 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_metop-b + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_metop-b + - obs space: + name: AMSU-A METOP-C + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_metop-c.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_metop-c.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_metop-c + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_metop-c.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_metop-c.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_metop-c.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_metop-c.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_metop-c.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_metop-c.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id008 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_metop-c + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_metop-c + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_metop-c + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_metop-c + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_metop-c + error parameter vector: *id008 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_metop-c + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_metop-c + - obs space: + name: AMSU-A NOAA-15 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_n15.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_n15.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_n15 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_n15.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_n15.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_n15.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_n15.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_n15.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_n15.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id009 + - 3.0 + - 2.0 + - 2.0 + - 0.6 + - 0.3 + - 0.23 + - 0.25 + - 0.275 + - 0.34 + - 0.4 + - 0.6 + - 1.0 + - 1.5 + - 5.0 + - 3.0 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_n15 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_n15 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_n15 + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_n15 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_n15 + error parameter vector: *id009 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_n15 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_n15 + - obs space: + name: AMSU-A NOAA-18 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_n18.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_n18.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_n18 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_n18.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_n18.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_n18.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_n18.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_n18.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_n18.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id010 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_n18 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_n18 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_n18 + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_n18 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_n18 + error parameter vector: *id010 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_n18 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_n18 + - obs space: + name: AMSU-A NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_n19.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_n19.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_n19 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_n19.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_n19.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_n19.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_n19.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_n19.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_n19.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id011 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_n19 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_n19 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_n19 + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_n19 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_n19 + error parameter vector: *id011 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_n19 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_n19 + - obs space: + name: ATMS NOAA-20 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/atms_n20.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.atms_n20.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: atms_n20 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/atms_n20.20231009T150000Z.satbias.nc4 + output file: cycle_dir/atms_n20.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: ATMS + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/atms_n20.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/atms_n20.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/atms_n20.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/atms_n20.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id012 + - 5.0 + - 5.0 + - 5.0 + - 3.0 + - 0.55 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 5.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7,16-22 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-7,16 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: atms_n20 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: atms_n20 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + use_biasterm: true + test_biasterm: ObsBiasTerm + sensor: atms_n20 + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: atms_n20 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: atms_n20 + error parameter vector: *id012 + obserr_bound_max: + - 4.5 + - 4.5 + - 3.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 2.0 + - 4.5 + - 4.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: atms_n20 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: atms_n20 + - obs space: + name: ATMS NPP + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/atms_npp.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.atms_npp.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: atms_npp + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/atms_npp.20231009T150000Z.satbias.nc4 + output file: cycle_dir/atms_npp.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: ATMS + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/atms_npp.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/atms_npp.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/atms_npp.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/atms_npp.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id013 + - 5.0 + - 5.0 + - 5.0 + - 3.0 + - 0.55 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 5.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7,16-22 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-7,16 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: atms_npp + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: atms_npp + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + use_biasterm: true + test_biasterm: ObsBiasTerm + sensor: atms_npp + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: atms_npp + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: atms_npp + error parameter vector: *id013 + obserr_bound_max: + - 4.5 + - 4.5 + - 3.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 2.0 + - 4.5 + - 4.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: atms_npp + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: atms_npp + - obs space: + name: AVHRR-3 METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/avhrr3_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.avhrr3_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: avhrr3_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/avhrr3_metop-b.20231009T150000Z.satbias.nc4 + output file: cycle_dir/avhrr3_metop-b.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/avhrr3_metop-b.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/avhrr3_metop-b.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/avhrr3_metop-b.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/avhrr3_metop-b.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.0 + - 1.08 + - 1.12 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 1e-05 + maxvalue: 1000.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: avhrr3_metop-b + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + cloud detection criteria: + - 0.6 + - 0.68 + - 0.72 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: avhrr3_metop-b + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: avhrr3_metop-b + - obs space: + name: AVHRR-3 NOAA-18 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/avhrr3_n18.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.avhrr3_n18.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: avhrr3_n18 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/avhrr3_n18.20231009T150000Z.satbias.nc4 + output file: cycle_dir/avhrr3_n18.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/avhrr3_n18.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/avhrr3_n18.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/avhrr3_n18.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/avhrr3_n18.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.0 + - 1.08 + - 1.12 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 1e-05 + maxvalue: 1000.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: avhrr3_n18 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + cloud detection criteria: + - 0.6 + - 0.68 + - 0.72 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: avhrr3_n18 + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: avhrr3_n18 + - obs space: + name: AVHRR-3 NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/avhrr3_n19.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.avhrr3_n19.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: avhrr3_n19 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/avhrr3_n19.20231009T150000Z.satbias.nc4 + output file: cycle_dir/avhrr3_n19.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/avhrr3_n19.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/avhrr3_n19.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/avhrr3_n19.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/avhrr3_n19.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.0 + - 1.08 + - 1.12 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 1e-05 + maxvalue: 1000.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: avhrr3_n19 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + cloud detection criteria: + - 0.6 + - 0.68 + - 0.72 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: avhrr3_n19 + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: avhrr3_n19 + - obs space: + name: CRIS-FSR NOAA-20 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/cris-fsr_n20.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.cris-fsr_n20.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: cris-fsr_n20 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/cris-fsr_n20.20231009T150000Z.satbias.nc4 + output file: cycle_dir/cris-fsr_n20.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/cris-fsr_n20.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/cris-fsr_n20.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/cris-fsr_n20.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/cris-fsr_n20.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/cris-fsr_108_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 0.823 + - 0.76 + - 0.736 + - 0.743 + - 0.856 + - 1.079 + - 0.888 + - 0.778 + - 0.671 + - 0.65 + - 0.643 + - 0.629 + - 0.629 + - 0.618 + - 0.638 + - 0.619 + - 0.61 + - 0.627 + - 0.601 + - 0.617 + - 0.608 + - 0.498 + - 0.5112 + - 0.4922 + - 0.4959 + - 0.4954 + - 0.4836 + - 0.514 + - 0.5005 + - 0.4917 + - 0.4881 + - 0.4656 + - 0.4793 + - 0.4638 + - 0.4557 + - 0.4666 + - 0.4468 + - 0.4534 + - 0.4471 + - 0.4448 + - 0.4469 + - 0.536 + - 0.4426 + - 0.4388 + - 0.534 + - 0.4368 + - 0.438 + - 0.531 + - 0.4379 + - 0.535 + - 0.4404 + - 0.4405 + - 0.4409 + - 0.4472 + - 0.4555 + - 0.4433 + - 0.4437 + - 0.4454 + - 0.4448 + - 0.4465 + - 0.4499 + - 0.4488 + - 0.54 + - 0.4534 + - 0.4472 + - 0.455 + - 0.4562 + - 0.452 + - 0.4639 + - 0.538 + - 0.4573 + - 0.4604 + - 0.4533 + - 0.4692 + - 0.566 + - 0.4457 + - 0.4457 + - 0.5154 + - 0.5084 + - 0.528 + - 0.552 + - 0.56 + - 0.567 + - 0.546 + - 0.495 + - 0.4809 + - 0.4732 + - 0.4861 + - 0.4632 + - 0.529 + - 0.4748 + - 0.5007 + - 0.5711 + - 0.595 + - 0.5469 + - 0.626 + - 0.541 + - 0.543 + - 0.533 + - 0.541 + - 0.4695 + - 0.53 + - 0.539 + - 0.529 + - 0.542 + - 0.4681 + - 0.536 + - 0.542 + - 0.535 + - 0.563 + - 0.4805 + - 0.647 + - 0.609 + - 0.553 + - 0.583 + - 0.576 + - 0.6294 + - 0.5885 + - 0.556 + - 0.578 + - 0.566 + - 0.601 + - 0.5627 + - 0.5675 + - 0.592 + - 0.5166 + - 0.589 + - 0.5291 + - 0.5892 + - 0.5976 + - 0.5834 + - 0.6512 + - 0.6748 + - 0.6615 + - 0.6003 + - 0.5669 + - 0.5587 + - 0.5507 + - 0.5871 + - 0.616 + - 0.637 + - 0.633 + - 0.639 + - 0.655 + - 0.641 + - 0.664 + - 0.648 + - 0.656 + - 0.663 + - 0.652 + - 0.681 + - 0.662 + - 0.673 + - 0.672 + - 0.68 + - 0.735 + - 0.732 + - 0.715 + - 0.674 + - 0.687 + - 0.702 + - 0.705 + - 0.715 + - 0.725 + - 0.707 + - 0.74 + - 0.74 + - 0.874 + - 0.737 + - 0.819 + - 0.76 + - 0.869 + - 0.9 + - 0.698 + - 0.823 + - 0.676 + - 0.682 + - 0.766 + - 0.68 + - 0.685 + - 0.694 + - 0.695 + - 0.689 + - 0.727 + - 0.695 + - 0.688 + - 0.677 + - 0.736 + - 0.651 + - 0.661 + - 0.6199 + - 0.6223 + - 0.6036 + - 0.6003 + - 0.5991 + - 0.598 + - 0.591 + - 0.5764 + - 0.577 + - 0.5593 + - 0.597 + - 0.576 + - 0.574 + - 0.578 + - 0.579 + - 0.575 + - 0.576 + - 0.568 + - 0.575 + - 0.569 + - 0.559 + - 0.568 + - 0.5401 + - 0.55 + - 0.5575 + - 0.578 + - 0.5635 + - 0.5786 + - 0.5807 + - 0.581 + - 0.573 + - 0.569 + - 0.567 + - 0.552 + - 0.55 + - 0.558 + - 0.552 + - 0.562 + - 0.574 + - 0.575 + - 0.629 + - 0.682 + - 0.756 + - 1.05 + - 1.122 + - 1.1402 + - 1.154 + - 1.131 + - 1.123 + - 1.159 + - 1.106 + - 1.116 + - 1.069 + - 1.077 + - 1.155 + - 1.162 + - 1.1402 + - 0.644 + - 1.208 + - 1.1402 + - 1.295 + - 1.258 + - 1.1402 + - 0.606 + - 0.603 + - 0.563 + - 0.592 + - 0.607 + - 0.611 + - 0.612 + - 0.618 + - 0.626 + - 0.629 + - 0.583 + - 0.8646 + - 0.626 + - 0.639 + - 0.559 + - 0.827 + - 0.612 + - 0.576 + - 0.58 + - 0.575 + - 0.688 + - 0.697 + - 0.743 + - 0.681 + - 0.832 + - 0.719 + - 0.785 + - 0.878 + - 0.9402 + - 1.0054 + - 1.073 + - 1.129 + - 1.035 + - 1.027 + - 0.9703 + - 1.195 + - 0.9153 + - 1.266 + - 1.153 + - 1.348 + - 1.18 + - 1.269 + - 1.311 + - 0.9914 + - 1.359 + - 1.166 + - 1.139 + - 1.2817 + - 1.398 + - 1.542 + - 1.229 + - 1.377 + - 1.28 + - 1.245 + - 1.1188 + - 1.193 + - 1.293 + - 1.275 + - 1.331 + - 1.34 + - 1.099 + - 1.048 + - 1.124 + - 1.225 + - 1.183 + - 1.196 + - 1.4 + - 1.333 + - 1.417 + - 1.326 + - 1.305 + - 1.0638 + - 1.268 + - 1.217 + - 1.289 + - 1.395 + - 1.232 + - 1.435 + - 1.298 + - 1.328 + - 1.262 + - 1.199 + - 1.391 + - 1.233 + - 1.329 + - 1.664 + - 1.509 + - 1.349 + - 1.481 + - 1.595 + - 1.485 + - 1.532 + - 1.504 + - 1.584 + - 1.609 + - 1.516 + - 1.489 + - 1.502 + - 1.544 + - 1.611 + - 1.539 + - 1.296 + - 1.288 + - 1.241 + - 1.32 + - 1.313 + - 1.301 + - 1.843 + - 1.747 + - 1.711 + - 1.771 + - 1.937 + - 1.575 + - 1.573 + - 1.5 + - 1.459 + - 1.402 + - 1.363 + - 2.201 + - 2.127 + - 2.177 + - 2.157 + - 2.192 + - 2.146 + - 2.151 + - 2.071 + - 1.986 + - 1.845 + - 1.687 + - 1.505 + - 1.373 + - 1.229 + - 1.113 + - 1.004 + - 0.936 + - 0.895 + - 0.871 + - 0.841 + - 0.821 + - 0.805 + - 0.8 + - 0.792 + - 0.797 + - 0.791 + - 0.783 + - 0.777 + - 0.785 + - 0.787 + - 0.789 + - 0.793 + - 0.794 + - 0.745 + - 0.75 + - 0.748 + - 0.749 + - 0.744 + - 0.733 + - 0.733 + - 0.741 + - 0.746 + - 0.746 + - 0.738 + - 0.743 + - 0.745 + - 0.749 + - 0.75 + - 0.75 + - 0.884 + - 0.906 + - 0.917 + - 0.924 + - 0.922 + - 0.928 + - 0.921 + - 0.938 + - 0.931 + - 0.943 + - 0.946 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, + 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, 2153, 2158, 2161, + 2168, 2171, 2175, 2182 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: cris-fsr_n20 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: &id014 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 31 + - 31 + - 30 + - 31 + - 30 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 31 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: cris-fsr_n20 + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 2.0 + - 0.4 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.4 + - 0.4 + - 0.4 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id014 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: cris-fsr_n20 + - obs space: + name: CRIS-FSR NPP + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/cris-fsr_npp.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.cris-fsr_npp.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: cris-fsr_npp + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/cris-fsr_npp.20231009T150000Z.satbias.nc4 + output file: cycle_dir/cris-fsr_npp.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/cris-fsr_npp.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/cris-fsr_npp.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/cris-fsr_npp.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/cris-fsr_npp.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/cris-fsr_108_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 0.823 + - 0.76 + - 0.736 + - 0.743 + - 0.856 + - 1.079 + - 0.888 + - 0.778 + - 0.671 + - 0.65 + - 0.643 + - 0.629 + - 0.629 + - 0.618 + - 0.638 + - 0.619 + - 0.61 + - 0.627 + - 0.601 + - 0.617 + - 0.608 + - 0.498 + - 0.5112 + - 0.4922 + - 0.4959 + - 0.4954 + - 0.4836 + - 0.514 + - 0.5005 + - 0.4917 + - 0.4881 + - 0.4656 + - 0.4793 + - 0.4638 + - 0.4557 + - 0.4666 + - 0.4468 + - 0.4534 + - 0.4471 + - 0.4448 + - 0.4469 + - 0.536 + - 0.4426 + - 0.4388 + - 0.534 + - 0.4368 + - 0.438 + - 0.531 + - 0.4379 + - 0.535 + - 0.4404 + - 0.4405 + - 0.4409 + - 0.4472 + - 0.4555 + - 0.4433 + - 0.4437 + - 0.4454 + - 0.4448 + - 0.4465 + - 0.4499 + - 0.4488 + - 0.54 + - 0.4534 + - 0.4472 + - 0.455 + - 0.4562 + - 0.452 + - 0.4639 + - 0.538 + - 0.4573 + - 0.4604 + - 0.4533 + - 0.4692 + - 0.566 + - 0.4457 + - 0.4457 + - 0.5154 + - 0.5084 + - 0.528 + - 0.552 + - 0.56 + - 0.567 + - 0.546 + - 0.495 + - 0.4809 + - 0.4732 + - 0.4861 + - 0.4632 + - 0.529 + - 0.4748 + - 0.5007 + - 0.5711 + - 0.595 + - 0.5469 + - 0.626 + - 0.541 + - 0.543 + - 0.533 + - 0.541 + - 0.4695 + - 0.53 + - 0.539 + - 0.529 + - 0.542 + - 0.4681 + - 0.536 + - 0.542 + - 0.535 + - 0.563 + - 0.4805 + - 0.647 + - 0.609 + - 0.553 + - 0.583 + - 0.576 + - 0.6294 + - 0.5885 + - 0.556 + - 0.578 + - 0.566 + - 0.601 + - 0.5627 + - 0.5675 + - 0.592 + - 0.5166 + - 0.589 + - 0.5291 + - 0.5892 + - 0.5976 + - 0.5834 + - 0.6512 + - 0.6748 + - 0.6615 + - 0.6003 + - 0.5669 + - 0.5587 + - 0.5507 + - 0.5871 + - 0.616 + - 0.637 + - 0.633 + - 0.639 + - 0.655 + - 0.641 + - 0.664 + - 0.648 + - 0.656 + - 0.663 + - 0.652 + - 0.681 + - 0.662 + - 0.673 + - 0.672 + - 0.68 + - 0.735 + - 0.732 + - 0.715 + - 0.674 + - 0.687 + - 0.702 + - 0.705 + - 0.715 + - 0.725 + - 0.707 + - 0.74 + - 0.74 + - 0.874 + - 0.737 + - 0.819 + - 0.76 + - 0.869 + - 0.9 + - 0.698 + - 0.823 + - 0.676 + - 0.682 + - 0.766 + - 0.68 + - 0.685 + - 0.694 + - 0.695 + - 0.689 + - 0.727 + - 0.695 + - 0.688 + - 0.677 + - 0.736 + - 0.651 + - 0.661 + - 0.6199 + - 0.6223 + - 0.6036 + - 0.6003 + - 0.5991 + - 0.598 + - 0.591 + - 0.5764 + - 0.577 + - 0.5593 + - 0.597 + - 0.576 + - 0.574 + - 0.578 + - 0.579 + - 0.575 + - 0.576 + - 0.568 + - 0.575 + - 0.569 + - 0.559 + - 0.568 + - 0.5401 + - 0.55 + - 0.5575 + - 0.578 + - 0.5635 + - 0.5786 + - 0.5807 + - 0.581 + - 0.573 + - 0.569 + - 0.567 + - 0.552 + - 0.55 + - 0.558 + - 0.552 + - 0.562 + - 0.574 + - 0.575 + - 0.629 + - 0.682 + - 0.756 + - 1.05 + - 1.122 + - 1.1402 + - 1.154 + - 1.131 + - 1.123 + - 1.159 + - 1.106 + - 1.116 + - 1.069 + - 1.077 + - 1.155 + - 1.162 + - 1.1402 + - 0.644 + - 1.208 + - 1.1402 + - 1.295 + - 1.258 + - 1.1402 + - 0.606 + - 0.603 + - 0.563 + - 0.592 + - 0.607 + - 0.611 + - 0.612 + - 0.618 + - 0.626 + - 0.629 + - 0.583 + - 0.8646 + - 0.626 + - 0.639 + - 0.559 + - 0.827 + - 0.612 + - 0.576 + - 0.58 + - 0.575 + - 0.688 + - 0.697 + - 0.743 + - 0.681 + - 0.832 + - 0.719 + - 0.785 + - 0.878 + - 0.9402 + - 1.0054 + - 1.073 + - 1.129 + - 1.035 + - 1.027 + - 0.9703 + - 1.195 + - 0.9153 + - 1.266 + - 1.153 + - 1.348 + - 1.18 + - 1.269 + - 1.311 + - 0.9914 + - 1.359 + - 1.166 + - 1.139 + - 1.2817 + - 1.398 + - 1.542 + - 1.229 + - 1.377 + - 1.28 + - 1.245 + - 1.1188 + - 1.193 + - 1.293 + - 1.275 + - 1.331 + - 1.34 + - 1.099 + - 1.048 + - 1.124 + - 1.225 + - 1.183 + - 1.196 + - 1.4 + - 1.333 + - 1.417 + - 1.326 + - 1.305 + - 1.0638 + - 1.268 + - 1.217 + - 1.289 + - 1.395 + - 1.232 + - 1.435 + - 1.298 + - 1.328 + - 1.262 + - 1.199 + - 1.391 + - 1.233 + - 1.329 + - 1.664 + - 1.509 + - 1.349 + - 1.481 + - 1.595 + - 1.485 + - 1.532 + - 1.504 + - 1.584 + - 1.609 + - 1.516 + - 1.489 + - 1.502 + - 1.544 + - 1.611 + - 1.539 + - 1.296 + - 1.288 + - 1.241 + - 1.32 + - 1.313 + - 1.301 + - 1.843 + - 1.747 + - 1.711 + - 1.771 + - 1.937 + - 1.575 + - 1.573 + - 1.5 + - 1.459 + - 1.402 + - 1.363 + - 2.201 + - 2.127 + - 2.177 + - 2.157 + - 2.192 + - 2.146 + - 2.151 + - 2.071 + - 1.986 + - 1.845 + - 1.687 + - 1.505 + - 1.373 + - 1.229 + - 1.113 + - 1.004 + - 0.936 + - 0.895 + - 0.871 + - 0.841 + - 0.821 + - 0.805 + - 0.8 + - 0.792 + - 0.797 + - 0.791 + - 0.783 + - 0.777 + - 0.785 + - 0.787 + - 0.789 + - 0.793 + - 0.794 + - 0.745 + - 0.75 + - 0.748 + - 0.749 + - 0.744 + - 0.733 + - 0.733 + - 0.741 + - 0.746 + - 0.746 + - 0.738 + - 0.743 + - 0.745 + - 0.749 + - 0.75 + - 0.75 + - 0.884 + - 0.906 + - 0.917 + - 0.924 + - 0.922 + - 0.928 + - 0.921 + - 0.938 + - 0.931 + - 0.943 + - 0.946 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, + 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, 2153, 2158, 2161, + 2168, 2171, 2175, 2182 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: cris-fsr_npp + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: &id015 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 31 + - 31 + - 30 + - 31 + - 30 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 31 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: cris-fsr_npp + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 2.0 + - 0.4 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.4 + - 0.4 + - 0.4 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id015 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: cris-fsr_npp + - obs space: + name: GMI GPM + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/gmi_gpm.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.gmi_gpm.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: gmi_gpm + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/gmi_gpm.20231009T150000Z.satbias.nc4 + output file: cycle_dir/gmi_gpm.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/gmi_gpm.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/gmi_gpm.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: cloudWaterContent + sensor: GMI_GPM + ch37v: 6 + ch37h: 7 + order: 2 + tlapse: cycle_dir/gmi_gpm.20231009T150000Z.tlapse.txt + - name: cloudWaterContent + sensor: GMI_GPM + ch37v: 6 + ch37h: 7 + tlapse: cycle_dir/gmi_gpm.20231009T150000Z.tlapse.txt + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/gmi_gpm.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/gmi_gpm.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-9 + minvalue: 50.0 + maxvalue: 320.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 10-13 + minvalue: 70.0 + maxvalue: 320.0 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 5 + maxvalue: 70 + - variable: + name: MetaData/latitude + minvalue: -55.0 + maxvalue: 55.0 + - variable: + name: MetaData/heightOfSurface + maxvalue: 2000 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + minvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/latitude + minvalue: -20.0 + maxvalue: 0.0 + - variable: + name: MetaData/longitude + minvalue: 25.0 + maxvalue: 40.0 + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/cloudWaterContent_obs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + maxvalue: 900 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - HofX + maxvalue: 900 + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/cloudWaterContent_hofx + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - HofX + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + minvalue: 0.0 + maxvalue: 0.05 + - variable: + name: ObsFunction/Emissivity_Diff_GMI + options: + channel: 2 + regression_constant_1: 0.1329 + regression_constant_2: 0.42468 + regression_coeff_1: + - -0.00548 + - 0.00772 + - 0.0053 + - -0.00425 + - 0.00053 + - 8e-05 + - -3e-05 + - -0.00144 + - 0.00059 + - -0.00016 + - 3e-05 + - -0.00011 + - 0.00017 + regression_coeff_2: + - 0.00289 + - -0.00142 + minvalue: 0.01 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + minvalue: 0.0 + maxvalue: 0.05 + - variable: + name: ObsFunction/Emissivity_Diff_GMI + options: + channel: 4 + regression_constant_1: 0.15627 + regression_constant_2: 0.83807 + regression_coeff_1: + - -0.01084 + - 0.01194 + - 0.01111 + - -0.00784 + - 0.0006 + - 8e-05 + - -3e-05 + - -0.00248 + - 0.00105 + - -8e-05 + - 0.0 + - -0.00013 + - 0.00016 + regression_coeff_2: + - 0.00048 + - -0.00207 + minvalue: 0.035 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + minvalue: 0.0 + maxvalue: 0.05 + - variable: + name: ObsFunction/Emissivity_Diff_GMI + options: + channel: 7 + regression_constant_1: 0.30306 + regression_constant_2: 1.24071 + regression_coeff_1: + - -0.01793 + - 0.0173 + - 0.01784 + - -0.01199 + - 0.00067 + - 0.00013 + - -4e-05 + - -0.00365 + - 0.00154 + - -4e-05 + - -1e-05 + - -0.00015 + - 0.00017 + regression_coeff_2: + - 0.00068 + - -0.00342 + minvalue: 0.05 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + - HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.3 + - 0.2 + - 0.3 + - 0.3 + x2: + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + err0: + - 2.7 + - 3.7 + - 3.5 + - 4.5 + - 4.0 + - 3.8 + - 300.0 + - 5.0 + - 11.5 + - 5.0 + - 5.0 + - 2.5 + - 3.0 + err1: + - 17.0 + - 23.0 + - 13.0 + - 25.0 + - 11.0 + - 13.0 + - 23.0 + - 10.0 + - 20.0 + - 15.0 + - 20.0 + - 8.0 + - 13.0 + err2: + - 25.0 + - 40.0 + - 40.0 + - 55.0 + - 35.0 + - 25.0 + - 500.0 + - 50.0 + - 50.0 + - 50.0 + - 50.0 + - 30.0 + - 40.0 + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 1,2,4,6 + threshold: 2.0 + absolute threshold: 30.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 9,10,11 + threshold: 2.0 + absolute threshold: 20.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 3,5,8 + threshold: 2.0 + absolute threshold: 15.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 12,13 + threshold: 2.0 + absolute threshold: 10.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 7 + threshold: 2.0 + absolute threshold: 5.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: gmi_gpm + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: gmi_gpm + - obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/gps.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - sequenceNumber + sort variable: impactHeightRO + sort order: ascending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.gps.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - bendingAngle + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + vertlayer: full + sr_steps: 2 + super_ref_qc: NBAM + obs filters: + - filter: BlackList + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 41,265,266,421,440,724,725,726,727,728,729 + - filter: Perform Action + filter variables: + - name: bendingAngle + where: + - variable: PreUseFlag/bendingAngle + minvalue: 1 + action: + name: reject + - filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266,267,268,269 + test variables: + - name: MetaData/impactHeightRO + maxvalue: 45000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 3-5 + test variables: + - name: MetaData/impactHeightRO + minvalue: 8000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 267,268,269 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 401 + test variables: + - name: MetaData/impactHeightRO + minvalue: 5000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 267,268,269 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 402-999 + test variables: + - name: MetaData/impactHeightRO + minvalue: 9000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 401 + test variables: + - name: MetaData/impactHeightRO + minvalue: 5000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 402-999 + test variables: + - name: MetaData/impactHeightRO + minvalue: 8000.1 + action: + name: reject + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: inflate error + inflation factor: 2.0 + where: + - variable: MetaData/satelliteIdentifier + is_in: 267,268,269 + - filter: Variable Assignment + assignments: + - name: JediAdjustObsError/bendingAngle + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/bendingAngle + defer to post: true + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: assign error + error parameter: 1.0 + where: + - variable: + name: JediAdjustObsError/bendingAngle + maxvalue: 1.0 + - variable: + name: JediAdjustObsError/bendingAngle + value: is_valid + - variable: + name: MetaData/satelliteIdentifier + is_in: 3,5,41,42,43,44,267,268,269,440,421,724,725,726,727,728,729, 750,751,752,753,754,755,821,825 + defer to post: true + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: assign error + error parameter: 10.0 + where: + - variable: + name: JediAdjustObsError/bendingAngle + minvalue: 10.0 + - variable: + name: JediAdjustObsError/bendingAngle + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: bendingAngle + threshold: 5 + action: + name: reject + defer to post: true + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: assign error + error function: JediAdjustObsError/bendingAngle + where: + - variable: + name: JediAdjustObsError/bendingAngle + value: is_valid + defer to post: true + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + defer to post: true + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate_GEOS + defer to post: true + observation_name: gps + - obs space: + name: IASI METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/iasi_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.iasi_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: iasi_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/iasi_metop-b.20231009T150000Z.satbias.nc4 + output file: cycle_dir/iasi_metop-b.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/iasi_metop-b.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/iasi_metop-b.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/iasi_metop-b.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/iasi_metop-b.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/iasi_metop_141_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id016 + - 0.727 + - 0.81 + - 0.75 + - 0.79 + - 0.7055 + - 0.74 + - 0.68 + - 0.72 + - 0.6526 + - 0.65 + - 0.665 + - 0.69 + - 0.6394 + - 0.64 + - 0.6528 + - 0.6065 + - 0.6246 + - 0.61 + - 0.6423 + - 0.5995 + - 0.59 + - 0.6069 + - 0.6 + - 0.5965 + - 0.64 + - 0.62 + - 0.589 + - 0.5865 + - 0.65 + - 0.5861 + - 0.61 + - 0.5874 + - 0.68 + - 0.606 + - 0.68 + - 4.38 + - 3.05 + - 2.31 + - 1.56 + - 1.33 + - 1.58 + - 0.93 + - 0.5832 + - 0.5587 + - 0.5867 + - 0.58 + - 0.5655 + - 0.5522 + - 0.5864 + - 0.5475 + - 0.5854 + - 0.5455 + - 0.5811 + - 0.5376 + - 0.5452 + - 0.5686 + - 0.5329 + - 0.5655 + - 0.5302 + - 0.545 + - 0.5628 + - 0.59 + - 0.5262 + - 0.559 + - 0.5264 + - 0.5442 + - 0.51 + - 0.5513 + - 0.5224 + - 0.5523 + - 0.5188 + - 0.5487 + - 0.5245 + - 0.58 + - 0.5437 + - 0.5343 + - 0.5364 + - 0.64 + - 0.5338 + - 0.72 + - 0.537 + - 0.75 + - 0.51 + - 0.65 + - 0.5274 + - 0.529 + - 0.5187 + - 0.5228 + - 1.12 + - 0.5222 + - 0.5109 + - 0.67 + - 0.5133 + - 0.5179 + - 0.507 + - 0.67 + - 0.5091 + - 0.62 + - 0.5093 + - 0.69 + - 0.5048 + - 0.5024 + - 0.78 + - 0.497 + - 0.5337 + - 0.4865 + - 0.4915 + - 0.4835 + - 0.4869 + - 0.87 + - 0.4824 + - 0.4852 + - 0.84 + - 0.84 + - 0.84 + - 0.5318 + - 0.8 + - 0.4772 + - 0.98 + - 0.488 + - 0.4978 + - 0.5157 + - 0.61 + - 0.5213 + - 0.4884 + - 0.79 + - 0.62 + - 0.66 + - 0.4691 + - 0.65 + - 0.4809 + - 0.468 + - 0.62 + - 0.4679 + - 0.6913 + - 0.4705 + - 0.4785 + - 0.47 + - 0.4773 + - 0.4703 + - 0.98 + - 0.4697 + - 0.4662 + - 0.65 + - 0.467 + - 0.4883 + - 0.4684 + - 0.4684 + - 0.4947 + - 0.5393 + - 0.5024 + - 0.4715 + - 0.621 + - 0.6136 + - 0.5316 + - 1.78 + - 0.5099 + - 1.14 + - 0.539 + - 1.79 + - 0.508 + - 0.5723 + - 1.94 + - 2.01 + - 0.49 + - 0.5647 + - 0.5022 + - 1.47 + - 0.5815 + - 0.6782 + - 2.13 + - 0.5445 + - 1.52 + - 0.5555 + - 1.96 + - 2.31 + - 2.33 + - 2.32 + - 2.31 + - 0.6994 + - 0.7006 + - 0.706 + - 0.9785 + - 0.7023 + - 0.6991 + - 0.6946 + - 2.28 + - 2.26 + - 2.26 + - 2.26 + - 0.6608 + - 0.6835 + - 0.6822 + - 2.24 + - 2.26 + - 0.6735 + - 2.28 + - 0.667 + - 0.7732 + - 0.6642 + - 0.648 + - 0.6629 + - 2.29 + - 2.29 + - 0.6799 + - 0.623 + - 2.32 + - 0.603 + - 0.6224 + - 2.32 + - 0.6187 + - 2.31 + - 2.31 + - 2.28 + - 2.29 + - 2.28 + - 2.26 + - 1.966 + - 2.27 + - 2.26 + - 2.25 + - 2.27 + - 2.24 + - 2.21 + - 2.24 + - 2.17 + - 2.18 + - 2.17 + - 2.21 + - 1.4 + - 2.16 + - 2.2 + - 2.13 + - 2.12 + - 2.13 + - 2.1 + - 2.12 + - 2.11 + - 2.09 + - 2.09 + - 2.08 + - 2.09 + - 2.04 + - 2.04 + - 1.966 + - 2.01 + - 2.05 + - 2.03 + - 2.06 + - 1.98 + - 1.95 + - 1.94 + - 1.91 + - 1.857 + - 1.76 + - 1.748 + - 1.83 + - 2.04 + - 1.748 + - 1.99 + - 2.075 + - 2.07 + - 2.02 + - 2.04 + - 2.1 + - 1.966 + - 2.18 + - 2.21 + - 2.24 + - 2.23 + - 2.23 + - 1.98 + - 2.2 + - 2.18 + - 2.18 + - 2.21 + - 2.23 + - 2.24 + - 2.24 + - 2.25 + - 1.8 + - 2.24 + - 1.73 + - 1.73 + - 2.27 + - 1.67 + - 2.21 + - 1.72 + - 2.23 + - 2.23 + - 2.23 + - 2.24 + - 2.23 + - 2.12 + - 2.17 + - 1.74 + - 2.02 + - 1.88 + - 1.67 + - 1.73 + - 1.83 + - 1.82 + - 1.73 + - 1.83 + - 2.19 + - 1.84 + - 1.89 + - 1.6 + - 1.71 + - 1.86 + - 1.85 + - 1.84 + - 1.87 + - 1.91 + - 1.52 + - 1.95 + - 1.87 + - 1.89 + - 1.91 + - 1.91 + - 1.93 + - 1.9 + - 1.91 + - 1.9 + - 1.89 + - 1.89 + - 1.91 + - 1.9 + - 1.91 + - 1.91 + - 1.91 + - 1.93 + - 1.94 + - 1.91 + - 1.92 + - 1.77 + - 1.91 + - 1.95 + - 1.19 + - 1.96 + - 1.98 + - 1.94 + - 1.55 + - 1.91 + - 1.92 + - 1.92 + - 1.97 + - 1.93 + - 1.99 + - 1.86 + - 1.12 + - 1.93 + - 1.92 + - 1.95 + - 1.85 + - 1.84 + - 1.91 + - 1.12 + - 1.82 + - 1.82 + - 1.95 + - 1.24 + - 1.94 + - 1.96 + - 1.21 + - 1.83 + - 1.96 + - 1.36 + - 1.96 + - 1.82 + - 1.92 + - 1.68 + - 1.93 + - 1.23 + - 1.96 + - 1.93 + - 1.86 + - 1.41 + - 1.16 + - 1.6 + - 1.25 + - 1.2 + - 1.65 + - 1.66 + - 1.87 + - 1.94 + - 1.96 + - 1.91 + - 1.25 + - 1.93 + - 1.91 + - 1.7 + - 0.99 + - 1.81 + - 1.92 + - 1.95 + - 1.5 + - 1.47 + - 1.15 + - 1.58 + - 1.18 + - 1.82 + - 1.13 + - 1.83 + - 1.91 + - 1.26 + - 1.27 + - 1.91 + - 1.45 + - 1.6 + - 1.29 + - 1.94 + - 1.94 + - 1.23 + - 1.95 + - 1.21 + - 1.94 + - 1.86 + - 1.9 + - 1.33 + - 1.75 + - 2.02 + - 1.98 + - 2.03 + - 1.83 + - 1.5 + - 2.04 + - 2.02 + - 1.9 + - 2.0 + - 2.02 + - 1.95 + - 1.93 + - 1.95 + - 1.95 + - 1.99 + - 2.0 + - 1.94 + - 1.96 + - 1.86 + - 1.92 + - 1.88 + - 1.86 + - 1.84 + - 1.87 + - 1.77 + - 1.89 + - 1.89 + - 1.88 + - 1.94 + - 1.82 + - 1.79 + - 1.86 + - 2.06 + - 2.33 + - 1.88 + - 1.86 + - 1.81 + - 1.8 + - 1.8 + - 1.86 + - 1.9 + - 2.0 + - 2.06 + - 2.1 + - 2.2 + - 2.0 + - 2.16 + - 1.98 + - 1.8 + - 1.8 + - 1.85 + - 1.75 + - 2.04 + - 2.19 + - 2.14 + - 2.19 + - 1.86 + - 2.1 + - 2.11 + - 2.18 + - 2.03 + - 2.28 + - 2.19 + - 2.26 + - 2.26 + - 2.21 + - 2.21 + - 2.26 + - 2.33 + - 2.27 + - 2.21 + - 2.12 + - 2.23 + - 2.26 + - 2.25 + - 1.88 + - 2.26 + - 2.24 + - 2.36 + - 2.29 + - 2.35 + - 2.3 + - 2.27 + - 2.08 + - 2.05 + - 2.27 + - 2.28 + - 2.27 + - 2.28 + - 1.97 + - 2.25 + - 2.25 + - 2.25 + - 2.31 + - 2.28 + - 2.27 + - 2.13 + - 2.24 + - 2.28 + - 2.28 + - 2.41 + - 2.34 + - 9.32 + - 2.28 + - 2.38 + - 2.27 + - 2.27 + - 2.39 + - 2.11 + - 2.09 + - 2.1 + - 2.06 + - 2.12 + - 2.08 + - 2.0 + - 1.93 + - 2.02 + - 2.55 + - 1.54 + - 1.64 + - 1.51 + - 1.55 + - 2.82 + - 2.92 + - 2.55 + - 2.37 + - 1.85 + - 1.6 + - 1.72 + - 1.74 + - 1.79 + - 1.9 + - 1.94 + - 2.0 + - 2.04 + - 2.08 + - 2.12 + - 2.13 + - 2.16 + - 2.18 + - 2.18 + - 2.2 + - 2.2 + - 2.41 + - 2.39 + - 2.38 + - 2.4 + - 2.42 + - 2.41 + - 2.43 + - 2.45 + - 2.43 + - 2.45 + - 2.43 + - 2.4 + - 2.44 + - 2.4 + - 2.42 + - 2.43 + - 2.45 + - 2.45 + - 2.45 + - 2.46 + - 2.45 + - 2.45 + - 2.43 + - 2.51 + - 2.48 + - 2.48 + - 2.53 + - 2.46 + - 2.49 + - 2.5 + - 2.5 + - 2.5 + - 2.52 + - 2.52 + - 2.54 + - 2.5 + - 2.48 + - 2.5 + - 2.55 + - 2.5 + - 2.48 + - 2.5 + - 2.5 + - 2.52 + - 2.52 + - 2.48 + - 2.5 + - 2.5 + - 2.52 + - 2.46 + - 2.53 + - 9.0 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, 7076, + 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, 7267, 7269, 7284, + 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, 7475, 7549, 7584, + 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, 7950, 7972, 7980, + 7995, 8007, 8015, 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: iasi_metop-b + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + error parameter vector: *id016 + use_flag: None + use_flag_clddet: &id017 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + error parameter vector: *id016 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: iasi_metop-b + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + error parameter vector: *id016 + obserr_bound_max: + - 3.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 4.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 4.0 + - 4.0 + - 3.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.75 + - 2.0 + - 0.75 + - 2.0 + - 2.0 + - 2.0 + - 0.75 + - 0.75 + - 0.75 + - 0.75 + - 2.0 + - 0.75 + - 0.75 + - 2.0 + - 0.75 + - 0.75 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 2.0 + - 2.5 + - 2.5 + - 3.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 3.5 + - 2.5 + - 2.5 + - 3.0 + - 3.5 + - 3.0 + - 4.0 + - 4.0 + - 0.75 + - 4.0 + - 4.0 + - 4.0 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.0 + - 4.5 + - 4.0 + - 4.0 + - 4.5 + - 2.5 + - 3.0 + - 2.5 + - 3.0 + - 2.5 + - 3.0 + - 2.0 + - 2.5 + - 2.5 + - 3.0 + - 3.0 + - 2.5 + - 3.0 + - 3.0 + - 3.0 + - 2.5 + - 2.5 + - 4.0 + - 4.5 + - 4.5 + - 5.0 + - 4.0 + - 4.0 + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 5.5 + - 5.5 + - 4.0 + - 5.0 + - 4.0 + - 4.5 + - 5.5 + - 5.5 + - 6.0 + - 4.5 + - 4.5 + - 4.0 + - 5.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.25 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.25 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.4 + - 6.0 + - 1.4 + - 6.0 + - 6.0 + - 1.4 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.5 + - 4.5 + - 6.0 + - 5.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 4.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 4.5 + - 6.0 + - 6.0 + - 4.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id017 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: iasi_metop-b + - obs space: + name: IASI METOP-C + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/iasi_metop-c.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.iasi_metop-c.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: iasi_metop-c + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/iasi_metop-c.20231009T150000Z.satbias.nc4 + output file: cycle_dir/iasi_metop-c.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/iasi_metop-c.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/iasi_metop-c.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/iasi_metop-c.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/iasi_metop-c.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/iasi_metop_141_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id018 + - 0.727 + - 0.81 + - 0.75 + - 0.79 + - 0.7055 + - 0.74 + - 0.68 + - 0.72 + - 0.6526 + - 0.65 + - 0.665 + - 0.69 + - 0.6394 + - 0.64 + - 0.6528 + - 0.6065 + - 0.6246 + - 0.61 + - 0.6423 + - 0.5995 + - 0.59 + - 0.6069 + - 0.6 + - 0.5965 + - 0.64 + - 0.62 + - 0.589 + - 0.5865 + - 0.65 + - 0.5861 + - 0.61 + - 0.5874 + - 0.68 + - 0.606 + - 0.68 + - 4.38 + - 3.05 + - 2.31 + - 1.56 + - 1.33 + - 1.58 + - 0.93 + - 0.5832 + - 0.5587 + - 0.5867 + - 0.58 + - 0.5655 + - 0.5522 + - 0.5864 + - 0.5475 + - 0.5854 + - 0.5455 + - 0.5811 + - 0.5376 + - 0.5452 + - 0.5686 + - 0.5329 + - 0.5655 + - 0.5302 + - 0.545 + - 0.5628 + - 0.59 + - 0.5262 + - 0.559 + - 0.5264 + - 0.5442 + - 0.51 + - 0.5513 + - 0.5224 + - 0.5523 + - 0.5188 + - 0.5487 + - 0.5245 + - 0.58 + - 0.5437 + - 0.5343 + - 0.5364 + - 0.64 + - 0.5338 + - 0.72 + - 0.537 + - 0.75 + - 0.51 + - 0.65 + - 0.5274 + - 0.529 + - 0.5187 + - 0.5228 + - 1.12 + - 0.5222 + - 0.5109 + - 0.67 + - 0.5133 + - 0.5179 + - 0.507 + - 0.67 + - 0.5091 + - 0.62 + - 0.5093 + - 0.69 + - 0.5048 + - 0.5024 + - 0.78 + - 0.497 + - 0.5337 + - 0.4865 + - 0.4915 + - 0.4835 + - 0.4869 + - 0.87 + - 0.4824 + - 0.4852 + - 0.84 + - 0.84 + - 0.84 + - 0.5318 + - 0.8 + - 0.4772 + - 0.98 + - 0.488 + - 0.4978 + - 0.5157 + - 0.61 + - 0.5213 + - 0.4884 + - 0.79 + - 0.62 + - 0.66 + - 0.4691 + - 0.65 + - 0.4809 + - 0.468 + - 0.62 + - 0.4679 + - 0.6913 + - 0.4705 + - 0.4785 + - 0.47 + - 0.4773 + - 0.4703 + - 0.98 + - 0.4697 + - 0.4662 + - 0.65 + - 0.467 + - 0.4883 + - 0.4684 + - 0.4684 + - 0.4947 + - 0.5393 + - 0.5024 + - 0.4715 + - 0.621 + - 0.6136 + - 0.5316 + - 1.78 + - 0.5099 + - 1.14 + - 0.539 + - 1.79 + - 0.508 + - 0.5723 + - 1.94 + - 2.01 + - 0.49 + - 0.5647 + - 0.5022 + - 1.47 + - 0.5815 + - 0.6782 + - 2.13 + - 0.5445 + - 1.52 + - 0.5555 + - 1.96 + - 2.31 + - 2.33 + - 2.32 + - 2.31 + - 0.6994 + - 0.7006 + - 0.706 + - 0.9785 + - 0.7023 + - 0.6991 + - 0.6946 + - 2.28 + - 2.26 + - 2.26 + - 2.26 + - 0.6608 + - 0.6835 + - 0.6822 + - 2.24 + - 2.26 + - 0.6735 + - 2.28 + - 0.667 + - 0.7732 + - 0.6642 + - 0.648 + - 0.6629 + - 2.29 + - 2.29 + - 0.6799 + - 0.623 + - 2.32 + - 0.603 + - 0.6224 + - 2.32 + - 0.6187 + - 2.31 + - 2.31 + - 2.28 + - 2.29 + - 2.28 + - 2.26 + - 1.966 + - 2.27 + - 2.26 + - 2.25 + - 2.27 + - 2.24 + - 2.21 + - 2.24 + - 2.17 + - 2.18 + - 2.17 + - 2.21 + - 1.4 + - 2.16 + - 2.2 + - 2.13 + - 2.12 + - 2.13 + - 2.1 + - 2.12 + - 2.11 + - 2.09 + - 2.09 + - 2.08 + - 2.09 + - 2.04 + - 2.04 + - 1.966 + - 2.01 + - 2.05 + - 2.03 + - 2.06 + - 1.98 + - 1.95 + - 1.94 + - 1.91 + - 1.857 + - 1.76 + - 1.748 + - 1.83 + - 2.04 + - 1.748 + - 1.99 + - 2.075 + - 2.07 + - 2.02 + - 2.04 + - 2.1 + - 1.966 + - 2.18 + - 2.21 + - 2.24 + - 2.23 + - 2.23 + - 1.98 + - 2.2 + - 2.18 + - 2.18 + - 2.21 + - 2.23 + - 2.24 + - 2.24 + - 2.25 + - 1.8 + - 2.24 + - 1.73 + - 1.73 + - 2.27 + - 1.67 + - 2.21 + - 1.72 + - 2.23 + - 2.23 + - 2.23 + - 2.24 + - 2.23 + - 2.12 + - 2.17 + - 1.74 + - 2.02 + - 1.88 + - 1.67 + - 1.73 + - 1.83 + - 1.82 + - 1.73 + - 1.83 + - 2.19 + - 1.84 + - 1.89 + - 1.6 + - 1.71 + - 1.86 + - 1.85 + - 1.84 + - 1.87 + - 1.91 + - 1.52 + - 1.95 + - 1.87 + - 1.89 + - 1.91 + - 1.91 + - 1.93 + - 1.9 + - 1.91 + - 1.9 + - 1.89 + - 1.89 + - 1.91 + - 1.9 + - 1.91 + - 1.91 + - 1.91 + - 1.93 + - 1.94 + - 1.91 + - 1.92 + - 1.77 + - 1.91 + - 1.95 + - 1.19 + - 1.96 + - 1.98 + - 1.94 + - 1.55 + - 1.91 + - 1.92 + - 1.92 + - 1.97 + - 1.93 + - 1.99 + - 1.86 + - 1.12 + - 1.93 + - 1.92 + - 1.95 + - 1.85 + - 1.84 + - 1.91 + - 1.12 + - 1.82 + - 1.82 + - 1.95 + - 1.24 + - 1.94 + - 1.96 + - 1.21 + - 1.83 + - 1.96 + - 1.36 + - 1.96 + - 1.82 + - 1.92 + - 1.68 + - 1.93 + - 1.23 + - 1.96 + - 1.93 + - 1.86 + - 1.41 + - 1.16 + - 1.6 + - 1.25 + - 1.2 + - 1.65 + - 1.66 + - 1.87 + - 1.94 + - 1.96 + - 1.91 + - 1.25 + - 1.93 + - 1.91 + - 1.7 + - 0.99 + - 1.81 + - 1.92 + - 1.95 + - 1.5 + - 1.47 + - 1.15 + - 1.58 + - 1.18 + - 1.82 + - 1.13 + - 1.83 + - 1.91 + - 1.26 + - 1.27 + - 1.91 + - 1.45 + - 1.6 + - 1.29 + - 1.94 + - 1.94 + - 1.23 + - 1.95 + - 1.21 + - 1.94 + - 1.86 + - 1.9 + - 1.33 + - 1.75 + - 2.02 + - 1.98 + - 2.03 + - 1.83 + - 1.5 + - 2.04 + - 2.02 + - 1.9 + - 2.0 + - 2.02 + - 1.95 + - 1.93 + - 1.95 + - 1.95 + - 1.99 + - 2.0 + - 1.94 + - 1.96 + - 1.86 + - 1.92 + - 1.88 + - 1.86 + - 1.84 + - 1.87 + - 1.77 + - 1.89 + - 1.89 + - 1.88 + - 1.94 + - 1.82 + - 1.79 + - 1.86 + - 2.06 + - 2.33 + - 1.88 + - 1.86 + - 1.81 + - 1.8 + - 1.8 + - 1.86 + - 1.9 + - 2.0 + - 2.06 + - 2.1 + - 2.2 + - 2.0 + - 2.16 + - 1.98 + - 1.8 + - 1.8 + - 1.85 + - 1.75 + - 2.04 + - 2.19 + - 2.14 + - 2.19 + - 1.86 + - 2.1 + - 2.11 + - 2.18 + - 2.03 + - 2.28 + - 2.19 + - 2.26 + - 2.26 + - 2.21 + - 2.21 + - 2.26 + - 2.33 + - 2.27 + - 2.21 + - 2.12 + - 2.23 + - 2.26 + - 2.25 + - 1.88 + - 2.26 + - 2.24 + - 2.36 + - 2.29 + - 2.35 + - 2.3 + - 2.27 + - 2.08 + - 2.05 + - 2.27 + - 2.28 + - 2.27 + - 2.28 + - 1.97 + - 2.25 + - 2.25 + - 2.25 + - 2.31 + - 2.28 + - 2.27 + - 2.13 + - 2.24 + - 2.28 + - 2.28 + - 2.41 + - 2.34 + - 9.32 + - 2.28 + - 2.38 + - 2.27 + - 2.27 + - 2.39 + - 2.11 + - 2.09 + - 2.1 + - 2.06 + - 2.12 + - 2.08 + - 2.0 + - 1.93 + - 2.02 + - 2.55 + - 1.54 + - 1.64 + - 1.51 + - 1.55 + - 2.82 + - 2.92 + - 2.55 + - 2.37 + - 1.85 + - 1.6 + - 1.72 + - 1.74 + - 1.79 + - 1.9 + - 1.94 + - 2.0 + - 2.04 + - 2.08 + - 2.12 + - 2.13 + - 2.16 + - 2.18 + - 2.18 + - 2.2 + - 2.2 + - 2.41 + - 2.39 + - 2.38 + - 2.4 + - 2.42 + - 2.41 + - 2.43 + - 2.45 + - 2.43 + - 2.45 + - 2.43 + - 2.4 + - 2.44 + - 2.4 + - 2.42 + - 2.43 + - 2.45 + - 2.45 + - 2.45 + - 2.46 + - 2.45 + - 2.45 + - 2.43 + - 2.51 + - 2.48 + - 2.48 + - 2.53 + - 2.46 + - 2.49 + - 2.5 + - 2.5 + - 2.5 + - 2.52 + - 2.52 + - 2.54 + - 2.5 + - 2.48 + - 2.5 + - 2.55 + - 2.5 + - 2.48 + - 2.5 + - 2.5 + - 2.52 + - 2.52 + - 2.48 + - 2.5 + - 2.5 + - 2.52 + - 2.46 + - 2.53 + - 9.0 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, 7076, + 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, 7267, 7269, 7284, + 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, 7475, 7549, 7584, + 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, 7950, 7972, 7980, + 7995, 8007, 8015, 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: iasi_metop-c + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + error parameter vector: *id018 + use_flag: None + use_flag_clddet: &id019 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + error parameter vector: *id018 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: iasi_metop-c + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + error parameter vector: *id018 + obserr_bound_max: + - 3.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 4.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 4.0 + - 4.0 + - 3.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.75 + - 2.0 + - 0.75 + - 2.0 + - 2.0 + - 2.0 + - 0.75 + - 0.75 + - 0.75 + - 0.75 + - 2.0 + - 0.75 + - 0.75 + - 2.0 + - 0.75 + - 0.75 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 2.0 + - 2.5 + - 2.5 + - 3.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 3.5 + - 2.5 + - 2.5 + - 3.0 + - 3.5 + - 3.0 + - 4.0 + - 4.0 + - 0.75 + - 4.0 + - 4.0 + - 4.0 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.0 + - 4.5 + - 4.0 + - 4.0 + - 4.5 + - 2.5 + - 3.0 + - 2.5 + - 3.0 + - 2.5 + - 3.0 + - 2.0 + - 2.5 + - 2.5 + - 3.0 + - 3.0 + - 2.5 + - 3.0 + - 3.0 + - 3.0 + - 2.5 + - 2.5 + - 4.0 + - 4.5 + - 4.5 + - 5.0 + - 4.0 + - 4.0 + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 5.5 + - 5.5 + - 4.0 + - 5.0 + - 4.0 + - 4.5 + - 5.5 + - 5.5 + - 6.0 + - 4.5 + - 4.5 + - 4.0 + - 5.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.25 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.25 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.4 + - 6.0 + - 1.4 + - 6.0 + - 6.0 + - 1.4 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.5 + - 4.5 + - 6.0 + - 5.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 4.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 4.5 + - 6.0 + - 6.0 + - 4.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id019 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: iasi_metop-c + - obs space: + name: MHS METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mhs_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mhs_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: mhs_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/mhs_metop-b.20231009T150000Z.satbias.nc4 + output file: cycle_dir/mhs_metop-b.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/mhs_metop-b.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/mhs_metop-b.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/mhs_metop-b.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/mhs_metop-b.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 550.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: mhs_metop-b + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: mhs_metop-b + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-b + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-b + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-b + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-b + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + observation_name: mhs_metop-b + - obs space: + name: MHS METOP-C + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mhs_metop-c.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mhs_metop-c.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: mhs_metop-c + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/mhs_metop-c.20231009T150000Z.satbias.nc4 + output file: cycle_dir/mhs_metop-c.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/mhs_metop-c.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/mhs_metop-c.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/mhs_metop-c.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/mhs_metop-c.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 550.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: mhs_metop-c + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: mhs_metop-c + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-c + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-c + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-c + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-c + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + observation_name: mhs_metop-c + - obs space: + name: MHS NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mhs_n19.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mhs_n19.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: mhs_n19 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/mhs_n19.20231009T150000Z.satbias.nc4 + output file: cycle_dir/mhs_n19.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/mhs_n19.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/mhs_n19.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/mhs_n19.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/mhs_n19.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 550.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: mhs_n19 + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: mhs_n19 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_n19 + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_n19 + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_n19 + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_n19 + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + observation_name: mhs_n19 + - obs space: + name: MLS55 AURA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mls55_aura.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mls55_aura.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - ozoneProfile + obs operator: + name: VertInterp + vertical coordinate: air_pressure + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + obs filters: + - filter: Bounds Check + filter variables: + - name: ozoneProfile + minvalue: 0 + maxvalue: 10000 + action: + name: reject + - filter: Background Check + filter variables: + - name: ozoneProfile + threshold: 5.0 + action: + name: reject + observation_name: mls55_aura + - obs space: + name: OMI AURA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/omi_aura.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.omi_aura.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - ozoneTotal + obs operator: + name: AtmVertInterpLay + geovals: + - mole_fraction_of_ozone_in_air + coefficients: + - 0.0078976797 + nlevels: + - 1 + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + obs filters: + - filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 5.0 + - filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: ozoneTotal + test variables: + - name: MetaData/solarZenithAngle + maxvalue: 84.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: ozoneTotal + test variables: + - name: MetaData/sensorScanPosition + minvalue: 3 + maxvalue: 24 + action: + name: reject + - filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 5.0 + action: + name: reject + observation_name: omi_aura + - obs space: + name: OMPSNM NPP + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/ompsnm_npp.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.ompsnm_npp.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - ozoneTotal + obs operator: + name: AtmVertInterpLay + geovals: + - mole_fraction_of_ozone_in_air + coefficients: + - 0.0078976797 + nlevels: + - 1 + obs filters: + - filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 5.216 + - filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: ozoneTotal + test variables: + - name: MetaData/solarZenithAngle + maxvalue: 84.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 5.0 + action: + name: reject + observation_name: ompsnm_npp + - obs space: + name: Pilot Balloon + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/pibal.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.pibal.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + vertical coordinate backup: air_pressure + observation vertical coordinate group backup: MetaData + observation vertical coordinate backup: pressure + interpolation method backup: log-linear + hofx scaling field: SurfaceWindScalingCombined + hofx scaling field group: DerivedVariables + linear obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + vertical coordinate backup: air_pressure + observation vertical coordinate group backup: MetaData + observation vertical coordinate backup: pressure + interpolation method backup: log-linear + hofx scaling field: SurfaceWindScalingCombined + hofx scaling field group: DerivedVariables + obs pre filters: + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_not_in: 221 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_not_in: 221 + action: + name: reject + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingCombined + SkipWhenNoObs: false + obs post filters: + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.2 + - 2.2 + - 2.3 + - 2.3 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.5 + - 2.7 + - 2.9 + - 3.1 + - 3.3 + - 3.5 + - 3.7 + - 3.9 + - 4.1 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + value: is_valid + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + value: is_valid + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 221 + cgross: + - 8.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windEastward + action: + name: reject + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 221 + cgross: + - 8.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windNorthward + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + test variables: + - name: ObsErrorData/windEastward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windEastward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windEastward + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windNorthward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windNorthward + defer to post: true + - filter: Bounds Check + filter variables: + - name: windNorthward + test variables: + - name: ObsErrorData/windNorthward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 221 + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 221 + defer to post: true + observation_name: pibal + - obs space: + name: Satellite Winds + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/satwind.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.satwind.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + linear obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + obs prior filters: + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + obs post filters: + - filter: PreQC + maxvalue: 3 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 80000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - filter: BlackList + where: + - variable: + name: ObsType/windEastward + is_in: 240, 241, 248, 249, 251, 255, 256, 260 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.45 + - 5 + - 5 + - 5 + - 5.075 + - 5.175 + - 5.3 + - 5.675 + - 6.05 + - 6.25 + - 6.625 + - 7 + - 7 + - 7 + - 7 + where: + - variable: + name: ObsType/windEastward + is_in: 244 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.9 + - 3.9 + - 4.0 + - 4.0 + - 4.1 + - 5 + - 6 + - 6.3 + - 6.6 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + where: + - variable: + name: ObsType/windEastward + is_in: 245, 246 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.9 + - 3.9 + - 4.0 + - 4.0 + - 4.1 + - 5 + - 6 + - 6.3 + - 6.6 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + where: + - variable: + name: ObsType/windEastward + is_in: 242, 243, 247, 252, 253 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.9 + - 3.9 + - 4.0 + - 4.5 + - 6.1 + - 6.0 + - 6.5 + - 7.3 + - 7.6 + - 7 + - 7.5 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + where: + - variable: + name: ObsType/windEastward + is_in: 254 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.9 + - 3.9 + - 4.0 + - 4.0 + - 4.1 + - 5.0 + - 7 + - 7.3 + - 7.6 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + where: + - variable: + name: ObsType/windEastward + is_in: 250 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.675 + - 5.45 + - 5.525 + - 5.8 + - 6.275 + - 6.575 + - 6.825 + - 7.05 + - 7.05 + - 7.05 + - 7.05 + where: + - variable: + name: ObsType/windEastward + is_in: 257 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 6.05 + - 5.625 + - 5.275 + - 5.375 + - 5.925 + - 6.475 + - 6.775 + - 7.05 + - 7.05 + - 7.05 + - 7.05 + where: + - variable: + name: ObsType/windEastward + is_in: 258 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6 + - 5.5 + - 5.5 + - 5.575 + - 6.025 + - 6.4 + - 6.775 + - 7.15 + - 7.15 + - 7.15 + - 7.15 + where: + - variable: + name: ObsType/windEastward + is_in: 259 + - filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/air_pressure_at_surface + value: MetaData/pressure + maxvalue: -11000 + where: + - variable: + name: ObsType/windEastward + is_in: 247 + - variable: + name: MetaData/surfaceQualifier + minvalue: 1 + - filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/air_pressure_at_surface + value: MetaData/pressure + maxvalue: -20000 + where: + - variable: + name: ObsType/windEastward + is_in: 244, 257, 258, 259, 260 + - variable: + name: MetaData/surfaceQualifier + minvalue: 1 + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/WindDirAngleDiff + maxvalue: 50.0 + where: + - variable: + name: ObsType/windEastward + is_in: 247 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/SatWindsLNVDCheck + maxvalue: 3 + where: + - variable: + name: ObsType/windEastward + is_in: 244, 247, 257-260 + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: BlackList + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 15.0 + - 15.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.1 + - 20.1 + - 20.1 + - 20.1 + - 20.1 + cgross: + - 2.5 + - 2.5 + - 2.5 + - 1.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 1.3 + - 2.5 + - 1.5 + - 1.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + wndtype: + - 240 + - 241 + - 242 + - 243 + - 244 + - 245 + - 246 + - 247 + - 248 + - 249 + - 250 + - 251 + - 252 + - 253 + - 254 + - 256 + - 257 + - 258 + - 259 + - 260 + variable: windEastward + action: + name: reject + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + cgross: + - 2.5 + - 2.5 + - 2.5 + - 1.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 1.3 + - 2.5 + - 1.5 + - 1.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 15.0 + - 15.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.1 + - 20.1 + - 20.1 + - 20.1 + - 20.1 + wndtype: + - 240 + - 241 + - 242 + - 243 + - 244 + - 245 + - 246 + - 247 + - 248 + - 249 + - 250 + - 251 + - 252 + - 253 + - 254 + - 256 + - 257 + - 258 + - 259 + - 260 + variable: windNorthward + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + defer to post: true + observation_name: satwind + - obs space: + name: Scatterometer Winds + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/scatwind.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.scatwind.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + linear obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + obs post filters: + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + errors: + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 290 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: BlackList + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 290 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/ScatWindsAmbiguityCheck + maxvalue: 1e-12 + where: + - variable: + name: ObsType/windEastward + is_in: 290 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windEastward + - name: HofX/windEastward + coefs: + - 1.0 + - -1.0 + minvalue: -5.0 + maxvalue: 5.0 + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windNorthward + - name: HofX/windNorthward + coefs: + - 1.0 + - -1.0 + minvalue: -5.0 + maxvalue: 5.0 + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 290 + cgross: + - 5.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windEastward + action: + name: reject + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 290 + cgross: + - 5.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windNorthward + action: + name: reject + defer to post: true + observation_name: scatwind + - obs space: + name: Surface Marine Stations + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sfcship.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sfcship.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + - virtualTemperature + - airTemperature + - specificHumidity + - stationPressure + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - name: Identity + variables: + - name: specificHumidity + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: Identity + variables: + - name: stationPressure + - name: specificHumidity + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + obs post filters: + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + minvalue: 1 + action: + name: reject + - filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + errors: + - 100 + - 100 + - 110 + - 120 + - 120 + - 100000000000.0 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + action: + name: inflate error + inflation factor: 0.7 + where: + - variable: ObsType/stationPressure + is_in: + - 180 + - variable: ObsSubType/stationPressure + is_in: + - 0 + - filter: Perform Action + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 300 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 300 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 4.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + - variable: ObsType/stationPressure + is_in: 180,183 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.8 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - variable: ObsType/stationPressure + is_in: 180,183 + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: + - 183 + action: + name: reject + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + errors: + - 1.75 + - 1.95 + - 2.25 + - 1.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - airTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/airTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + - filter: Bounds Check + filter variables: + - name: airTemperature + test variables: + - name: ObsErrorData/airTemperature + maxvalue: 100000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: + - 183 + action: + name: reject + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + errors: + - 1.75 + - 1.95 + - 2.25 + - 1.0 + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - virtualTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: GsiAdjustObsError/virtualTemperature + where: + - variable: + name: GsiAdjustObsError/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: virtualTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/virtualTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/virtualTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/virtualTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/virtualTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: PreUseFlag/virtualTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: TempObsErrorData/virtualTemperature + where: + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: virtualTemperature + defer to post: true + - filter: Bounds Check + filter variables: + - name: virtualTemperature + test variables: + - name: ObsErrorData/virtualTemperature + maxvalue: 100000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: PreUseFlag/specificHumidity + minvalue: 1 + action: + name: reject + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: + - 183 + action: + name: reject + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorSatSpecHumidity + options: + variable: specificHumidity + input_error_name: GsiInputObsError + - filter: BlackList + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: specificHumidity + inflation factor: 8 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: specificHumidity + threshold: 8.0 + action: + name: reject + where: + - variable: PreQC/specificHumidity + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: specificHumidity + threshold: 5.6 + action: + name: reject + where: + - variable: PreQC/specificHumidity + is_in: + - 3 + defer to post: true + - filter: BlackList + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: specificHumidity + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_in: + - 284 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_in: + - 284 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: + - 280 + action: + name: assign error + error parameter: 2.5 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: + - 282 + action: + name: assign error + error parameter: 2.2 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + minvalue: 1 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + minvalue: 1 + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + threshold: 6.0 + action: + name: reject + where: + - variable: PreQC/windEastward + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windEastward + threshold: 4.2 + action: + name: reject + where: + - variable: PreQC/windEastward + is_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + threshold: 6.0 + action: + name: reject + where: + - variable: PreQC/windNorthward + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + threshold: 4.2 + action: + name: reject + where: + - variable: PreQC/windNorthward + is_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + satwndtype: + - 280 + - 282 + - 284 + error_min: + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + cgross: + - 6.0 + - 6.0 + - 6.0 + wndtype: + - 280 + - 282 + - 284 + variable: windEastward + action: + name: reject + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + satwndtype: + - 280 + - 282 + - 284 + error_min: + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + cgross: + - 6.0 + - 6.0 + - 6.0 + wndtype: + - 280 + - 282 + - 284 + variable: windNorthward + action: + name: reject + defer to post: true + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + defer to post: true + observation_name: sfcship + - obs space: + name: Surface Land Stations + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sfc.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sfc.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + - virtualTemperature + - airTemperature + - specificHumidity + - stationPressure + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: specificHumidity + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: stationPressure + - name: specificHumidity + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + obs post filters: + - filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 49999 + action: + name: reject + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + minvalue: 1 + action: + name: reject + - filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 181 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 70000 + - 65000 + - 60000 + - 55000 + errors: + - 100 + - 120 + - 120 + - 100000000000.0 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 187 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + errors: + - 100 + - 130 + - 160 + - 100000000000.0 + - filter: Perform Action + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 300 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 300 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 4.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + - variable: ObsType/stationPressure + is_in: 187 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.8 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - variable: ObsType/stationPressure + is_in: 187 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 3.6 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + - variable: ObsType/stationPressure + is_in: + - 181 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.52 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - variable: ObsType/stationPressure + is_in: + - 181 + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + defer to post: true + - filter: Bounds Check + filter variables: + - name: stationPressure + test variables: + - name: ObsErrorData/stationPressure + maxvalue: 100000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: 181,187 + action: + name: reject + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: 181,187 + action: + name: reject + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: 181,187 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_in: 281,287 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_in: 281,287 + action: + name: reject + observation_name: sfc + - obs space: + name: Radiosondes + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sondes.20231009T210000Z.nc4 + missing file action: error + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sondes.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + - virtualTemperature + - airTemperature + - specificHumidity + - stationPressure + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + - name: VertInterp + variables: + - name: airTemperature + - name: virtualTemperature + - name: specificHumidity + - name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + - name: VertInterp + variables: + - name: airTemperature + - name: virtualTemperature + - name: specificHumidity + - name: Identity + variables: + - name: stationPressure + obs prior filters: + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + obs post filters: + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + minvalue: 1 + action: + name: reject + - filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 60000 + - 55000 + errors: + - 100 + - 100000000000.0 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 300 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 300 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 4.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.8 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: PreUseFlag/specificHumidity + minvalue: 1 + action: + name: reject + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + action: + name: assign error + error function: + name: ObsFunction/ObsErrorSatSpecHumidity + options: + variable: specificHumidity + input_error_name: GsiInputObsError + - filter: BlackList + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: specificHumidity + inflation factor: 8.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/specificHumidity + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/specificHumidity + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: specificHumidity + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 70000 + - 65000 + - 60000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + errors: + - 1.3 + - 1.1 + - 0.9 + - 0.7 + - 0.6 + - 0.6 + - 0.55 + - 0.5 + - 0.5 + - 0.55 + - 0.65 + - 1.1 + - 1.2 + - 1.2 + - 1.4 + - 1.6 + - 1.85 + - 2.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - airTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/airTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 8.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 5.6 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 70000 + - 65000 + - 60000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + errors: + - 1.3 + - 1.1 + - 0.9 + - 0.7 + - 0.6 + - 0.6 + - 0.55 + - 0.5 + - 0.5 + - 0.55 + - 0.65 + - 1.1 + - 1.2 + - 1.2 + - 1.4 + - 1.6 + - 1.85 + - 2.0 + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - virtualTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: GsiAdjustObsError/virtualTemperature + where: + - variable: + name: GsiAdjustObsError/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: virtualTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/virtualTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/virtualTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/virtualTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/virtualTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: PreUseFlag/virtualTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 8.0 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 5.6 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: TempObsErrorData/virtualTemperature + where: + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: virtualTemperature + defer to post: true + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.2 + - 2.2 + - 2.3 + - 2.3 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.5 + - 2.7 + - 2.9 + - 3.1 + - 3.3 + - 3.5 + - 3.7 + - 3.9 + - 4.1 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + value: is_valid + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + value: is_valid + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 220,221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 220,221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 220 + - 221 + cgross: + - 8.0 + - 8.0 + error_min: + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + variable: windEastward + action: + name: reject + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 220 + - 221 + cgross: + - 8.0 + - 8.0 + error_min: + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + variable: windNorthward + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + test variables: + - name: ObsErrorData/windEastward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windEastward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windEastward + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windNorthward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windNorthward + defer to post: true + - filter: Bounds Check + filter variables: + - name: windNorthward + test variables: + - name: ObsErrorData/windNorthward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 220 + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 220 + defer to post: true + observation_name: sondes + - obs space: + name: ssmis_f17 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/ssmis_f17.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.ssmis_f17.20231009T210000Z.nc4 + simulated variables: + - brightnessTemperature + channels: None + io pool: + max pool size: 6 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs options: + Sensor_ID: ssmis_f17 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + obs bias: + input file: cycle_dir/ssmis_f17.20231009T150000Z.satbias.nc4 + output file: cycle_dir/ssmis_f17.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/ssmis_f17.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/ssmis_f17.20231009T150000Z.tlapse.txt + - name: cosineOfLatitudeTimesOrbitNode + - name: sineOfLatitude + - name: emissivityJacobian + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/ssmis_f17.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/ssmis_f17.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 1.0 + - 1.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 2.4 + - 1.27 + - 1.44 + - 3.0 + - 1.34 + - 1.74 + - 3.75 + - 3.0 + - 3.0 + - 2.0 + - 6.4 + - 1.0 + - 1.0 + obs post filters: + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/geopotential_height_at_surface + minvalue: 2000.0 + min_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 1-3,8-18 + where: + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/ice_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/surface_snow_area_fraction + maxvalue: 0.99 + max_exclusive: true + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 1-2, 12-16 + reference: ObsValue/brightnessTemperature_2 + value: HofX/brightnessTemperature_2 + minvalue: -1.5 + maxvalue: 1.5 + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + action: + name: reject + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 9 + reference: ObsValue/brightnessTemperature_9 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: + - -0.485934 + - 0.485934 + - 0.473806 + - -0.473806 + intercept: 271.252327 + maxvalue: 2 + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 10 + reference: ObsValue/brightnessTemperature_10 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: + - -0.413688 + - 0.413688 + - 0.361549 + - -0.361549 + intercept: 272.280341 + maxvalue: 2 + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 11 + reference: ObsValue/brightnessTemperature_11 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: + - -0.400882 + - 0.400882 + - 0.27051 + - -0.27051 + intercept: 278.824902 + maxvalue: 2 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: ssmis_f17 + obserr_demisf: + - 0.01 + - 0.01 + - 0.01 + - 0.01 + - 0.01 + obserr_dtempf: + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor1 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_1 + coefs: + - 2.25 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_1 + threshold: DerivedMetaData/errorInflationFactor1 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor2 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_2 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_2 + threshold: DerivedMetaData/errorInflationFactor2 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor3 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_3 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_3 + threshold: DerivedMetaData/errorInflationFactor3 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor4 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_4 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_4 + threshold: DerivedMetaData/errorInflationFactor4 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor5 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_5 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_5 + threshold: DerivedMetaData/errorInflationFactor5 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor6 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_6 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_6 + threshold: DerivedMetaData/errorInflationFactor6 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor7 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_7 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_7 + threshold: DerivedMetaData/errorInflationFactor7 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor8 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_8 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_8 + threshold: DerivedMetaData/errorInflationFactor8 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor9 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_9 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_9 + threshold: DerivedMetaData/errorInflationFactor9 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_10 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_10 + threshold: DerivedMetaData/errorInflationFactor10 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_11 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_11 + threshold: DerivedMetaData/errorInflationFactor11 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor12 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_12 + coefs: + - 3.6 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_12 + threshold: DerivedMetaData/errorInflationFactor12 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor13 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_13 + coefs: + - 1.905 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_13 + threshold: DerivedMetaData/errorInflationFactor13 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor14 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_14 + coefs: + - 2.16 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_14 + threshold: DerivedMetaData/errorInflationFactor14 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor15 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_15 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_15 + threshold: DerivedMetaData/errorInflationFactor15 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor16 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_16 + coefs: + - 2.01 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_16 + threshold: DerivedMetaData/errorInflationFactor16 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor17 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_17 + coefs: + - 2.61 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_17 + threshold: DerivedMetaData/errorInflationFactor17 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor18 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_18 + coefs: + - 5.625 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_18 + threshold: DerivedMetaData/errorInflationFactor18 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor19 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_19 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_19 + threshold: DerivedMetaData/errorInflationFactor19 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor20 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_20 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_20 + threshold: DerivedMetaData/errorInflationFactor20 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor21 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_21 + coefs: + - 3.0 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_21 + threshold: DerivedMetaData/errorInflationFactor21 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor22 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_22 + coefs: + - 9.6 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_22 + threshold: DerivedMetaData/errorInflationFactor22 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor23 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_23 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_23 + threshold: DerivedMetaData/errorInflationFactor23 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor24 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_24 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_24 + threshold: DerivedMetaData/errorInflationFactor24 + absolute threshold: 6.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: ssmis_f17 +variational: + minimizer: + algorithm: DRPCG + iterations: + - geometry: + fms initialization: + namelist filename: ./fv3-jedi/fv3files/fmsmpp.nml + field table filename: ./fv3-jedi/fv3files/field_table_gmao + akbk: ./fv3-jedi/fv3files/akbk72.nc4 + layout: + - 4 + - 4 + npx: 91 + npy: 91 + npz: 72 + gradient norm reduction: 0.001 + ninner: 10 + linear model: + name: Identity + increment variables: *id003 + variable change: Identity + tstep: PT1H + diagnostics: + departures: ombg + online diagnostics: + write increment: true + increment: + state component: + filetype: auxgrid + gridtype: latlon + datapath: ./ + filename: experiment_id.increment-iter1. + field io names: + eastward_wind: ua + northward_wind: va + air_temperature: t + air_pressure_levels: pe + water_vapor_mixing_ratio_wrt_moist_air: q + cloud_liquid_ice: qi + cloud_liquid_water: ql + rain_water: qr + snow_water: qs + mole_fraction_of_ozone_in_air: o3ppmv + geopotential_height_times_gravity_at_surface: phis + skin_temperature_at_surface: ts + air_pressure_at_surface: ps +final: + diagnostics: + departures: oman + prints: + frequency: PT3H +output: + filetype: cube sphere history + provider: geos + datapath: cycle_dir + filename: experiment_id.analysis.%yyyy%mm%dd_%hh%MM%ssz.nc4 + first: PT0H + frequency: PT1H + field io names: *id001 diff --git a/src/swell/test/jedi_configs/jedi_3dfgat_marine_cycle_config.yaml b/src/swell/test/jedi_configs/jedi_3dfgat_marine_cycle_config.yaml new file mode 100644 index 000000000..b6e13450e --- /dev/null +++ b/src/swell/test/jedi_configs/jedi_3dfgat_marine_cycle_config.yaml @@ -0,0 +1,959 @@ +cost function: + cost type: 3D-FGAT + jb evaluation: false + time window: + begin: '2021-07-01T09:00:00Z' + end: '2021-07-01T15:00:00Z' + bound to include: begin + geometry: + mom6_input_nml: soca/input.nml + fields metadata: soca/fields_metadata.yaml + geom_grid_file: INPUT/soca_gridspec.nc + analysis variables: &id001 + - sea_water_salinity + - sea_water_potential_temperature + - sea_surface_height_above_geoid + - sea_water_cell_thickness + - sea_ice_area_fraction + - sea_ice_thickness + - sea_ice_snow_thickness + model: + name: PseudoModel + tstep: PT3H + states: + - date: '2021-07-01T12:00:00Z' + ocn_filename: ocn.fc.2021-07-01T09:00:00Z.PT3H.nc + ice_filename: ice.fc.2021-07-01T09:00:00Z.PT3H.nc + basename: ./ + read_from_file: 1 + - date: '2021-07-01T15:00:00Z' + ocn_filename: ocn.fc.2021-07-01T09:00:00Z.PT6H.nc + ice_filename: ice.fc.2021-07-01T09:00:00Z.PT6H.nc + basename: ./ + read_from_file: 1 + background: + date: '2021-07-01T09:00:00Z' + read_from_file: 1 + basename: ./ + ocn_filename: MOM6.res.20210701T090000Z.nc + ice_filename: cice.res.20210701T090000Z.nc + state variables: + - sea_ice_area_fraction + - sea_ice_thickness + - sea_ice_snow_thickness + - sea_water_salinity + - sea_water_potential_temperature + - sea_surface_height_above_geoid + - sea_water_cell_thickness + - ocean_mixed_layer_thickness + - sea_water_depth + background error: + covariance model: SABER + saber central block: + saber block name: diffusion + active variables: + - sea_ice_area_fraction + - sea_ice_thickness + - sea_ice_snow_thickness + - sea_water_potential_temperature + - sea_water_salinity + - sea_surface_height_above_geoid + read: + groups: + - variables: + - sea_water_potential_temperature + - sea_water_salinity + horizontal: + filepath: background_error_model/new_corr_1p0 + vertical: + levels: 50 + filepath: background_error_model/vt.20210701T090000Z + - variables: + - sea_surface_height_above_geoid + - sea_ice_area_fraction + - sea_ice_thickness + - sea_ice_snow_thickness + horizontal: + filepath: background_error_model/new_corr_1p5 + date: '2021-07-01T09:00:00Z' + saber outer blocks: + - saber block name: SOCABkgErrFilt + ocean_depth_min: 100 + rescale_bkgerr: 1.0 + efold_z: 2500.0 + - saber block name: SOCAParametricOceanStdDev + temperature: + sst: + filepath: cycle_dir/soca/godas_sst_bgerr.nc + variable: sst_bgerr + unbalanced salinity: {} + unbalanced ssh: {} + linear variable change: + input variables: *id001 + output variables: *id001 + linear variable changes: + - linear variable change name: BalanceSOCA + ksshts: + nlayers: 2 + observations: + observers: + - obs space: + name: adt_cryosat2n + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_cryosat2n.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_cryosat2n.20210701T090000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_ice_area_fraction + maxvalue: 1e-05 + observation_name: adt_cryosat2n + - obs space: + name: adt_jason3 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_jason3.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_jason3.20210701T090000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_ice_area_fraction + maxvalue: 1e-05 + observation_name: adt_jason3 + - obs space: + name: adt_saral + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_saral.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_saral.20210701T090000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_ice_area_fraction + maxvalue: 1e-05 + observation_name: adt_saral + - obs space: + name: adt_sentinel3a + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_sentinel3a.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_sentinel3a.20210701T090000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_ice_area_fraction + maxvalue: 1e-05 + observation_name: adt_sentinel3a + - obs space: + name: adt_sentinel3b + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_sentinel3b.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_sentinel3b.20210701T090000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_ice_area_fraction + maxvalue: 1e-05 + observation_name: adt_sentinel3b + - obs space: + name: insitu_profile_argo + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/insitu_profile_argo.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.insitu_profile_argo.20210701T090000Z.nc4 + simulated variables: + - waterTemperature + - salinity + obs operator: + name: Composite + components: + - name: InsituTemperature + variables: + - name: waterTemperature + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + variables: + - name: salinity + vertical coordinate: sea_water_depth + observation vertical coordinate: depth + interpolation method: linear + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: ObsError/salinity + minvalue: 0.0001 + - filter: Bounds Check + where: + - variable: + name: ObsValue/salinity + minvalue: 1.0 + maxvalue: 40.0 + - filter: Background Check + threshold: 3.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 2.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: insitu_profile_argo + - obs space: + name: icec_amsr2_north + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/icec_amsr2_north.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.icec_amsr2_north.20210701T090000Z.nc4 + simulated variables: + - seaIceFraction + io pool: + max pool size: 1 + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: 0.0 + maxvalue: 1.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + maxvalue: 2.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + maxvalue: 0.0 + action: + name: inflate error + inflation factor: 2.0 + - filter: Gaussian Thinning + horizontal_mesh: 25 + use_reduced_horizontal_grid: true + select_median: true + action: + name: reduce obs space + observation_name: icec_amsr2_north + - obs space: + name: icec_amsr2_south + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/icec_amsr2_south.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.icec_amsr2_south.20210701T090000Z.nc4 + simulated variables: + - seaIceFraction + io pool: + max pool size: 1 + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: 0.0 + maxvalue: 1.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + maxvalue: 2.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + maxvalue: 0.0 + action: + name: inflate error + inflation factor: 2.0 + - filter: Gaussian Thinning + horizontal_mesh: 25 + use_reduced_horizontal_grid: true + select_median: true + action: + name: reduce obs space + observation_name: icec_amsr2_south + - obs space: + name: icec_nsidc_nh + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/icec_nsidc_nh.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.icec_nsidc_nh.20210701T090000Z.nc4 + simulated variables: + - seaIceFraction + io pool: + max pool size: 1 + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Bounds Check + minvalue: 0.0 + maxvalue: 1.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + maxvalue: 2.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + maxvalue: 0.0 + action: + name: inflate error + inflation factor: 2.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Domain Check + action: + name: passivate + where: + - variable: + name: GeoVaLs/sea_ice_area_fraction + minvalue: 1e-05 + observation_name: icec_nsidc_nh + - obs space: + name: icec_nsidc_sh + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/icec_nsidc_sh.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.icec_nsidc_sh.20210701T090000Z.nc4 + simulated variables: + - seaIceFraction + io pool: + max pool size: 1 + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Bounds Check + minvalue: 0.0 + maxvalue: 1.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + maxvalue: 2.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + maxvalue: 0.0 + action: + name: inflate error + inflation factor: 2.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Domain Check + action: + name: passivate + where: + - variable: + name: GeoVaLs/sea_ice_area_fraction + minvalue: 1e-05 + observation_name: icec_nsidc_sh + - obs space: + name: sst_ostia + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sst_ostia.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sst_ostia.20210701T090000Z.nc4 + simulated variables: + - seaSurfaceTemperature + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 + - filter: Domain Check + where: + - variable: + name: ObsError/seaSurfaceTemperature + minvalue: 0.001 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Domain Check + action: + name: passivate + where: + - variable: + name: GeoVaLs/sea_area_fraction + maxvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_ice_area_fraction + maxvalue: 1e-05 + observation_name: sst_ostia + - obs space: + name: sss_smos + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sss_smos.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sss_smos.20210701T090000Z.nc4 + simulated variables: + - seaSurfaceSalinity + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: 0.1 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 2.5 + - filter: Domain Check + action: + name: passivate + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 10.0 + - filter: Domain Check + action: + name: reject + where: + - variable: + name: ObsError/seaSurfaceSalinity + maxvalue: 0.87 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: sss_smos + - obs space: + name: sss_smapv5 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sss_smapv5.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sss_smapv5.20210701T090000Z.nc4 + simulated variables: + - seaSurfaceSalinity + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: 0.1 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 2.5 + - filter: Domain Check + action: + name: passivate + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 10.0 + - filter: Domain Check + action: + name: reject + where: + - variable: + name: ObsError/seaSurfaceSalinity + maxvalue: 0.87 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: sss_smapv5 + - obs space: + name: sst_abi_g16_l3c + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sst_abi_g16_l3c.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sst_abi_g16_l3c.20210701T090000Z.nc4 + simulated variables: + - seaSurfaceTemperature + get values: + time interpolation: linear + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: -1.0 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 5.0 + - filter: Domain Check + where: + - variable: + name: ObsError/seaSurfaceTemperature + minvalue: 1e-08 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Perform Action + action: + name: assign error + error function: + name: ObsFunction/LinearCombination + options: + variables: + - ObsError/seaSurfaceTemperature + coefs: + - 1.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_ice_area_fraction + maxvalue: 1e-05 + observation_name: sst_abi_g16_l3c + - obs space: + name: sst_gmi_l3u + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sst_gmi_l3u.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sst_gmi_l3u.20210701T090000Z.nc4 + simulated variables: + - seaSurfaceTemperature + get values: + time interpolation: linear + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: -1.0 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 5.0 + - filter: Domain Check + where: + - variable: + name: ObsError/seaSurfaceTemperature + minvalue: 1e-08 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Perform Action + action: + name: assign error + error function: + name: ObsFunction/LinearCombination + options: + variables: + - ObsError/seaSurfaceTemperature + coefs: + - 1.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_ice_area_fraction + maxvalue: 1e-05 + observation_name: sst_gmi_l3u + - obs space: + name: sst_viirs_n20_l3u + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sst_viirs_n20_l3u.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sst_viirs_n20_l3u.20210701T090000Z.nc4 + simulated variables: + - seaSurfaceTemperature + get values: + time interpolation: linear + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: -1.0 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 5.0 + - filter: Domain Check + where: + - variable: + name: ObsError/seaSurfaceTemperature + minvalue: 1e-08 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Perform Action + action: + name: assign error + error function: + name: ObsFunction/LinearCombination + options: + variables: + - ObsError/seaSurfaceTemperature + coefs: + - 1.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_ice_area_fraction + maxvalue: 1e-05 + observation_name: sst_viirs_n20_l3u + - obs space: + name: temp_profile_xbt + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/temp_profile_xbt.20210701T090000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.temp_profile_xbt.20210701T090000Z.nc4 + simulated variables: + - waterTemperature + obs operator: + name: InsituTemperature + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: ObsError/waterTemperature + minvalue: 0.001 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 3.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 3.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: temp_profile_xbt +variational: + minimizer: + algorithm: RPCG + iterations: + - geometry: + mom6_input_nml: soca/input.nml + fields metadata: soca/fields_metadata.yaml + geom_grid_file: INPUT/soca_gridspec.nc + gradient norm reduction: 1e-10 + ninner: 10 + diagnostics: + departures: ombg + online diagnostics: + write increment: true + increment: + state component: + datadir: ./ + date: '2021-07-01T09:00:00Z' + exp: experiment_id + type: incr +final: + diagnostics: + departures: oman + prints: + frequency: PT3H +output: + datadir: ./ + exp: experiment_id + type: an diff --git a/src/swell/test/jedi_configs/jedi_3dvar_atmos_config.yaml b/src/swell/test/jedi_configs/jedi_3dvar_atmos_config.yaml new file mode 100644 index 000000000..47475c2d4 --- /dev/null +++ b/src/swell/test/jedi_configs/jedi_3dvar_atmos_config.yaml @@ -0,0 +1,19206 @@ +cost function: + cost type: 3D-Var + jb evaluation: false + time window: + begin: '2023-10-09T21:00:00Z' + end: '2023-10-10T03:00:00Z' + bound to include: begin + geometry: + fms initialization: + namelist filename: ./fv3-jedi/fv3files/fmsmpp.nml + field table filename: ./fv3-jedi/fv3files/field_table_gmao + akbk: ./fv3-jedi/fv3files/akbk72.nc4 + layout: + - 4 + - 4 + npx: '91' + npy: '91' + npz: '72' + analysis variables: &id002 + - eastward_wind + - northward_wind + - air_temperature + - water_vapor_mixing_ratio_wrt_moist_air + - air_pressure_at_surface + - air_pressure_levels + - cloud_liquid_ice + - cloud_liquid_water + - rain_water + - snow_water + - mole_fraction_of_ozone_in_air + - geopotential_height_times_gravity_at_surface + - fraction_of_ocean + - fraction_of_lake + - fraction_of_ice + - skin_temperature_at_surface + background: + datetime: '2023-10-10T00:00:00Z' + filetype: cube sphere history + provider: geos + compute edge pressure from surface pressure: true + max allowable geometry difference: 0.001 + datapath: cycle_dir + filenames: + - bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4 + - fv3-jedi/bkg/geos.crtmsrf.91.nc4 + state variables: + - eastward_wind + - northward_wind + - air_temperature + - air_pressure_at_surface + - air_pressure_levels + - water_vapor_mixing_ratio_wrt_moist_air + - cloud_liquid_ice + - cloud_liquid_water + - rain_water + - snow_water + - mole_fraction_of_ozone_in_air + - geopotential_height_times_gravity_at_surface + - initial_mass_fraction_of_large_scale_cloud_condensate + - initial_mass_fraction_of_convective_cloud_condensate + - convective_cloud_area_fraction + - fraction_of_ocean + - fraction_of_land + - isotropic_variance_of_filtered_topography + - surface_velocity_scale + - surface_buoyancy_scale + - planetary_boundary_layer_height + - surface_exchange_coefficient_for_momentum + - surface_exchange_coefficient_for_heat + - surface_exchange_coefficient_for_moisture + - KCBL_before_moist + - surface_temp_before_moist + - lower_index_where_Kh_greater_than_2 + - upper_index_where_Kh_greater_than_2 + - fraction_of_lake + - fraction_of_ice + - vtype + - stype + - vfrac + - sheleg + - skin_temperature_at_surface + - soilt + - soilm + - eastward_wind_at_surface + - northward_wind_at_surface + field io names: &id019 + eastward_wind: ua + northward_wind: va + air_temperature: t + air_pressure_at_surface: ps + air_pressure_levels: pe + water_vapor_mixing_ratio_wrt_moist_air: q + cloud_liquid_ice: qi + cloud_liquid_water: ql + rain_water: qr + snow_water: qs + mole_fraction_of_ozone_in_air: o3ppmv + geopotential_height_times_gravity_at_surface: phis + initial_mass_fraction_of_large_scale_cloud_condensate: qls + initial_mass_fraction_of_convective_cloud_condensate: qcn + convective_cloud_area_fraction: cfcn + fraction_of_ocean: frocean + fraction_of_land: frland + isotropic_variance_of_filtered_topography: varflt + surface_velocity_scale: ustar + surface_buoyancy_scale: bstar + planetary_boundary_layer_height: zpbl + surface_exchange_coefficient_for_momentum: cm + surface_exchange_coefficient_for_heat: ct + surface_exchange_coefficient_for_moisture: cq + KCBL_before_moist: kcbl + surface_temp_before_moist: tsm + lower_index_where_Kh_greater_than_2: khl + upper_index_where_Kh_greater_than_2: khu + fraction_of_lake: frlake + fraction_of_ice: frseaice + skin_temperature_at_surface: ts + eastward_wind_at_surface: u10m + northward_wind_at_surface: v10m + background error: + covariance model: SABER + covariance type: gsi hybrid covariance + saber central block: + saber block name: gsi hybrid covariance + read: + gsi akbk: ./fv3-jedi/fv3files/akbk72.nc4 + gsi error covariance file: ./fv3-jedi/gsibec/gsi-coeffs-gmao-global-l72x144y91.nc4 + gsi berror namelist file: ./fv3-jedi/gsibec/cli_gsibec_configuration_l72x144y91.nml + processor layout x direction: 4 + processor layout y direction: 24 + debugging mode: false + saber outer blocks: + - saber block name: interpolation + inner geometry: + function space: StructuredColumns + custom grid matching gsi: + type: latlon + lats: 91 + lons: 144 + custom partitioner matching gsi: + bands: 24 + halo: 1 + forward interpolator: + local interpolator type: oops unstructured grid interpolator + inverse interpolator: + local interpolator type: oops unstructured grid interpolator + state variables to inverse: &id001 + - eastward_wind + - northward_wind + - air_temperature + - air_pressure_at_surface + - air_pressure_levels + - water_vapor_mixing_ratio_wrt_moist_air + - cloud_liquid_ice + - cloud_liquid_water + - rain_water + - snow_water + - mole_fraction_of_ozone_in_air + - fraction_of_ocean + - fraction_of_lake + - fraction_of_ice + - geopotential_height_times_gravity_at_surface + - skin_temperature_at_surface + linear variable change: + linear variable change name: Control2Analysis + input variables: *id001 + output variables: *id002 + observations: + get values: + variable change: + variable change name: Model2GeoVaLs + hydrometeor effective radii method: gsi + tropopause pressure method: gsi + observers: + - obs space: + name: Aircraft Temperature + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/aircraft_temperature.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.aircraft_temperature.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - airTemperature + obs bias: + input file: cycle_dir/aircraft_temperature.20231009T150000Z.acftbias + output file: cycle_dir/aircraft_temperature.20231009T210000Z.acftbias + bc by record: true + variational bc: + predictors: + - name: constant + - name: obsMetadataPredictor + variable: instantaneousAltitudeRate + - name: obsMetadataPredictor + variable: instantaneousAltitudeRate + order: 2 + covariance: + minimal required obs number: 3 + variance range: + - 1e-06 + - 1.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/aircraft_temperature.20231009T150000Z.acftbias_cov + inflation: + ratio: 1.005 + ratio for small dataset: 2.0 + obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - airTemperature + linear obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - airTemperature + obs filters: + - filter: Variable Assignment + where: + - variable: + name: ObsType/airTemperature + is_in: 130 + assignments: + - name: MetaData/stationIdentification + value: 'KX130 ' + - filter: Variable Assignment + where: + - variable: + name: MetaData/instantaneousAltitudeRate + minvalue: 50.0 + assignments: + - name: MetaData/instantaneousAltitudeRate + value: 0.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 2.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + errors: + - 2.5 + - 2.3 + - 2.1 + - 1.9 + - 1.7 + where: + - variable: + name: ObsType/airTemperature + is_in: 130 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + errors: + - 1.4706 + - 1.3529 + - 1.2353 + - 1.1176 + - 1.0 + where: + - variable: + name: ObsType/airTemperature + is_in: 131,133 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 70000 + - 65000 + - 60000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 5000 + errors: + - 1.5 + - 1.3 + - 1.1 + - 0.9 + - 0.8 + - 0.8 + - 0.75 + - 0.7 + - 0.7 + - 0.75 + - 0.85 + - 1.3 + - 1.5 + - 1.5 + where: + - variable: + name: ObsType/airTemperature + is_in: 132 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 60000 + - 40000 + errors: + - 1.5 + - 1.35 + - 1.25 + - 1.1 + - 1.0 + - 1.3 + - 1.7 + where: + - variable: + name: ObsType/airTemperature + is_in: 134 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + errors: + - 1.4706 + - 1.3529 + - 1.2353 + - 1.1176 + - 1000000000.0 + where: + - variable: + name: ObsType/airTemperature + is_in: 135 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: + - airTemperature + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreQC/airTemperature + is_in: 4-15 + action: + name: passivate + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: + name: ObsType/airTemperature + is_in: 134, 135 + action: + name: passivate + defer to post: true + - filter: Perform Action + action: + name: inflate error + inflation factor: 10.0 + filter variables: + - name: airTemperature + where: + - variable: + name: MetaData/pressure + maxvalue: 110000 + minvalue: 50000 + - variable: + name: ObsType/airTemperature + is_in: 130 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + observation_name: aircraft_temperature + - obs space: + name: Aircraft Wind + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/aircraft_wind.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.aircraft_wind.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + obs prior filters: + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + obs post filters: + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: PreQC/windEastward + is_in: 4-15 + action: + name: passivate + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: ObsType/windEastward + is_in: 234, 235 + action: + name: passivate + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 80000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 1.4 + - 1.5 + - 1.6 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.3 + - 2.6 + - 2.8 + - 3.0 + - 3.2 + - 2.7 + - 2.4 + - 2.1 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.6 + where: + - variable: + name: ObsType/windEastward + is_in: 230 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.0 + where: + - variable: + name: ObsType/windEastward + is_in: 231 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 2.5 + where: + - variable: + name: ObsType/windEastward + is_in: 233 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.0 + where: + - variable: + name: ObsType/windEastward + is_in: 234, 235 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + errors: + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.2 + - 2.2 + - 2.3 + - 2.3 + - 2.4 + - 2.4 + - 2.5 + - 2.7 + - 2.9 + - 3.1 + where: + - variable: + name: ObsType/windEastward + is_in: 232 + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: + - windEastward + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: + - windNorthward + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 230 + - 231 + - 232 + - 233 + - 234 + - 235 + cgross: + - 6.0 + - 6.5 + - 7.0 + - 7.5 + - 7.5 + - 7.5 + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + variable: windEastward + action: + name: reject + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 230 + - 231 + - 232 + - 233 + - 234 + - 235 + cgross: + - 6.0 + - 6.5 + - 7.0 + - 7.5 + - 7.5 + - 7.5 + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + variable: windNorthward + action: + name: reject + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + defer to post: true + observation_name: aircraft_wind + - obs space: + name: AIRS AQUA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/airs_aqua.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.airs_aqua.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: airs_aqua + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/airs_aqua.20231009T150000Z.satbias.nc4 + output file: cycle_dir/airs_aqua.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/airs_aqua.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/airs_aqua.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/airs_aqua.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/airs_aqua.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/airs_aqua_119_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.2 + - 1.2 + - 1.3136 + - 1.4 + - 1.4 + - 1.2639 + - 1.4 + - 1.4 + - 1.1802 + - 1.2517 + - 1.1719 + - 1.2 + - 1.1728 + - 1.1442 + - 1.2 + - 1.2 + - 1.15 + - 1.0801 + - 1.15 + - 1.15 + - 1.0396 + - 1.15 + - 1.15 + - 1.15 + - 1.15 + - 1.15 + - 1.15 + - 1.15 + - 0.9946 + - 1.05 + - 0.9217 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.9591 + - 0.9465 + - 0.9593 + - 0.9337 + - 1.0 + - 0.9861 + - 1.0017 + - 1.1 + - 1.0083 + - 1.0024 + - 1.1 + - 0.9967 + - 1.0094 + - 0.9412 + - 1.1 + - 0.998 + - 0.9807 + - 0.857 + - 0.8727 + - 0.8114 + - 0.879 + - 0.871 + - 0.8853 + - 0.7937 + - 0.8243 + - 0.8 + - 0.8016 + - 0.8 + - 0.7781 + - 0.7475 + - 0.85 + - 0.7405 + - 0.715 + - 0.7416 + - 0.7465 + - 0.9 + - 0.7198 + - 0.7157 + - 0.9 + - 0.727 + - 0.7246 + - 0.704 + - 0.7039 + - 0.66 + - 0.6694 + - 0.6669 + - 0.7031 + - 0.6977 + - 0.6488 + - 0.6653 + - 0.9 + - 0.6265 + - 0.622 + - 0.6308 + - 0.6297 + - 0.621 + - 0.6225 + - 0.6229 + - 0.6234 + - 0.6238 + - 0.6332 + - 0.6425 + - 0.7028 + - 0.6152 + - 0.9 + - 0.7257 + - 0.7288 + - 1.15 + - 0.9 + - 0.6673 + - 0.7473 + - 0.6767 + - 0.7056 + - 0.9 + - 0.95 + - 0.7271 + - 0.95 + - 0.725 + - 0.7601 + - 0.6973 + - 0.7573 + - 0.6011 + - 0.606 + - 0.9 + - 0.6635 + - 0.586 + - 0.5766 + - 0.75 + - 2.0386 + - 0.75 + - 1.0 + - 0.9 + - 0.9 + - 0.9 + - 0.9 + - 0.9 + - 0.9 + - 1.0 + - 1.3386 + - 1.0 + - 1.0 + - 0.85 + - 0.95 + - 1.7386 + - 0.95 + - 0.9 + - 0.8 + - 1.7386 + - 0.75 + - 0.75 + - 0.75 + - 0.8 + - 0.75 + - 0.8 + - 0.9 + - 0.75 + - 0.8 + - 0.8 + - 1.1 + - 0.75 + - 1.1 + - 0.75 + - 0.5991 + - 0.5348 + - 0.6541 + - 0.7421 + - 0.6192 + - 0.8186 + - 1.0616 + - 0.8848 + - 1.024 + - 2.5 + - 1.0249 + - 1.0795 + - 1.2199 + - 2.5 + - 2.5 + - 1.3103 + - 1.3603 + - 2.5 + - 2.5 + - 2.5 + - 1.323 + - 2.5 + - 2.5 + - 2.5 + - 1.4406 + - 2.5 + - 2.5 + - 1.3965 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 1.6997 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 1.6264 + - 2.5 + - 2.5 + - 2.5 + - 1.3436 + - 2.5 + - 2.5 + - 0.5727 + - 0.6838 + - 0.5994 + - 0.5178 + - 0.5145 + - 0.547 + - 0.5572 + - 0.5002 + - 0.4974 + - 0.55 + - 0.4953 + - 0.4883 + - 0.4948 + - 0.5446 + - 0.5777 + - 1.5 + - 1.5 + - 3.0 + - 3.0 + - 2.5 + - 2.5 + - 2.0 + - 1.0 + - 1.5 + - 1.5 + - 1.8 + - 0.6 + - 0.7 + - 0.65 + - 0.675 + - 0.7 + - 0.75 + - 0.775 + - 0.8 + - 0.8 + - 0.85 + - 0.85 + - 0.85 + - 0.7 + - 0.7 + - 0.7 + - 0.7 + - 0.7 + - 0.7 + - 0.7 + - 0.725 + - 0.75 + - 0.775 + - 0.8 + - 0.825 + - 0.8 + - 0.8 + - 0.8 + - 0.75 + - 0.8 + - 0.8 + - 0.8 + - 0.8 + - 0.8 + - 0.85 + - 0.8 + - 0.8 + - 2.5 + - 0.75 + - 0.75 + - 0.75 + - 0.75 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 2122, 2123, 2128, 2134, 2141, 2145, 2149, 2153, 2164, 2189, 2197, + 2209, 2226, 2234, 2280, 2318, 2321, 2325, 2328, 2333, 2339, 2348, 2353, + 2355, 2357, 2363, 2370, 2371, 2377 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: airs_aqua + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: &id003 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: airs_aqua + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 1.0 + - 1.0 + - 3.0 + - 1.0 + - 3.0 + - 1.0 + - 1.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 3.0 + - 1.0 + - 1.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 3.0 + - 3.0 + - 3.5 + - 3.0 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 1.7 + - 3.0 + - 1.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.5 + - 1.25 + - 3.5 + - 3.5 + - 3.0 + - 3.0 + - 1.5 + - 3.0 + - 3.0 + - 3.0 + - 1.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.5 + - 3.0 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.5 + - 3.0 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id003 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: airs_aqua + - obs space: + name: AMSR2 GCOM-W1 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsr2_gcom-w1.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsr2_gcom-w1.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: amsr2_gcom-w1 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsr2_gcom-w1.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsr2_gcom-w1.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsr2_gcom-w1.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsr2_gcom-w1.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsr2_gcom-w1.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsr2_gcom-w1.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 340.0 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.999 + - variable: + name: GeoVaLs/skin_temperature_at_surface_where_sea + minvalue: 275 + - variable: + name: GeoVaLs/wind_speed_at_surface + maxvalue: 12 + - variable: + name: MetaData/latitude + minvalue: -60.0 + maxvalue: 60.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/TotalColumnVaporGuess + minvalue: 10.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SunGlintAngle + minvalue: 20.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: &id004 + - 0.48 + - 3.0737 + - 0.7433 + - 3.643 + - 3.5304 + - 4.427 + - 5.1448 + - 5.0785 + - 4.9763 + - 9.3215 + - 2.5789 + - 5.5274 + - 0.6641 + - 1.3674 + clwret_types: + - ObsValue + maxvalue: 1.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id004 + clwret_types: + - HofX + maxvalue: 1.0 + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: None + value: + name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id004 + clwret_types: + - ObsValue + reference: + name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id004 + clwret_types: + - HofX + minvalue: -0.5 + maxvalue: 0.5 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id004 + clwret_types: + - ObsValue + - HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.1 + - 0.1 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 0.6 + - 0.6 + - 0.6 + - 0.6 + - 0.6 + - 0.5 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + err0: + - 0.8 + - 0.9 + - 0.8 + - 0.9 + - 1.0 + - 1.1 + - 2.0 + - 3.5 + - 3.0 + - 4.8 + - 5.0 + - 6.0 + - 4.5 + - 6.3 + err1: + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 18.5 + - 20.0 + - 40.0 + - 20.0 + - 25.0 + - 30.0 + - 30.0 + - 30.0 + - 20.0 + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: None + threshold: 2.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 7-10 + absolute threshold: 30 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 11-14 + absolute threshold: 50 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: amsr2_gcom-w1 + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsr2_gcom-w1 + - obs space: + name: AMSU-A AQUA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_aqua.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_aqua.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_aqua + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_aqua.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_aqua.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_aqua.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_aqua.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_aqua.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_aqua.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id005 + - 2.5 + - 2.0 + - 2.0 + - 0.5 + - 0.4 + - 0.4 + - 0.5 + - 0.3 + - 0.35 + - 0.35 + - 0.45 + - 1.0 + - 1.5 + - 3.75 + - 6.3 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_aqua + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_aqua + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_aqua + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_aqua + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_aqua + error parameter vector: *id005 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 3.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 3.0 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_aqua + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_aqua + - obs space: + name: AMSU-A METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_metop-b.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_metop-b.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_metop-b.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_metop-b.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_metop-b.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_metop-b.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id006 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_metop-b + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_metop-b + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_metop-b + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_metop-b + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_metop-b + error parameter vector: *id006 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_metop-b + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_metop-b + - obs space: + name: AMSU-A METOP-C + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_metop-c.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_metop-c.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_metop-c + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_metop-c.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_metop-c.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_metop-c.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_metop-c.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_metop-c.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_metop-c.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id007 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_metop-c + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_metop-c + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_metop-c + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_metop-c + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_metop-c + error parameter vector: *id007 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_metop-c + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_metop-c + - obs space: + name: AMSU-A NOAA-15 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_n15.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_n15.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_n15 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_n15.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_n15.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_n15.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_n15.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_n15.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_n15.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id008 + - 3.0 + - 2.0 + - 2.0 + - 0.6 + - 0.3 + - 0.23 + - 0.25 + - 0.275 + - 0.34 + - 0.4 + - 0.6 + - 1.0 + - 1.5 + - 5.0 + - 3.0 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_n15 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_n15 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_n15 + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_n15 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_n15 + error parameter vector: *id008 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_n15 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_n15 + - obs space: + name: AMSU-A NOAA-18 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_n18.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_n18.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_n18 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_n18.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_n18.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_n18.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_n18.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_n18.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_n18.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id009 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_n18 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_n18 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_n18 + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_n18 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_n18 + error parameter vector: *id009 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_n18 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_n18 + - obs space: + name: AMSU-A NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_n19.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_n19.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_n19 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_n19.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_n19.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_n19.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_n19.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_n19.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_n19.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id010 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_n19 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_n19 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_n19 + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_n19 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_n19 + error parameter vector: *id010 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_n19 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_n19 + - obs space: + name: ATMS NOAA-20 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/atms_n20.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.atms_n20.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: atms_n20 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/atms_n20.20231009T150000Z.satbias.nc4 + output file: cycle_dir/atms_n20.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: ATMS + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/atms_n20.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/atms_n20.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/atms_n20.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/atms_n20.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id011 + - 5.0 + - 5.0 + - 5.0 + - 3.0 + - 0.55 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 5.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7,16-22 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-7,16 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: atms_n20 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: atms_n20 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + use_biasterm: true + test_biasterm: ObsBiasTerm + sensor: atms_n20 + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: atms_n20 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: atms_n20 + error parameter vector: *id011 + obserr_bound_max: + - 4.5 + - 4.5 + - 3.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 2.0 + - 4.5 + - 4.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: atms_n20 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: atms_n20 + - obs space: + name: ATMS NPP + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/atms_npp.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.atms_npp.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: atms_npp + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/atms_npp.20231009T150000Z.satbias.nc4 + output file: cycle_dir/atms_npp.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: ATMS + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/atms_npp.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/atms_npp.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/atms_npp.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/atms_npp.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id012 + - 5.0 + - 5.0 + - 5.0 + - 3.0 + - 0.55 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 5.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7,16-22 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-7,16 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: atms_npp + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: atms_npp + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + use_biasterm: true + test_biasterm: ObsBiasTerm + sensor: atms_npp + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: atms_npp + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: atms_npp + error parameter vector: *id012 + obserr_bound_max: + - 4.5 + - 4.5 + - 3.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 2.0 + - 4.5 + - 4.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: atms_npp + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: atms_npp + - obs space: + name: AVHRR-3 METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/avhrr3_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.avhrr3_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: avhrr3_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/avhrr3_metop-b.20231009T150000Z.satbias.nc4 + output file: cycle_dir/avhrr3_metop-b.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/avhrr3_metop-b.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/avhrr3_metop-b.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/avhrr3_metop-b.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/avhrr3_metop-b.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.0 + - 1.08 + - 1.12 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 1e-05 + maxvalue: 1000.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: avhrr3_metop-b + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + cloud detection criteria: + - 0.6 + - 0.68 + - 0.72 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: avhrr3_metop-b + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: avhrr3_metop-b + - obs space: + name: AVHRR-3 NOAA-18 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/avhrr3_n18.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.avhrr3_n18.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: avhrr3_n18 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/avhrr3_n18.20231009T150000Z.satbias.nc4 + output file: cycle_dir/avhrr3_n18.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/avhrr3_n18.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/avhrr3_n18.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/avhrr3_n18.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/avhrr3_n18.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.0 + - 1.08 + - 1.12 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 1e-05 + maxvalue: 1000.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: avhrr3_n18 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + cloud detection criteria: + - 0.6 + - 0.68 + - 0.72 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: avhrr3_n18 + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: avhrr3_n18 + - obs space: + name: AVHRR-3 NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/avhrr3_n19.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.avhrr3_n19.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: avhrr3_n19 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/avhrr3_n19.20231009T150000Z.satbias.nc4 + output file: cycle_dir/avhrr3_n19.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/avhrr3_n19.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/avhrr3_n19.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/avhrr3_n19.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/avhrr3_n19.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.0 + - 1.08 + - 1.12 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 1e-05 + maxvalue: 1000.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: avhrr3_n19 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + cloud detection criteria: + - 0.6 + - 0.68 + - 0.72 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: avhrr3_n19 + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: avhrr3_n19 + - obs space: + name: CRIS-FSR NOAA-20 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/cris-fsr_n20.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.cris-fsr_n20.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: cris-fsr_n20 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/cris-fsr_n20.20231009T150000Z.satbias.nc4 + output file: cycle_dir/cris-fsr_n20.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/cris-fsr_n20.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/cris-fsr_n20.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/cris-fsr_n20.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/cris-fsr_n20.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/cris-fsr_108_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 0.823 + - 0.76 + - 0.736 + - 0.743 + - 0.856 + - 1.079 + - 0.888 + - 0.778 + - 0.671 + - 0.65 + - 0.643 + - 0.629 + - 0.629 + - 0.618 + - 0.638 + - 0.619 + - 0.61 + - 0.627 + - 0.601 + - 0.617 + - 0.608 + - 0.498 + - 0.5112 + - 0.4922 + - 0.4959 + - 0.4954 + - 0.4836 + - 0.514 + - 0.5005 + - 0.4917 + - 0.4881 + - 0.4656 + - 0.4793 + - 0.4638 + - 0.4557 + - 0.4666 + - 0.4468 + - 0.4534 + - 0.4471 + - 0.4448 + - 0.4469 + - 0.536 + - 0.4426 + - 0.4388 + - 0.534 + - 0.4368 + - 0.438 + - 0.531 + - 0.4379 + - 0.535 + - 0.4404 + - 0.4405 + - 0.4409 + - 0.4472 + - 0.4555 + - 0.4433 + - 0.4437 + - 0.4454 + - 0.4448 + - 0.4465 + - 0.4499 + - 0.4488 + - 0.54 + - 0.4534 + - 0.4472 + - 0.455 + - 0.4562 + - 0.452 + - 0.4639 + - 0.538 + - 0.4573 + - 0.4604 + - 0.4533 + - 0.4692 + - 0.566 + - 0.4457 + - 0.4457 + - 0.5154 + - 0.5084 + - 0.528 + - 0.552 + - 0.56 + - 0.567 + - 0.546 + - 0.495 + - 0.4809 + - 0.4732 + - 0.4861 + - 0.4632 + - 0.529 + - 0.4748 + - 0.5007 + - 0.5711 + - 0.595 + - 0.5469 + - 0.626 + - 0.541 + - 0.543 + - 0.533 + - 0.541 + - 0.4695 + - 0.53 + - 0.539 + - 0.529 + - 0.542 + - 0.4681 + - 0.536 + - 0.542 + - 0.535 + - 0.563 + - 0.4805 + - 0.647 + - 0.609 + - 0.553 + - 0.583 + - 0.576 + - 0.6294 + - 0.5885 + - 0.556 + - 0.578 + - 0.566 + - 0.601 + - 0.5627 + - 0.5675 + - 0.592 + - 0.5166 + - 0.589 + - 0.5291 + - 0.5892 + - 0.5976 + - 0.5834 + - 0.6512 + - 0.6748 + - 0.6615 + - 0.6003 + - 0.5669 + - 0.5587 + - 0.5507 + - 0.5871 + - 0.616 + - 0.637 + - 0.633 + - 0.639 + - 0.655 + - 0.641 + - 0.664 + - 0.648 + - 0.656 + - 0.663 + - 0.652 + - 0.681 + - 0.662 + - 0.673 + - 0.672 + - 0.68 + - 0.735 + - 0.732 + - 0.715 + - 0.674 + - 0.687 + - 0.702 + - 0.705 + - 0.715 + - 0.725 + - 0.707 + - 0.74 + - 0.74 + - 0.874 + - 0.737 + - 0.819 + - 0.76 + - 0.869 + - 0.9 + - 0.698 + - 0.823 + - 0.676 + - 0.682 + - 0.766 + - 0.68 + - 0.685 + - 0.694 + - 0.695 + - 0.689 + - 0.727 + - 0.695 + - 0.688 + - 0.677 + - 0.736 + - 0.651 + - 0.661 + - 0.6199 + - 0.6223 + - 0.6036 + - 0.6003 + - 0.5991 + - 0.598 + - 0.591 + - 0.5764 + - 0.577 + - 0.5593 + - 0.597 + - 0.576 + - 0.574 + - 0.578 + - 0.579 + - 0.575 + - 0.576 + - 0.568 + - 0.575 + - 0.569 + - 0.559 + - 0.568 + - 0.5401 + - 0.55 + - 0.5575 + - 0.578 + - 0.5635 + - 0.5786 + - 0.5807 + - 0.581 + - 0.573 + - 0.569 + - 0.567 + - 0.552 + - 0.55 + - 0.558 + - 0.552 + - 0.562 + - 0.574 + - 0.575 + - 0.629 + - 0.682 + - 0.756 + - 1.05 + - 1.122 + - 1.1402 + - 1.154 + - 1.131 + - 1.123 + - 1.159 + - 1.106 + - 1.116 + - 1.069 + - 1.077 + - 1.155 + - 1.162 + - 1.1402 + - 0.644 + - 1.208 + - 1.1402 + - 1.295 + - 1.258 + - 1.1402 + - 0.606 + - 0.603 + - 0.563 + - 0.592 + - 0.607 + - 0.611 + - 0.612 + - 0.618 + - 0.626 + - 0.629 + - 0.583 + - 0.8646 + - 0.626 + - 0.639 + - 0.559 + - 0.827 + - 0.612 + - 0.576 + - 0.58 + - 0.575 + - 0.688 + - 0.697 + - 0.743 + - 0.681 + - 0.832 + - 0.719 + - 0.785 + - 0.878 + - 0.9402 + - 1.0054 + - 1.073 + - 1.129 + - 1.035 + - 1.027 + - 0.9703 + - 1.195 + - 0.9153 + - 1.266 + - 1.153 + - 1.348 + - 1.18 + - 1.269 + - 1.311 + - 0.9914 + - 1.359 + - 1.166 + - 1.139 + - 1.2817 + - 1.398 + - 1.542 + - 1.229 + - 1.377 + - 1.28 + - 1.245 + - 1.1188 + - 1.193 + - 1.293 + - 1.275 + - 1.331 + - 1.34 + - 1.099 + - 1.048 + - 1.124 + - 1.225 + - 1.183 + - 1.196 + - 1.4 + - 1.333 + - 1.417 + - 1.326 + - 1.305 + - 1.0638 + - 1.268 + - 1.217 + - 1.289 + - 1.395 + - 1.232 + - 1.435 + - 1.298 + - 1.328 + - 1.262 + - 1.199 + - 1.391 + - 1.233 + - 1.329 + - 1.664 + - 1.509 + - 1.349 + - 1.481 + - 1.595 + - 1.485 + - 1.532 + - 1.504 + - 1.584 + - 1.609 + - 1.516 + - 1.489 + - 1.502 + - 1.544 + - 1.611 + - 1.539 + - 1.296 + - 1.288 + - 1.241 + - 1.32 + - 1.313 + - 1.301 + - 1.843 + - 1.747 + - 1.711 + - 1.771 + - 1.937 + - 1.575 + - 1.573 + - 1.5 + - 1.459 + - 1.402 + - 1.363 + - 2.201 + - 2.127 + - 2.177 + - 2.157 + - 2.192 + - 2.146 + - 2.151 + - 2.071 + - 1.986 + - 1.845 + - 1.687 + - 1.505 + - 1.373 + - 1.229 + - 1.113 + - 1.004 + - 0.936 + - 0.895 + - 0.871 + - 0.841 + - 0.821 + - 0.805 + - 0.8 + - 0.792 + - 0.797 + - 0.791 + - 0.783 + - 0.777 + - 0.785 + - 0.787 + - 0.789 + - 0.793 + - 0.794 + - 0.745 + - 0.75 + - 0.748 + - 0.749 + - 0.744 + - 0.733 + - 0.733 + - 0.741 + - 0.746 + - 0.746 + - 0.738 + - 0.743 + - 0.745 + - 0.749 + - 0.75 + - 0.75 + - 0.884 + - 0.906 + - 0.917 + - 0.924 + - 0.922 + - 0.928 + - 0.921 + - 0.938 + - 0.931 + - 0.943 + - 0.946 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, + 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, 2153, 2158, 2161, + 2168, 2171, 2175, 2182 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: cris-fsr_n20 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: &id013 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 31 + - 31 + - 30 + - 31 + - 30 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 31 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: cris-fsr_n20 + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 2.0 + - 0.4 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.4 + - 0.4 + - 0.4 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id013 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: cris-fsr_n20 + - obs space: + name: CRIS-FSR NPP + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/cris-fsr_npp.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.cris-fsr_npp.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: cris-fsr_npp + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/cris-fsr_npp.20231009T150000Z.satbias.nc4 + output file: cycle_dir/cris-fsr_npp.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/cris-fsr_npp.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/cris-fsr_npp.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/cris-fsr_npp.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/cris-fsr_npp.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/cris-fsr_108_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 0.823 + - 0.76 + - 0.736 + - 0.743 + - 0.856 + - 1.079 + - 0.888 + - 0.778 + - 0.671 + - 0.65 + - 0.643 + - 0.629 + - 0.629 + - 0.618 + - 0.638 + - 0.619 + - 0.61 + - 0.627 + - 0.601 + - 0.617 + - 0.608 + - 0.498 + - 0.5112 + - 0.4922 + - 0.4959 + - 0.4954 + - 0.4836 + - 0.514 + - 0.5005 + - 0.4917 + - 0.4881 + - 0.4656 + - 0.4793 + - 0.4638 + - 0.4557 + - 0.4666 + - 0.4468 + - 0.4534 + - 0.4471 + - 0.4448 + - 0.4469 + - 0.536 + - 0.4426 + - 0.4388 + - 0.534 + - 0.4368 + - 0.438 + - 0.531 + - 0.4379 + - 0.535 + - 0.4404 + - 0.4405 + - 0.4409 + - 0.4472 + - 0.4555 + - 0.4433 + - 0.4437 + - 0.4454 + - 0.4448 + - 0.4465 + - 0.4499 + - 0.4488 + - 0.54 + - 0.4534 + - 0.4472 + - 0.455 + - 0.4562 + - 0.452 + - 0.4639 + - 0.538 + - 0.4573 + - 0.4604 + - 0.4533 + - 0.4692 + - 0.566 + - 0.4457 + - 0.4457 + - 0.5154 + - 0.5084 + - 0.528 + - 0.552 + - 0.56 + - 0.567 + - 0.546 + - 0.495 + - 0.4809 + - 0.4732 + - 0.4861 + - 0.4632 + - 0.529 + - 0.4748 + - 0.5007 + - 0.5711 + - 0.595 + - 0.5469 + - 0.626 + - 0.541 + - 0.543 + - 0.533 + - 0.541 + - 0.4695 + - 0.53 + - 0.539 + - 0.529 + - 0.542 + - 0.4681 + - 0.536 + - 0.542 + - 0.535 + - 0.563 + - 0.4805 + - 0.647 + - 0.609 + - 0.553 + - 0.583 + - 0.576 + - 0.6294 + - 0.5885 + - 0.556 + - 0.578 + - 0.566 + - 0.601 + - 0.5627 + - 0.5675 + - 0.592 + - 0.5166 + - 0.589 + - 0.5291 + - 0.5892 + - 0.5976 + - 0.5834 + - 0.6512 + - 0.6748 + - 0.6615 + - 0.6003 + - 0.5669 + - 0.5587 + - 0.5507 + - 0.5871 + - 0.616 + - 0.637 + - 0.633 + - 0.639 + - 0.655 + - 0.641 + - 0.664 + - 0.648 + - 0.656 + - 0.663 + - 0.652 + - 0.681 + - 0.662 + - 0.673 + - 0.672 + - 0.68 + - 0.735 + - 0.732 + - 0.715 + - 0.674 + - 0.687 + - 0.702 + - 0.705 + - 0.715 + - 0.725 + - 0.707 + - 0.74 + - 0.74 + - 0.874 + - 0.737 + - 0.819 + - 0.76 + - 0.869 + - 0.9 + - 0.698 + - 0.823 + - 0.676 + - 0.682 + - 0.766 + - 0.68 + - 0.685 + - 0.694 + - 0.695 + - 0.689 + - 0.727 + - 0.695 + - 0.688 + - 0.677 + - 0.736 + - 0.651 + - 0.661 + - 0.6199 + - 0.6223 + - 0.6036 + - 0.6003 + - 0.5991 + - 0.598 + - 0.591 + - 0.5764 + - 0.577 + - 0.5593 + - 0.597 + - 0.576 + - 0.574 + - 0.578 + - 0.579 + - 0.575 + - 0.576 + - 0.568 + - 0.575 + - 0.569 + - 0.559 + - 0.568 + - 0.5401 + - 0.55 + - 0.5575 + - 0.578 + - 0.5635 + - 0.5786 + - 0.5807 + - 0.581 + - 0.573 + - 0.569 + - 0.567 + - 0.552 + - 0.55 + - 0.558 + - 0.552 + - 0.562 + - 0.574 + - 0.575 + - 0.629 + - 0.682 + - 0.756 + - 1.05 + - 1.122 + - 1.1402 + - 1.154 + - 1.131 + - 1.123 + - 1.159 + - 1.106 + - 1.116 + - 1.069 + - 1.077 + - 1.155 + - 1.162 + - 1.1402 + - 0.644 + - 1.208 + - 1.1402 + - 1.295 + - 1.258 + - 1.1402 + - 0.606 + - 0.603 + - 0.563 + - 0.592 + - 0.607 + - 0.611 + - 0.612 + - 0.618 + - 0.626 + - 0.629 + - 0.583 + - 0.8646 + - 0.626 + - 0.639 + - 0.559 + - 0.827 + - 0.612 + - 0.576 + - 0.58 + - 0.575 + - 0.688 + - 0.697 + - 0.743 + - 0.681 + - 0.832 + - 0.719 + - 0.785 + - 0.878 + - 0.9402 + - 1.0054 + - 1.073 + - 1.129 + - 1.035 + - 1.027 + - 0.9703 + - 1.195 + - 0.9153 + - 1.266 + - 1.153 + - 1.348 + - 1.18 + - 1.269 + - 1.311 + - 0.9914 + - 1.359 + - 1.166 + - 1.139 + - 1.2817 + - 1.398 + - 1.542 + - 1.229 + - 1.377 + - 1.28 + - 1.245 + - 1.1188 + - 1.193 + - 1.293 + - 1.275 + - 1.331 + - 1.34 + - 1.099 + - 1.048 + - 1.124 + - 1.225 + - 1.183 + - 1.196 + - 1.4 + - 1.333 + - 1.417 + - 1.326 + - 1.305 + - 1.0638 + - 1.268 + - 1.217 + - 1.289 + - 1.395 + - 1.232 + - 1.435 + - 1.298 + - 1.328 + - 1.262 + - 1.199 + - 1.391 + - 1.233 + - 1.329 + - 1.664 + - 1.509 + - 1.349 + - 1.481 + - 1.595 + - 1.485 + - 1.532 + - 1.504 + - 1.584 + - 1.609 + - 1.516 + - 1.489 + - 1.502 + - 1.544 + - 1.611 + - 1.539 + - 1.296 + - 1.288 + - 1.241 + - 1.32 + - 1.313 + - 1.301 + - 1.843 + - 1.747 + - 1.711 + - 1.771 + - 1.937 + - 1.575 + - 1.573 + - 1.5 + - 1.459 + - 1.402 + - 1.363 + - 2.201 + - 2.127 + - 2.177 + - 2.157 + - 2.192 + - 2.146 + - 2.151 + - 2.071 + - 1.986 + - 1.845 + - 1.687 + - 1.505 + - 1.373 + - 1.229 + - 1.113 + - 1.004 + - 0.936 + - 0.895 + - 0.871 + - 0.841 + - 0.821 + - 0.805 + - 0.8 + - 0.792 + - 0.797 + - 0.791 + - 0.783 + - 0.777 + - 0.785 + - 0.787 + - 0.789 + - 0.793 + - 0.794 + - 0.745 + - 0.75 + - 0.748 + - 0.749 + - 0.744 + - 0.733 + - 0.733 + - 0.741 + - 0.746 + - 0.746 + - 0.738 + - 0.743 + - 0.745 + - 0.749 + - 0.75 + - 0.75 + - 0.884 + - 0.906 + - 0.917 + - 0.924 + - 0.922 + - 0.928 + - 0.921 + - 0.938 + - 0.931 + - 0.943 + - 0.946 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, + 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, 2153, 2158, 2161, + 2168, 2171, 2175, 2182 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: cris-fsr_npp + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: &id014 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 31 + - 31 + - 30 + - 31 + - 30 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 31 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: cris-fsr_npp + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 2.0 + - 0.4 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.4 + - 0.4 + - 0.4 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id014 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: cris-fsr_npp + - obs space: + name: GMI GPM + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/gmi_gpm.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.gmi_gpm.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: gmi_gpm + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/gmi_gpm.20231009T150000Z.satbias.nc4 + output file: cycle_dir/gmi_gpm.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/gmi_gpm.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/gmi_gpm.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: cloudWaterContent + sensor: GMI_GPM + ch37v: 6 + ch37h: 7 + order: 2 + tlapse: cycle_dir/gmi_gpm.20231009T150000Z.tlapse.txt + - name: cloudWaterContent + sensor: GMI_GPM + ch37v: 6 + ch37h: 7 + tlapse: cycle_dir/gmi_gpm.20231009T150000Z.tlapse.txt + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/gmi_gpm.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/gmi_gpm.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-9 + minvalue: 50.0 + maxvalue: 320.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 10-13 + minvalue: 70.0 + maxvalue: 320.0 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 5 + maxvalue: 70 + - variable: + name: MetaData/latitude + minvalue: -55.0 + maxvalue: 55.0 + - variable: + name: MetaData/heightOfSurface + maxvalue: 2000 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + minvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/latitude + minvalue: -20.0 + maxvalue: 0.0 + - variable: + name: MetaData/longitude + minvalue: 25.0 + maxvalue: 40.0 + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/cloudWaterContent_obs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + maxvalue: 900 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - HofX + maxvalue: 900 + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/cloudWaterContent_hofx + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - HofX + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + minvalue: 0.0 + maxvalue: 0.05 + - variable: + name: ObsFunction/Emissivity_Diff_GMI + options: + channel: 2 + regression_constant_1: 0.1329 + regression_constant_2: 0.42468 + regression_coeff_1: + - -0.00548 + - 0.00772 + - 0.0053 + - -0.00425 + - 0.00053 + - 8e-05 + - -3e-05 + - -0.00144 + - 0.00059 + - -0.00016 + - 3e-05 + - -0.00011 + - 0.00017 + regression_coeff_2: + - 0.00289 + - -0.00142 + minvalue: 0.01 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + minvalue: 0.0 + maxvalue: 0.05 + - variable: + name: ObsFunction/Emissivity_Diff_GMI + options: + channel: 4 + regression_constant_1: 0.15627 + regression_constant_2: 0.83807 + regression_coeff_1: + - -0.01084 + - 0.01194 + - 0.01111 + - -0.00784 + - 0.0006 + - 8e-05 + - -3e-05 + - -0.00248 + - 0.00105 + - -8e-05 + - 0.0 + - -0.00013 + - 0.00016 + regression_coeff_2: + - 0.00048 + - -0.00207 + minvalue: 0.035 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + minvalue: 0.0 + maxvalue: 0.05 + - variable: + name: ObsFunction/Emissivity_Diff_GMI + options: + channel: 7 + regression_constant_1: 0.30306 + regression_constant_2: 1.24071 + regression_coeff_1: + - -0.01793 + - 0.0173 + - 0.01784 + - -0.01199 + - 0.00067 + - 0.00013 + - -4e-05 + - -0.00365 + - 0.00154 + - -4e-05 + - -1e-05 + - -0.00015 + - 0.00017 + regression_coeff_2: + - 0.00068 + - -0.00342 + minvalue: 0.05 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + - HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.3 + - 0.2 + - 0.3 + - 0.3 + x2: + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + err0: + - 2.7 + - 3.7 + - 3.5 + - 4.5 + - 4.0 + - 3.8 + - 300.0 + - 5.0 + - 11.5 + - 5.0 + - 5.0 + - 2.5 + - 3.0 + err1: + - 17.0 + - 23.0 + - 13.0 + - 25.0 + - 11.0 + - 13.0 + - 23.0 + - 10.0 + - 20.0 + - 15.0 + - 20.0 + - 8.0 + - 13.0 + err2: + - 25.0 + - 40.0 + - 40.0 + - 55.0 + - 35.0 + - 25.0 + - 500.0 + - 50.0 + - 50.0 + - 50.0 + - 50.0 + - 30.0 + - 40.0 + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 1,2,4,6 + threshold: 2.0 + absolute threshold: 30.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 9,10,11 + threshold: 2.0 + absolute threshold: 20.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 3,5,8 + threshold: 2.0 + absolute threshold: 15.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 12,13 + threshold: 2.0 + absolute threshold: 10.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 7 + threshold: 2.0 + absolute threshold: 5.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: gmi_gpm + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: gmi_gpm + - obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/gps.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - sequenceNumber + sort variable: impactHeightRO + sort order: ascending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.gps.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - bendingAngle + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + vertlayer: full + sr_steps: 2 + super_ref_qc: NBAM + obs filters: + - filter: BlackList + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 41,265,266,421,440,724,725,726,727,728,729 + - filter: Perform Action + filter variables: + - name: bendingAngle + where: + - variable: PreUseFlag/bendingAngle + minvalue: 1 + action: + name: reject + - filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266,267,268,269 + test variables: + - name: MetaData/impactHeightRO + maxvalue: 45000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 3-5 + test variables: + - name: MetaData/impactHeightRO + minvalue: 8000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 267,268,269 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 401 + test variables: + - name: MetaData/impactHeightRO + minvalue: 5000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 267,268,269 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 402-999 + test variables: + - name: MetaData/impactHeightRO + minvalue: 9000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 401 + test variables: + - name: MetaData/impactHeightRO + minvalue: 5000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 402-999 + test variables: + - name: MetaData/impactHeightRO + minvalue: 8000.1 + action: + name: reject + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: inflate error + inflation factor: 2.0 + where: + - variable: MetaData/satelliteIdentifier + is_in: 267,268,269 + - filter: Variable Assignment + assignments: + - name: JediAdjustObsError/bendingAngle + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/bendingAngle + defer to post: true + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: assign error + error parameter: 1.0 + where: + - variable: + name: JediAdjustObsError/bendingAngle + maxvalue: 1.0 + - variable: + name: JediAdjustObsError/bendingAngle + value: is_valid + - variable: + name: MetaData/satelliteIdentifier + is_in: 3,5,41,42,43,44,267,268,269,440,421,724,725,726,727,728,729, 750,751,752,753,754,755,821,825 + defer to post: true + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: assign error + error parameter: 10.0 + where: + - variable: + name: JediAdjustObsError/bendingAngle + minvalue: 10.0 + - variable: + name: JediAdjustObsError/bendingAngle + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: bendingAngle + threshold: 5 + action: + name: reject + defer to post: true + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: assign error + error function: JediAdjustObsError/bendingAngle + where: + - variable: + name: JediAdjustObsError/bendingAngle + value: is_valid + defer to post: true + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + defer to post: true + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate_GEOS + defer to post: true + observation_name: gps + - obs space: + name: IASI METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/iasi_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.iasi_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: iasi_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/iasi_metop-b.20231009T150000Z.satbias.nc4 + output file: cycle_dir/iasi_metop-b.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/iasi_metop-b.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/iasi_metop-b.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/iasi_metop-b.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/iasi_metop-b.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/iasi_metop_141_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id015 + - 0.727 + - 0.81 + - 0.75 + - 0.79 + - 0.7055 + - 0.74 + - 0.68 + - 0.72 + - 0.6526 + - 0.65 + - 0.665 + - 0.69 + - 0.6394 + - 0.64 + - 0.6528 + - 0.6065 + - 0.6246 + - 0.61 + - 0.6423 + - 0.5995 + - 0.59 + - 0.6069 + - 0.6 + - 0.5965 + - 0.64 + - 0.62 + - 0.589 + - 0.5865 + - 0.65 + - 0.5861 + - 0.61 + - 0.5874 + - 0.68 + - 0.606 + - 0.68 + - 4.38 + - 3.05 + - 2.31 + - 1.56 + - 1.33 + - 1.58 + - 0.93 + - 0.5832 + - 0.5587 + - 0.5867 + - 0.58 + - 0.5655 + - 0.5522 + - 0.5864 + - 0.5475 + - 0.5854 + - 0.5455 + - 0.5811 + - 0.5376 + - 0.5452 + - 0.5686 + - 0.5329 + - 0.5655 + - 0.5302 + - 0.545 + - 0.5628 + - 0.59 + - 0.5262 + - 0.559 + - 0.5264 + - 0.5442 + - 0.51 + - 0.5513 + - 0.5224 + - 0.5523 + - 0.5188 + - 0.5487 + - 0.5245 + - 0.58 + - 0.5437 + - 0.5343 + - 0.5364 + - 0.64 + - 0.5338 + - 0.72 + - 0.537 + - 0.75 + - 0.51 + - 0.65 + - 0.5274 + - 0.529 + - 0.5187 + - 0.5228 + - 1.12 + - 0.5222 + - 0.5109 + - 0.67 + - 0.5133 + - 0.5179 + - 0.507 + - 0.67 + - 0.5091 + - 0.62 + - 0.5093 + - 0.69 + - 0.5048 + - 0.5024 + - 0.78 + - 0.497 + - 0.5337 + - 0.4865 + - 0.4915 + - 0.4835 + - 0.4869 + - 0.87 + - 0.4824 + - 0.4852 + - 0.84 + - 0.84 + - 0.84 + - 0.5318 + - 0.8 + - 0.4772 + - 0.98 + - 0.488 + - 0.4978 + - 0.5157 + - 0.61 + - 0.5213 + - 0.4884 + - 0.79 + - 0.62 + - 0.66 + - 0.4691 + - 0.65 + - 0.4809 + - 0.468 + - 0.62 + - 0.4679 + - 0.6913 + - 0.4705 + - 0.4785 + - 0.47 + - 0.4773 + - 0.4703 + - 0.98 + - 0.4697 + - 0.4662 + - 0.65 + - 0.467 + - 0.4883 + - 0.4684 + - 0.4684 + - 0.4947 + - 0.5393 + - 0.5024 + - 0.4715 + - 0.621 + - 0.6136 + - 0.5316 + - 1.78 + - 0.5099 + - 1.14 + - 0.539 + - 1.79 + - 0.508 + - 0.5723 + - 1.94 + - 2.01 + - 0.49 + - 0.5647 + - 0.5022 + - 1.47 + - 0.5815 + - 0.6782 + - 2.13 + - 0.5445 + - 1.52 + - 0.5555 + - 1.96 + - 2.31 + - 2.33 + - 2.32 + - 2.31 + - 0.6994 + - 0.7006 + - 0.706 + - 0.9785 + - 0.7023 + - 0.6991 + - 0.6946 + - 2.28 + - 2.26 + - 2.26 + - 2.26 + - 0.6608 + - 0.6835 + - 0.6822 + - 2.24 + - 2.26 + - 0.6735 + - 2.28 + - 0.667 + - 0.7732 + - 0.6642 + - 0.648 + - 0.6629 + - 2.29 + - 2.29 + - 0.6799 + - 0.623 + - 2.32 + - 0.603 + - 0.6224 + - 2.32 + - 0.6187 + - 2.31 + - 2.31 + - 2.28 + - 2.29 + - 2.28 + - 2.26 + - 1.966 + - 2.27 + - 2.26 + - 2.25 + - 2.27 + - 2.24 + - 2.21 + - 2.24 + - 2.17 + - 2.18 + - 2.17 + - 2.21 + - 1.4 + - 2.16 + - 2.2 + - 2.13 + - 2.12 + - 2.13 + - 2.1 + - 2.12 + - 2.11 + - 2.09 + - 2.09 + - 2.08 + - 2.09 + - 2.04 + - 2.04 + - 1.966 + - 2.01 + - 2.05 + - 2.03 + - 2.06 + - 1.98 + - 1.95 + - 1.94 + - 1.91 + - 1.857 + - 1.76 + - 1.748 + - 1.83 + - 2.04 + - 1.748 + - 1.99 + - 2.075 + - 2.07 + - 2.02 + - 2.04 + - 2.1 + - 1.966 + - 2.18 + - 2.21 + - 2.24 + - 2.23 + - 2.23 + - 1.98 + - 2.2 + - 2.18 + - 2.18 + - 2.21 + - 2.23 + - 2.24 + - 2.24 + - 2.25 + - 1.8 + - 2.24 + - 1.73 + - 1.73 + - 2.27 + - 1.67 + - 2.21 + - 1.72 + - 2.23 + - 2.23 + - 2.23 + - 2.24 + - 2.23 + - 2.12 + - 2.17 + - 1.74 + - 2.02 + - 1.88 + - 1.67 + - 1.73 + - 1.83 + - 1.82 + - 1.73 + - 1.83 + - 2.19 + - 1.84 + - 1.89 + - 1.6 + - 1.71 + - 1.86 + - 1.85 + - 1.84 + - 1.87 + - 1.91 + - 1.52 + - 1.95 + - 1.87 + - 1.89 + - 1.91 + - 1.91 + - 1.93 + - 1.9 + - 1.91 + - 1.9 + - 1.89 + - 1.89 + - 1.91 + - 1.9 + - 1.91 + - 1.91 + - 1.91 + - 1.93 + - 1.94 + - 1.91 + - 1.92 + - 1.77 + - 1.91 + - 1.95 + - 1.19 + - 1.96 + - 1.98 + - 1.94 + - 1.55 + - 1.91 + - 1.92 + - 1.92 + - 1.97 + - 1.93 + - 1.99 + - 1.86 + - 1.12 + - 1.93 + - 1.92 + - 1.95 + - 1.85 + - 1.84 + - 1.91 + - 1.12 + - 1.82 + - 1.82 + - 1.95 + - 1.24 + - 1.94 + - 1.96 + - 1.21 + - 1.83 + - 1.96 + - 1.36 + - 1.96 + - 1.82 + - 1.92 + - 1.68 + - 1.93 + - 1.23 + - 1.96 + - 1.93 + - 1.86 + - 1.41 + - 1.16 + - 1.6 + - 1.25 + - 1.2 + - 1.65 + - 1.66 + - 1.87 + - 1.94 + - 1.96 + - 1.91 + - 1.25 + - 1.93 + - 1.91 + - 1.7 + - 0.99 + - 1.81 + - 1.92 + - 1.95 + - 1.5 + - 1.47 + - 1.15 + - 1.58 + - 1.18 + - 1.82 + - 1.13 + - 1.83 + - 1.91 + - 1.26 + - 1.27 + - 1.91 + - 1.45 + - 1.6 + - 1.29 + - 1.94 + - 1.94 + - 1.23 + - 1.95 + - 1.21 + - 1.94 + - 1.86 + - 1.9 + - 1.33 + - 1.75 + - 2.02 + - 1.98 + - 2.03 + - 1.83 + - 1.5 + - 2.04 + - 2.02 + - 1.9 + - 2.0 + - 2.02 + - 1.95 + - 1.93 + - 1.95 + - 1.95 + - 1.99 + - 2.0 + - 1.94 + - 1.96 + - 1.86 + - 1.92 + - 1.88 + - 1.86 + - 1.84 + - 1.87 + - 1.77 + - 1.89 + - 1.89 + - 1.88 + - 1.94 + - 1.82 + - 1.79 + - 1.86 + - 2.06 + - 2.33 + - 1.88 + - 1.86 + - 1.81 + - 1.8 + - 1.8 + - 1.86 + - 1.9 + - 2.0 + - 2.06 + - 2.1 + - 2.2 + - 2.0 + - 2.16 + - 1.98 + - 1.8 + - 1.8 + - 1.85 + - 1.75 + - 2.04 + - 2.19 + - 2.14 + - 2.19 + - 1.86 + - 2.1 + - 2.11 + - 2.18 + - 2.03 + - 2.28 + - 2.19 + - 2.26 + - 2.26 + - 2.21 + - 2.21 + - 2.26 + - 2.33 + - 2.27 + - 2.21 + - 2.12 + - 2.23 + - 2.26 + - 2.25 + - 1.88 + - 2.26 + - 2.24 + - 2.36 + - 2.29 + - 2.35 + - 2.3 + - 2.27 + - 2.08 + - 2.05 + - 2.27 + - 2.28 + - 2.27 + - 2.28 + - 1.97 + - 2.25 + - 2.25 + - 2.25 + - 2.31 + - 2.28 + - 2.27 + - 2.13 + - 2.24 + - 2.28 + - 2.28 + - 2.41 + - 2.34 + - 9.32 + - 2.28 + - 2.38 + - 2.27 + - 2.27 + - 2.39 + - 2.11 + - 2.09 + - 2.1 + - 2.06 + - 2.12 + - 2.08 + - 2.0 + - 1.93 + - 2.02 + - 2.55 + - 1.54 + - 1.64 + - 1.51 + - 1.55 + - 2.82 + - 2.92 + - 2.55 + - 2.37 + - 1.85 + - 1.6 + - 1.72 + - 1.74 + - 1.79 + - 1.9 + - 1.94 + - 2.0 + - 2.04 + - 2.08 + - 2.12 + - 2.13 + - 2.16 + - 2.18 + - 2.18 + - 2.2 + - 2.2 + - 2.41 + - 2.39 + - 2.38 + - 2.4 + - 2.42 + - 2.41 + - 2.43 + - 2.45 + - 2.43 + - 2.45 + - 2.43 + - 2.4 + - 2.44 + - 2.4 + - 2.42 + - 2.43 + - 2.45 + - 2.45 + - 2.45 + - 2.46 + - 2.45 + - 2.45 + - 2.43 + - 2.51 + - 2.48 + - 2.48 + - 2.53 + - 2.46 + - 2.49 + - 2.5 + - 2.5 + - 2.5 + - 2.52 + - 2.52 + - 2.54 + - 2.5 + - 2.48 + - 2.5 + - 2.55 + - 2.5 + - 2.48 + - 2.5 + - 2.5 + - 2.52 + - 2.52 + - 2.48 + - 2.5 + - 2.5 + - 2.52 + - 2.46 + - 2.53 + - 9.0 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, 7076, + 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, 7267, 7269, 7284, + 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, 7475, 7549, 7584, + 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, 7950, 7972, 7980, + 7995, 8007, 8015, 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: iasi_metop-b + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + error parameter vector: *id015 + use_flag: None + use_flag_clddet: &id016 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + error parameter vector: *id015 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: iasi_metop-b + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + error parameter vector: *id015 + obserr_bound_max: + - 3.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 4.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 4.0 + - 4.0 + - 3.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.75 + - 2.0 + - 0.75 + - 2.0 + - 2.0 + - 2.0 + - 0.75 + - 0.75 + - 0.75 + - 0.75 + - 2.0 + - 0.75 + - 0.75 + - 2.0 + - 0.75 + - 0.75 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 2.0 + - 2.5 + - 2.5 + - 3.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 3.5 + - 2.5 + - 2.5 + - 3.0 + - 3.5 + - 3.0 + - 4.0 + - 4.0 + - 0.75 + - 4.0 + - 4.0 + - 4.0 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.0 + - 4.5 + - 4.0 + - 4.0 + - 4.5 + - 2.5 + - 3.0 + - 2.5 + - 3.0 + - 2.5 + - 3.0 + - 2.0 + - 2.5 + - 2.5 + - 3.0 + - 3.0 + - 2.5 + - 3.0 + - 3.0 + - 3.0 + - 2.5 + - 2.5 + - 4.0 + - 4.5 + - 4.5 + - 5.0 + - 4.0 + - 4.0 + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 5.5 + - 5.5 + - 4.0 + - 5.0 + - 4.0 + - 4.5 + - 5.5 + - 5.5 + - 6.0 + - 4.5 + - 4.5 + - 4.0 + - 5.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.25 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.25 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.4 + - 6.0 + - 1.4 + - 6.0 + - 6.0 + - 1.4 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.5 + - 4.5 + - 6.0 + - 5.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 4.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 4.5 + - 6.0 + - 6.0 + - 4.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id016 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: iasi_metop-b + - obs space: + name: IASI METOP-C + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/iasi_metop-c.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.iasi_metop-c.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: iasi_metop-c + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/iasi_metop-c.20231009T150000Z.satbias.nc4 + output file: cycle_dir/iasi_metop-c.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/iasi_metop-c.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/iasi_metop-c.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/iasi_metop-c.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/iasi_metop-c.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/iasi_metop_141_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id017 + - 0.727 + - 0.81 + - 0.75 + - 0.79 + - 0.7055 + - 0.74 + - 0.68 + - 0.72 + - 0.6526 + - 0.65 + - 0.665 + - 0.69 + - 0.6394 + - 0.64 + - 0.6528 + - 0.6065 + - 0.6246 + - 0.61 + - 0.6423 + - 0.5995 + - 0.59 + - 0.6069 + - 0.6 + - 0.5965 + - 0.64 + - 0.62 + - 0.589 + - 0.5865 + - 0.65 + - 0.5861 + - 0.61 + - 0.5874 + - 0.68 + - 0.606 + - 0.68 + - 4.38 + - 3.05 + - 2.31 + - 1.56 + - 1.33 + - 1.58 + - 0.93 + - 0.5832 + - 0.5587 + - 0.5867 + - 0.58 + - 0.5655 + - 0.5522 + - 0.5864 + - 0.5475 + - 0.5854 + - 0.5455 + - 0.5811 + - 0.5376 + - 0.5452 + - 0.5686 + - 0.5329 + - 0.5655 + - 0.5302 + - 0.545 + - 0.5628 + - 0.59 + - 0.5262 + - 0.559 + - 0.5264 + - 0.5442 + - 0.51 + - 0.5513 + - 0.5224 + - 0.5523 + - 0.5188 + - 0.5487 + - 0.5245 + - 0.58 + - 0.5437 + - 0.5343 + - 0.5364 + - 0.64 + - 0.5338 + - 0.72 + - 0.537 + - 0.75 + - 0.51 + - 0.65 + - 0.5274 + - 0.529 + - 0.5187 + - 0.5228 + - 1.12 + - 0.5222 + - 0.5109 + - 0.67 + - 0.5133 + - 0.5179 + - 0.507 + - 0.67 + - 0.5091 + - 0.62 + - 0.5093 + - 0.69 + - 0.5048 + - 0.5024 + - 0.78 + - 0.497 + - 0.5337 + - 0.4865 + - 0.4915 + - 0.4835 + - 0.4869 + - 0.87 + - 0.4824 + - 0.4852 + - 0.84 + - 0.84 + - 0.84 + - 0.5318 + - 0.8 + - 0.4772 + - 0.98 + - 0.488 + - 0.4978 + - 0.5157 + - 0.61 + - 0.5213 + - 0.4884 + - 0.79 + - 0.62 + - 0.66 + - 0.4691 + - 0.65 + - 0.4809 + - 0.468 + - 0.62 + - 0.4679 + - 0.6913 + - 0.4705 + - 0.4785 + - 0.47 + - 0.4773 + - 0.4703 + - 0.98 + - 0.4697 + - 0.4662 + - 0.65 + - 0.467 + - 0.4883 + - 0.4684 + - 0.4684 + - 0.4947 + - 0.5393 + - 0.5024 + - 0.4715 + - 0.621 + - 0.6136 + - 0.5316 + - 1.78 + - 0.5099 + - 1.14 + - 0.539 + - 1.79 + - 0.508 + - 0.5723 + - 1.94 + - 2.01 + - 0.49 + - 0.5647 + - 0.5022 + - 1.47 + - 0.5815 + - 0.6782 + - 2.13 + - 0.5445 + - 1.52 + - 0.5555 + - 1.96 + - 2.31 + - 2.33 + - 2.32 + - 2.31 + - 0.6994 + - 0.7006 + - 0.706 + - 0.9785 + - 0.7023 + - 0.6991 + - 0.6946 + - 2.28 + - 2.26 + - 2.26 + - 2.26 + - 0.6608 + - 0.6835 + - 0.6822 + - 2.24 + - 2.26 + - 0.6735 + - 2.28 + - 0.667 + - 0.7732 + - 0.6642 + - 0.648 + - 0.6629 + - 2.29 + - 2.29 + - 0.6799 + - 0.623 + - 2.32 + - 0.603 + - 0.6224 + - 2.32 + - 0.6187 + - 2.31 + - 2.31 + - 2.28 + - 2.29 + - 2.28 + - 2.26 + - 1.966 + - 2.27 + - 2.26 + - 2.25 + - 2.27 + - 2.24 + - 2.21 + - 2.24 + - 2.17 + - 2.18 + - 2.17 + - 2.21 + - 1.4 + - 2.16 + - 2.2 + - 2.13 + - 2.12 + - 2.13 + - 2.1 + - 2.12 + - 2.11 + - 2.09 + - 2.09 + - 2.08 + - 2.09 + - 2.04 + - 2.04 + - 1.966 + - 2.01 + - 2.05 + - 2.03 + - 2.06 + - 1.98 + - 1.95 + - 1.94 + - 1.91 + - 1.857 + - 1.76 + - 1.748 + - 1.83 + - 2.04 + - 1.748 + - 1.99 + - 2.075 + - 2.07 + - 2.02 + - 2.04 + - 2.1 + - 1.966 + - 2.18 + - 2.21 + - 2.24 + - 2.23 + - 2.23 + - 1.98 + - 2.2 + - 2.18 + - 2.18 + - 2.21 + - 2.23 + - 2.24 + - 2.24 + - 2.25 + - 1.8 + - 2.24 + - 1.73 + - 1.73 + - 2.27 + - 1.67 + - 2.21 + - 1.72 + - 2.23 + - 2.23 + - 2.23 + - 2.24 + - 2.23 + - 2.12 + - 2.17 + - 1.74 + - 2.02 + - 1.88 + - 1.67 + - 1.73 + - 1.83 + - 1.82 + - 1.73 + - 1.83 + - 2.19 + - 1.84 + - 1.89 + - 1.6 + - 1.71 + - 1.86 + - 1.85 + - 1.84 + - 1.87 + - 1.91 + - 1.52 + - 1.95 + - 1.87 + - 1.89 + - 1.91 + - 1.91 + - 1.93 + - 1.9 + - 1.91 + - 1.9 + - 1.89 + - 1.89 + - 1.91 + - 1.9 + - 1.91 + - 1.91 + - 1.91 + - 1.93 + - 1.94 + - 1.91 + - 1.92 + - 1.77 + - 1.91 + - 1.95 + - 1.19 + - 1.96 + - 1.98 + - 1.94 + - 1.55 + - 1.91 + - 1.92 + - 1.92 + - 1.97 + - 1.93 + - 1.99 + - 1.86 + - 1.12 + - 1.93 + - 1.92 + - 1.95 + - 1.85 + - 1.84 + - 1.91 + - 1.12 + - 1.82 + - 1.82 + - 1.95 + - 1.24 + - 1.94 + - 1.96 + - 1.21 + - 1.83 + - 1.96 + - 1.36 + - 1.96 + - 1.82 + - 1.92 + - 1.68 + - 1.93 + - 1.23 + - 1.96 + - 1.93 + - 1.86 + - 1.41 + - 1.16 + - 1.6 + - 1.25 + - 1.2 + - 1.65 + - 1.66 + - 1.87 + - 1.94 + - 1.96 + - 1.91 + - 1.25 + - 1.93 + - 1.91 + - 1.7 + - 0.99 + - 1.81 + - 1.92 + - 1.95 + - 1.5 + - 1.47 + - 1.15 + - 1.58 + - 1.18 + - 1.82 + - 1.13 + - 1.83 + - 1.91 + - 1.26 + - 1.27 + - 1.91 + - 1.45 + - 1.6 + - 1.29 + - 1.94 + - 1.94 + - 1.23 + - 1.95 + - 1.21 + - 1.94 + - 1.86 + - 1.9 + - 1.33 + - 1.75 + - 2.02 + - 1.98 + - 2.03 + - 1.83 + - 1.5 + - 2.04 + - 2.02 + - 1.9 + - 2.0 + - 2.02 + - 1.95 + - 1.93 + - 1.95 + - 1.95 + - 1.99 + - 2.0 + - 1.94 + - 1.96 + - 1.86 + - 1.92 + - 1.88 + - 1.86 + - 1.84 + - 1.87 + - 1.77 + - 1.89 + - 1.89 + - 1.88 + - 1.94 + - 1.82 + - 1.79 + - 1.86 + - 2.06 + - 2.33 + - 1.88 + - 1.86 + - 1.81 + - 1.8 + - 1.8 + - 1.86 + - 1.9 + - 2.0 + - 2.06 + - 2.1 + - 2.2 + - 2.0 + - 2.16 + - 1.98 + - 1.8 + - 1.8 + - 1.85 + - 1.75 + - 2.04 + - 2.19 + - 2.14 + - 2.19 + - 1.86 + - 2.1 + - 2.11 + - 2.18 + - 2.03 + - 2.28 + - 2.19 + - 2.26 + - 2.26 + - 2.21 + - 2.21 + - 2.26 + - 2.33 + - 2.27 + - 2.21 + - 2.12 + - 2.23 + - 2.26 + - 2.25 + - 1.88 + - 2.26 + - 2.24 + - 2.36 + - 2.29 + - 2.35 + - 2.3 + - 2.27 + - 2.08 + - 2.05 + - 2.27 + - 2.28 + - 2.27 + - 2.28 + - 1.97 + - 2.25 + - 2.25 + - 2.25 + - 2.31 + - 2.28 + - 2.27 + - 2.13 + - 2.24 + - 2.28 + - 2.28 + - 2.41 + - 2.34 + - 9.32 + - 2.28 + - 2.38 + - 2.27 + - 2.27 + - 2.39 + - 2.11 + - 2.09 + - 2.1 + - 2.06 + - 2.12 + - 2.08 + - 2.0 + - 1.93 + - 2.02 + - 2.55 + - 1.54 + - 1.64 + - 1.51 + - 1.55 + - 2.82 + - 2.92 + - 2.55 + - 2.37 + - 1.85 + - 1.6 + - 1.72 + - 1.74 + - 1.79 + - 1.9 + - 1.94 + - 2.0 + - 2.04 + - 2.08 + - 2.12 + - 2.13 + - 2.16 + - 2.18 + - 2.18 + - 2.2 + - 2.2 + - 2.41 + - 2.39 + - 2.38 + - 2.4 + - 2.42 + - 2.41 + - 2.43 + - 2.45 + - 2.43 + - 2.45 + - 2.43 + - 2.4 + - 2.44 + - 2.4 + - 2.42 + - 2.43 + - 2.45 + - 2.45 + - 2.45 + - 2.46 + - 2.45 + - 2.45 + - 2.43 + - 2.51 + - 2.48 + - 2.48 + - 2.53 + - 2.46 + - 2.49 + - 2.5 + - 2.5 + - 2.5 + - 2.52 + - 2.52 + - 2.54 + - 2.5 + - 2.48 + - 2.5 + - 2.55 + - 2.5 + - 2.48 + - 2.5 + - 2.5 + - 2.52 + - 2.52 + - 2.48 + - 2.5 + - 2.5 + - 2.52 + - 2.46 + - 2.53 + - 9.0 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, 7076, + 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, 7267, 7269, 7284, + 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, 7475, 7549, 7584, + 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, 7950, 7972, 7980, + 7995, 8007, 8015, 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: iasi_metop-c + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + error parameter vector: *id017 + use_flag: None + use_flag_clddet: &id018 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + error parameter vector: *id017 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: iasi_metop-c + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + error parameter vector: *id017 + obserr_bound_max: + - 3.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 4.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 4.0 + - 4.0 + - 3.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.75 + - 2.0 + - 0.75 + - 2.0 + - 2.0 + - 2.0 + - 0.75 + - 0.75 + - 0.75 + - 0.75 + - 2.0 + - 0.75 + - 0.75 + - 2.0 + - 0.75 + - 0.75 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 2.0 + - 2.5 + - 2.5 + - 3.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 3.5 + - 2.5 + - 2.5 + - 3.0 + - 3.5 + - 3.0 + - 4.0 + - 4.0 + - 0.75 + - 4.0 + - 4.0 + - 4.0 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.0 + - 4.5 + - 4.0 + - 4.0 + - 4.5 + - 2.5 + - 3.0 + - 2.5 + - 3.0 + - 2.5 + - 3.0 + - 2.0 + - 2.5 + - 2.5 + - 3.0 + - 3.0 + - 2.5 + - 3.0 + - 3.0 + - 3.0 + - 2.5 + - 2.5 + - 4.0 + - 4.5 + - 4.5 + - 5.0 + - 4.0 + - 4.0 + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 5.5 + - 5.5 + - 4.0 + - 5.0 + - 4.0 + - 4.5 + - 5.5 + - 5.5 + - 6.0 + - 4.5 + - 4.5 + - 4.0 + - 5.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.25 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.25 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.4 + - 6.0 + - 1.4 + - 6.0 + - 6.0 + - 1.4 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.5 + - 4.5 + - 6.0 + - 5.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 4.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 4.5 + - 6.0 + - 6.0 + - 4.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id018 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: iasi_metop-c + - obs space: + name: MHS METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mhs_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mhs_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: mhs_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/mhs_metop-b.20231009T150000Z.satbias.nc4 + output file: cycle_dir/mhs_metop-b.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/mhs_metop-b.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/mhs_metop-b.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/mhs_metop-b.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/mhs_metop-b.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 550.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: mhs_metop-b + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: mhs_metop-b + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-b + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-b + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-b + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-b + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + observation_name: mhs_metop-b + - obs space: + name: MHS METOP-C + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mhs_metop-c.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mhs_metop-c.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: mhs_metop-c + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/mhs_metop-c.20231009T150000Z.satbias.nc4 + output file: cycle_dir/mhs_metop-c.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/mhs_metop-c.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/mhs_metop-c.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/mhs_metop-c.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/mhs_metop-c.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 550.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: mhs_metop-c + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: mhs_metop-c + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-c + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-c + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-c + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-c + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + observation_name: mhs_metop-c + - obs space: + name: MHS NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mhs_n19.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mhs_n19.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: mhs_n19 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/mhs_n19.20231009T150000Z.satbias.nc4 + output file: cycle_dir/mhs_n19.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/mhs_n19.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/mhs_n19.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/mhs_n19.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/mhs_n19.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 550.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: mhs_n19 + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: mhs_n19 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_n19 + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_n19 + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_n19 + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_n19 + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + observation_name: mhs_n19 + - obs space: + name: MLS55 AURA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mls55_aura.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mls55_aura.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - ozoneProfile + obs operator: + name: VertInterp + vertical coordinate: air_pressure + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + obs filters: + - filter: Bounds Check + filter variables: + - name: ozoneProfile + minvalue: 0 + maxvalue: 10000 + action: + name: reject + - filter: Background Check + filter variables: + - name: ozoneProfile + threshold: 5.0 + action: + name: reject + observation_name: mls55_aura + - obs space: + name: OMI AURA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/omi_aura.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.omi_aura.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - ozoneTotal + obs operator: + name: AtmVertInterpLay + geovals: + - mole_fraction_of_ozone_in_air + coefficients: + - 0.0078976797 + nlevels: + - 1 + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + obs filters: + - filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 5.0 + - filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: ozoneTotal + test variables: + - name: MetaData/solarZenithAngle + maxvalue: 84.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: ozoneTotal + test variables: + - name: MetaData/sensorScanPosition + minvalue: 3 + maxvalue: 24 + action: + name: reject + - filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 5.0 + action: + name: reject + observation_name: omi_aura + - obs space: + name: OMPSNM NPP + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/ompsnm_npp.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.ompsnm_npp.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - ozoneTotal + obs operator: + name: AtmVertInterpLay + geovals: + - mole_fraction_of_ozone_in_air + coefficients: + - 0.0078976797 + nlevels: + - 1 + obs filters: + - filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 5.216 + - filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: ozoneTotal + test variables: + - name: MetaData/solarZenithAngle + maxvalue: 84.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 5.0 + action: + name: reject + observation_name: ompsnm_npp + - obs space: + name: Pilot Balloon + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/pibal.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.pibal.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + vertical coordinate backup: air_pressure + observation vertical coordinate group backup: MetaData + observation vertical coordinate backup: pressure + interpolation method backup: log-linear + hofx scaling field: SurfaceWindScalingCombined + hofx scaling field group: DerivedVariables + linear obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + vertical coordinate backup: air_pressure + observation vertical coordinate group backup: MetaData + observation vertical coordinate backup: pressure + interpolation method backup: log-linear + hofx scaling field: SurfaceWindScalingCombined + hofx scaling field group: DerivedVariables + obs pre filters: + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_not_in: 221 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_not_in: 221 + action: + name: reject + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingCombined + SkipWhenNoObs: false + obs post filters: + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.2 + - 2.2 + - 2.3 + - 2.3 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.5 + - 2.7 + - 2.9 + - 3.1 + - 3.3 + - 3.5 + - 3.7 + - 3.9 + - 4.1 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + value: is_valid + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + value: is_valid + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 221 + cgross: + - 8.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windEastward + action: + name: reject + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 221 + cgross: + - 8.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windNorthward + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + test variables: + - name: ObsErrorData/windEastward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windEastward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windEastward + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windNorthward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windNorthward + defer to post: true + - filter: Bounds Check + filter variables: + - name: windNorthward + test variables: + - name: ObsErrorData/windNorthward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 221 + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 221 + defer to post: true + observation_name: pibal + - obs space: + name: Satellite Winds + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/satwind.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.satwind.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + linear obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + obs prior filters: + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + obs post filters: + - filter: PreQC + maxvalue: 3 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 80000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - filter: BlackList + where: + - variable: + name: ObsType/windEastward + is_in: 240, 241, 248, 249, 251, 255, 256, 260 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.45 + - 5 + - 5 + - 5 + - 5.075 + - 5.175 + - 5.3 + - 5.675 + - 6.05 + - 6.25 + - 6.625 + - 7 + - 7 + - 7 + - 7 + where: + - variable: + name: ObsType/windEastward + is_in: 244 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.9 + - 3.9 + - 4.0 + - 4.0 + - 4.1 + - 5 + - 6 + - 6.3 + - 6.6 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + where: + - variable: + name: ObsType/windEastward + is_in: 245, 246 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.9 + - 3.9 + - 4.0 + - 4.0 + - 4.1 + - 5 + - 6 + - 6.3 + - 6.6 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + where: + - variable: + name: ObsType/windEastward + is_in: 242, 243, 247, 252, 253 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.9 + - 3.9 + - 4.0 + - 4.5 + - 6.1 + - 6.0 + - 6.5 + - 7.3 + - 7.6 + - 7 + - 7.5 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + where: + - variable: + name: ObsType/windEastward + is_in: 254 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.9 + - 3.9 + - 4.0 + - 4.0 + - 4.1 + - 5.0 + - 7 + - 7.3 + - 7.6 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + where: + - variable: + name: ObsType/windEastward + is_in: 250 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.675 + - 5.45 + - 5.525 + - 5.8 + - 6.275 + - 6.575 + - 6.825 + - 7.05 + - 7.05 + - 7.05 + - 7.05 + where: + - variable: + name: ObsType/windEastward + is_in: 257 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 6.05 + - 5.625 + - 5.275 + - 5.375 + - 5.925 + - 6.475 + - 6.775 + - 7.05 + - 7.05 + - 7.05 + - 7.05 + where: + - variable: + name: ObsType/windEastward + is_in: 258 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6 + - 5.5 + - 5.5 + - 5.575 + - 6.025 + - 6.4 + - 6.775 + - 7.15 + - 7.15 + - 7.15 + - 7.15 + where: + - variable: + name: ObsType/windEastward + is_in: 259 + - filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/air_pressure_at_surface + value: MetaData/pressure + maxvalue: -11000 + where: + - variable: + name: ObsType/windEastward + is_in: 247 + - variable: + name: MetaData/surfaceQualifier + minvalue: 1 + - filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/air_pressure_at_surface + value: MetaData/pressure + maxvalue: -20000 + where: + - variable: + name: ObsType/windEastward + is_in: 244, 257, 258, 259, 260 + - variable: + name: MetaData/surfaceQualifier + minvalue: 1 + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/WindDirAngleDiff + maxvalue: 50.0 + where: + - variable: + name: ObsType/windEastward + is_in: 247 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/SatWindsLNVDCheck + maxvalue: 3 + where: + - variable: + name: ObsType/windEastward + is_in: 244, 247, 257-260 + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: BlackList + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 15.0 + - 15.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.1 + - 20.1 + - 20.1 + - 20.1 + - 20.1 + cgross: + - 2.5 + - 2.5 + - 2.5 + - 1.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 1.3 + - 2.5 + - 1.5 + - 1.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + wndtype: + - 240 + - 241 + - 242 + - 243 + - 244 + - 245 + - 246 + - 247 + - 248 + - 249 + - 250 + - 251 + - 252 + - 253 + - 254 + - 256 + - 257 + - 258 + - 259 + - 260 + variable: windEastward + action: + name: reject + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + cgross: + - 2.5 + - 2.5 + - 2.5 + - 1.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 1.3 + - 2.5 + - 1.5 + - 1.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 15.0 + - 15.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.1 + - 20.1 + - 20.1 + - 20.1 + - 20.1 + wndtype: + - 240 + - 241 + - 242 + - 243 + - 244 + - 245 + - 246 + - 247 + - 248 + - 249 + - 250 + - 251 + - 252 + - 253 + - 254 + - 256 + - 257 + - 258 + - 259 + - 260 + variable: windNorthward + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + defer to post: true + observation_name: satwind + - obs space: + name: Scatterometer Winds + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/scatwind.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.scatwind.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + linear obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + obs post filters: + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + errors: + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 290 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: BlackList + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 290 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/ScatWindsAmbiguityCheck + maxvalue: 1e-12 + where: + - variable: + name: ObsType/windEastward + is_in: 290 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windEastward + - name: HofX/windEastward + coefs: + - 1.0 + - -1.0 + minvalue: -5.0 + maxvalue: 5.0 + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windNorthward + - name: HofX/windNorthward + coefs: + - 1.0 + - -1.0 + minvalue: -5.0 + maxvalue: 5.0 + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 290 + cgross: + - 5.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windEastward + action: + name: reject + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 290 + cgross: + - 5.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windNorthward + action: + name: reject + defer to post: true + observation_name: scatwind + - obs space: + name: Surface Marine Stations + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sfcship.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sfcship.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + - virtualTemperature + - airTemperature + - specificHumidity + - stationPressure + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - name: Identity + variables: + - name: specificHumidity + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: Identity + variables: + - name: stationPressure + - name: specificHumidity + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + obs post filters: + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + minvalue: 1 + action: + name: reject + - filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + errors: + - 100 + - 100 + - 110 + - 120 + - 120 + - 100000000000.0 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + action: + name: inflate error + inflation factor: 0.7 + where: + - variable: ObsType/stationPressure + is_in: + - 180 + - variable: ObsSubType/stationPressure + is_in: + - 0 + - filter: Perform Action + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 300 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 300 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 4.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + - variable: ObsType/stationPressure + is_in: 180,183 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.8 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - variable: ObsType/stationPressure + is_in: 180,183 + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: + - 183 + action: + name: reject + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + errors: + - 1.75 + - 1.95 + - 2.25 + - 1.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - airTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/airTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + - filter: Bounds Check + filter variables: + - name: airTemperature + test variables: + - name: ObsErrorData/airTemperature + maxvalue: 100000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: + - 183 + action: + name: reject + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + errors: + - 1.75 + - 1.95 + - 2.25 + - 1.0 + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - virtualTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: GsiAdjustObsError/virtualTemperature + where: + - variable: + name: GsiAdjustObsError/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: virtualTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/virtualTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/virtualTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/virtualTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/virtualTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: PreUseFlag/virtualTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: TempObsErrorData/virtualTemperature + where: + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: virtualTemperature + defer to post: true + - filter: Bounds Check + filter variables: + - name: virtualTemperature + test variables: + - name: ObsErrorData/virtualTemperature + maxvalue: 100000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: PreUseFlag/specificHumidity + minvalue: 1 + action: + name: reject + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: + - 183 + action: + name: reject + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorSatSpecHumidity + options: + variable: specificHumidity + input_error_name: GsiInputObsError + - filter: BlackList + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: specificHumidity + inflation factor: 8 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: specificHumidity + threshold: 8.0 + action: + name: reject + where: + - variable: PreQC/specificHumidity + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: specificHumidity + threshold: 5.6 + action: + name: reject + where: + - variable: PreQC/specificHumidity + is_in: + - 3 + defer to post: true + - filter: BlackList + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: specificHumidity + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_in: + - 284 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_in: + - 284 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: + - 280 + action: + name: assign error + error parameter: 2.5 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: + - 282 + action: + name: assign error + error parameter: 2.2 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + minvalue: 1 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + minvalue: 1 + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + threshold: 6.0 + action: + name: reject + where: + - variable: PreQC/windEastward + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windEastward + threshold: 4.2 + action: + name: reject + where: + - variable: PreQC/windEastward + is_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + threshold: 6.0 + action: + name: reject + where: + - variable: PreQC/windNorthward + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + threshold: 4.2 + action: + name: reject + where: + - variable: PreQC/windNorthward + is_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + satwndtype: + - 280 + - 282 + - 284 + error_min: + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + cgross: + - 6.0 + - 6.0 + - 6.0 + wndtype: + - 280 + - 282 + - 284 + variable: windEastward + action: + name: reject + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + satwndtype: + - 280 + - 282 + - 284 + error_min: + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + cgross: + - 6.0 + - 6.0 + - 6.0 + wndtype: + - 280 + - 282 + - 284 + variable: windNorthward + action: + name: reject + defer to post: true + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + defer to post: true + observation_name: sfcship + - obs space: + name: Surface Land Stations + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sfc.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sfc.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + - virtualTemperature + - airTemperature + - specificHumidity + - stationPressure + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: specificHumidity + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: stationPressure + - name: specificHumidity + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + obs post filters: + - filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 49999 + action: + name: reject + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + minvalue: 1 + action: + name: reject + - filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 181 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 70000 + - 65000 + - 60000 + - 55000 + errors: + - 100 + - 120 + - 120 + - 100000000000.0 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 187 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + errors: + - 100 + - 130 + - 160 + - 100000000000.0 + - filter: Perform Action + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 300 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 300 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 4.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + - variable: ObsType/stationPressure + is_in: 187 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.8 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - variable: ObsType/stationPressure + is_in: 187 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 3.6 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + - variable: ObsType/stationPressure + is_in: + - 181 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.52 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - variable: ObsType/stationPressure + is_in: + - 181 + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + defer to post: true + - filter: Bounds Check + filter variables: + - name: stationPressure + test variables: + - name: ObsErrorData/stationPressure + maxvalue: 100000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: 181,187 + action: + name: reject + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: 181,187 + action: + name: reject + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: 181,187 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_in: 281,287 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_in: 281,287 + action: + name: reject + observation_name: sfc + - obs space: + name: Radiosondes + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sondes.20231009T210000Z.nc4 + missing file action: error + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sondes.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + - virtualTemperature + - airTemperature + - specificHumidity + - stationPressure + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + - name: VertInterp + variables: + - name: airTemperature + - name: virtualTemperature + - name: specificHumidity + - name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + - name: VertInterp + variables: + - name: airTemperature + - name: virtualTemperature + - name: specificHumidity + - name: Identity + variables: + - name: stationPressure + obs prior filters: + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + obs post filters: + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + minvalue: 1 + action: + name: reject + - filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 60000 + - 55000 + errors: + - 100 + - 100000000000.0 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 300 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 300 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 4.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.8 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: PreUseFlag/specificHumidity + minvalue: 1 + action: + name: reject + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + action: + name: assign error + error function: + name: ObsFunction/ObsErrorSatSpecHumidity + options: + variable: specificHumidity + input_error_name: GsiInputObsError + - filter: BlackList + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: specificHumidity + inflation factor: 8.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/specificHumidity + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/specificHumidity + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: specificHumidity + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 70000 + - 65000 + - 60000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + errors: + - 1.3 + - 1.1 + - 0.9 + - 0.7 + - 0.6 + - 0.6 + - 0.55 + - 0.5 + - 0.5 + - 0.55 + - 0.65 + - 1.1 + - 1.2 + - 1.2 + - 1.4 + - 1.6 + - 1.85 + - 2.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - airTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/airTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 8.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 5.6 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 70000 + - 65000 + - 60000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + errors: + - 1.3 + - 1.1 + - 0.9 + - 0.7 + - 0.6 + - 0.6 + - 0.55 + - 0.5 + - 0.5 + - 0.55 + - 0.65 + - 1.1 + - 1.2 + - 1.2 + - 1.4 + - 1.6 + - 1.85 + - 2.0 + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - virtualTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: GsiAdjustObsError/virtualTemperature + where: + - variable: + name: GsiAdjustObsError/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: virtualTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/virtualTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/virtualTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/virtualTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/virtualTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: PreUseFlag/virtualTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 8.0 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 5.6 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: TempObsErrorData/virtualTemperature + where: + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: virtualTemperature + defer to post: true + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.2 + - 2.2 + - 2.3 + - 2.3 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.5 + - 2.7 + - 2.9 + - 3.1 + - 3.3 + - 3.5 + - 3.7 + - 3.9 + - 4.1 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + value: is_valid + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + value: is_valid + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 220,221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 220,221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 220 + - 221 + cgross: + - 8.0 + - 8.0 + error_min: + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + variable: windEastward + action: + name: reject + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 220 + - 221 + cgross: + - 8.0 + - 8.0 + error_min: + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + variable: windNorthward + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + test variables: + - name: ObsErrorData/windEastward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windEastward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windEastward + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windNorthward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windNorthward + defer to post: true + - filter: Bounds Check + filter variables: + - name: windNorthward + test variables: + - name: ObsErrorData/windNorthward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 220 + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 220 + defer to post: true + observation_name: sondes + - obs space: + name: ssmis_f17 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/ssmis_f17.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.ssmis_f17.20231009T210000Z.nc4 + simulated variables: + - brightnessTemperature + channels: None + io pool: + max pool size: 6 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs options: + Sensor_ID: ssmis_f17 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + obs bias: + input file: cycle_dir/ssmis_f17.20231009T150000Z.satbias.nc4 + output file: cycle_dir/ssmis_f17.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/ssmis_f17.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/ssmis_f17.20231009T150000Z.tlapse.txt + - name: cosineOfLatitudeTimesOrbitNode + - name: sineOfLatitude + - name: emissivityJacobian + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/ssmis_f17.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/ssmis_f17.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 1.0 + - 1.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 2.4 + - 1.27 + - 1.44 + - 3.0 + - 1.34 + - 1.74 + - 3.75 + - 3.0 + - 3.0 + - 2.0 + - 6.4 + - 1.0 + - 1.0 + obs post filters: + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/geopotential_height_at_surface + minvalue: 2000.0 + min_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 1-3,8-18 + where: + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/ice_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/surface_snow_area_fraction + maxvalue: 0.99 + max_exclusive: true + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 1-2, 12-16 + reference: ObsValue/brightnessTemperature_2 + value: HofX/brightnessTemperature_2 + minvalue: -1.5 + maxvalue: 1.5 + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + action: + name: reject + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 9 + reference: ObsValue/brightnessTemperature_9 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: + - -0.485934 + - 0.485934 + - 0.473806 + - -0.473806 + intercept: 271.252327 + maxvalue: 2 + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 10 + reference: ObsValue/brightnessTemperature_10 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: + - -0.413688 + - 0.413688 + - 0.361549 + - -0.361549 + intercept: 272.280341 + maxvalue: 2 + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 11 + reference: ObsValue/brightnessTemperature_11 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: + - -0.400882 + - 0.400882 + - 0.27051 + - -0.27051 + intercept: 278.824902 + maxvalue: 2 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: ssmis_f17 + obserr_demisf: + - 0.01 + - 0.01 + - 0.01 + - 0.01 + - 0.01 + obserr_dtempf: + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor1 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_1 + coefs: + - 2.25 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_1 + threshold: DerivedMetaData/errorInflationFactor1 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor2 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_2 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_2 + threshold: DerivedMetaData/errorInflationFactor2 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor3 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_3 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_3 + threshold: DerivedMetaData/errorInflationFactor3 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor4 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_4 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_4 + threshold: DerivedMetaData/errorInflationFactor4 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor5 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_5 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_5 + threshold: DerivedMetaData/errorInflationFactor5 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor6 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_6 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_6 + threshold: DerivedMetaData/errorInflationFactor6 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor7 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_7 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_7 + threshold: DerivedMetaData/errorInflationFactor7 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor8 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_8 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_8 + threshold: DerivedMetaData/errorInflationFactor8 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor9 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_9 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_9 + threshold: DerivedMetaData/errorInflationFactor9 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_10 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_10 + threshold: DerivedMetaData/errorInflationFactor10 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_11 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_11 + threshold: DerivedMetaData/errorInflationFactor11 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor12 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_12 + coefs: + - 3.6 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_12 + threshold: DerivedMetaData/errorInflationFactor12 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor13 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_13 + coefs: + - 1.905 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_13 + threshold: DerivedMetaData/errorInflationFactor13 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor14 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_14 + coefs: + - 2.16 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_14 + threshold: DerivedMetaData/errorInflationFactor14 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor15 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_15 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_15 + threshold: DerivedMetaData/errorInflationFactor15 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor16 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_16 + coefs: + - 2.01 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_16 + threshold: DerivedMetaData/errorInflationFactor16 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor17 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_17 + coefs: + - 2.61 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_17 + threshold: DerivedMetaData/errorInflationFactor17 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor18 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_18 + coefs: + - 5.625 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_18 + threshold: DerivedMetaData/errorInflationFactor18 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor19 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_19 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_19 + threshold: DerivedMetaData/errorInflationFactor19 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor20 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_20 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_20 + threshold: DerivedMetaData/errorInflationFactor20 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor21 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_21 + coefs: + - 3.0 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_21 + threshold: DerivedMetaData/errorInflationFactor21 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor22 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_22 + coefs: + - 9.6 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_22 + threshold: DerivedMetaData/errorInflationFactor22 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor23 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_23 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_23 + threshold: DerivedMetaData/errorInflationFactor23 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor24 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_24 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_24 + threshold: DerivedMetaData/errorInflationFactor24 + absolute threshold: 6.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: ssmis_f17 +variational: + minimizer: + algorithm: DRPCG + iterations: + - geometry: + fms initialization: + namelist filename: ./fv3-jedi/fv3files/fmsmpp.nml + field table filename: ./fv3-jedi/fv3files/field_table_gmao + akbk: ./fv3-jedi/fv3files/akbk72.nc4 + layout: + - 4 + - 4 + npx: 91 + npy: 91 + npz: 72 + gradient norm reduction: 0.0001 + ninner: 5 + diagnostics: + departures: ombg + online diagnostics: + write increment: true + increment: + state component: + filetype: auxgrid + gridtype: latlon + datapath: ./ + filename: experiment_id.increment-iter1. + field io names: + eastward_wind: ua + northward_wind: va + air_temperature: t + air_pressure_levels: pe + water_vapor_mixing_ratio_wrt_moist_air: q + cloud_liquid_ice: qi + cloud_liquid_water: ql + rain_water: qr + snow_water: qs + mole_fraction_of_ozone_in_air: o3ppmv + geopotential_height_times_gravity_at_surface: phis + skin_temperature_at_surface: ts + air_pressure_at_surface: ps +final: + diagnostics: + departures: oman + prints: + frequency: PT3H +output: + filetype: cube sphere history + provider: geos + datapath: cycle_dir + filename: experiment_id.analysis.%yyyy%mm%dd_%hh%MM%ssz.nc4 + first: PT0H + frequency: PT1H + field io names: *id019 diff --git a/src/swell/test/jedi_configs/jedi_3dvar_cf_config.yaml b/src/swell/test/jedi_configs/jedi_3dvar_cf_config.yaml new file mode 100644 index 000000000..a319e2227 --- /dev/null +++ b/src/swell/test/jedi_configs/jedi_3dvar_cf_config.yaml @@ -0,0 +1,179 @@ +cost function: + cost type: 3D-Var + jb evaluation: false + time window: + begin: '2023-08-05T15:00:00Z' + end: '2023-08-05T21:00:00Z' + bound to include: begin + geometry: + fms initialization: + namelist filename: cycle_dir/fmsmpp.nml + field table filename: cycle_dir/field_table_gmao + akbk: cycle_dir/akbk72.nc4 + layout: + - 2 + - 2 + npx: 91 + npy: 91 + npz: 72 + analysis variables: + - volume_mixing_ratio_of_no2 + background: + datetime: '2023-08-05T18:00:00Z' + filetype: cube sphere history + provider: geos + max allowable geometry difference: 0.1 + datapath: cycle_dir + filename: bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4 + state variables: + - air_pressure_thickness + - air_pressure_at_surface + - volume_mixing_ratio_of_no2 + - volume_mixing_ratio_of_no + - volume_mixing_ratio_of_o3 + - volume_mixing_ratio_of_co + field io names: &id001 + eastward_wind: ua + northward_wind: va + air_temperature: T + air_pressure_at_surface: PS + air_pressure_thickness: DELP + volume_mixing_ratio_of_no2: NO2 + volume_mixing_ratio_of_no: "NO" + volume_mixing_ratio_of_o3: O3 + volume_mixing_ratio_of_co: CO + background error: + covariance model: SABER + saber central block: + saber block name: ID + observations: + get values: + observers: + - obs space: + name: tempo_no2_tropo + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/tempo_no2_tropo.20230805T150000Z.nc4 + obsdataout: + engine: + type: H5File + allow overwrite: true + obsfile: cycle_dir/experiment_id.tempo_no2_tropo.20230805T150000Z.nc4 + simulated variables: + - nitrogendioxideColumn + observed variables: + - nitrogendioxideColumn + monitoring only: false + obs filters: + - filter: RejectList + where: + - variable: + name: MetaData/cloud_fraction + minvalue: 0.15 + - filter: RejectList + where: + - variable: + name: MetaData/solar_zenith_angle + minvalue: 70 + obs operator: + name: ColumnRetrieval + nlayers_retrieval: 72 + tracer variables: + - volume_mixing_ratio_of_no2 + isApriori: false + isAveragingKernel: true + stretchVertices: topbottom + observation_name: tempo_no2_tropo + - obs space: + name: tropomi_s5p_no2_tropo + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/tropomi_s5p_no2_tropo.20230805T150000Z.nc4 + obsdataout: + engine: + type: H5File + allow overwrite: true + obsfile: cycle_dir/experiment_id.tropomi_s5p_no2_tropo.20230805T150000Z.nc4 + simulated variables: + - nitrogendioxideColumn + observed variables: + - nitrogendioxideColumn + obs filters: + - filter: Domain Check + filter variables: + - name: nitrogendioxideColumn + where: + - variable: + name: MetaData/latitude + minvalue: -65 + maxvalue: 65 + obs operator: + name: ColumnRetrieval + nlayers_retrieval: 34 + tracer variables: + - volume_mixing_ratio_of_no2 + isApriori: false + isAveragingKernel: true + stretchVertices: topbottom + observation_name: tropomi_s5p_no2_tropo +variational: + minimizer: + algorithm: DRPCG + iterations: + - geometry: + fms initialization: + namelist filename: cycle_dir/fmsmpp.nml + field table filename: cycle_dir/field_table_gmao + akbk: cycle_dir/akbk72.nc4 + layout: + - 2 + - 2 + npx: 91 + npy: 91 + npz: 72 + gradient norm reduction: 1e-10 + ninner: 5 + diagnostics: + departures: ombg + online diagnostics: + write increment: true + increment: + state component: + filetype: auxgrid + gridtype: latlon + datapath: ./ + filename: experiment_id.increment-iter1. + field io names: *id001 +final: + diagnostics: + departures: oman + prints: + frequency: PT3H + increment: + geometry: + fms initialization: + namelist filename: cycle_dir/fmsmpp.nml + field table filename: cycle_dir/field_table_gmao + akbk: cycle_dir/akbk72.nc4 + layout: + - 2 + - 2 + npx: 91 + npy: 91 + npz: 72 + output: + state component: + states: + - filetype: cube sphere history + datapath: cycle_dir + filename: experiment_id.inc.%yyyy%mm%dd_%hh%MM%ssz.nc4 +output: + filetype: cube sphere history + provider: geos + datapath: cycle_dir + filename: experiment_id.analysis.%yyyy%mm%dd_%hh%MM%ssz.nc4 + first: PT0H + frequency: PT1H + field io names: *id001 diff --git a/src/swell/test/jedi_configs/jedi_3dvar_marine_config.yaml b/src/swell/test/jedi_configs/jedi_3dvar_marine_config.yaml new file mode 100644 index 000000000..3626dc230 --- /dev/null +++ b/src/swell/test/jedi_configs/jedi_3dvar_marine_config.yaml @@ -0,0 +1,694 @@ +cost function: + cost type: 3D-Var + jb evaluation: false + time window: + begin: '2021-07-01T00:00:00Z' + end: '2021-07-02T00:00:00Z' + bound to include: begin + geometry: + mom6_input_nml: soca/input.nml + fields metadata: soca/fields_metadata.yaml + geom_grid_file: INPUT/soca_gridspec.nc + analysis variables: &id001 + - sea_water_salinity + - sea_water_potential_temperature + - sea_surface_height_above_geoid + - sea_water_cell_thickness + background: + date: '2021-07-01T12:00:00Z' + read_from_file: 1 + basename: ./ + ocn_filename: MOM6.res.20210701T120000Z.nc + state variables: + - sea_water_salinity + - sea_water_potential_temperature + - sea_surface_height_above_geoid + - sea_water_cell_thickness + - ocean_mixed_layer_thickness + - sea_water_depth + background error: + covariance model: SABER + saber central block: + saber block name: diffusion + active variables: + - sea_water_potential_temperature + - sea_water_salinity + - sea_surface_height_above_geoid + read: + groups: + - variables: + - sea_water_potential_temperature + - sea_water_salinity + horizontal: + filepath: background_error_model/new_corr_1p0 + vertical: + levels: 50 + filepath: background_error_model/vt.20210701T120000Z + - variables: + - sea_surface_height_above_geoid + horizontal: + filepath: background_error_model/new_corr_1p5 + date: '2021-07-01T12:00:00Z' + saber outer blocks: + - saber block name: SOCABkgErrFilt + ocean_depth_min: 100 + rescale_bkgerr: 1.0 + efold_z: 2500.0 + - saber block name: SOCAParametricOceanStdDev + temperature: + sst: + filepath: cycle_dir/soca/godas_sst_bgerr.nc + variable: sst_bgerr + unbalanced salinity: {} + unbalanced ssh: {} + linear variable change: + input variables: *id001 + output variables: *id001 + linear variable changes: + - linear variable change name: BalanceSOCA + ksshts: + nlayers: 2 + observations: + get values: + observers: + - obs space: + name: adt_cryosat2n + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_cryosat2n.20210701T000000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_cryosat2n.20210701T000000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: adt_cryosat2n + - obs space: + name: adt_jason3 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_jason3.20210701T000000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_jason3.20210701T000000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: adt_jason3 + - obs space: + name: adt_saral + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_saral.20210701T000000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_saral.20210701T000000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: adt_saral + - obs space: + name: adt_sentinel3a + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_sentinel3a.20210701T000000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_sentinel3a.20210701T000000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: adt_sentinel3a + - obs space: + name: adt_sentinel3b + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_sentinel3b.20210701T000000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_sentinel3b.20210701T000000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: adt_sentinel3b + - obs space: + name: insitu_profile_argo + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/insitu_profile_argo.20210701T000000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.insitu_profile_argo.20210701T000000Z.nc4 + simulated variables: + - waterTemperature + - salinity + obs operator: + name: Composite + components: + - name: InsituTemperature + variables: + - name: waterTemperature + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + variables: + - name: salinity + vertical coordinate: sea_water_depth + observation vertical coordinate: depth + interpolation method: linear + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: ObsError/salinity + minvalue: 0.0001 + - filter: Bounds Check + where: + - variable: + name: ObsValue/salinity + minvalue: 1.0 + maxvalue: 40.0 + - filter: Background Check + threshold: 3.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 2.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: insitu_profile_argo + - obs space: + name: sst_ostia + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sst_ostia.20210701T000000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sst_ostia.20210701T000000Z.nc4 + simulated variables: + - seaSurfaceTemperature + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 + - filter: Domain Check + where: + - variable: + name: ObsError/seaSurfaceTemperature + minvalue: 0.001 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Domain Check + action: + name: passivate + where: + - variable: + name: GeoVaLs/sea_area_fraction + maxvalue: 0.9 + observation_name: sst_ostia + - obs space: + name: sss_smos + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sss_smos.20210701T000000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sss_smos.20210701T000000Z.nc4 + simulated variables: + - seaSurfaceSalinity + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: 0.1 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 2.5 + - filter: Domain Check + action: + name: passivate + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 10.0 + - filter: Domain Check + action: + name: reject + where: + - variable: + name: ObsError/seaSurfaceSalinity + maxvalue: 0.87 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: sss_smos + - obs space: + name: sss_smapv5 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sss_smapv5.20210701T000000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sss_smapv5.20210701T000000Z.nc4 + simulated variables: + - seaSurfaceSalinity + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: 0.1 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 2.5 + - filter: Domain Check + action: + name: passivate + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 10.0 + - filter: Domain Check + action: + name: reject + where: + - variable: + name: ObsError/seaSurfaceSalinity + maxvalue: 0.87 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: sss_smapv5 + - obs space: + name: sst_abi_g16_l3c + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sst_abi_g16_l3c.20210701T000000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sst_abi_g16_l3c.20210701T000000Z.nc4 + simulated variables: + - seaSurfaceTemperature + get values: + time interpolation: linear + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: -1.0 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 5.0 + - filter: Domain Check + where: + - variable: + name: ObsError/seaSurfaceTemperature + minvalue: 1e-08 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Perform Action + action: + name: assign error + error function: + name: ObsFunction/LinearCombination + options: + variables: + - ObsError/seaSurfaceTemperature + coefs: + - 1.0 + observation_name: sst_abi_g16_l3c + - obs space: + name: sst_gmi_l3u + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sst_gmi_l3u.20210701T000000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sst_gmi_l3u.20210701T000000Z.nc4 + simulated variables: + - seaSurfaceTemperature + get values: + time interpolation: linear + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: -1.0 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 5.0 + - filter: Domain Check + where: + - variable: + name: ObsError/seaSurfaceTemperature + minvalue: 1e-08 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Perform Action + action: + name: assign error + error function: + name: ObsFunction/LinearCombination + options: + variables: + - ObsError/seaSurfaceTemperature + coefs: + - 1.0 + observation_name: sst_gmi_l3u + - obs space: + name: sst_viirs_n20_l3u + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sst_viirs_n20_l3u.20210701T000000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sst_viirs_n20_l3u.20210701T000000Z.nc4 + simulated variables: + - seaSurfaceTemperature + get values: + time interpolation: linear + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: -1.0 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 5.0 + - filter: Domain Check + where: + - variable: + name: ObsError/seaSurfaceTemperature + minvalue: 1e-08 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Perform Action + action: + name: assign error + error function: + name: ObsFunction/LinearCombination + options: + variables: + - ObsError/seaSurfaceTemperature + coefs: + - 1.0 + observation_name: sst_viirs_n20_l3u + - obs space: + name: temp_profile_xbt + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/temp_profile_xbt.20210701T000000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.temp_profile_xbt.20210701T000000Z.nc4 + simulated variables: + - waterTemperature + obs operator: + name: InsituTemperature + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: ObsError/waterTemperature + minvalue: 0.001 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 3.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 3.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: temp_profile_xbt +variational: + minimizer: + algorithm: RPCG + iterations: + - geometry: + mom6_input_nml: soca/input.nml + fields metadata: soca/fields_metadata.yaml + geom_grid_file: INPUT/soca_gridspec.nc + gradient norm reduction: 1e-10 + ninner: 5 + diagnostics: + departures: ombg + online diagnostics: + write increment: true + increment: + state component: + datadir: ./ + date: '2021-07-01T00:00:00Z' + exp: experiment_id + type: incr +final: + diagnostics: + departures: oman + prints: + frequency: PT3H +output: + datadir: ./ + exp: experiment_id + type: an diff --git a/src/swell/test/jedi_configs/jedi_3dvar_marine_cycle_config.yaml b/src/swell/test/jedi_configs/jedi_3dvar_marine_cycle_config.yaml new file mode 100644 index 000000000..c2b9d4ba0 --- /dev/null +++ b/src/swell/test/jedi_configs/jedi_3dvar_marine_cycle_config.yaml @@ -0,0 +1,694 @@ +cost function: + cost type: 3D-Var + jb evaluation: false + time window: + begin: '2021-07-01T09:00:00Z' + end: '2021-07-01T15:00:00Z' + bound to include: begin + geometry: + mom6_input_nml: soca/input.nml + fields metadata: soca/fields_metadata.yaml + geom_grid_file: INPUT/soca_gridspec.nc + analysis variables: &id001 + - sea_water_salinity + - sea_water_potential_temperature + - sea_surface_height_above_geoid + - sea_water_cell_thickness + background: + date: '2021-07-01T12:00:00Z' + read_from_file: 1 + basename: ./ + ocn_filename: MOM6.res.20210701T120000Z.nc + state variables: + - sea_water_salinity + - sea_water_potential_temperature + - sea_surface_height_above_geoid + - sea_water_cell_thickness + - ocean_mixed_layer_thickness + - sea_water_depth + background error: + covariance model: SABER + saber central block: + saber block name: diffusion + active variables: + - sea_water_potential_temperature + - sea_water_salinity + - sea_surface_height_above_geoid + read: + groups: + - variables: + - sea_water_potential_temperature + - sea_water_salinity + horizontal: + filepath: background_error_model/new_corr_1p0 + vertical: + levels: 50 + filepath: background_error_model/vt.20210701T120000Z + - variables: + - sea_surface_height_above_geoid + horizontal: + filepath: background_error_model/new_corr_1p5 + date: '2021-07-01T12:00:00Z' + saber outer blocks: + - saber block name: SOCABkgErrFilt + ocean_depth_min: 100 + rescale_bkgerr: 1.0 + efold_z: 2500.0 + - saber block name: SOCAParametricOceanStdDev + temperature: + sst: + filepath: cycle_dir/soca/godas_sst_bgerr.nc + variable: sst_bgerr + unbalanced salinity: {} + unbalanced ssh: {} + linear variable change: + input variables: *id001 + output variables: *id001 + linear variable changes: + - linear variable change name: BalanceSOCA + ksshts: + nlayers: 2 + observations: + get values: + observers: + - obs space: + name: adt_cryosat2n + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_cryosat2n.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_cryosat2n.20210701T090000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: adt_cryosat2n + - obs space: + name: adt_jason3 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_jason3.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_jason3.20210701T090000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: adt_jason3 + - obs space: + name: adt_saral + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_saral.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_saral.20210701T090000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: adt_saral + - obs space: + name: adt_sentinel3a + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_sentinel3a.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_sentinel3a.20210701T090000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: adt_sentinel3a + - obs space: + name: adt_sentinel3b + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/adt_sentinel3b.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.adt_sentinel3b.20210701T090000Z.nc4 + simulated variables: + - absoluteDynamicTopography + obs operator: + name: ADT + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 7.5 + - filter: Background Check + absolute threshold: 0.2 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_floor_depth_below_sea_surface + minvalue: 500 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: adt_sentinel3b + - obs space: + name: insitu_profile_argo + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/insitu_profile_argo.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.insitu_profile_argo.20210701T090000Z.nc4 + simulated variables: + - waterTemperature + - salinity + obs operator: + name: Composite + components: + - name: InsituTemperature + variables: + - name: waterTemperature + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + variables: + - name: salinity + vertical coordinate: sea_water_depth + observation vertical coordinate: depth + interpolation method: linear + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Domain Check + where: + - variable: + name: ObsError/salinity + minvalue: 0.0001 + - filter: Bounds Check + where: + - variable: + name: ObsValue/salinity + minvalue: 1.0 + maxvalue: 40.0 + - filter: Background Check + threshold: 3.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 2.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: insitu_profile_argo + - obs space: + name: sst_ostia + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sst_ostia.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sst_ostia.20210701T090000Z.nc4 + simulated variables: + - seaSurfaceTemperature + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 5.0 + - filter: Domain Check + where: + - variable: + name: ObsError/seaSurfaceTemperature + minvalue: 0.001 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Domain Check + action: + name: passivate + where: + - variable: + name: GeoVaLs/sea_area_fraction + maxvalue: 0.9 + observation_name: sst_ostia + - obs space: + name: sss_smos + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sss_smos.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sss_smos.20210701T090000Z.nc4 + simulated variables: + - seaSurfaceSalinity + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: 0.1 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 2.5 + - filter: Domain Check + action: + name: passivate + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 10.0 + - filter: Domain Check + action: + name: reject + where: + - variable: + name: ObsError/seaSurfaceSalinity + maxvalue: 0.87 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: sss_smos + - obs space: + name: sss_smapv5 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sss_smapv5.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sss_smapv5.20210701T090000Z.nc4 + simulated variables: + - seaSurfaceSalinity + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: 0.1 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 2.5 + - filter: Domain Check + action: + name: passivate + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 10.0 + - filter: Domain Check + action: + name: reject + where: + - variable: + name: ObsError/seaSurfaceSalinity + maxvalue: 0.87 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: sss_smapv5 + - obs space: + name: sst_abi_g16_l3c + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sst_abi_g16_l3c.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sst_abi_g16_l3c.20210701T090000Z.nc4 + simulated variables: + - seaSurfaceTemperature + get values: + time interpolation: linear + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: -1.0 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 5.0 + - filter: Domain Check + where: + - variable: + name: ObsError/seaSurfaceTemperature + minvalue: 1e-08 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Perform Action + action: + name: assign error + error function: + name: ObsFunction/LinearCombination + options: + variables: + - ObsError/seaSurfaceTemperature + coefs: + - 1.0 + observation_name: sst_abi_g16_l3c + - obs space: + name: sst_gmi_l3u + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sst_gmi_l3u.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sst_gmi_l3u.20210701T090000Z.nc4 + simulated variables: + - seaSurfaceTemperature + get values: + time interpolation: linear + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: -1.0 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 5.0 + - filter: Domain Check + where: + - variable: + name: ObsError/seaSurfaceTemperature + minvalue: 1e-08 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Perform Action + action: + name: assign error + error function: + name: ObsFunction/LinearCombination + options: + variables: + - ObsError/seaSurfaceTemperature + coefs: + - 1.0 + observation_name: sst_gmi_l3u + - obs space: + name: sst_viirs_n20_l3u + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sst_viirs_n20_l3u.20210701T090000Z.nc4 + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sst_viirs_n20_l3u.20210701T090000Z.nc4 + simulated variables: + - seaSurfaceTemperature + get values: + time interpolation: linear + obs operator: + name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_marine/observations/obsop_name_map.yaml + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_area_fraction + minvalue: 0.9 + - filter: Bounds Check + minvalue: -1.0 + maxvalue: 40.0 + - filter: Background Check + absolute threshold: 5.0 + - filter: Domain Check + where: + - variable: + name: ObsError/seaSurfaceTemperature + minvalue: 1e-08 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + - filter: Perform Action + action: + name: assign error + error function: + name: ObsFunction/LinearCombination + options: + variables: + - ObsError/seaSurfaceTemperature + coefs: + - 1.0 + observation_name: sst_viirs_n20_l3u + - obs space: + name: temp_profile_xbt + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/temp_profile_xbt.20210701T090000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.temp_profile_xbt.20210701T090000Z.nc4 + simulated variables: + - waterTemperature + obs operator: + name: InsituTemperature + obs error: + covariance model: diagonal + obs filters: + - filter: Domain Check + where: + - variable: + name: ObsError/waterTemperature + minvalue: 0.001 + - filter: Bounds Check + minvalue: -2.0 + maxvalue: 36.0 + - filter: Background Check + threshold: 3.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/sea_surface_temperature + minvalue: 3.0 + - filter: Domain Check + where: + - variable: + name: GeoVaLs/distance_from_coast + minvalue: 100000.0 + observation_name: temp_profile_xbt +variational: + minimizer: + algorithm: RPCG + iterations: + - geometry: + mom6_input_nml: soca/input.nml + fields metadata: soca/fields_metadata.yaml + geom_grid_file: INPUT/soca_gridspec.nc + gradient norm reduction: 1e-10 + ninner: 10 + diagnostics: + departures: ombg + online diagnostics: + write increment: true + increment: + state component: + datadir: ./ + date: '2021-07-01T09:00:00Z' + exp: experiment_id + type: incr +final: + diagnostics: + departures: oman + prints: + frequency: PT3H +output: + datadir: ./ + exp: experiment_id + type: an diff --git a/src/swell/test/jedi_configs/jedi_hofx_cf_config.yaml b/src/swell/test/jedi_configs/jedi_hofx_cf_config.yaml new file mode 100644 index 000000000..59b2f3dd7 --- /dev/null +++ b/src/swell/test/jedi_configs/jedi_hofx_cf_config.yaml @@ -0,0 +1,111 @@ +time window: + begin: '2023-08-05T15:00:00Z' + end: '2023-08-05T21:00:00Z' + bound to include: begin +geometry: + fms initialization: + namelist filename: cycle_dir/fmsmpp.nml + field table filename: cycle_dir/field_table_gmao + akbk: cycle_dir/akbk72.nc4 + layout: + - 2 + - 2 + npx: 91 + npy: 91 + npz: 72 +state: + datetime: '2023-08-05T18:00:00Z' + filetype: cube sphere history + provider: geos + max allowable geometry difference: 0.1 + datapath: cycle_dir + filename: bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4 + state variables: + - air_pressure_thickness + - air_pressure_at_surface + - volume_mixing_ratio_of_no2 + - volume_mixing_ratio_of_no + - volume_mixing_ratio_of_o3 + - volume_mixing_ratio_of_co + field io names: + eastward_wind: ua + northward_wind: va + air_temperature: T + air_pressure_at_surface: PS + air_pressure_thickness: DELP + volume_mixing_ratio_of_no2: NO2 + volume_mixing_ratio_of_no: "NO" + volume_mixing_ratio_of_o3: O3 + volume_mixing_ratio_of_co: CO +observations: + get values: + observers: + - obs space: + name: tempo_no2_tropo + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/tempo_no2_tropo.20230805T150000Z.nc4 + obsdataout: + engine: + type: H5File + allow overwrite: true + obsfile: cycle_dir/experiment_id.tempo_no2_tropo.20230805T150000Z.nc4 + simulated variables: + - nitrogendioxideColumn + observed variables: + - nitrogendioxideColumn + monitoring only: false + obs filters: + - filter: RejectList + where: + - variable: + name: MetaData/cloud_fraction + minvalue: 0.15 + - filter: RejectList + where: + - variable: + name: MetaData/solar_zenith_angle + minvalue: 70 + obs operator: + name: ColumnRetrieval + nlayers_retrieval: 72 + tracer variables: + - volume_mixing_ratio_of_no2 + isApriori: false + isAveragingKernel: true + stretchVertices: topbottom + observation_name: tempo_no2_tropo + - obs space: + name: tropomi_s5p_no2_tropo + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/tropomi_s5p_no2_tropo.20230805T150000Z.nc4 + obsdataout: + engine: + type: H5File + allow overwrite: true + obsfile: cycle_dir/experiment_id.tropomi_s5p_no2_tropo.20230805T150000Z.nc4 + simulated variables: + - nitrogendioxideColumn + observed variables: + - nitrogendioxideColumn + obs filters: + - filter: Domain Check + filter variables: + - name: nitrogendioxideColumn + where: + - variable: + name: MetaData/latitude + minvalue: -65 + maxvalue: 65 + obs operator: + name: ColumnRetrieval + nlayers_retrieval: 34 + tracer variables: + - volume_mixing_ratio_of_no2 + isApriori: false + isAveragingKernel: true + stretchVertices: topbottom + observation_name: tropomi_s5p_no2_tropo diff --git a/src/swell/test/jedi_configs/jedi_hofx_config.yaml b/src/swell/test/jedi_configs/jedi_hofx_config.yaml new file mode 100644 index 000000000..be7a4c0f9 --- /dev/null +++ b/src/swell/test/jedi_configs/jedi_hofx_config.yaml @@ -0,0 +1,19233 @@ +time window: + begin: '2023-10-09T21:00:00Z' + end: '2023-10-10T03:00:00Z' + bound to include: begin +geometry: + fms initialization: + namelist filename: ./fv3-jedi/fv3files/fmsmpp.nml + field table filename: ./fv3-jedi/fv3files/field_table_gmao + akbk: ./fv3-jedi/fv3files/akbk72.nc4 + layout: + - 2 + - 2 + npx: '91' + npy: '91' + npz: '72' +initial condition: + datetime: '2023-10-09T21:00:00Z' + filetype: cube sphere history + provider: geos + compute edge pressure from surface pressure: true + max allowable geometry difference: 0.001 + datapath: cycle_dir + filenames: + - bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4 + - fv3-jedi/bkg/geos.crtmsrf.91.nc4 + state variables: + - eastward_wind + - northward_wind + - air_temperature + - air_pressure_at_surface + - air_pressure_levels + - water_vapor_mixing_ratio_wrt_moist_air + - cloud_liquid_ice + - cloud_liquid_water + - rain_water + - snow_water + - mole_fraction_of_ozone_in_air + - geopotential_height_times_gravity_at_surface + - initial_mass_fraction_of_large_scale_cloud_condensate + - initial_mass_fraction_of_convective_cloud_condensate + - convective_cloud_area_fraction + - fraction_of_ocean + - fraction_of_land + - isotropic_variance_of_filtered_topography + - surface_velocity_scale + - surface_buoyancy_scale + - planetary_boundary_layer_height + - surface_exchange_coefficient_for_momentum + - surface_exchange_coefficient_for_heat + - surface_exchange_coefficient_for_moisture + - KCBL_before_moist + - surface_temp_before_moist + - lower_index_where_Kh_greater_than_2 + - upper_index_where_Kh_greater_than_2 + - fraction_of_lake + - fraction_of_ice + - vtype + - stype + - vfrac + - sheleg + - skin_temperature_at_surface + - soilt + - soilm + - eastward_wind_at_surface + - northward_wind_at_surface + field io names: &id017 + eastward_wind: ua + northward_wind: va + air_temperature: t + air_pressure_at_surface: ps + air_pressure_levels: pe + water_vapor_mixing_ratio_wrt_moist_air: q + cloud_liquid_ice: qi + cloud_liquid_water: ql + rain_water: qr + snow_water: qs + mole_fraction_of_ozone_in_air: o3ppmv + geopotential_height_times_gravity_at_surface: phis + initial_mass_fraction_of_large_scale_cloud_condensate: qls + initial_mass_fraction_of_convective_cloud_condensate: qcn + convective_cloud_area_fraction: cfcn + fraction_of_ocean: frocean + fraction_of_land: frland + isotropic_variance_of_filtered_topography: varflt + surface_velocity_scale: ustar + surface_buoyancy_scale: bstar + planetary_boundary_layer_height: zpbl + surface_exchange_coefficient_for_momentum: cm + surface_exchange_coefficient_for_heat: ct + surface_exchange_coefficient_for_moisture: cq + KCBL_before_moist: kcbl + surface_temp_before_moist: tsm + lower_index_where_Kh_greater_than_2: khl + upper_index_where_Kh_greater_than_2: khu + fraction_of_lake: frlake + fraction_of_ice: frseaice + skin_temperature_at_surface: ts + eastward_wind_at_surface: u10m + northward_wind_at_surface: v10m +observations: + get values: + variable change: + variable change name: Model2GeoVaLs + hydrometeor effective radii method: gsi + tropopause pressure method: gsi + observers: + - obs space: + name: Aircraft Temperature + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/aircraft_temperature.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.aircraft_temperature.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - airTemperature + obs bias: + input file: cycle_dir/aircraft_temperature.20231009T150000Z.acftbias + output file: cycle_dir/aircraft_temperature.20231009T210000Z.acftbias + bc by record: true + variational bc: + predictors: + - name: constant + - name: obsMetadataPredictor + variable: instantaneousAltitudeRate + - name: obsMetadataPredictor + variable: instantaneousAltitudeRate + order: 2 + covariance: + minimal required obs number: 3 + variance range: + - 1e-06 + - 1.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/aircraft_temperature.20231009T150000Z.acftbias_cov + inflation: + ratio: 1.005 + ratio for small dataset: 2.0 + obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - airTemperature + linear obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - airTemperature + obs filters: + - filter: Variable Assignment + where: + - variable: + name: ObsType/airTemperature + is_in: 130 + assignments: + - name: MetaData/stationIdentification + value: 'KX130 ' + - filter: Variable Assignment + where: + - variable: + name: MetaData/instantaneousAltitudeRate + minvalue: 50.0 + assignments: + - name: MetaData/instantaneousAltitudeRate + value: 0.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 2.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + errors: + - 2.5 + - 2.3 + - 2.1 + - 1.9 + - 1.7 + where: + - variable: + name: ObsType/airTemperature + is_in: 130 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + errors: + - 1.4706 + - 1.3529 + - 1.2353 + - 1.1176 + - 1.0 + where: + - variable: + name: ObsType/airTemperature + is_in: 131,133 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 70000 + - 65000 + - 60000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 5000 + errors: + - 1.5 + - 1.3 + - 1.1 + - 0.9 + - 0.8 + - 0.8 + - 0.75 + - 0.7 + - 0.7 + - 0.75 + - 0.85 + - 1.3 + - 1.5 + - 1.5 + where: + - variable: + name: ObsType/airTemperature + is_in: 132 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 60000 + - 40000 + errors: + - 1.5 + - 1.35 + - 1.25 + - 1.1 + - 1.0 + - 1.3 + - 1.7 + where: + - variable: + name: ObsType/airTemperature + is_in: 134 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + errors: + - 1.4706 + - 1.3529 + - 1.2353 + - 1.1176 + - 1000000000.0 + where: + - variable: + name: ObsType/airTemperature + is_in: 135 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: + - airTemperature + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreQC/airTemperature + is_in: 4-15 + action: + name: passivate + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: + name: ObsType/airTemperature + is_in: 134, 135 + action: + name: passivate + defer to post: true + - filter: Perform Action + action: + name: inflate error + inflation factor: 10.0 + filter variables: + - name: airTemperature + where: + - variable: + name: MetaData/pressure + maxvalue: 110000 + minvalue: 50000 + - variable: + name: ObsType/airTemperature + is_in: 130 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + - filter: GOMsaver + filename: cycle_dir/aircraft_temperature-geovals.20231009T210000Z.nc4 + observation_name: aircraft_temperature + get values: + time interpolation: linear + - obs space: + name: Aircraft Wind + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/aircraft_wind.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.aircraft_wind.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + obs prior filters: + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + - filter: GOMsaver + filename: cycle_dir/aircraft_wind-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: PreQC/windEastward + is_in: 4-15 + action: + name: passivate + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: ObsType/windEastward + is_in: 234, 235 + action: + name: passivate + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 80000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 1.4 + - 1.5 + - 1.6 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.3 + - 2.6 + - 2.8 + - 3.0 + - 3.2 + - 2.7 + - 2.4 + - 2.1 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.6 + where: + - variable: + name: ObsType/windEastward + is_in: 230 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.0 + where: + - variable: + name: ObsType/windEastward + is_in: 231 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 2.5 + where: + - variable: + name: ObsType/windEastward + is_in: 233 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.0 + where: + - variable: + name: ObsType/windEastward + is_in: 234, 235 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + errors: + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.2 + - 2.2 + - 2.3 + - 2.3 + - 2.4 + - 2.4 + - 2.5 + - 2.7 + - 2.9 + - 3.1 + where: + - variable: + name: ObsType/windEastward + is_in: 232 + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: + - windEastward + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: + - windNorthward + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 230 + - 231 + - 232 + - 233 + - 234 + - 235 + cgross: + - 6.0 + - 6.5 + - 7.0 + - 7.5 + - 7.5 + - 7.5 + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + variable: windEastward + action: + name: reject + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 230 + - 231 + - 232 + - 233 + - 234 + - 235 + cgross: + - 6.0 + - 6.5 + - 7.0 + - 7.5 + - 7.5 + - 7.5 + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + variable: windNorthward + action: + name: reject + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + defer to post: true + observation_name: aircraft_wind + get values: + time interpolation: linear + - obs space: + name: AIRS AQUA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/airs_aqua.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.airs_aqua.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: airs_aqua + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/airs_aqua.20231009T150000Z.satbias.nc4 + output file: cycle_dir/airs_aqua.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/airs_aqua.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/airs_aqua.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/airs_aqua.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/airs_aqua.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/airs_aqua_119_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.2 + - 1.2 + - 1.3136 + - 1.4 + - 1.4 + - 1.2639 + - 1.4 + - 1.4 + - 1.1802 + - 1.2517 + - 1.1719 + - 1.2 + - 1.1728 + - 1.1442 + - 1.2 + - 1.2 + - 1.15 + - 1.0801 + - 1.15 + - 1.15 + - 1.0396 + - 1.15 + - 1.15 + - 1.15 + - 1.15 + - 1.15 + - 1.15 + - 1.15 + - 0.9946 + - 1.05 + - 0.9217 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.9591 + - 0.9465 + - 0.9593 + - 0.9337 + - 1.0 + - 0.9861 + - 1.0017 + - 1.1 + - 1.0083 + - 1.0024 + - 1.1 + - 0.9967 + - 1.0094 + - 0.9412 + - 1.1 + - 0.998 + - 0.9807 + - 0.857 + - 0.8727 + - 0.8114 + - 0.879 + - 0.871 + - 0.8853 + - 0.7937 + - 0.8243 + - 0.8 + - 0.8016 + - 0.8 + - 0.7781 + - 0.7475 + - 0.85 + - 0.7405 + - 0.715 + - 0.7416 + - 0.7465 + - 0.9 + - 0.7198 + - 0.7157 + - 0.9 + - 0.727 + - 0.7246 + - 0.704 + - 0.7039 + - 0.66 + - 0.6694 + - 0.6669 + - 0.7031 + - 0.6977 + - 0.6488 + - 0.6653 + - 0.9 + - 0.6265 + - 0.622 + - 0.6308 + - 0.6297 + - 0.621 + - 0.6225 + - 0.6229 + - 0.6234 + - 0.6238 + - 0.6332 + - 0.6425 + - 0.7028 + - 0.6152 + - 0.9 + - 0.7257 + - 0.7288 + - 1.15 + - 0.9 + - 0.6673 + - 0.7473 + - 0.6767 + - 0.7056 + - 0.9 + - 0.95 + - 0.7271 + - 0.95 + - 0.725 + - 0.7601 + - 0.6973 + - 0.7573 + - 0.6011 + - 0.606 + - 0.9 + - 0.6635 + - 0.586 + - 0.5766 + - 0.75 + - 2.0386 + - 0.75 + - 1.0 + - 0.9 + - 0.9 + - 0.9 + - 0.9 + - 0.9 + - 0.9 + - 1.0 + - 1.3386 + - 1.0 + - 1.0 + - 0.85 + - 0.95 + - 1.7386 + - 0.95 + - 0.9 + - 0.8 + - 1.7386 + - 0.75 + - 0.75 + - 0.75 + - 0.8 + - 0.75 + - 0.8 + - 0.9 + - 0.75 + - 0.8 + - 0.8 + - 1.1 + - 0.75 + - 1.1 + - 0.75 + - 0.5991 + - 0.5348 + - 0.6541 + - 0.7421 + - 0.6192 + - 0.8186 + - 1.0616 + - 0.8848 + - 1.024 + - 2.5 + - 1.0249 + - 1.0795 + - 1.2199 + - 2.5 + - 2.5 + - 1.3103 + - 1.3603 + - 2.5 + - 2.5 + - 2.5 + - 1.323 + - 2.5 + - 2.5 + - 2.5 + - 1.4406 + - 2.5 + - 2.5 + - 1.3965 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 1.6997 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 1.6264 + - 2.5 + - 2.5 + - 2.5 + - 1.3436 + - 2.5 + - 2.5 + - 0.5727 + - 0.6838 + - 0.5994 + - 0.5178 + - 0.5145 + - 0.547 + - 0.5572 + - 0.5002 + - 0.4974 + - 0.55 + - 0.4953 + - 0.4883 + - 0.4948 + - 0.5446 + - 0.5777 + - 1.5 + - 1.5 + - 3.0 + - 3.0 + - 2.5 + - 2.5 + - 2.0 + - 1.0 + - 1.5 + - 1.5 + - 1.8 + - 0.6 + - 0.7 + - 0.65 + - 0.675 + - 0.7 + - 0.75 + - 0.775 + - 0.8 + - 0.8 + - 0.85 + - 0.85 + - 0.85 + - 0.7 + - 0.7 + - 0.7 + - 0.7 + - 0.7 + - 0.7 + - 0.7 + - 0.725 + - 0.75 + - 0.775 + - 0.8 + - 0.825 + - 0.8 + - 0.8 + - 0.8 + - 0.75 + - 0.8 + - 0.8 + - 0.8 + - 0.8 + - 0.8 + - 0.85 + - 0.8 + - 0.8 + - 2.5 + - 0.75 + - 0.75 + - 0.75 + - 0.75 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + - filter: GOMsaver + filename: cycle_dir/airs_aqua-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 2122, 2123, 2128, 2134, 2141, 2145, 2149, 2153, 2164, 2189, 2197, + 2209, 2226, 2234, 2280, 2318, 2321, 2325, 2328, 2333, 2339, 2348, 2353, + 2355, 2357, 2363, 2370, 2371, 2377 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: airs_aqua + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: &id001 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: airs_aqua + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 1.0 + - 1.0 + - 3.0 + - 1.0 + - 3.0 + - 1.0 + - 1.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 3.0 + - 1.0 + - 1.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 3.0 + - 3.0 + - 3.5 + - 3.0 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 1.7 + - 3.0 + - 1.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.5 + - 1.25 + - 3.5 + - 3.5 + - 3.0 + - 3.0 + - 1.5 + - 3.0 + - 3.0 + - 3.0 + - 1.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.5 + - 3.0 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.5 + - 3.0 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.5 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id001 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: airs_aqua + get values: + time interpolation: linear + - obs space: + name: AMSR2 GCOM-W1 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsr2_gcom-w1.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsr2_gcom-w1.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: amsr2_gcom-w1 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsr2_gcom-w1.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsr2_gcom-w1.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsr2_gcom-w1.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsr2_gcom-w1.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsr2_gcom-w1.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsr2_gcom-w1.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 340.0 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.999 + - variable: + name: GeoVaLs/skin_temperature_at_surface_where_sea + minvalue: 275 + - variable: + name: GeoVaLs/wind_speed_at_surface + maxvalue: 12 + - variable: + name: MetaData/latitude + minvalue: -60.0 + maxvalue: 60.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/TotalColumnVaporGuess + minvalue: 10.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SunGlintAngle + minvalue: 20.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: &id002 + - 0.48 + - 3.0737 + - 0.7433 + - 3.643 + - 3.5304 + - 4.427 + - 5.1448 + - 5.0785 + - 4.9763 + - 9.3215 + - 2.5789 + - 5.5274 + - 0.6641 + - 1.3674 + clwret_types: + - ObsValue + maxvalue: 1.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id002 + clwret_types: + - HofX + maxvalue: 1.0 + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: None + value: + name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id002 + clwret_types: + - ObsValue + reference: + name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id002 + clwret_types: + - HofX + minvalue: -0.5 + maxvalue: 0.5 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id002 + clwret_types: + - ObsValue + - HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.1 + - 0.1 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 0.6 + - 0.6 + - 0.6 + - 0.6 + - 0.6 + - 0.5 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + err0: + - 0.8 + - 0.9 + - 0.8 + - 0.9 + - 1.0 + - 1.1 + - 2.0 + - 3.5 + - 3.0 + - 4.8 + - 5.0 + - 6.0 + - 4.5 + - 6.3 + err1: + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 18.5 + - 20.0 + - 40.0 + - 20.0 + - 25.0 + - 30.0 + - 30.0 + - 30.0 + - 20.0 + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: None + threshold: 2.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 7-10 + absolute threshold: 30 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 11-14 + absolute threshold: 50 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: amsr2_gcom-w1 + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: GOMsaver + filename: cycle_dir/amsr2_gcom-w1-geovals.20231009T210000Z.nc4 + observation_name: amsr2_gcom-w1 + get values: + time interpolation: linear + - obs space: + name: AMSU-A AQUA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_aqua.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_aqua.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_aqua + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_aqua.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_aqua.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_aqua.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_aqua.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_aqua.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_aqua.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id003 + - 2.5 + - 2.0 + - 2.0 + - 0.5 + - 0.4 + - 0.4 + - 0.5 + - 0.3 + - 0.35 + - 0.35 + - 0.45 + - 1.0 + - 1.5 + - 3.75 + - 6.3 + - filter: GOMsaver + filename: cycle_dir/amsua_aqua-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_aqua + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_aqua + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_aqua + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_aqua + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_aqua + error parameter vector: *id003 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 3.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 3.0 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_aqua + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_aqua + get values: + time interpolation: linear + - obs space: + name: AMSU-A METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_metop-b.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_metop-b.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_metop-b.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_metop-b.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_metop-b.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_metop-b.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id004 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + - filter: GOMsaver + filename: cycle_dir/amsua_metop-b-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_metop-b + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_metop-b + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_metop-b + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_metop-b + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_metop-b + error parameter vector: *id004 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_metop-b + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_metop-b + get values: + time interpolation: linear + - obs space: + name: AMSU-A METOP-C + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_metop-c.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_metop-c.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_metop-c + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_metop-c.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_metop-c.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_metop-c.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_metop-c.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_metop-c.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_metop-c.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id005 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + - filter: GOMsaver + filename: cycle_dir/amsua_metop-c-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_metop-c + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_metop-c + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_metop-c + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_metop-c + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_metop-c + error parameter vector: *id005 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_metop-c + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_metop-c + get values: + time interpolation: linear + - obs space: + name: AMSU-A NOAA-15 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_n15.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_n15.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_n15 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_n15.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_n15.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_n15.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_n15.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_n15.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_n15.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id006 + - 3.0 + - 2.0 + - 2.0 + - 0.6 + - 0.3 + - 0.23 + - 0.25 + - 0.275 + - 0.34 + - 0.4 + - 0.6 + - 1.0 + - 1.5 + - 5.0 + - 3.0 + - filter: GOMsaver + filename: cycle_dir/amsua_n15-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_n15 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_n15 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_n15 + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_n15 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_n15 + error parameter vector: *id006 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_n15 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_n15 + get values: + time interpolation: linear + - obs space: + name: AMSU-A NOAA-18 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_n18.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_n18.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_n18 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_n18.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_n18.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_n18.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_n18.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_n18.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_n18.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id007 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + - filter: GOMsaver + filename: cycle_dir/amsua_n18-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_n18 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_n18 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_n18 + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_n18 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_n18 + error parameter vector: *id007 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_n18 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_n18 + get values: + time interpolation: linear + - obs space: + name: AMSU-A NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_n19.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_n19.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_n19 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_n19.20231009T150000Z.satbias.nc4 + output file: cycle_dir/amsua_n19.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_n19.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_n19.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_n19.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_n19.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id008 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + - filter: GOMsaver + filename: cycle_dir/amsua_n19-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_n19 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_n19 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_n19 + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_n19 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_n19 + error parameter vector: *id008 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_n19 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_n19 + get values: + time interpolation: linear + - obs space: + name: ATMS NOAA-20 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/atms_n20.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.atms_n20.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: atms_n20 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/atms_n20.20231009T150000Z.satbias.nc4 + output file: cycle_dir/atms_n20.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: ATMS + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/atms_n20.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/atms_n20.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/atms_n20.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/atms_n20.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id009 + - 5.0 + - 5.0 + - 5.0 + - 3.0 + - 0.55 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 5.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - filter: GOMsaver + filename: cycle_dir/atms_n20-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7,16-22 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-7,16 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: atms_n20 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: atms_n20 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + use_biasterm: true + test_biasterm: ObsBiasTerm + sensor: atms_n20 + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: atms_n20 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: atms_n20 + error parameter vector: *id009 + obserr_bound_max: + - 4.5 + - 4.5 + - 3.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 2.0 + - 4.5 + - 4.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: atms_n20 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: atms_n20 + get values: + time interpolation: linear + - obs space: + name: ATMS NPP + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/atms_npp.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.atms_npp.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: atms_npp + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/atms_npp.20231009T150000Z.satbias.nc4 + output file: cycle_dir/atms_npp.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: ATMS + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/atms_npp.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/atms_npp.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/atms_npp.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/atms_npp.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id010 + - 5.0 + - 5.0 + - 5.0 + - 3.0 + - 0.55 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 5.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - filter: GOMsaver + filename: cycle_dir/atms_npp-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7,16-22 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-7,16 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: atms_npp + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: atms_npp + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + use_biasterm: true + test_biasterm: ObsBiasTerm + sensor: atms_npp + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: atms_npp + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: atms_npp + error parameter vector: *id010 + obserr_bound_max: + - 4.5 + - 4.5 + - 3.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 2.0 + - 4.5 + - 4.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: atms_npp + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: atms_npp + get values: + time interpolation: linear + - obs space: + name: AVHRR-3 METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/avhrr3_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.avhrr3_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: avhrr3_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/avhrr3_metop-b.20231009T150000Z.satbias.nc4 + output file: cycle_dir/avhrr3_metop-b.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/avhrr3_metop-b.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/avhrr3_metop-b.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/avhrr3_metop-b.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/avhrr3_metop-b.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.0 + - 1.08 + - 1.12 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + - filter: GOMsaver + filename: cycle_dir/avhrr3_metop-b-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 1e-05 + maxvalue: 1000.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: avhrr3_metop-b + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + cloud detection criteria: + - 0.6 + - 0.68 + - 0.72 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: avhrr3_metop-b + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: avhrr3_metop-b + get values: + time interpolation: linear + - obs space: + name: AVHRR-3 NOAA-18 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/avhrr3_n18.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.avhrr3_n18.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: avhrr3_n18 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/avhrr3_n18.20231009T150000Z.satbias.nc4 + output file: cycle_dir/avhrr3_n18.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/avhrr3_n18.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/avhrr3_n18.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/avhrr3_n18.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/avhrr3_n18.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.0 + - 1.08 + - 1.12 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + - filter: GOMsaver + filename: cycle_dir/avhrr3_n18-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 1e-05 + maxvalue: 1000.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: avhrr3_n18 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + cloud detection criteria: + - 0.6 + - 0.68 + - 0.72 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: avhrr3_n18 + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: avhrr3_n18 + get values: + time interpolation: linear + - obs space: + name: AVHRR-3 NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/avhrr3_n19.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.avhrr3_n19.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: avhrr3_n19 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/avhrr3_n19.20231009T150000Z.satbias.nc4 + output file: cycle_dir/avhrr3_n19.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/avhrr3_n19.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/avhrr3_n19.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/avhrr3_n19.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/avhrr3_n19.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.0 + - 1.08 + - 1.12 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + - filter: GOMsaver + filename: cycle_dir/avhrr3_n19-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 1e-05 + maxvalue: 1000.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: avhrr3_n19 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + cloud detection criteria: + - 0.6 + - 0.68 + - 0.72 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: avhrr3_n19 + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: avhrr3_n19 + get values: + time interpolation: linear + - obs space: + name: CRIS-FSR NOAA-20 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/cris-fsr_n20.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.cris-fsr_n20.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: cris-fsr_n20 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/cris-fsr_n20.20231009T150000Z.satbias.nc4 + output file: cycle_dir/cris-fsr_n20.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/cris-fsr_n20.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/cris-fsr_n20.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/cris-fsr_n20.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/cris-fsr_n20.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/cris-fsr_108_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 0.823 + - 0.76 + - 0.736 + - 0.743 + - 0.856 + - 1.079 + - 0.888 + - 0.778 + - 0.671 + - 0.65 + - 0.643 + - 0.629 + - 0.629 + - 0.618 + - 0.638 + - 0.619 + - 0.61 + - 0.627 + - 0.601 + - 0.617 + - 0.608 + - 0.498 + - 0.5112 + - 0.4922 + - 0.4959 + - 0.4954 + - 0.4836 + - 0.514 + - 0.5005 + - 0.4917 + - 0.4881 + - 0.4656 + - 0.4793 + - 0.4638 + - 0.4557 + - 0.4666 + - 0.4468 + - 0.4534 + - 0.4471 + - 0.4448 + - 0.4469 + - 0.536 + - 0.4426 + - 0.4388 + - 0.534 + - 0.4368 + - 0.438 + - 0.531 + - 0.4379 + - 0.535 + - 0.4404 + - 0.4405 + - 0.4409 + - 0.4472 + - 0.4555 + - 0.4433 + - 0.4437 + - 0.4454 + - 0.4448 + - 0.4465 + - 0.4499 + - 0.4488 + - 0.54 + - 0.4534 + - 0.4472 + - 0.455 + - 0.4562 + - 0.452 + - 0.4639 + - 0.538 + - 0.4573 + - 0.4604 + - 0.4533 + - 0.4692 + - 0.566 + - 0.4457 + - 0.4457 + - 0.5154 + - 0.5084 + - 0.528 + - 0.552 + - 0.56 + - 0.567 + - 0.546 + - 0.495 + - 0.4809 + - 0.4732 + - 0.4861 + - 0.4632 + - 0.529 + - 0.4748 + - 0.5007 + - 0.5711 + - 0.595 + - 0.5469 + - 0.626 + - 0.541 + - 0.543 + - 0.533 + - 0.541 + - 0.4695 + - 0.53 + - 0.539 + - 0.529 + - 0.542 + - 0.4681 + - 0.536 + - 0.542 + - 0.535 + - 0.563 + - 0.4805 + - 0.647 + - 0.609 + - 0.553 + - 0.583 + - 0.576 + - 0.6294 + - 0.5885 + - 0.556 + - 0.578 + - 0.566 + - 0.601 + - 0.5627 + - 0.5675 + - 0.592 + - 0.5166 + - 0.589 + - 0.5291 + - 0.5892 + - 0.5976 + - 0.5834 + - 0.6512 + - 0.6748 + - 0.6615 + - 0.6003 + - 0.5669 + - 0.5587 + - 0.5507 + - 0.5871 + - 0.616 + - 0.637 + - 0.633 + - 0.639 + - 0.655 + - 0.641 + - 0.664 + - 0.648 + - 0.656 + - 0.663 + - 0.652 + - 0.681 + - 0.662 + - 0.673 + - 0.672 + - 0.68 + - 0.735 + - 0.732 + - 0.715 + - 0.674 + - 0.687 + - 0.702 + - 0.705 + - 0.715 + - 0.725 + - 0.707 + - 0.74 + - 0.74 + - 0.874 + - 0.737 + - 0.819 + - 0.76 + - 0.869 + - 0.9 + - 0.698 + - 0.823 + - 0.676 + - 0.682 + - 0.766 + - 0.68 + - 0.685 + - 0.694 + - 0.695 + - 0.689 + - 0.727 + - 0.695 + - 0.688 + - 0.677 + - 0.736 + - 0.651 + - 0.661 + - 0.6199 + - 0.6223 + - 0.6036 + - 0.6003 + - 0.5991 + - 0.598 + - 0.591 + - 0.5764 + - 0.577 + - 0.5593 + - 0.597 + - 0.576 + - 0.574 + - 0.578 + - 0.579 + - 0.575 + - 0.576 + - 0.568 + - 0.575 + - 0.569 + - 0.559 + - 0.568 + - 0.5401 + - 0.55 + - 0.5575 + - 0.578 + - 0.5635 + - 0.5786 + - 0.5807 + - 0.581 + - 0.573 + - 0.569 + - 0.567 + - 0.552 + - 0.55 + - 0.558 + - 0.552 + - 0.562 + - 0.574 + - 0.575 + - 0.629 + - 0.682 + - 0.756 + - 1.05 + - 1.122 + - 1.1402 + - 1.154 + - 1.131 + - 1.123 + - 1.159 + - 1.106 + - 1.116 + - 1.069 + - 1.077 + - 1.155 + - 1.162 + - 1.1402 + - 0.644 + - 1.208 + - 1.1402 + - 1.295 + - 1.258 + - 1.1402 + - 0.606 + - 0.603 + - 0.563 + - 0.592 + - 0.607 + - 0.611 + - 0.612 + - 0.618 + - 0.626 + - 0.629 + - 0.583 + - 0.8646 + - 0.626 + - 0.639 + - 0.559 + - 0.827 + - 0.612 + - 0.576 + - 0.58 + - 0.575 + - 0.688 + - 0.697 + - 0.743 + - 0.681 + - 0.832 + - 0.719 + - 0.785 + - 0.878 + - 0.9402 + - 1.0054 + - 1.073 + - 1.129 + - 1.035 + - 1.027 + - 0.9703 + - 1.195 + - 0.9153 + - 1.266 + - 1.153 + - 1.348 + - 1.18 + - 1.269 + - 1.311 + - 0.9914 + - 1.359 + - 1.166 + - 1.139 + - 1.2817 + - 1.398 + - 1.542 + - 1.229 + - 1.377 + - 1.28 + - 1.245 + - 1.1188 + - 1.193 + - 1.293 + - 1.275 + - 1.331 + - 1.34 + - 1.099 + - 1.048 + - 1.124 + - 1.225 + - 1.183 + - 1.196 + - 1.4 + - 1.333 + - 1.417 + - 1.326 + - 1.305 + - 1.0638 + - 1.268 + - 1.217 + - 1.289 + - 1.395 + - 1.232 + - 1.435 + - 1.298 + - 1.328 + - 1.262 + - 1.199 + - 1.391 + - 1.233 + - 1.329 + - 1.664 + - 1.509 + - 1.349 + - 1.481 + - 1.595 + - 1.485 + - 1.532 + - 1.504 + - 1.584 + - 1.609 + - 1.516 + - 1.489 + - 1.502 + - 1.544 + - 1.611 + - 1.539 + - 1.296 + - 1.288 + - 1.241 + - 1.32 + - 1.313 + - 1.301 + - 1.843 + - 1.747 + - 1.711 + - 1.771 + - 1.937 + - 1.575 + - 1.573 + - 1.5 + - 1.459 + - 1.402 + - 1.363 + - 2.201 + - 2.127 + - 2.177 + - 2.157 + - 2.192 + - 2.146 + - 2.151 + - 2.071 + - 1.986 + - 1.845 + - 1.687 + - 1.505 + - 1.373 + - 1.229 + - 1.113 + - 1.004 + - 0.936 + - 0.895 + - 0.871 + - 0.841 + - 0.821 + - 0.805 + - 0.8 + - 0.792 + - 0.797 + - 0.791 + - 0.783 + - 0.777 + - 0.785 + - 0.787 + - 0.789 + - 0.793 + - 0.794 + - 0.745 + - 0.75 + - 0.748 + - 0.749 + - 0.744 + - 0.733 + - 0.733 + - 0.741 + - 0.746 + - 0.746 + - 0.738 + - 0.743 + - 0.745 + - 0.749 + - 0.75 + - 0.75 + - 0.884 + - 0.906 + - 0.917 + - 0.924 + - 0.922 + - 0.928 + - 0.921 + - 0.938 + - 0.931 + - 0.943 + - 0.946 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + - filter: GOMsaver + filename: cycle_dir/cris-fsr_n20-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, + 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, 2153, 2158, 2161, + 2168, 2171, 2175, 2182 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: cris-fsr_n20 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: &id011 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 31 + - 31 + - 30 + - 31 + - 30 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 31 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: cris-fsr_n20 + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 2.0 + - 0.4 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.4 + - 0.4 + - 0.4 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id011 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: cris-fsr_n20 + get values: + time interpolation: linear + - obs space: + name: CRIS-FSR NPP + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/cris-fsr_npp.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.cris-fsr_npp.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: cris-fsr_npp + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/cris-fsr_npp.20231009T150000Z.satbias.nc4 + output file: cycle_dir/cris-fsr_npp.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/cris-fsr_npp.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/cris-fsr_npp.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/cris-fsr_npp.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/cris-fsr_npp.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/cris-fsr_108_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 0.823 + - 0.76 + - 0.736 + - 0.743 + - 0.856 + - 1.079 + - 0.888 + - 0.778 + - 0.671 + - 0.65 + - 0.643 + - 0.629 + - 0.629 + - 0.618 + - 0.638 + - 0.619 + - 0.61 + - 0.627 + - 0.601 + - 0.617 + - 0.608 + - 0.498 + - 0.5112 + - 0.4922 + - 0.4959 + - 0.4954 + - 0.4836 + - 0.514 + - 0.5005 + - 0.4917 + - 0.4881 + - 0.4656 + - 0.4793 + - 0.4638 + - 0.4557 + - 0.4666 + - 0.4468 + - 0.4534 + - 0.4471 + - 0.4448 + - 0.4469 + - 0.536 + - 0.4426 + - 0.4388 + - 0.534 + - 0.4368 + - 0.438 + - 0.531 + - 0.4379 + - 0.535 + - 0.4404 + - 0.4405 + - 0.4409 + - 0.4472 + - 0.4555 + - 0.4433 + - 0.4437 + - 0.4454 + - 0.4448 + - 0.4465 + - 0.4499 + - 0.4488 + - 0.54 + - 0.4534 + - 0.4472 + - 0.455 + - 0.4562 + - 0.452 + - 0.4639 + - 0.538 + - 0.4573 + - 0.4604 + - 0.4533 + - 0.4692 + - 0.566 + - 0.4457 + - 0.4457 + - 0.5154 + - 0.5084 + - 0.528 + - 0.552 + - 0.56 + - 0.567 + - 0.546 + - 0.495 + - 0.4809 + - 0.4732 + - 0.4861 + - 0.4632 + - 0.529 + - 0.4748 + - 0.5007 + - 0.5711 + - 0.595 + - 0.5469 + - 0.626 + - 0.541 + - 0.543 + - 0.533 + - 0.541 + - 0.4695 + - 0.53 + - 0.539 + - 0.529 + - 0.542 + - 0.4681 + - 0.536 + - 0.542 + - 0.535 + - 0.563 + - 0.4805 + - 0.647 + - 0.609 + - 0.553 + - 0.583 + - 0.576 + - 0.6294 + - 0.5885 + - 0.556 + - 0.578 + - 0.566 + - 0.601 + - 0.5627 + - 0.5675 + - 0.592 + - 0.5166 + - 0.589 + - 0.5291 + - 0.5892 + - 0.5976 + - 0.5834 + - 0.6512 + - 0.6748 + - 0.6615 + - 0.6003 + - 0.5669 + - 0.5587 + - 0.5507 + - 0.5871 + - 0.616 + - 0.637 + - 0.633 + - 0.639 + - 0.655 + - 0.641 + - 0.664 + - 0.648 + - 0.656 + - 0.663 + - 0.652 + - 0.681 + - 0.662 + - 0.673 + - 0.672 + - 0.68 + - 0.735 + - 0.732 + - 0.715 + - 0.674 + - 0.687 + - 0.702 + - 0.705 + - 0.715 + - 0.725 + - 0.707 + - 0.74 + - 0.74 + - 0.874 + - 0.737 + - 0.819 + - 0.76 + - 0.869 + - 0.9 + - 0.698 + - 0.823 + - 0.676 + - 0.682 + - 0.766 + - 0.68 + - 0.685 + - 0.694 + - 0.695 + - 0.689 + - 0.727 + - 0.695 + - 0.688 + - 0.677 + - 0.736 + - 0.651 + - 0.661 + - 0.6199 + - 0.6223 + - 0.6036 + - 0.6003 + - 0.5991 + - 0.598 + - 0.591 + - 0.5764 + - 0.577 + - 0.5593 + - 0.597 + - 0.576 + - 0.574 + - 0.578 + - 0.579 + - 0.575 + - 0.576 + - 0.568 + - 0.575 + - 0.569 + - 0.559 + - 0.568 + - 0.5401 + - 0.55 + - 0.5575 + - 0.578 + - 0.5635 + - 0.5786 + - 0.5807 + - 0.581 + - 0.573 + - 0.569 + - 0.567 + - 0.552 + - 0.55 + - 0.558 + - 0.552 + - 0.562 + - 0.574 + - 0.575 + - 0.629 + - 0.682 + - 0.756 + - 1.05 + - 1.122 + - 1.1402 + - 1.154 + - 1.131 + - 1.123 + - 1.159 + - 1.106 + - 1.116 + - 1.069 + - 1.077 + - 1.155 + - 1.162 + - 1.1402 + - 0.644 + - 1.208 + - 1.1402 + - 1.295 + - 1.258 + - 1.1402 + - 0.606 + - 0.603 + - 0.563 + - 0.592 + - 0.607 + - 0.611 + - 0.612 + - 0.618 + - 0.626 + - 0.629 + - 0.583 + - 0.8646 + - 0.626 + - 0.639 + - 0.559 + - 0.827 + - 0.612 + - 0.576 + - 0.58 + - 0.575 + - 0.688 + - 0.697 + - 0.743 + - 0.681 + - 0.832 + - 0.719 + - 0.785 + - 0.878 + - 0.9402 + - 1.0054 + - 1.073 + - 1.129 + - 1.035 + - 1.027 + - 0.9703 + - 1.195 + - 0.9153 + - 1.266 + - 1.153 + - 1.348 + - 1.18 + - 1.269 + - 1.311 + - 0.9914 + - 1.359 + - 1.166 + - 1.139 + - 1.2817 + - 1.398 + - 1.542 + - 1.229 + - 1.377 + - 1.28 + - 1.245 + - 1.1188 + - 1.193 + - 1.293 + - 1.275 + - 1.331 + - 1.34 + - 1.099 + - 1.048 + - 1.124 + - 1.225 + - 1.183 + - 1.196 + - 1.4 + - 1.333 + - 1.417 + - 1.326 + - 1.305 + - 1.0638 + - 1.268 + - 1.217 + - 1.289 + - 1.395 + - 1.232 + - 1.435 + - 1.298 + - 1.328 + - 1.262 + - 1.199 + - 1.391 + - 1.233 + - 1.329 + - 1.664 + - 1.509 + - 1.349 + - 1.481 + - 1.595 + - 1.485 + - 1.532 + - 1.504 + - 1.584 + - 1.609 + - 1.516 + - 1.489 + - 1.502 + - 1.544 + - 1.611 + - 1.539 + - 1.296 + - 1.288 + - 1.241 + - 1.32 + - 1.313 + - 1.301 + - 1.843 + - 1.747 + - 1.711 + - 1.771 + - 1.937 + - 1.575 + - 1.573 + - 1.5 + - 1.459 + - 1.402 + - 1.363 + - 2.201 + - 2.127 + - 2.177 + - 2.157 + - 2.192 + - 2.146 + - 2.151 + - 2.071 + - 1.986 + - 1.845 + - 1.687 + - 1.505 + - 1.373 + - 1.229 + - 1.113 + - 1.004 + - 0.936 + - 0.895 + - 0.871 + - 0.841 + - 0.821 + - 0.805 + - 0.8 + - 0.792 + - 0.797 + - 0.791 + - 0.783 + - 0.777 + - 0.785 + - 0.787 + - 0.789 + - 0.793 + - 0.794 + - 0.745 + - 0.75 + - 0.748 + - 0.749 + - 0.744 + - 0.733 + - 0.733 + - 0.741 + - 0.746 + - 0.746 + - 0.738 + - 0.743 + - 0.745 + - 0.749 + - 0.75 + - 0.75 + - 0.884 + - 0.906 + - 0.917 + - 0.924 + - 0.922 + - 0.928 + - 0.921 + - 0.938 + - 0.931 + - 0.943 + - 0.946 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + - filter: GOMsaver + filename: cycle_dir/cris-fsr_npp-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, + 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, 2153, 2158, 2161, + 2168, 2171, 2175, 2182 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: cris-fsr_npp + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: &id012 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 31 + - 31 + - 30 + - 31 + - 30 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 31 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 31 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 31 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + - 30 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: cris-fsr_npp + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 0.4 + - 2.0 + - 0.4 + - 2.0 + - 0.4 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.4 + - 0.4 + - 0.4 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 1.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id012 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: cris-fsr_npp + get values: + time interpolation: linear + - obs space: + name: GMI GPM + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/gmi_gpm.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.gmi_gpm.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: gmi_gpm + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/gmi_gpm.20231009T150000Z.satbias.nc4 + output file: cycle_dir/gmi_gpm.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/gmi_gpm.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/gmi_gpm.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: cloudWaterContent + sensor: GMI_GPM + ch37v: 6 + ch37h: 7 + order: 2 + tlapse: cycle_dir/gmi_gpm.20231009T150000Z.tlapse.txt + - name: cloudWaterContent + sensor: GMI_GPM + ch37v: 6 + ch37h: 7 + tlapse: cycle_dir/gmi_gpm.20231009T150000Z.tlapse.txt + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/gmi_gpm.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/gmi_gpm.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-9 + minvalue: 50.0 + maxvalue: 320.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 10-13 + minvalue: 70.0 + maxvalue: 320.0 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 5 + maxvalue: 70 + - variable: + name: MetaData/latitude + minvalue: -55.0 + maxvalue: 55.0 + - variable: + name: MetaData/heightOfSurface + maxvalue: 2000 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + minvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/latitude + minvalue: -20.0 + maxvalue: 0.0 + - variable: + name: MetaData/longitude + minvalue: 25.0 + maxvalue: 40.0 + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/cloudWaterContent_obs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + maxvalue: 900 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - HofX + maxvalue: 900 + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/cloudWaterContent_hofx + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - HofX + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + minvalue: 0.0 + maxvalue: 0.05 + - variable: + name: ObsFunction/Emissivity_Diff_GMI + options: + channel: 2 + regression_constant_1: 0.1329 + regression_constant_2: 0.42468 + regression_coeff_1: + - -0.00548 + - 0.00772 + - 0.0053 + - -0.00425 + - 0.00053 + - 8e-05 + - -3e-05 + - -0.00144 + - 0.00059 + - -0.00016 + - 3e-05 + - -0.00011 + - 0.00017 + regression_coeff_2: + - 0.00289 + - -0.00142 + minvalue: 0.01 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + minvalue: 0.0 + maxvalue: 0.05 + - variable: + name: ObsFunction/Emissivity_Diff_GMI + options: + channel: 4 + regression_constant_1: 0.15627 + regression_constant_2: 0.83807 + regression_coeff_1: + - -0.01084 + - 0.01194 + - 0.01111 + - -0.00784 + - 0.0006 + - 8e-05 + - -3e-05 + - -0.00248 + - 0.00105 + - -8e-05 + - 0.0 + - -0.00013 + - 0.00016 + regression_coeff_2: + - 0.00048 + - -0.00207 + minvalue: 0.035 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + minvalue: 0.0 + maxvalue: 0.05 + - variable: + name: ObsFunction/Emissivity_Diff_GMI + options: + channel: 7 + regression_constant_1: 0.30306 + regression_constant_2: 1.24071 + regression_coeff_1: + - -0.01793 + - 0.0173 + - 0.01784 + - -0.01199 + - 0.00067 + - 0.00013 + - -4e-05 + - -0.00365 + - 0.00154 + - -4e-05 + - -1e-05 + - -0.00015 + - 0.00017 + regression_coeff_2: + - 0.00068 + - -0.00342 + minvalue: 0.05 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: + - ObsValue + - HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.2 + - 0.3 + - 0.2 + - 0.3 + - 0.3 + x2: + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + err0: + - 2.7 + - 3.7 + - 3.5 + - 4.5 + - 4.0 + - 3.8 + - 300.0 + - 5.0 + - 11.5 + - 5.0 + - 5.0 + - 2.5 + - 3.0 + err1: + - 17.0 + - 23.0 + - 13.0 + - 25.0 + - 11.0 + - 13.0 + - 23.0 + - 10.0 + - 20.0 + - 15.0 + - 20.0 + - 8.0 + - 13.0 + err2: + - 25.0 + - 40.0 + - 40.0 + - 55.0 + - 35.0 + - 25.0 + - 500.0 + - 50.0 + - 50.0 + - 50.0 + - 50.0 + - 30.0 + - 40.0 + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 1,2,4,6 + threshold: 2.0 + absolute threshold: 30.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 9,10,11 + threshold: 2.0 + absolute threshold: 20.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 3,5,8 + threshold: 2.0 + absolute threshold: 15.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 12,13 + threshold: 2.0 + absolute threshold: 10.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 7 + threshold: 2.0 + absolute threshold: 5.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: gmi_gpm + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: GOMsaver + filename: cycle_dir/gmi_gpm-geovals.20231009T210000Z.nc4 + observation_name: gmi_gpm + get values: + time interpolation: linear + - obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/gps.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - sequenceNumber + sort variable: impactHeightRO + sort order: ascending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.gps.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - bendingAngle + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + vertlayer: full + sr_steps: 2 + super_ref_qc: NBAM + obs filters: + - filter: BlackList + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 41,265,266,421,440,724,725,726,727,728,729 + - filter: Perform Action + filter variables: + - name: bendingAngle + where: + - variable: PreUseFlag/bendingAngle + minvalue: 1 + action: + name: reject + - filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266,267,268,269 + test variables: + - name: MetaData/impactHeightRO + maxvalue: 45000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 3-5 + test variables: + - name: MetaData/impactHeightRO + minvalue: 8000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 267,268,269 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 401 + test variables: + - name: MetaData/impactHeightRO + minvalue: 5000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 267,268,269 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 402-999 + test variables: + - name: MetaData/impactHeightRO + minvalue: 9000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 401 + test variables: + - name: MetaData/impactHeightRO + minvalue: 5000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 402-999 + test variables: + - name: MetaData/impactHeightRO + minvalue: 8000.1 + action: + name: reject + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: inflate error + inflation factor: 2.0 + where: + - variable: MetaData/satelliteIdentifier + is_in: 267,268,269 + - filter: Variable Assignment + assignments: + - name: JediAdjustObsError/bendingAngle + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/bendingAngle + defer to post: true + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: assign error + error parameter: 1.0 + where: + - variable: + name: JediAdjustObsError/bendingAngle + maxvalue: 1.0 + - variable: + name: JediAdjustObsError/bendingAngle + value: is_valid + - variable: + name: MetaData/satelliteIdentifier + is_in: 3,5,41,42,43,44,267,268,269,440,421,724,725,726,727,728,729, 750,751,752,753,754,755,821,825 + defer to post: true + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: assign error + error parameter: 10.0 + where: + - variable: + name: JediAdjustObsError/bendingAngle + minvalue: 10.0 + - variable: + name: JediAdjustObsError/bendingAngle + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: bendingAngle + threshold: 5 + action: + name: reject + defer to post: true + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: assign error + error function: JediAdjustObsError/bendingAngle + where: + - variable: + name: JediAdjustObsError/bendingAngle + value: is_valid + defer to post: true + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + defer to post: true + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate_GEOS + defer to post: true + - filter: GOMsaver + filename: cycle_dir/gps-geovals.20231009T210000Z.nc4 + observation_name: gps + get values: + time interpolation: linear + - obs space: + name: IASI METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/iasi_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.iasi_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: iasi_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/iasi_metop-b.20231009T150000Z.satbias.nc4 + output file: cycle_dir/iasi_metop-b.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/iasi_metop-b.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/iasi_metop-b.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/iasi_metop-b.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/iasi_metop-b.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/iasi_metop_141_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id013 + - 0.727 + - 0.81 + - 0.75 + - 0.79 + - 0.7055 + - 0.74 + - 0.68 + - 0.72 + - 0.6526 + - 0.65 + - 0.665 + - 0.69 + - 0.6394 + - 0.64 + - 0.6528 + - 0.6065 + - 0.6246 + - 0.61 + - 0.6423 + - 0.5995 + - 0.59 + - 0.6069 + - 0.6 + - 0.5965 + - 0.64 + - 0.62 + - 0.589 + - 0.5865 + - 0.65 + - 0.5861 + - 0.61 + - 0.5874 + - 0.68 + - 0.606 + - 0.68 + - 4.38 + - 3.05 + - 2.31 + - 1.56 + - 1.33 + - 1.58 + - 0.93 + - 0.5832 + - 0.5587 + - 0.5867 + - 0.58 + - 0.5655 + - 0.5522 + - 0.5864 + - 0.5475 + - 0.5854 + - 0.5455 + - 0.5811 + - 0.5376 + - 0.5452 + - 0.5686 + - 0.5329 + - 0.5655 + - 0.5302 + - 0.545 + - 0.5628 + - 0.59 + - 0.5262 + - 0.559 + - 0.5264 + - 0.5442 + - 0.51 + - 0.5513 + - 0.5224 + - 0.5523 + - 0.5188 + - 0.5487 + - 0.5245 + - 0.58 + - 0.5437 + - 0.5343 + - 0.5364 + - 0.64 + - 0.5338 + - 0.72 + - 0.537 + - 0.75 + - 0.51 + - 0.65 + - 0.5274 + - 0.529 + - 0.5187 + - 0.5228 + - 1.12 + - 0.5222 + - 0.5109 + - 0.67 + - 0.5133 + - 0.5179 + - 0.507 + - 0.67 + - 0.5091 + - 0.62 + - 0.5093 + - 0.69 + - 0.5048 + - 0.5024 + - 0.78 + - 0.497 + - 0.5337 + - 0.4865 + - 0.4915 + - 0.4835 + - 0.4869 + - 0.87 + - 0.4824 + - 0.4852 + - 0.84 + - 0.84 + - 0.84 + - 0.5318 + - 0.8 + - 0.4772 + - 0.98 + - 0.488 + - 0.4978 + - 0.5157 + - 0.61 + - 0.5213 + - 0.4884 + - 0.79 + - 0.62 + - 0.66 + - 0.4691 + - 0.65 + - 0.4809 + - 0.468 + - 0.62 + - 0.4679 + - 0.6913 + - 0.4705 + - 0.4785 + - 0.47 + - 0.4773 + - 0.4703 + - 0.98 + - 0.4697 + - 0.4662 + - 0.65 + - 0.467 + - 0.4883 + - 0.4684 + - 0.4684 + - 0.4947 + - 0.5393 + - 0.5024 + - 0.4715 + - 0.621 + - 0.6136 + - 0.5316 + - 1.78 + - 0.5099 + - 1.14 + - 0.539 + - 1.79 + - 0.508 + - 0.5723 + - 1.94 + - 2.01 + - 0.49 + - 0.5647 + - 0.5022 + - 1.47 + - 0.5815 + - 0.6782 + - 2.13 + - 0.5445 + - 1.52 + - 0.5555 + - 1.96 + - 2.31 + - 2.33 + - 2.32 + - 2.31 + - 0.6994 + - 0.7006 + - 0.706 + - 0.9785 + - 0.7023 + - 0.6991 + - 0.6946 + - 2.28 + - 2.26 + - 2.26 + - 2.26 + - 0.6608 + - 0.6835 + - 0.6822 + - 2.24 + - 2.26 + - 0.6735 + - 2.28 + - 0.667 + - 0.7732 + - 0.6642 + - 0.648 + - 0.6629 + - 2.29 + - 2.29 + - 0.6799 + - 0.623 + - 2.32 + - 0.603 + - 0.6224 + - 2.32 + - 0.6187 + - 2.31 + - 2.31 + - 2.28 + - 2.29 + - 2.28 + - 2.26 + - 1.966 + - 2.27 + - 2.26 + - 2.25 + - 2.27 + - 2.24 + - 2.21 + - 2.24 + - 2.17 + - 2.18 + - 2.17 + - 2.21 + - 1.4 + - 2.16 + - 2.2 + - 2.13 + - 2.12 + - 2.13 + - 2.1 + - 2.12 + - 2.11 + - 2.09 + - 2.09 + - 2.08 + - 2.09 + - 2.04 + - 2.04 + - 1.966 + - 2.01 + - 2.05 + - 2.03 + - 2.06 + - 1.98 + - 1.95 + - 1.94 + - 1.91 + - 1.857 + - 1.76 + - 1.748 + - 1.83 + - 2.04 + - 1.748 + - 1.99 + - 2.075 + - 2.07 + - 2.02 + - 2.04 + - 2.1 + - 1.966 + - 2.18 + - 2.21 + - 2.24 + - 2.23 + - 2.23 + - 1.98 + - 2.2 + - 2.18 + - 2.18 + - 2.21 + - 2.23 + - 2.24 + - 2.24 + - 2.25 + - 1.8 + - 2.24 + - 1.73 + - 1.73 + - 2.27 + - 1.67 + - 2.21 + - 1.72 + - 2.23 + - 2.23 + - 2.23 + - 2.24 + - 2.23 + - 2.12 + - 2.17 + - 1.74 + - 2.02 + - 1.88 + - 1.67 + - 1.73 + - 1.83 + - 1.82 + - 1.73 + - 1.83 + - 2.19 + - 1.84 + - 1.89 + - 1.6 + - 1.71 + - 1.86 + - 1.85 + - 1.84 + - 1.87 + - 1.91 + - 1.52 + - 1.95 + - 1.87 + - 1.89 + - 1.91 + - 1.91 + - 1.93 + - 1.9 + - 1.91 + - 1.9 + - 1.89 + - 1.89 + - 1.91 + - 1.9 + - 1.91 + - 1.91 + - 1.91 + - 1.93 + - 1.94 + - 1.91 + - 1.92 + - 1.77 + - 1.91 + - 1.95 + - 1.19 + - 1.96 + - 1.98 + - 1.94 + - 1.55 + - 1.91 + - 1.92 + - 1.92 + - 1.97 + - 1.93 + - 1.99 + - 1.86 + - 1.12 + - 1.93 + - 1.92 + - 1.95 + - 1.85 + - 1.84 + - 1.91 + - 1.12 + - 1.82 + - 1.82 + - 1.95 + - 1.24 + - 1.94 + - 1.96 + - 1.21 + - 1.83 + - 1.96 + - 1.36 + - 1.96 + - 1.82 + - 1.92 + - 1.68 + - 1.93 + - 1.23 + - 1.96 + - 1.93 + - 1.86 + - 1.41 + - 1.16 + - 1.6 + - 1.25 + - 1.2 + - 1.65 + - 1.66 + - 1.87 + - 1.94 + - 1.96 + - 1.91 + - 1.25 + - 1.93 + - 1.91 + - 1.7 + - 0.99 + - 1.81 + - 1.92 + - 1.95 + - 1.5 + - 1.47 + - 1.15 + - 1.58 + - 1.18 + - 1.82 + - 1.13 + - 1.83 + - 1.91 + - 1.26 + - 1.27 + - 1.91 + - 1.45 + - 1.6 + - 1.29 + - 1.94 + - 1.94 + - 1.23 + - 1.95 + - 1.21 + - 1.94 + - 1.86 + - 1.9 + - 1.33 + - 1.75 + - 2.02 + - 1.98 + - 2.03 + - 1.83 + - 1.5 + - 2.04 + - 2.02 + - 1.9 + - 2.0 + - 2.02 + - 1.95 + - 1.93 + - 1.95 + - 1.95 + - 1.99 + - 2.0 + - 1.94 + - 1.96 + - 1.86 + - 1.92 + - 1.88 + - 1.86 + - 1.84 + - 1.87 + - 1.77 + - 1.89 + - 1.89 + - 1.88 + - 1.94 + - 1.82 + - 1.79 + - 1.86 + - 2.06 + - 2.33 + - 1.88 + - 1.86 + - 1.81 + - 1.8 + - 1.8 + - 1.86 + - 1.9 + - 2.0 + - 2.06 + - 2.1 + - 2.2 + - 2.0 + - 2.16 + - 1.98 + - 1.8 + - 1.8 + - 1.85 + - 1.75 + - 2.04 + - 2.19 + - 2.14 + - 2.19 + - 1.86 + - 2.1 + - 2.11 + - 2.18 + - 2.03 + - 2.28 + - 2.19 + - 2.26 + - 2.26 + - 2.21 + - 2.21 + - 2.26 + - 2.33 + - 2.27 + - 2.21 + - 2.12 + - 2.23 + - 2.26 + - 2.25 + - 1.88 + - 2.26 + - 2.24 + - 2.36 + - 2.29 + - 2.35 + - 2.3 + - 2.27 + - 2.08 + - 2.05 + - 2.27 + - 2.28 + - 2.27 + - 2.28 + - 1.97 + - 2.25 + - 2.25 + - 2.25 + - 2.31 + - 2.28 + - 2.27 + - 2.13 + - 2.24 + - 2.28 + - 2.28 + - 2.41 + - 2.34 + - 9.32 + - 2.28 + - 2.38 + - 2.27 + - 2.27 + - 2.39 + - 2.11 + - 2.09 + - 2.1 + - 2.06 + - 2.12 + - 2.08 + - 2.0 + - 1.93 + - 2.02 + - 2.55 + - 1.54 + - 1.64 + - 1.51 + - 1.55 + - 2.82 + - 2.92 + - 2.55 + - 2.37 + - 1.85 + - 1.6 + - 1.72 + - 1.74 + - 1.79 + - 1.9 + - 1.94 + - 2.0 + - 2.04 + - 2.08 + - 2.12 + - 2.13 + - 2.16 + - 2.18 + - 2.18 + - 2.2 + - 2.2 + - 2.41 + - 2.39 + - 2.38 + - 2.4 + - 2.42 + - 2.41 + - 2.43 + - 2.45 + - 2.43 + - 2.45 + - 2.43 + - 2.4 + - 2.44 + - 2.4 + - 2.42 + - 2.43 + - 2.45 + - 2.45 + - 2.45 + - 2.46 + - 2.45 + - 2.45 + - 2.43 + - 2.51 + - 2.48 + - 2.48 + - 2.53 + - 2.46 + - 2.49 + - 2.5 + - 2.5 + - 2.5 + - 2.52 + - 2.52 + - 2.54 + - 2.5 + - 2.48 + - 2.5 + - 2.55 + - 2.5 + - 2.48 + - 2.5 + - 2.5 + - 2.52 + - 2.52 + - 2.48 + - 2.5 + - 2.5 + - 2.52 + - 2.46 + - 2.53 + - 9.0 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + - filter: GOMsaver + filename: cycle_dir/iasi_metop-b-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, 7076, + 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, 7267, 7269, 7284, + 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, 7475, 7549, 7584, + 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, 7950, 7972, 7980, + 7995, 8007, 8015, 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: iasi_metop-b + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + error parameter vector: *id013 + use_flag: None + use_flag_clddet: &id014 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + error parameter vector: *id013 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: iasi_metop-b + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + error parameter vector: *id013 + obserr_bound_max: + - 3.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 4.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 4.0 + - 4.0 + - 3.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.75 + - 2.0 + - 0.75 + - 2.0 + - 2.0 + - 2.0 + - 0.75 + - 0.75 + - 0.75 + - 0.75 + - 2.0 + - 0.75 + - 0.75 + - 2.0 + - 0.75 + - 0.75 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 2.0 + - 2.5 + - 2.5 + - 3.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 3.5 + - 2.5 + - 2.5 + - 3.0 + - 3.5 + - 3.0 + - 4.0 + - 4.0 + - 0.75 + - 4.0 + - 4.0 + - 4.0 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.0 + - 4.5 + - 4.0 + - 4.0 + - 4.5 + - 2.5 + - 3.0 + - 2.5 + - 3.0 + - 2.5 + - 3.0 + - 2.0 + - 2.5 + - 2.5 + - 3.0 + - 3.0 + - 2.5 + - 3.0 + - 3.0 + - 3.0 + - 2.5 + - 2.5 + - 4.0 + - 4.5 + - 4.5 + - 5.0 + - 4.0 + - 4.0 + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 5.5 + - 5.5 + - 4.0 + - 5.0 + - 4.0 + - 4.5 + - 5.5 + - 5.5 + - 6.0 + - 4.5 + - 4.5 + - 4.0 + - 5.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.25 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.25 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.4 + - 6.0 + - 1.4 + - 6.0 + - 6.0 + - 1.4 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.5 + - 4.5 + - 6.0 + - 5.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 4.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 4.5 + - 6.0 + - 6.0 + - 4.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id014 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: iasi_metop-b + get values: + time interpolation: linear + - obs space: + name: IASI METOP-C + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/iasi_metop-c.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.iasi_metop-c.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: iasi_metop-c + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/iasi_metop-c.20231009T150000Z.satbias.nc4 + output file: cycle_dir/iasi_metop-c.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/iasi_metop-c.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/iasi_metop-c.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/iasi_metop-c.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/iasi_metop-c.20231009T210000Z.satbias_cov.nc4 + obs error: + covariance model: cross variable covariances + input file: fv3-jedi/rcov/iasi_metop_141_jedi_rcov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id015 + - 0.727 + - 0.81 + - 0.75 + - 0.79 + - 0.7055 + - 0.74 + - 0.68 + - 0.72 + - 0.6526 + - 0.65 + - 0.665 + - 0.69 + - 0.6394 + - 0.64 + - 0.6528 + - 0.6065 + - 0.6246 + - 0.61 + - 0.6423 + - 0.5995 + - 0.59 + - 0.6069 + - 0.6 + - 0.5965 + - 0.64 + - 0.62 + - 0.589 + - 0.5865 + - 0.65 + - 0.5861 + - 0.61 + - 0.5874 + - 0.68 + - 0.606 + - 0.68 + - 4.38 + - 3.05 + - 2.31 + - 1.56 + - 1.33 + - 1.58 + - 0.93 + - 0.5832 + - 0.5587 + - 0.5867 + - 0.58 + - 0.5655 + - 0.5522 + - 0.5864 + - 0.5475 + - 0.5854 + - 0.5455 + - 0.5811 + - 0.5376 + - 0.5452 + - 0.5686 + - 0.5329 + - 0.5655 + - 0.5302 + - 0.545 + - 0.5628 + - 0.59 + - 0.5262 + - 0.559 + - 0.5264 + - 0.5442 + - 0.51 + - 0.5513 + - 0.5224 + - 0.5523 + - 0.5188 + - 0.5487 + - 0.5245 + - 0.58 + - 0.5437 + - 0.5343 + - 0.5364 + - 0.64 + - 0.5338 + - 0.72 + - 0.537 + - 0.75 + - 0.51 + - 0.65 + - 0.5274 + - 0.529 + - 0.5187 + - 0.5228 + - 1.12 + - 0.5222 + - 0.5109 + - 0.67 + - 0.5133 + - 0.5179 + - 0.507 + - 0.67 + - 0.5091 + - 0.62 + - 0.5093 + - 0.69 + - 0.5048 + - 0.5024 + - 0.78 + - 0.497 + - 0.5337 + - 0.4865 + - 0.4915 + - 0.4835 + - 0.4869 + - 0.87 + - 0.4824 + - 0.4852 + - 0.84 + - 0.84 + - 0.84 + - 0.5318 + - 0.8 + - 0.4772 + - 0.98 + - 0.488 + - 0.4978 + - 0.5157 + - 0.61 + - 0.5213 + - 0.4884 + - 0.79 + - 0.62 + - 0.66 + - 0.4691 + - 0.65 + - 0.4809 + - 0.468 + - 0.62 + - 0.4679 + - 0.6913 + - 0.4705 + - 0.4785 + - 0.47 + - 0.4773 + - 0.4703 + - 0.98 + - 0.4697 + - 0.4662 + - 0.65 + - 0.467 + - 0.4883 + - 0.4684 + - 0.4684 + - 0.4947 + - 0.5393 + - 0.5024 + - 0.4715 + - 0.621 + - 0.6136 + - 0.5316 + - 1.78 + - 0.5099 + - 1.14 + - 0.539 + - 1.79 + - 0.508 + - 0.5723 + - 1.94 + - 2.01 + - 0.49 + - 0.5647 + - 0.5022 + - 1.47 + - 0.5815 + - 0.6782 + - 2.13 + - 0.5445 + - 1.52 + - 0.5555 + - 1.96 + - 2.31 + - 2.33 + - 2.32 + - 2.31 + - 0.6994 + - 0.7006 + - 0.706 + - 0.9785 + - 0.7023 + - 0.6991 + - 0.6946 + - 2.28 + - 2.26 + - 2.26 + - 2.26 + - 0.6608 + - 0.6835 + - 0.6822 + - 2.24 + - 2.26 + - 0.6735 + - 2.28 + - 0.667 + - 0.7732 + - 0.6642 + - 0.648 + - 0.6629 + - 2.29 + - 2.29 + - 0.6799 + - 0.623 + - 2.32 + - 0.603 + - 0.6224 + - 2.32 + - 0.6187 + - 2.31 + - 2.31 + - 2.28 + - 2.29 + - 2.28 + - 2.26 + - 1.966 + - 2.27 + - 2.26 + - 2.25 + - 2.27 + - 2.24 + - 2.21 + - 2.24 + - 2.17 + - 2.18 + - 2.17 + - 2.21 + - 1.4 + - 2.16 + - 2.2 + - 2.13 + - 2.12 + - 2.13 + - 2.1 + - 2.12 + - 2.11 + - 2.09 + - 2.09 + - 2.08 + - 2.09 + - 2.04 + - 2.04 + - 1.966 + - 2.01 + - 2.05 + - 2.03 + - 2.06 + - 1.98 + - 1.95 + - 1.94 + - 1.91 + - 1.857 + - 1.76 + - 1.748 + - 1.83 + - 2.04 + - 1.748 + - 1.99 + - 2.075 + - 2.07 + - 2.02 + - 2.04 + - 2.1 + - 1.966 + - 2.18 + - 2.21 + - 2.24 + - 2.23 + - 2.23 + - 1.98 + - 2.2 + - 2.18 + - 2.18 + - 2.21 + - 2.23 + - 2.24 + - 2.24 + - 2.25 + - 1.8 + - 2.24 + - 1.73 + - 1.73 + - 2.27 + - 1.67 + - 2.21 + - 1.72 + - 2.23 + - 2.23 + - 2.23 + - 2.24 + - 2.23 + - 2.12 + - 2.17 + - 1.74 + - 2.02 + - 1.88 + - 1.67 + - 1.73 + - 1.83 + - 1.82 + - 1.73 + - 1.83 + - 2.19 + - 1.84 + - 1.89 + - 1.6 + - 1.71 + - 1.86 + - 1.85 + - 1.84 + - 1.87 + - 1.91 + - 1.52 + - 1.95 + - 1.87 + - 1.89 + - 1.91 + - 1.91 + - 1.93 + - 1.9 + - 1.91 + - 1.9 + - 1.89 + - 1.89 + - 1.91 + - 1.9 + - 1.91 + - 1.91 + - 1.91 + - 1.93 + - 1.94 + - 1.91 + - 1.92 + - 1.77 + - 1.91 + - 1.95 + - 1.19 + - 1.96 + - 1.98 + - 1.94 + - 1.55 + - 1.91 + - 1.92 + - 1.92 + - 1.97 + - 1.93 + - 1.99 + - 1.86 + - 1.12 + - 1.93 + - 1.92 + - 1.95 + - 1.85 + - 1.84 + - 1.91 + - 1.12 + - 1.82 + - 1.82 + - 1.95 + - 1.24 + - 1.94 + - 1.96 + - 1.21 + - 1.83 + - 1.96 + - 1.36 + - 1.96 + - 1.82 + - 1.92 + - 1.68 + - 1.93 + - 1.23 + - 1.96 + - 1.93 + - 1.86 + - 1.41 + - 1.16 + - 1.6 + - 1.25 + - 1.2 + - 1.65 + - 1.66 + - 1.87 + - 1.94 + - 1.96 + - 1.91 + - 1.25 + - 1.93 + - 1.91 + - 1.7 + - 0.99 + - 1.81 + - 1.92 + - 1.95 + - 1.5 + - 1.47 + - 1.15 + - 1.58 + - 1.18 + - 1.82 + - 1.13 + - 1.83 + - 1.91 + - 1.26 + - 1.27 + - 1.91 + - 1.45 + - 1.6 + - 1.29 + - 1.94 + - 1.94 + - 1.23 + - 1.95 + - 1.21 + - 1.94 + - 1.86 + - 1.9 + - 1.33 + - 1.75 + - 2.02 + - 1.98 + - 2.03 + - 1.83 + - 1.5 + - 2.04 + - 2.02 + - 1.9 + - 2.0 + - 2.02 + - 1.95 + - 1.93 + - 1.95 + - 1.95 + - 1.99 + - 2.0 + - 1.94 + - 1.96 + - 1.86 + - 1.92 + - 1.88 + - 1.86 + - 1.84 + - 1.87 + - 1.77 + - 1.89 + - 1.89 + - 1.88 + - 1.94 + - 1.82 + - 1.79 + - 1.86 + - 2.06 + - 2.33 + - 1.88 + - 1.86 + - 1.81 + - 1.8 + - 1.8 + - 1.86 + - 1.9 + - 2.0 + - 2.06 + - 2.1 + - 2.2 + - 2.0 + - 2.16 + - 1.98 + - 1.8 + - 1.8 + - 1.85 + - 1.75 + - 2.04 + - 2.19 + - 2.14 + - 2.19 + - 1.86 + - 2.1 + - 2.11 + - 2.18 + - 2.03 + - 2.28 + - 2.19 + - 2.26 + - 2.26 + - 2.21 + - 2.21 + - 2.26 + - 2.33 + - 2.27 + - 2.21 + - 2.12 + - 2.23 + - 2.26 + - 2.25 + - 1.88 + - 2.26 + - 2.24 + - 2.36 + - 2.29 + - 2.35 + - 2.3 + - 2.27 + - 2.08 + - 2.05 + - 2.27 + - 2.28 + - 2.27 + - 2.28 + - 1.97 + - 2.25 + - 2.25 + - 2.25 + - 2.31 + - 2.28 + - 2.27 + - 2.13 + - 2.24 + - 2.28 + - 2.28 + - 2.41 + - 2.34 + - 9.32 + - 2.28 + - 2.38 + - 2.27 + - 2.27 + - 2.39 + - 2.11 + - 2.09 + - 2.1 + - 2.06 + - 2.12 + - 2.08 + - 2.0 + - 1.93 + - 2.02 + - 2.55 + - 1.54 + - 1.64 + - 1.51 + - 1.55 + - 2.82 + - 2.92 + - 2.55 + - 2.37 + - 1.85 + - 1.6 + - 1.72 + - 1.74 + - 1.79 + - 1.9 + - 1.94 + - 2.0 + - 2.04 + - 2.08 + - 2.12 + - 2.13 + - 2.16 + - 2.18 + - 2.18 + - 2.2 + - 2.2 + - 2.41 + - 2.39 + - 2.38 + - 2.4 + - 2.42 + - 2.41 + - 2.43 + - 2.45 + - 2.43 + - 2.45 + - 2.43 + - 2.4 + - 2.44 + - 2.4 + - 2.42 + - 2.43 + - 2.45 + - 2.45 + - 2.45 + - 2.46 + - 2.45 + - 2.45 + - 2.43 + - 2.51 + - 2.48 + - 2.48 + - 2.53 + - 2.46 + - 2.49 + - 2.5 + - 2.5 + - 2.5 + - 2.52 + - 2.52 + - 2.54 + - 2.5 + - 2.48 + - 2.5 + - 2.55 + - 2.5 + - 2.48 + - 2.5 + - 2.5 + - 2.52 + - 2.52 + - 2.48 + - 2.5 + - 2.5 + - 2.52 + - 2.46 + - 2.53 + - 9.0 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + - filter: GOMsaver + filename: cycle_dir/iasi_metop-c-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, 7076, + 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, 7267, 7269, 7284, + 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, 7475, 7549, 7584, + 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, 7950, 7972, 7980, + 7995, 8007, 8015, 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: iasi_metop-c + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: None + options: + channels: None + error parameter vector: *id015 + use_flag: None + use_flag_clddet: &id016 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 31 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 31 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + error parameter vector: *id015 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: iasi_metop-c + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + error parameter vector: *id015 + obserr_bound_max: + - 3.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 4.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 4.0 + - 4.0 + - 3.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 0.75 + - 2.0 + - 0.75 + - 2.0 + - 2.0 + - 2.0 + - 0.75 + - 0.75 + - 0.75 + - 0.75 + - 2.0 + - 0.75 + - 0.75 + - 2.0 + - 0.75 + - 0.75 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 2.0 + - 2.5 + - 2.5 + - 3.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 3.5 + - 2.5 + - 2.5 + - 3.0 + - 3.5 + - 3.0 + - 4.0 + - 4.0 + - 0.75 + - 4.0 + - 4.0 + - 4.0 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.5 + - 4.0 + - 4.5 + - 4.0 + - 4.0 + - 4.5 + - 2.5 + - 3.0 + - 2.5 + - 3.0 + - 2.5 + - 3.0 + - 2.0 + - 2.5 + - 2.5 + - 3.0 + - 3.0 + - 2.5 + - 3.0 + - 3.0 + - 3.0 + - 2.5 + - 2.5 + - 4.0 + - 4.5 + - 4.5 + - 5.0 + - 4.0 + - 4.0 + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 5.5 + - 5.5 + - 4.0 + - 5.0 + - 4.0 + - 4.5 + - 5.5 + - 5.5 + - 6.0 + - 4.5 + - 4.5 + - 4.0 + - 5.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.25 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.25 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.4 + - 6.0 + - 1.4 + - 6.0 + - 6.0 + - 1.4 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 1.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.5 + - 4.5 + - 6.0 + - 5.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 4.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 4.5 + - 6.0 + - 6.0 + - 4.5 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 5.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + - 6.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SurfTypeCheckRad + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: *id016 + maxvalue: 1e-12 + defer to post: true + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: iasi_metop-c + get values: + time interpolation: linear + - obs space: + name: MHS METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mhs_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mhs_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: mhs_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/mhs_metop-b.20231009T150000Z.satbias.nc4 + output file: cycle_dir/mhs_metop-b.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/mhs_metop-b.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/mhs_metop-b.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/mhs_metop-b.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/mhs_metop-b.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 550.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: mhs_metop-b + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: mhs_metop-b + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-b + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-b + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-b + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-b + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: GOMsaver + filename: cycle_dir/mhs_metop-b-geovals.20231009T210000Z.nc4 + observation_name: mhs_metop-b + get values: + time interpolation: linear + - obs space: + name: MHS METOP-C + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mhs_metop-c.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mhs_metop-c.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: mhs_metop-c + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/mhs_metop-c.20231009T150000Z.satbias.nc4 + output file: cycle_dir/mhs_metop-c.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/mhs_metop-c.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/mhs_metop-c.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/mhs_metop-c.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/mhs_metop-c.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 550.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: mhs_metop-c + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: mhs_metop-c + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-c + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-c + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-c + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-c + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: GOMsaver + filename: cycle_dir/mhs_metop-c-geovals.20231009T210000Z.nc4 + observation_name: mhs_metop-c + get values: + time interpolation: linear + - obs space: + name: MHS NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mhs_n19.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mhs_n19.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: mhs_n19 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/mhs_n19.20231009T150000Z.satbias.nc4 + output file: cycle_dir/mhs_n19.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/mhs_n19.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/mhs_n19.20231009T150000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/mhs_n19.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/mhs_n19.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 550.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: mhs_n19 + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: mhs_n19 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_n19 + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_n19 + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_n19 + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_n19 + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: GOMsaver + filename: cycle_dir/mhs_n19-geovals.20231009T210000Z.nc4 + observation_name: mhs_n19 + get values: + time interpolation: linear + - obs space: + name: MLS55 AURA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mls55_aura.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mls55_aura.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - ozoneProfile + obs operator: + name: VertInterp + vertical coordinate: air_pressure + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + obs filters: + - filter: Bounds Check + filter variables: + - name: ozoneProfile + minvalue: 0 + maxvalue: 10000 + action: + name: reject + - filter: Background Check + filter variables: + - name: ozoneProfile + threshold: 5.0 + action: + name: reject + - filter: GOMsaver + filename: cycle_dir/mls55_aura-geovals.20231009T210000Z.nc4 + observation_name: mls55_aura + get values: + time interpolation: linear + - obs space: + name: OMI AURA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/omi_aura.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.omi_aura.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - ozoneTotal + obs operator: + name: AtmVertInterpLay + geovals: + - mole_fraction_of_ozone_in_air + coefficients: + - 0.0078976797 + nlevels: + - 1 + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + obs filters: + - filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 5.0 + - filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: ozoneTotal + test variables: + - name: MetaData/solarZenithAngle + maxvalue: 84.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: ozoneTotal + test variables: + - name: MetaData/sensorScanPosition + minvalue: 3 + maxvalue: 24 + action: + name: reject + - filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 5.0 + action: + name: reject + - filter: GOMsaver + filename: cycle_dir/omi_aura-geovals.20231009T210000Z.nc4 + observation_name: omi_aura + get values: + time interpolation: linear + - obs space: + name: OMPSNM NPP + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/ompsnm_npp.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.ompsnm_npp.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - ozoneTotal + obs operator: + name: AtmVertInterpLay + geovals: + - mole_fraction_of_ozone_in_air + coefficients: + - 0.0078976797 + nlevels: + - 1 + obs filters: + - filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 5.216 + - filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: ozoneTotal + test variables: + - name: MetaData/solarZenithAngle + maxvalue: 84.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 5.0 + action: + name: reject + - filter: GOMsaver + filename: cycle_dir/ompsnm_npp-geovals.20231009T210000Z.nc4 + observation_name: ompsnm_npp + get values: + time interpolation: linear + - obs space: + name: Pilot Balloon + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/pibal.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.pibal.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + vertical coordinate backup: air_pressure + observation vertical coordinate group backup: MetaData + observation vertical coordinate backup: pressure + interpolation method backup: log-linear + hofx scaling field: SurfaceWindScalingCombined + hofx scaling field group: DerivedVariables + linear obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + vertical coordinate backup: air_pressure + observation vertical coordinate group backup: MetaData + observation vertical coordinate backup: pressure + interpolation method backup: log-linear + hofx scaling field: SurfaceWindScalingCombined + hofx scaling field group: DerivedVariables + obs pre filters: + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_not_in: 221 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_not_in: 221 + action: + name: reject + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingCombined + SkipWhenNoObs: false + - filter: GOMsaver + filename: cycle_dir/pibal-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.2 + - 2.2 + - 2.3 + - 2.3 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.5 + - 2.7 + - 2.9 + - 3.1 + - 3.3 + - 3.5 + - 3.7 + - 3.9 + - 4.1 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + value: is_valid + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + value: is_valid + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 221 + cgross: + - 8.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windEastward + action: + name: reject + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 221 + cgross: + - 8.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windNorthward + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + test variables: + - name: ObsErrorData/windEastward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windEastward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windEastward + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windNorthward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windNorthward + defer to post: true + - filter: Bounds Check + filter variables: + - name: windNorthward + test variables: + - name: ObsErrorData/windNorthward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 221 + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 221 + defer to post: true + observation_name: pibal + get values: + time interpolation: linear + - obs space: + name: Satellite Winds + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/satwind.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.satwind.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + linear obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + obs prior filters: + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + - filter: GOMsaver + filename: cycle_dir/satwind-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: PreQC + maxvalue: 3 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 80000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - 2 + - filter: BlackList + where: + - variable: + name: ObsType/windEastward + is_in: 240, 241, 248, 249, 251, 255, 256, 260 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.45 + - 5 + - 5 + - 5 + - 5.075 + - 5.175 + - 5.3 + - 5.675 + - 6.05 + - 6.25 + - 6.625 + - 7 + - 7 + - 7 + - 7 + where: + - variable: + name: ObsType/windEastward + is_in: 244 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.9 + - 3.9 + - 4.0 + - 4.0 + - 4.1 + - 5 + - 6 + - 6.3 + - 6.6 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + where: + - variable: + name: ObsType/windEastward + is_in: 245, 246 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.9 + - 3.9 + - 4.0 + - 4.0 + - 4.1 + - 5 + - 6 + - 6.3 + - 6.6 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + where: + - variable: + name: ObsType/windEastward + is_in: 242, 243, 247, 252, 253 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.9 + - 3.9 + - 4.0 + - 4.5 + - 6.1 + - 6.0 + - 6.5 + - 7.3 + - 7.6 + - 7 + - 7.5 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + - 7 + where: + - variable: + name: ObsType/windEastward + is_in: 254 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.8 + - 3.9 + - 3.9 + - 4.0 + - 4.0 + - 4.1 + - 5.0 + - 7 + - 7.3 + - 7.6 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + - 8 + where: + - variable: + name: ObsType/windEastward + is_in: 250 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.9 + - 5.675 + - 5.45 + - 5.525 + - 5.8 + - 6.275 + - 6.575 + - 6.825 + - 7.05 + - 7.05 + - 7.05 + - 7.05 + where: + - variable: + name: ObsType/windEastward + is_in: 257 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 6.05 + - 5.625 + - 5.275 + - 5.375 + - 5.925 + - 6.475 + - 6.775 + - 7.05 + - 7.05 + - 7.05 + - 7.05 + where: + - variable: + name: ObsType/windEastward + is_in: 258 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6.5 + - 6 + - 5.5 + - 5.5 + - 5.575 + - 6.025 + - 6.4 + - 6.775 + - 7.15 + - 7.15 + - 7.15 + - 7.15 + where: + - variable: + name: ObsType/windEastward + is_in: 259 + - filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/air_pressure_at_surface + value: MetaData/pressure + maxvalue: -11000 + where: + - variable: + name: ObsType/windEastward + is_in: 247 + - variable: + name: MetaData/surfaceQualifier + minvalue: 1 + - filter: Difference Check + filter variables: + - name: windEastward + - name: windNorthward + reference: GeoVaLs/air_pressure_at_surface + value: MetaData/pressure + maxvalue: -20000 + where: + - variable: + name: ObsType/windEastward + is_in: 244, 257, 258, 259, 260 + - variable: + name: MetaData/surfaceQualifier + minvalue: 1 + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/WindDirAngleDiff + maxvalue: 50.0 + where: + - variable: + name: ObsType/windEastward + is_in: 247 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/SatWindsLNVDCheck + maxvalue: 3 + where: + - variable: + name: ObsType/windEastward + is_in: 244, 247, 257-260 + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: BlackList + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 15.0 + - 15.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.1 + - 20.1 + - 20.1 + - 20.1 + - 20.1 + cgross: + - 2.5 + - 2.5 + - 2.5 + - 1.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 1.3 + - 2.5 + - 1.5 + - 1.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + wndtype: + - 240 + - 241 + - 242 + - 243 + - 244 + - 245 + - 246 + - 247 + - 248 + - 249 + - 250 + - 251 + - 252 + - 253 + - 254 + - 256 + - 257 + - 258 + - 259 + - 260 + variable: windEastward + action: + name: reject + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + cgross: + - 2.5 + - 2.5 + - 2.5 + - 1.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 1.3 + - 2.5 + - 1.5 + - 1.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 15.0 + - 15.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.0 + - 20.1 + - 20.1 + - 20.1 + - 20.1 + - 20.1 + wndtype: + - 240 + - 241 + - 242 + - 243 + - 244 + - 245 + - 246 + - 247 + - 248 + - 249 + - 250 + - 251 + - 252 + - 253 + - 254 + - 256 + - 257 + - 258 + - 259 + - 260 + variable: windNorthward + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 240-260 + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 240-260 + defer to post: true + observation_name: satwind + get values: + time interpolation: linear + - obs space: + name: Scatterometer Winds + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/scatwind.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.scatwind.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + linear obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + - filter: GOMsaver + filename: cycle_dir/scatwind-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + errors: + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 290 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: BlackList + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 290 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/ScatWindsAmbiguityCheck + maxvalue: 1e-12 + where: + - variable: + name: ObsType/windEastward + is_in: 290 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windEastward + - name: HofX/windEastward + coefs: + - 1.0 + - -1.0 + minvalue: -5.0 + maxvalue: 5.0 + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windNorthward + - name: HofX/windNorthward + coefs: + - 1.0 + - -1.0 + minvalue: -5.0 + maxvalue: 5.0 + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 290 + cgross: + - 5.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windEastward + action: + name: reject + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 290 + cgross: + - 5.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windNorthward + action: + name: reject + defer to post: true + observation_name: scatwind + get values: + time interpolation: linear + - obs space: + name: Surface Marine Stations + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sfcship.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sfcship.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + - virtualTemperature + - airTemperature + - specificHumidity + - stationPressure + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - name: Identity + variables: + - name: specificHumidity + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: Identity + variables: + - name: stationPressure + - name: specificHumidity + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + - filter: GOMsaver + filename: cycle_dir/sfcship-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + minvalue: 1 + action: + name: reject + - filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + errors: + - 100 + - 100 + - 110 + - 120 + - 120 + - 100000000000.0 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + action: + name: inflate error + inflation factor: 0.7 + where: + - variable: ObsType/stationPressure + is_in: + - 180 + - variable: ObsSubType/stationPressure + is_in: + - 0 + - filter: Perform Action + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 300 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 300 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 4.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + - variable: ObsType/stationPressure + is_in: 180,183 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.8 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - variable: ObsType/stationPressure + is_in: 180,183 + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: + - 183 + action: + name: reject + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + errors: + - 1.75 + - 1.95 + - 2.25 + - 1.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - airTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/airTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + - filter: Bounds Check + filter variables: + - name: airTemperature + test variables: + - name: ObsErrorData/airTemperature + maxvalue: 100000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: + - 183 + action: + name: reject + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + errors: + - 1.75 + - 1.95 + - 2.25 + - 1.0 + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - virtualTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: GsiAdjustObsError/virtualTemperature + where: + - variable: + name: GsiAdjustObsError/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: virtualTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/virtualTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/virtualTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/virtualTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/virtualTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: PreUseFlag/virtualTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: TempObsErrorData/virtualTemperature + where: + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: virtualTemperature + defer to post: true + - filter: Bounds Check + filter variables: + - name: virtualTemperature + test variables: + - name: ObsErrorData/virtualTemperature + maxvalue: 100000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: PreUseFlag/specificHumidity + minvalue: 1 + action: + name: reject + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: + - 183 + action: + name: reject + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorSatSpecHumidity + options: + variable: specificHumidity + input_error_name: GsiInputObsError + - filter: BlackList + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: specificHumidity + inflation factor: 8 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: specificHumidity + threshold: 8.0 + action: + name: reject + where: + - variable: PreQC/specificHumidity + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: specificHumidity + threshold: 5.6 + action: + name: reject + where: + - variable: PreQC/specificHumidity + is_in: + - 3 + defer to post: true + - filter: BlackList + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: specificHumidity + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_in: + - 284 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_in: + - 284 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: + - 280 + action: + name: assign error + error parameter: 2.5 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: + - 282 + action: + name: assign error + error parameter: 2.2 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + minvalue: 1 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + minvalue: 1 + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + threshold: 6.0 + action: + name: reject + where: + - variable: PreQC/windEastward + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windEastward + threshold: 4.2 + action: + name: reject + where: + - variable: PreQC/windEastward + is_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + threshold: 6.0 + action: + name: reject + where: + - variable: PreQC/windNorthward + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + threshold: 4.2 + action: + name: reject + where: + - variable: PreQC/windNorthward + is_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + satwndtype: + - 280 + - 282 + - 284 + error_min: + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + cgross: + - 6.0 + - 6.0 + - 6.0 + wndtype: + - 280 + - 282 + - 284 + variable: windEastward + action: + name: reject + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + satwndtype: + - 280 + - 282 + - 284 + error_min: + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + cgross: + - 6.0 + - 6.0 + - 6.0 + wndtype: + - 280 + - 282 + - 284 + variable: windNorthward + action: + name: reject + defer to post: true + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + defer to post: true + observation_name: sfcship + get values: + time interpolation: linear + - obs space: + name: Surface Land Stations + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sfc.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sfc.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + - virtualTemperature + - airTemperature + - specificHumidity + - stationPressure + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: specificHumidity + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: stationPressure + - name: specificHumidity + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + - filter: GOMsaver + filename: cycle_dir/sfc-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 49999 + action: + name: reject + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + minvalue: 1 + action: + name: reject + - filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 181 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 70000 + - 65000 + - 60000 + - 55000 + errors: + - 100 + - 120 + - 120 + - 100000000000.0 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 187 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + errors: + - 100 + - 130 + - 160 + - 100000000000.0 + - filter: Perform Action + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 300 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 300 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 4.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + - variable: ObsType/stationPressure + is_in: 187 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.8 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - variable: ObsType/stationPressure + is_in: 187 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 3.6 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + - variable: ObsType/stationPressure + is_in: + - 181 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.52 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - variable: ObsType/stationPressure + is_in: + - 181 + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + defer to post: true + - filter: Bounds Check + filter variables: + - name: stationPressure + test variables: + - name: ObsErrorData/stationPressure + maxvalue: 100000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: 181,187 + action: + name: reject + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: 181,187 + action: + name: reject + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: 181,187 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_in: 281,287 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_in: 281,287 + action: + name: reject + observation_name: sfc + get values: + time interpolation: linear + - obs space: + name: Radiosondes + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sondes.20231009T210000Z.nc4 + missing file action: error + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sondes.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + - virtualTemperature + - airTemperature + - specificHumidity + - stationPressure + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + - name: VertInterp + variables: + - name: airTemperature + - name: virtualTemperature + - name: specificHumidity + - name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + - name: VertInterp + variables: + - name: airTemperature + - name: virtualTemperature + - name: specificHumidity + - name: Identity + variables: + - name: stationPressure + obs prior filters: + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + - filter: GOMsaver + filename: cycle_dir/sondes-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + minvalue: 1 + action: + name: reject + - filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 60000 + - 55000 + errors: + - 100 + - 100000000000.0 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 300 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 300 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 4.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.8 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: PreUseFlag/specificHumidity + minvalue: 1 + action: + name: reject + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + action: + name: assign error + error function: + name: ObsFunction/ObsErrorSatSpecHumidity + options: + variable: specificHumidity + input_error_name: GsiInputObsError + - filter: BlackList + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: specificHumidity + inflation factor: 8.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/specificHumidity + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/specificHumidity + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: specificHumidity + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 70000 + - 65000 + - 60000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + errors: + - 1.3 + - 1.1 + - 0.9 + - 0.7 + - 0.6 + - 0.6 + - 0.55 + - 0.5 + - 0.5 + - 0.55 + - 0.65 + - 1.1 + - 1.2 + - 1.2 + - 1.4 + - 1.6 + - 1.85 + - 2.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - airTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/airTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 8.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 5.6 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 70000 + - 65000 + - 60000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + errors: + - 1.3 + - 1.1 + - 0.9 + - 0.7 + - 0.6 + - 0.6 + - 0.55 + - 0.5 + - 0.5 + - 0.55 + - 0.65 + - 1.1 + - 1.2 + - 1.2 + - 1.4 + - 1.6 + - 1.85 + - 2.0 + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - virtualTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: GsiAdjustObsError/virtualTemperature + where: + - variable: + name: GsiAdjustObsError/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: virtualTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/virtualTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/virtualTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/virtualTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/virtualTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: PreUseFlag/virtualTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 8.0 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 5.6 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: TempObsErrorData/virtualTemperature + where: + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: virtualTemperature + defer to post: true + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.2 + - 2.2 + - 2.3 + - 2.3 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.5 + - 2.7 + - 2.9 + - 3.1 + - 3.3 + - 3.5 + - 3.7 + - 3.9 + - 4.1 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + value: is_valid + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + value: is_valid + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 220,221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 220,221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 220 + - 221 + cgross: + - 8.0 + - 8.0 + error_min: + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + variable: windEastward + action: + name: reject + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 220 + - 221 + cgross: + - 8.0 + - 8.0 + error_min: + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + variable: windNorthward + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + test variables: + - name: ObsErrorData/windEastward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windEastward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windEastward + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windNorthward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windNorthward + defer to post: true + - filter: Bounds Check + filter variables: + - name: windNorthward + test variables: + - name: ObsErrorData/windNorthward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 220 + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 220 + defer to post: true + observation_name: sondes + get values: + time interpolation: linear + - obs space: + name: ssmis_f17 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/ssmis_f17.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.ssmis_f17.20231009T210000Z.nc4 + simulated variables: + - brightnessTemperature + channels: None + io pool: + max pool size: 6 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs options: + Sensor_ID: ssmis_f17 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + obs bias: + input file: cycle_dir/ssmis_f17.20231009T150000Z.satbias.nc4 + output file: cycle_dir/ssmis_f17.20231009T210000Z.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/ssmis_f17.20231009T150000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/ssmis_f17.20231009T150000Z.tlapse.txt + - name: cosineOfLatitudeTimesOrbitNode + - name: sineOfLatitude + - name: emissivityJacobian + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/ssmis_f17.20231009T150000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/ssmis_f17.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 1.0 + - 1.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 2.4 + - 1.27 + - 1.44 + - 3.0 + - 1.34 + - 1.74 + - 3.75 + - 3.0 + - 3.0 + - 2.0 + - 6.4 + - 1.0 + - 1.0 + - filter: GOMsaver + filename: cycle_dir/ssmis_f17-geovals.20231009T210000Z.nc4 + obs post filters: + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/geopotential_height_at_surface + minvalue: 2000.0 + min_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 1-3,8-18 + where: + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/ice_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/surface_snow_area_fraction + maxvalue: 0.99 + max_exclusive: true + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 1-2, 12-16 + reference: ObsValue/brightnessTemperature_2 + value: HofX/brightnessTemperature_2 + minvalue: -1.5 + maxvalue: 1.5 + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + action: + name: reject + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 9 + reference: ObsValue/brightnessTemperature_9 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: + - -0.485934 + - 0.485934 + - 0.473806 + - -0.473806 + intercept: 271.252327 + maxvalue: 2 + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 10 + reference: ObsValue/brightnessTemperature_10 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: + - -0.413688 + - 0.413688 + - 0.361549 + - -0.361549 + intercept: 272.280341 + maxvalue: 2 + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 11 + reference: ObsValue/brightnessTemperature_11 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: + - -0.400882 + - 0.400882 + - 0.27051 + - -0.27051 + intercept: 278.824902 + maxvalue: 2 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: ssmis_f17 + obserr_demisf: + - 0.01 + - 0.01 + - 0.01 + - 0.01 + - 0.01 + obserr_dtempf: + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor1 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_1 + coefs: + - 2.25 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_1 + threshold: DerivedMetaData/errorInflationFactor1 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor2 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_2 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_2 + threshold: DerivedMetaData/errorInflationFactor2 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor3 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_3 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_3 + threshold: DerivedMetaData/errorInflationFactor3 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor4 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_4 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_4 + threshold: DerivedMetaData/errorInflationFactor4 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor5 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_5 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_5 + threshold: DerivedMetaData/errorInflationFactor5 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor6 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_6 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_6 + threshold: DerivedMetaData/errorInflationFactor6 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor7 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_7 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_7 + threshold: DerivedMetaData/errorInflationFactor7 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor8 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_8 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_8 + threshold: DerivedMetaData/errorInflationFactor8 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor9 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_9 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_9 + threshold: DerivedMetaData/errorInflationFactor9 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_10 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_10 + threshold: DerivedMetaData/errorInflationFactor10 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_11 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_11 + threshold: DerivedMetaData/errorInflationFactor11 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor12 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_12 + coefs: + - 3.6 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_12 + threshold: DerivedMetaData/errorInflationFactor12 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor13 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_13 + coefs: + - 1.905 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_13 + threshold: DerivedMetaData/errorInflationFactor13 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor14 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_14 + coefs: + - 2.16 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_14 + threshold: DerivedMetaData/errorInflationFactor14 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor15 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_15 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_15 + threshold: DerivedMetaData/errorInflationFactor15 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor16 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_16 + coefs: + - 2.01 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_16 + threshold: DerivedMetaData/errorInflationFactor16 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor17 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_17 + coefs: + - 2.61 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_17 + threshold: DerivedMetaData/errorInflationFactor17 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor18 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_18 + coefs: + - 5.625 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_18 + threshold: DerivedMetaData/errorInflationFactor18 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor19 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_19 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_19 + threshold: DerivedMetaData/errorInflationFactor19 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor20 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_20 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_20 + threshold: DerivedMetaData/errorInflationFactor20 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor21 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_21 + coefs: + - 3.0 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_21 + threshold: DerivedMetaData/errorInflationFactor21 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor22 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_22 + coefs: + - 9.6 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_22 + threshold: DerivedMetaData/errorInflationFactor22 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor23 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_23 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_23 + threshold: DerivedMetaData/errorInflationFactor23 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor24 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_24 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_24 + threshold: DerivedMetaData/errorInflationFactor24 + absolute threshold: 6.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: ssmis_f17 + get values: + time interpolation: linear +model: + name: PSEUDO + tstep: PT1H + filetype: cube sphere history + provider: geos + compute edge pressure from surface pressure: true + max allowable geometry difference: 0.001 + datapath: cycle_dir + filenames: + - bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4 + - fv3-jedi/bkg/geos.crtmsrf.91.nc4 + field io names: *id017 +forecast length: PT6H diff --git a/src/swell/test/jedi_configs/jedi_localensembleda_config.yaml b/src/swell/test/jedi_configs/jedi_localensembleda_config.yaml new file mode 100644 index 000000000..92964d5b5 --- /dev/null +++ b/src/swell/test/jedi_configs/jedi_localensembleda_config.yaml @@ -0,0 +1,9498 @@ +geometry: + fms initialization: + namelist filename: ./fv3-jedi/fv3files/fmsmpp.nml + field table filename: ./fv3-jedi/fv3files/field_table_gmao + akbk: ./fv3-jedi/fv3files/akbk72.nc4 + layout: + - 4 + - 4 + npx: '91' + npy: '91' + npz: '72' +time window: + begin: '2023-10-09T21:00:00Z' + end: '2023-10-10T03:00:00Z' + bound to include: begin +increment variables: +- eastward_wind +- northward_wind +- air_temperature +- air_pressure_at_surface +- air_pressure_levels +- water_vapor_mixing_ratio_wrt_moist_air +- cloud_liquid_ice +- cloud_liquid_water +- mole_fraction_of_ozone_in_air +background: + date: '2023-10-09T21:00:00Z' + members from template: + template: + datetime: '2023-10-10T00:00:00Z' + filetype: cube sphere history + provider: geos + compute edge pressure from surface pressure: true + max allowable geometry difference: 0.001 + datapath: . + filenames: + - ebkg/mem%mem%/geos.mem%mem%.%yyyy%mm%dd_%hh%MM%ssz.nc4 + - fv3-jedi/bkg/geos.crtmsrf.91.nc4 + state variables: + - eastward_wind + - northward_wind + - air_temperature + - air_pressure_at_surface + - air_pressure_levels + - water_vapor_mixing_ratio_wrt_moist_air + - cloud_liquid_ice + - cloud_liquid_water + - rain_water + - snow_water + - mole_fraction_of_ozone_in_air + - geopotential_height_times_gravity_at_surface + - initial_mass_fraction_of_large_scale_cloud_condensate + - initial_mass_fraction_of_convective_cloud_condensate + - convective_cloud_area_fraction + - fraction_of_ocean + - fraction_of_land + - isotropic_variance_of_filtered_topography + - surface_velocity_scale + - surface_buoyancy_scale + - planetary_boundary_layer_height + - surface_exchange_coefficient_for_momentum + - surface_exchange_coefficient_for_heat + - surface_exchange_coefficient_for_moisture + - KCBL_before_moist + - surface_temp_before_moist + - lower_index_where_Kh_greater_than_2 + - upper_index_where_Kh_greater_than_2 + - fraction_of_lake + - fraction_of_ice + - vtype + - stype + - vfrac + - sheleg + - skin_temperature_at_surface + - soilt + - soilm + - eastward_wind_at_surface + - northward_wind_at_surface + field io names: + eastward_wind: ua + northward_wind: va + air_temperature: t + air_pressure_at_surface: ps + air_pressure_levels: pe + water_vapor_mixing_ratio_wrt_moist_air: q + cloud_liquid_ice: qi + cloud_liquid_water: ql + rain_water: qr + snow_water: qs + mole_fraction_of_ozone_in_air: o3ppmv + geopotential_height_times_gravity_at_surface: phis + initial_mass_fraction_of_large_scale_cloud_condensate: qls + initial_mass_fraction_of_convective_cloud_condensate: qcn + convective_cloud_area_fraction: cfcn + fraction_of_ocean: frocean + fraction_of_land: frland + isotropic_variance_of_filtered_topography: varflt + surface_velocity_scale: ustar + surface_buoyancy_scale: bstar + planetary_boundary_layer_height: zpbl + surface_exchange_coefficient_for_momentum: cm + surface_exchange_coefficient_for_heat: ct + surface_exchange_coefficient_for_moisture: cq + KCBL_before_moist: kcbl + surface_temp_before_moist: tsm + lower_index_where_Kh_greater_than_2: khl + upper_index_where_Kh_greater_than_2: khu + fraction_of_lake: frlake + fraction_of_ice: frseaice + skin_temperature_at_surface: ts + eastward_wind_at_surface: u10m + northward_wind_at_surface: v10m + pattern: '%mem%' + nmembers: 16 + zero padding: 3 +observations: + get values: + variable change: + variable change name: Model2GeoVaLs + hydrometeor effective radii method: gsi + tropopause pressure method: gsi + observers: + - obs space: + name: Aircraft Temperature + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/aircraft_temperature.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.aircraft_temperature.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - airTemperature + distribution: + name: Halo + halo size: 5000000.0 + obs bias: + input file: cycle_dir/aircraft_temperature.20231009T210000Z.acftbias + output file: cycle_dir/aircraft_temperature.20231009T210000Z.acftbias + bc by record: true + static bc: + predictors: + - name: constant + - name: obsMetadataPredictor + variable: instantaneousAltitudeRate + - name: obsMetadataPredictor + variable: instantaneousAltitudeRate + order: 2 + covariance: + minimal required obs number: 3 + variance range: + - 1e-06 + - 1.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/aircraft_temperature.20231009T210000Z.acftbias_cov + inflation: + ratio: 1.005 + ratio for small dataset: 2.0 + obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - airTemperature + linear obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - airTemperature + obs filters: + - filter: Variable Assignment + where: + - variable: + name: ObsType/airTemperature + is_in: 130 + assignments: + - name: MetaData/stationIdentification + value: 'KX130 ' + - filter: Variable Assignment + where: + - variable: + name: MetaData/instantaneousAltitudeRate + minvalue: 50.0 + assignments: + - name: MetaData/instantaneousAltitudeRate + value: 0.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 2.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + errors: + - 2.5 + - 2.3 + - 2.1 + - 1.9 + - 1.7 + where: + - variable: + name: ObsType/airTemperature + is_in: 130 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + errors: + - 1.4706 + - 1.3529 + - 1.2353 + - 1.1176 + - 1.0 + where: + - variable: + name: ObsType/airTemperature + is_in: 131,133 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 70000 + - 65000 + - 60000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 5000 + errors: + - 1.5 + - 1.3 + - 1.1 + - 0.9 + - 0.8 + - 0.8 + - 0.75 + - 0.7 + - 0.7 + - 0.75 + - 0.85 + - 1.3 + - 1.5 + - 1.5 + where: + - variable: + name: ObsType/airTemperature + is_in: 132 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 60000 + - 40000 + errors: + - 1.5 + - 1.35 + - 1.25 + - 1.1 + - 1.0 + - 1.3 + - 1.7 + where: + - variable: + name: ObsType/airTemperature + is_in: 134 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + errors: + - 1.4706 + - 1.3529 + - 1.2353 + - 1.1176 + - 1000000000.0 + where: + - variable: + name: ObsType/airTemperature + is_in: 135 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: + - airTemperature + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreQC/airTemperature + is_in: 4-15 + action: + name: passivate + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: + name: ObsType/airTemperature + is_in: 134, 135 + action: + name: passivate + defer to post: true + - filter: Perform Action + action: + name: inflate error + inflation factor: 10.0 + filter variables: + - name: airTemperature + where: + - variable: + name: MetaData/pressure + maxvalue: 110000 + minvalue: 50000 + - variable: + name: ObsType/airTemperature + is_in: 130 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + observation_name: aircraft_temperature + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: Aircraft Wind + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/aircraft_wind.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.aircraft_wind.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + obs prior filters: + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + obs post filters: + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: PreQC/windEastward + is_in: 4-15 + action: + name: passivate + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: ObsType/windEastward + is_in: 234, 235 + action: + name: passivate + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 100000 + - 95000 + - 80000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + errors: + - 1.4 + - 1.5 + - 1.6 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.3 + - 2.6 + - 2.8 + - 3.0 + - 3.2 + - 2.7 + - 2.4 + - 2.1 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.6 + where: + - variable: + name: ObsType/windEastward + is_in: 230 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.0 + where: + - variable: + name: ObsType/windEastward + is_in: 231 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 2.5 + where: + - variable: + name: ObsType/windEastward + is_in: 233 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.0 + where: + - variable: + name: ObsType/windEastward + is_in: 234, 235 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + errors: + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.2 + - 2.2 + - 2.3 + - 2.3 + - 2.4 + - 2.4 + - 2.5 + - 2.7 + - 2.9 + - 3.1 + where: + - variable: + name: ObsType/windEastward + is_in: 232 + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: + - windEastward + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: + - windNorthward + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 230 + - 231 + - 232 + - 233 + - 234 + - 235 + cgross: + - 6.0 + - 6.5 + - 7.0 + - 7.5 + - 7.5 + - 7.5 + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + variable: windEastward + action: + name: reject + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 230 + - 231 + - 232 + - 233 + - 234 + - 235 + cgross: + - 6.0 + - 6.5 + - 7.0 + - 7.5 + - 7.5 + - 7.5 + error_min: + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + - 6.1 + variable: windNorthward + action: + name: reject + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + defer to post: true + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + defer to post: true + observation_name: aircraft_wind + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: Radiosondes + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sondes.20231009T210000Z.nc4 + missing file action: error + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sondes.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + - virtualTemperature + - airTemperature + - specificHumidity + - stationPressure + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + - name: VertInterp + variables: + - name: airTemperature + - name: virtualTemperature + - name: specificHumidity + - name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + hofx scaling field: SurfaceWindScalingPressure + hofx scaling field group: DerivedVariables + - name: VertInterp + variables: + - name: airTemperature + - name: virtualTemperature + - name: specificHumidity + - name: Identity + variables: + - name: stationPressure + obs prior filters: + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + obs post filters: + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + minvalue: 1 + action: + name: reject + - filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 60000 + - 55000 + errors: + - 100 + - 100000000000.0 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 300 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 300 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 4.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.8 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: PreUseFlag/specificHumidity + minvalue: 1 + action: + name: reject + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + action: + name: assign error + error function: + name: ObsFunction/ObsErrorSatSpecHumidity + options: + variable: specificHumidity + input_error_name: GsiInputObsError + - filter: BlackList + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: specificHumidity + inflation factor: 8.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/specificHumidity + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/specificHumidity + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: specificHumidity + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 70000 + - 65000 + - 60000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + errors: + - 1.3 + - 1.1 + - 0.9 + - 0.7 + - 0.6 + - 0.6 + - 0.55 + - 0.5 + - 0.5 + - 0.55 + - 0.65 + - 1.1 + - 1.2 + - 1.2 + - 1.4 + - 1.6 + - 1.85 + - 2.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - airTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/airTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 8.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 5.6 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 70000 + - 65000 + - 60000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + errors: + - 1.3 + - 1.1 + - 0.9 + - 0.7 + - 0.6 + - 0.6 + - 0.55 + - 0.5 + - 0.5 + - 0.55 + - 0.65 + - 1.1 + - 1.2 + - 1.2 + - 1.4 + - 1.6 + - 1.85 + - 2.0 + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - virtualTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: GsiAdjustObsError/virtualTemperature + where: + - variable: + name: GsiAdjustObsError/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: virtualTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/virtualTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/virtualTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/virtualTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/virtualTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: PreUseFlag/virtualTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 8.0 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 5.6 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: TempObsErrorData/virtualTemperature + where: + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: virtualTemperature + defer to post: true + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.2 + - 2.2 + - 2.3 + - 2.3 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.5 + - 2.7 + - 2.9 + - 3.1 + - 3.3 + - 3.5 + - 3.7 + - 3.9 + - 4.1 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + value: is_valid + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + value: is_valid + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 220,221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 220,221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 220 + - 221 + cgross: + - 8.0 + - 8.0 + error_min: + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + variable: windEastward + action: + name: reject + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 220 + - 221 + cgross: + - 8.0 + - 8.0 + error_min: + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + variable: windNorthward + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + test variables: + - name: ObsErrorData/windEastward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windEastward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windEastward + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windNorthward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windNorthward + defer to post: true + - filter: Bounds Check + filter variables: + - name: windNorthward + test variables: + - name: ObsErrorData/windNorthward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 220 + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 220 + defer to post: true + observation_name: sondes + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: gnssrobndnbam + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/gps.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - sequenceNumber + sort variable: impactHeightRO + sort order: ascending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.gps.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - bendingAngle + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: GnssroBndNBAM + obs options: + use_compress: 1 + vertlayer: full + sr_steps: 2 + super_ref_qc: NBAM + obs filters: + - filter: BlackList + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 41,265,266,421,440,724,725,726,727,728,729 + - filter: Perform Action + filter variables: + - name: bendingAngle + where: + - variable: PreUseFlag/bendingAngle + minvalue: 1 + action: + name: reject + - filter: Domain Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/impactHeightRO + minvalue: 0 + maxvalue: 55000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266,267,268,269 + test variables: + - name: MetaData/impactHeightRO + maxvalue: 45000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 3-5 + test variables: + - name: MetaData/impactHeightRO + minvalue: 8000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 267,268,269 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 401 + test variables: + - name: MetaData/impactHeightRO + minvalue: 5000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 267,268,269 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 402-999 + test variables: + - name: MetaData/impactHeightRO + minvalue: 9000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 401 + test variables: + - name: MetaData/impactHeightRO + minvalue: 5000.1 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: bendingAngle + where: + - variable: + name: MetaData/satelliteIdentifier + is_in: 265,266 + - variable: + name: MetaData/satelliteConstellationRO + is_in: 402-999 + test variables: + - name: MetaData/impactHeightRO + minvalue: 8000.1 + action: + name: reject + - filter: ROobserror + filter variables: + - name: bendingAngle + errmodel: NBAM + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: inflate error + inflation factor: 2.0 + where: + - variable: MetaData/satelliteIdentifier + is_in: 267,268,269 + - filter: Variable Assignment + assignments: + - name: JediAdjustObsError/bendingAngle + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/bendingAngle + defer to post: true + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: assign error + error parameter: 1.0 + where: + - variable: + name: JediAdjustObsError/bendingAngle + maxvalue: 1.0 + - variable: + name: JediAdjustObsError/bendingAngle + value: is_valid + - variable: + name: MetaData/satelliteIdentifier + is_in: 3,5,41,42,43,44,267,268,269,440,421,724,725,726,727,728,729, 750,751,752,753,754,755,821,825 + defer to post: true + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: assign error + error parameter: 10.0 + where: + - variable: + name: JediAdjustObsError/bendingAngle + minvalue: 10.0 + - variable: + name: JediAdjustObsError/bendingAngle + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: bendingAngle + threshold: 5 + action: + name: reject + defer to post: true + - filter: Perform Action + filter variables: + - name: bendingAngle + action: + name: assign error + error function: JediAdjustObsError/bendingAngle + where: + - variable: + name: JediAdjustObsError/bendingAngle + value: is_valid + defer to post: true + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + defer to post: true + - filter: Background Check RONBAM + filter variables: + - name: bendingAngle + action: + name: RONBAMErrInflate_GEOS + defer to post: true + observation_name: gps + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: AMSU-A AQUA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_aqua.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_aqua.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_aqua + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_aqua.20231009T210000Z.satbias.nc4 + output file: cycle_dir/amsua_aqua.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_aqua.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_aqua.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_aqua.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_aqua.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id001 + - 2.5 + - 2.0 + - 2.0 + - 0.5 + - 0.4 + - 0.4 + - 0.5 + - 0.3 + - 0.35 + - 0.35 + - 0.45 + - 1.0 + - 1.5 + - 3.75 + - 6.3 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_aqua + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_aqua + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_aqua + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_aqua + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_aqua + error parameter vector: *id001 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 3.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 3.0 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_aqua + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_aqua + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: AMSU-A NOAA-15 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_n15.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_n15.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_n15 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_n15.20231009T210000Z.satbias.nc4 + output file: cycle_dir/amsua_n15.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_n15.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_n15.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_n15.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_n15.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id002 + - 3.0 + - 2.0 + - 2.0 + - 0.6 + - 0.3 + - 0.23 + - 0.25 + - 0.275 + - 0.34 + - 0.4 + - 0.6 + - 1.0 + - 1.5 + - 5.0 + - 3.0 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_n15 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_n15 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_n15 + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_n15 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_n15 + error parameter vector: *id002 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_n15 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_n15 + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: AMSU-A NOAA-18 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_n18.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_n18.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_n18 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_n18.20231009T210000Z.satbias.nc4 + output file: cycle_dir/amsua_n18.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_n18.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_n18.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_n18.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_n18.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id003 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_n18 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_n18 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_n18 + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_n18 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_n18 + error parameter vector: *id003 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_n18 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_n18 + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: AMSU-A NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_n19.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_n19.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_n19 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_n19.20231009T210000Z.satbias.nc4 + output file: cycle_dir/amsua_n19.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_n19.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_n19.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_n19.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_n19.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id004 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_n19 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_n19 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_n19 + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_n19 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_n19 + error parameter vector: *id004 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_n19 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_n19 + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: AMSR2 GCOM-W1 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsr2_gcom-w1.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsr2_gcom-w1.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: amsr2_gcom-w1 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsr2_gcom-w1.20231009T210000Z.satbias.nc4 + output file: cycle_dir/amsr2_gcom-w1.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsr2_gcom-w1.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsr2_gcom-w1.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsr2_gcom-w1.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsr2_gcom-w1.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 340.0 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.999 + - variable: + name: GeoVaLs/skin_temperature_at_surface_where_sea + minvalue: 275 + - variable: + name: GeoVaLs/wind_speed_at_surface + maxvalue: 12 + - variable: + name: MetaData/latitude + minvalue: -60.0 + maxvalue: 60.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/TotalColumnVaporGuess + minvalue: 10.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/SunGlintAngle + minvalue: 20.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: &id005 + - 0.48 + - 3.0737 + - 0.7433 + - 3.643 + - 3.5304 + - 4.427 + - 5.1448 + - 5.0785 + - 4.9763 + - 9.3215 + - 2.5789 + - 5.5274 + - 0.6641 + - 1.3674 + clwret_types: + - ObsValue + maxvalue: 1.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id005 + clwret_types: + - HofX + maxvalue: 1.0 + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: None + value: + name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id005 + clwret_types: + - ObsValue + reference: + name: ObsFunction/CLWRetMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id005 + clwret_types: + - HofX + minvalue: -0.5 + maxvalue: 0.5 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch18v: 7 + clwret_ch18h: 8 + clwret_ch36v: 11 + clwret_ch36h: 12 + sys_bias: *id005 + clwret_types: + - ObsValue + - HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.1 + - 0.1 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 0.6 + - 0.6 + - 0.6 + - 0.6 + - 0.6 + - 0.5 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + err0: + - 0.8 + - 0.9 + - 0.8 + - 0.9 + - 1.0 + - 1.1 + - 2.0 + - 3.5 + - 3.0 + - 4.8 + - 5.0 + - 6.0 + - 4.5 + - 6.3 + err1: + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 5.0 + - 18.5 + - 20.0 + - 40.0 + - 20.0 + - 25.0 + - 30.0 + - 30.0 + - 30.0 + - 20.0 + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: None + threshold: 2.0 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 7-10 + absolute threshold: 30 + action: + name: reject + - filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 11-14 + absolute threshold: 50 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: amsr2_gcom-w1 + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsr2_gcom-w1 + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: ATMS NOAA-20 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/atms_n20.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.atms_n20.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: atms_n20 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/atms_n20.20231009T210000Z.satbias.nc4 + output file: cycle_dir/atms_n20.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: ATMS + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/atms_n20.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/atms_n20.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/atms_n20.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/atms_n20.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id006 + - 5.0 + - 5.0 + - 5.0 + - 3.0 + - 0.55 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 5.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7,16-22 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-7,16 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: atms_n20 + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: atms_n20 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + use_biasterm: true + test_biasterm: ObsBiasTerm + sensor: atms_n20 + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: atms_n20 + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: atms_n20 + error parameter vector: *id006 + obserr_bound_max: + - 4.5 + - 4.5 + - 3.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 2.0 + - 4.5 + - 4.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: atms_n20 + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: atms_n20 + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: ATMS NPP + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/atms_npp.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.atms_npp.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: atms_npp + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/atms_npp.20231009T210000Z.satbias.nc4 + output file: cycle_dir/atms_npp.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: ATMS + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/atms_npp.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/atms_npp.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/atms_npp.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/atms_npp.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id007 + - 5.0 + - 5.0 + - 5.0 + - 3.0 + - 0.55 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.3 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 5.0 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7,16-22 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-7,16 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: atms_npp + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: atms_npp + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + use_biasterm: true + test_biasterm: ObsBiasTerm + sensor: atms_npp + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: atms_npp + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: atms_npp + error parameter vector: *id007 + obserr_bound_max: + - 4.5 + - 4.5 + - 3.0 + - 3.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 1.0 + - 2.0 + - 4.5 + - 4.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: atms_npp + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: atms_npp + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: AVHRR-3 METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/avhrr3_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.avhrr3_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: avhrr3_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/avhrr3_metop-b.20231009T210000Z.satbias.nc4 + output file: cycle_dir/avhrr3_metop-b.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/avhrr3_metop-b.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/avhrr3_metop-b.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/avhrr3_metop-b.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/avhrr3_metop-b.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.0 + - 1.08 + - 1.12 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 1e-05 + maxvalue: 1000.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: avhrr3_metop-b + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + cloud detection criteria: + - 0.6 + - 0.68 + - 0.72 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: avhrr3_metop-b + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: avhrr3_metop-b + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: AVHRR-3 NOAA-18 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/avhrr3_n18.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.avhrr3_n18.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: avhrr3_n18 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/avhrr3_n18.20231009T210000Z.satbias.nc4 + output file: cycle_dir/avhrr3_n18.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/avhrr3_n18.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/avhrr3_n18.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/avhrr3_n18.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/avhrr3_n18.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.0 + - 1.08 + - 1.12 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 1e-05 + maxvalue: 1000.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: avhrr3_n18 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + cloud detection criteria: + - 0.6 + - 0.68 + - 0.72 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: avhrr3_n18 + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: avhrr3_n18 + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: AVHRR-3 NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/avhrr3_n19.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.avhrr3_n19.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: avhrr3_n19 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/avhrr3_n19.20231009T210000Z.satbias.nc4 + output file: cycle_dir/avhrr3_n19.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/avhrr3_n19.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/avhrr3_n19.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/avhrr3_n19.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/avhrr3_n19.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.0 + - 1.08 + - 1.12 + - filter: Variable Assignment + assignments: + - name: ObsError/brightnessTemperature + channels: None + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature + channels: None + obs post filters: + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 1e-05 + maxvalue: 1000.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: avhrr3_n19 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: None + options: + channels: None + use_flag: None + use_flag_clddet: + - 1 + - 1 + - 1 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + cloud detection criteria: + - 0.6 + - 0.68 + - 0.72 + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: avhrr3_n19 + obserr_demisf: + - 0.01 + - 0.02 + - 0.03 + - 0.02 + - 0.03 + obserr_dtempf: + - 0.5 + - 2.0 + - 4.0 + - 2.0 + - 4.0 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: None + options: + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.5 + - 0.04 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_max: + - 3.0 + - 3.0 + - 3.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: avhrr3_n19 + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: Scatterometer Winds + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/scatwind.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.scatwind.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + linear obs operator: + name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + obs post filters: + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + errors: + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 290 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: BlackList + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 290 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/ScatWindsAmbiguityCheck + maxvalue: 1e-12 + where: + - variable: + name: ObsType/windEastward + is_in: 290 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windEastward + - name: HofX/windEastward + coefs: + - 1.0 + - -1.0 + minvalue: -5.0 + maxvalue: 5.0 + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windNorthward + - name: HofX/windNorthward + coefs: + - 1.0 + - -1.0 + minvalue: -5.0 + maxvalue: 5.0 + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 290 + cgross: + - 5.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windEastward + action: + name: reject + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 290 + cgross: + - 5.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windNorthward + action: + name: reject + defer to post: true + observation_name: scatwind + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: Surface Marine Stations + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sfcship.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sfcship.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + - virtualTemperature + - airTemperature + - specificHumidity + - stationPressure + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - name: Identity + variables: + - name: specificHumidity + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: Identity + variables: + - name: stationPressure + - name: specificHumidity + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + obs post filters: + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + minvalue: 1 + action: + name: reject + - filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + errors: + - 100 + - 100 + - 110 + - 120 + - 120 + - 100000000000.0 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + action: + name: inflate error + inflation factor: 0.7 + where: + - variable: ObsType/stationPressure + is_in: + - 180 + - variable: ObsSubType/stationPressure + is_in: + - 0 + - filter: Perform Action + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 300 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 300 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 4.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + - variable: ObsType/stationPressure + is_in: 180,183 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.8 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - variable: ObsType/stationPressure + is_in: 180,183 + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: + - 183 + action: + name: reject + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + errors: + - 1.75 + - 1.95 + - 2.25 + - 1.0 + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - airTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/airTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: airTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + - filter: Bounds Check + filter variables: + - name: airTemperature + test variables: + - name: ObsErrorData/airTemperature + maxvalue: 100000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: + - 183 + action: + name: reject + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + errors: + - 1.75 + - 1.95 + - 2.25 + - 1.0 + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + test QCthreshold: 3 + inflate variables: + - virtualTemperature + pressure: MetaData/pressure + distance threshold: -1.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 7 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: MetaData/pressure + minvalue: 0 + maxvalue: 9999 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: GsiAdjustObsError/virtualTemperature + where: + - variable: + name: GsiAdjustObsError/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: virtualTemperature + inflation factor: 8.0 + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/virtualTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/virtualTemperature + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/virtualTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/virtualTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: PreUseFlag/virtualTemperature + minvalue: 1 + action: + name: reject + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: virtualTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/virtualTemperature + is_in: + - 3 + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: assign error + error function: TempObsErrorData/virtualTemperature + where: + - variable: + name: TempObsErrorData/virtualTemperature + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: virtualTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: virtualTemperature + defer to post: true + - filter: Bounds Check + filter variables: + - name: virtualTemperature + test variables: + - name: ObsErrorData/virtualTemperature + maxvalue: 100000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: PreUseFlag/specificHumidity + minvalue: 1 + action: + name: reject + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: + - 183 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: + - 183 + action: + name: reject + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: + - 180 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorSatSpecHumidity + options: + variable: specificHumidity + input_error_name: GsiInputObsError + - filter: BlackList + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: specificHumidity + inflation factor: 8 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: specificHumidity + threshold: 8.0 + action: + name: reject + where: + - variable: PreQC/specificHumidity + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: specificHumidity + threshold: 5.6 + action: + name: reject + where: + - variable: PreQC/specificHumidity + is_in: + - 3 + defer to post: true + - filter: BlackList + filter variables: + - name: specificHumidity + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: specificHumidity + defer to post: true + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_in: + - 284 + action: + name: assign error + error parameter: 100000000000.0 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_in: + - 284 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: + - 280 + action: + name: assign error + error parameter: 2.5 + - filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: ObsType/windEastward + is_in: + - 282 + action: + name: assign error + error parameter: 2.2 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + minvalue: 1 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + minvalue: 1 + action: + name: reject + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + threshold: 6.0 + action: + name: reject + where: + - variable: PreQC/windEastward + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windEastward + threshold: 4.2 + action: + name: reject + where: + - variable: PreQC/windEastward + is_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + threshold: 6.0 + action: + name: reject + where: + - variable: PreQC/windNorthward + is_not_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + threshold: 4.2 + action: + name: reject + where: + - variable: PreQC/windNorthward + is_in: + - 3 + defer to post: true + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + satwndtype: + - 280 + - 282 + - 284 + error_min: + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + cgross: + - 6.0 + - 6.0 + - 6.0 + wndtype: + - 280 + - 282 + - 284 + variable: windEastward + action: + name: reject + defer to post: true + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + satwndtype: + - 280 + - 282 + - 284 + error_min: + - 1.4 + - 1.4 + - 1.4 + error_max: + - 6.1 + - 6.1 + - 6.1 + cgross: + - 6.0 + - 6.0 + - 6.0 + wndtype: + - 280 + - 282 + - 284 + variable: windNorthward + action: + name: reject + defer to post: true + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + defer to post: true + observation_name: sfcship + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: Surface Land Stations + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/sfc.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.sfc.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + - virtualTemperature + - airTemperature + - specificHumidity + - stationPressure + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: SfcCorrected + variables: + - name: stationPressure + correction scheme to use: GSL + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: specificHumidity + linear obs operator: + name: Composite + components: + - name: VertInterp + variables: + - name: windEastward + - name: windNorthward + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + hofx scaling field: SurfaceWindScalingHeight + hofx scaling field group: DerivedVariables + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: airTemperature + - name: virtualTemperature + - name: Identity + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + variables: + - name: stationPressure + - name: specificHumidity + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + obs post filters: + - filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 49999 + action: + name: reject + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + minvalue: 1 + action: + name: reject + - filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 181 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 70000 + - 65000 + - 60000 + - 55000 + errors: + - 100 + - 120 + - 120 + - 100000000000.0 + - filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: + - 187 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: ObsValue/stationPressure + xvals: + - 80000 + - 75000 + - 70000 + - 65000 + errors: + - 100 + - 130 + - 160 + - 100000000000.0 + - filter: Perform Action + action: + name: inflate error + inflation factor: 1.2 + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_geomz: geopotential_height + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + station_altitude: height + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/stationPressure + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 100 + where: + - variable: + name: TempObsErrorData/stationPressure + maxvalue: 100 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error parameter: 300 + where: + - variable: + name: TempObsErrorData/stationPressure + minvalue: 300 + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 4.0 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + - variable: ObsType/stationPressure + is_in: 187 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.8 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - variable: ObsType/stationPressure + is_in: 187 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 3.6 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_not_in: + - 3 + - variable: ObsType/stationPressure + is_in: + - 181 + defer to post: true + - filter: Background Check + filter variables: + - name: stationPressure + threshold: 2.52 + action: + name: reject + where: + - variable: PreQC/stationPressure + is_in: + - 3 + - variable: ObsType/stationPressure + is_in: + - 181 + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: assign error + error function: TempObsErrorData/stationPressure + where: + - variable: + name: TempObsErrorData/stationPressure + value: is_valid + defer to post: true + - filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + defer to post: true + - filter: Bounds Check + filter variables: + - name: stationPressure + test variables: + - name: ObsErrorData/stationPressure + maxvalue: 100000.0 + defer to post: true + - filter: Perform Action + filter variables: + - name: specificHumidity + where: + - variable: ObsType/specificHumidity + is_in: 181,187 + action: + name: reject + - filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: ObsType/airTemperature + is_in: 181,187 + action: + name: reject + - filter: Perform Action + filter variables: + - name: virtualTemperature + where: + - variable: ObsType/virtualTemperature + is_in: 181,187 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_in: 281,287 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_in: 281,287 + action: + name: reject + observation_name: sfc + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: MHS METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mhs_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mhs_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: mhs_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/mhs_metop-b.20231009T210000Z.satbias.nc4 + output file: cycle_dir/mhs_metop-b.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/mhs_metop-b.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/mhs_metop-b.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/mhs_metop-b.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/mhs_metop-b.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 550.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: mhs_metop-b + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: mhs_metop-b + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-b + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-b + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-b + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-b + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + observation_name: mhs_metop-b + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: MHS METOP-C + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mhs_metop-c.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mhs_metop-c.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: mhs_metop-c + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/mhs_metop-c.20231009T210000Z.satbias.nc4 + output file: cycle_dir/mhs_metop-c.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/mhs_metop-c.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/mhs_metop-c.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/mhs_metop-c.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/mhs_metop-c.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 550.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: mhs_metop-c + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: mhs_metop-c + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-c + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-c + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_metop-c + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_metop-c + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + observation_name: mhs_metop-c + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: MHS NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mhs_n19.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mhs_n19.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + Clouds: + - Water + - Ice + - Rain + - Snow + Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: mhs_n19 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Clouds: + - Water + - Ice + - Rain + - Snow + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/mhs_n19.20231009T210000Z.satbias.nc4 + output file: cycle_dir/mhs_n19.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/mhs_n19.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/mhs_n19.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/mhs_n19.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/mhs_n19.20231009T210000Z.satbias_cov.nc4 + obs filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 50.0 + maxvalue: 550.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + sensor: mhs_n19 + use_flag: None + minvalue: 1e-12 + action: + name: reject + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + test_bias: ObsBiasData + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: mhs_n19 + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_n19 + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_n19 + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 700.0 + - 700.0 + - 30.0 + - 50.0 + - 60.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: mhs_n19 + channels: None + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 0.0 + - 1.0 + - 0.0 + - 1.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: mhs_n19 + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: None + options: + channels: None + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: + - ObsValue + - HofX + bias_application: HofX + x0: + - 0.05 + - 0.05 + - 0.05 + - 0.05 + - 0.05 + x1: + - 25.0 + - 25.0 + - 25.0 + - 25.0 + - 25.0 + err0: + - 300.0 + - 300.0 + - 2.5 + - 2.0 + - 2.0 + err1: + - 350.0 + - 350.0 + - 15.0 + - 25.0 + - 30.0 + obserr_bound_max: + - 5.0 + - 5.0 + - 10.0 + - 10.0 + - 10.0 + action: + name: reject + observation_name: mhs_n19 + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: MLS55 AURA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/mls55_aura.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.mls55_aura.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - ozoneProfile + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: VertInterp + vertical coordinate: air_pressure + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + obs filters: + - filter: Bounds Check + filter variables: + - name: ozoneProfile + minvalue: 0 + maxvalue: 10000 + action: + name: reject + - filter: Background Check + filter variables: + - name: ozoneProfile + threshold: 5.0 + action: + name: reject + observation_name: mls55_aura + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: OMI AURA + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/omi_aura.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.omi_aura.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - ozoneTotal + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: AtmVertInterpLay + geovals: + - mole_fraction_of_ozone_in_air + coefficients: + - 0.0078976797 + nlevels: + - 1 + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + obs filters: + - filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 5.0 + - filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: ozoneTotal + test variables: + - name: MetaData/solarZenithAngle + maxvalue: 84.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: ozoneTotal + test variables: + - name: MetaData/sensorScanPosition + minvalue: 3 + maxvalue: 24 + action: + name: reject + - filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 5.0 + action: + name: reject + observation_name: omi_aura + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: OMPSNM NPP + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/ompsnm_npp.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.ompsnm_npp.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - ozoneTotal + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: AtmVertInterpLay + geovals: + - mole_fraction_of_ozone_in_air + coefficients: + - 0.0078976797 + nlevels: + - 1 + obs filters: + - filter: Perform Action + filter variables: + - name: ozoneTotal + action: + name: assign error + error parameter: 5.216 + - filter: Bounds Check + filter variables: + - name: ozoneTotal + minvalue: 0 + maxvalue: 1000 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: ozoneTotal + test variables: + - name: MetaData/solarZenithAngle + maxvalue: 84.0 + action: + name: reject + - filter: Background Check + filter variables: + - name: ozoneTotal + threshold: 5.0 + action: + name: reject + observation_name: ompsnm_npp + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: Pilot Balloon + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/pibal.20231009T210000Z.nc4 + missing file action: warn + obsgrouping: + group variables: + - stationIdentification + - releaseTime + sort variable: pressure + sort order: descending + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.pibal.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - windEastward + - windNorthward + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + vertical coordinate backup: air_pressure + observation vertical coordinate group backup: MetaData + observation vertical coordinate backup: pressure + interpolation method backup: log-linear + hofx scaling field: SurfaceWindScalingCombined + hofx scaling field group: DerivedVariables + linear obs operator: + name: Composite + components: + - name: VertInterp + observation alias file: experiment_root/experiment_id/configuration/jedi/interfaces/geos_atmosphere/observations/obsop_name_map.yaml + vertical coordinate: geopotential_height + observation vertical coordinate group: DerivedVariables + observation vertical coordinate: adjustedHeight + interpolation method: linear + vertical coordinate backup: air_pressure + observation vertical coordinate group backup: MetaData + observation vertical coordinate backup: pressure + interpolation method backup: log-linear + hofx scaling field: SurfaceWindScalingCombined + hofx scaling field group: DerivedVariables + obs pre filters: + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: ObsType/windEastward + is_not_in: 221 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: ObsType/windNorthward + is_not_in: 221 + action: + name: reject + obs prior filters: + - filter: Variable Transforms + Transform: AdjustedHeightCoordinate + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingHeight + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingPressure + SkipWhenNoObs: false + - filter: Variable Transforms + Transform: SurfaceWindScalingCombined + SkipWhenNoObs: false + obs post filters: + - filter: Bounds Check + filter variables: + - name: windEastward + - name: windNorthward + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: + - 110000 + - 105000 + - 100000 + - 95000 + - 90000 + - 85000 + - 80000 + - 75000 + - 70000 + - 65000 + - 60000 + - 55000 + - 50000 + - 45000 + - 40000 + - 35000 + - 30000 + - 25000 + - 20000 + - 15000 + - 10000 + - 7500 + - 5000 + - 4000 + - 3000 + - 2000 + - 1000 + - 500 + - 400 + - 300 + - 200 + - 100 + - 0 + errors: + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - 2.0 + - 2.1 + - 2.2 + - 2.2 + - 2.3 + - 2.3 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.4 + - 2.5 + - 2.7 + - 2.9 + - 3.1 + - 3.3 + - 3.5 + - 3.7 + - 3.9 + - 4.1 + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + - filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + value: is_valid + - filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + value: is_valid + - filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 221 + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError + geovar_sfc_geomz: height_above_mean_sea_level_at_surface + - filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 221 + cgross: + - 8.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windEastward + action: + name: reject + - filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: + - 221 + cgross: + - 8.0 + error_min: + - 1.4 + error_max: + - 6.1 + variable: windNorthward + action: + name: reject + - filter: Bounds Check + filter variables: + - name: windEastward + test variables: + - name: ObsErrorData/windEastward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windEastward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windEastward + defer to post: true + - filter: Variable Assignment + assignments: + - name: TempObsErrorData/windNorthward + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/windNorthward + defer to post: true + - filter: Bounds Check + filter variables: + - name: windNorthward + test variables: + - name: ObsErrorData/windNorthward + maxvalue: 100.0 + action: + name: reject + defer to post: true + - filter: BlackList + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + where: + - variable: + name: ObsType/windEastward + is_in: 221 + defer to post: true + - filter: BlackList + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 221 + defer to post: true + observation_name: pibal + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: ssmis_f17 + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/ssmis_f17.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.ssmis_f17.20231009T210000Z.nc4 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + io pool: + max pool size: 6 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs options: + Sensor_ID: ssmis_f17 + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + obs bias: + input file: cycle_dir/ssmis_f17.20231009T210000Z.satbias.nc4 + output file: cycle_dir/ssmis_f17.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: cycle_dir/ssmis_f17.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/ssmis_f17.20231009T210000Z.tlapse.txt + - name: cosineOfLatitudeTimesOrbitNode + - name: sineOfLatitude + - name: emissivityJacobian + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/ssmis_f17.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/ssmis_f17.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: + - 1.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 1.0 + - 1.0 + - 3.0 + - 3.0 + - 3.0 + - 3.0 + - 2.4 + - 1.27 + - 1.44 + - 3.0 + - 1.34 + - 1.74 + - 3.75 + - 3.0 + - 3.0 + - 2.0 + - 6.4 + - 1.0 + - 1.0 + obs post filters: + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: reject + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: None + where: + - variable: + name: GeoVaLs/geopotential_height_at_surface + minvalue: 2000.0 + min_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 1-3,8-18 + where: + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/ice_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/surface_snow_area_fraction + maxvalue: 0.99 + max_exclusive: true + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 1-2, 12-16 + reference: ObsValue/brightnessTemperature_2 + value: HofX/brightnessTemperature_2 + minvalue: -1.5 + maxvalue: 1.5 + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + action: + name: reject + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 9 + reference: ObsValue/brightnessTemperature_9 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: + - -0.485934 + - 0.485934 + - 0.473806 + - -0.473806 + intercept: 271.252327 + maxvalue: 2 + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 10 + reference: ObsValue/brightnessTemperature_10 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: + - -0.413688 + - 0.413688 + - 0.361549 + - -0.361549 + intercept: 272.280341 + maxvalue: 2 + - filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 11 + reference: ObsValue/brightnessTemperature_11 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: + - -0.400882 + - 0.400882 + - 0.27051 + - -0.27051 + intercept: 278.824902 + maxvalue: 2 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: None + options: + channels: None + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: ssmis_f17 + obserr_demisf: + - 0.01 + - 0.01 + - 0.01 + - 0.01 + - 0.01 + obserr_dtempf: + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - 0.5 + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor1 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_1 + coefs: + - 2.25 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_1 + threshold: DerivedMetaData/errorInflationFactor1 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor2 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_2 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_2 + threshold: DerivedMetaData/errorInflationFactor2 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor3 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_3 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_3 + threshold: DerivedMetaData/errorInflationFactor3 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor4 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_4 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_4 + threshold: DerivedMetaData/errorInflationFactor4 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor5 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_5 + coefs: + - 0.75 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_5 + threshold: DerivedMetaData/errorInflationFactor5 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor6 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_6 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_6 + threshold: DerivedMetaData/errorInflationFactor6 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor7 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_7 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_7 + threshold: DerivedMetaData/errorInflationFactor7 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor8 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_8 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_8 + threshold: DerivedMetaData/errorInflationFactor8 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor9 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_9 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_9 + threshold: DerivedMetaData/errorInflationFactor9 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_10 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_10 + threshold: DerivedMetaData/errorInflationFactor10 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_11 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_11 + threshold: DerivedMetaData/errorInflationFactor11 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor12 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_12 + coefs: + - 3.6 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_12 + threshold: DerivedMetaData/errorInflationFactor12 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor13 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_13 + coefs: + - 1.905 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_13 + threshold: DerivedMetaData/errorInflationFactor13 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor14 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_14 + coefs: + - 2.16 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_14 + threshold: DerivedMetaData/errorInflationFactor14 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor15 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_15 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_15 + threshold: DerivedMetaData/errorInflationFactor15 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor16 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_16 + coefs: + - 2.01 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_16 + threshold: DerivedMetaData/errorInflationFactor16 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor17 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_17 + coefs: + - 2.61 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_17 + threshold: DerivedMetaData/errorInflationFactor17 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor18 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_18 + coefs: + - 5.625 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_18 + threshold: DerivedMetaData/errorInflationFactor18 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor19 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_19 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_19 + threshold: DerivedMetaData/errorInflationFactor19 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor20 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_20 + coefs: + - 4.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_20 + threshold: DerivedMetaData/errorInflationFactor20 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor21 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_21 + coefs: + - 3.0 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_21 + threshold: DerivedMetaData/errorInflationFactor21 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor22 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_22 + coefs: + - 9.6 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_22 + threshold: DerivedMetaData/errorInflationFactor22 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor23 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_23 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_23 + threshold: DerivedMetaData/errorInflationFactor23 + absolute threshold: 6.0 + action: + name: reject + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor24 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_24 + coefs: + - 1.5 + exponents: + - -1 + - filter: Background Check + filter variables: + - name: brightnessTemperature_24 + threshold: DerivedMetaData/errorInflationFactor24 + absolute threshold: 6.0 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: ssmis_f17 + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: AMSU-A METOP-B + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_metop-b.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_metop-b.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_metop-b + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_metop-b.20231009T210000Z.satbias.nc4 + output file: cycle_dir/amsua_metop-b.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_metop-b.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_metop-b.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_metop-b.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_metop-b.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id008 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_metop-b + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_metop-b + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_metop-b + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_metop-b + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_metop-b + error parameter vector: *id008 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_metop-b + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_metop-b + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 + - obs space: + name: AMSU-A METOP-C + obsdatain: + engine: + type: H5File + obsfile: cycle_dir/amsua_metop-c.20231009T210000Z.nc4 + missing file action: warn + obsdataout: + engine: + type: H5File + obsfile: cycle_dir/experiment_id.amsua_metop-c.20231009T210000Z.nc4 + io pool: + max pool size: 6 + simulated variables: + - brightnessTemperature + channels: None + distribution: + name: Halo + halo size: 5000000.0 + obs operator: + name: CRTM + Absorbers: + - H2O + - O3 + - CO2 + obs options: + Sensor_ID: amsua_metop-c + EndianType: little_endian + CoefficientPath: /discover/nobackup/projects/gmao/advda/SwellStaticFiles/jedi/crtm_coefficients/2.4.1// + linear obs operator: + Absorbers: + - H2O + - O3 + Surfaces: + - Water_Temperature + - Land_Temperature + - Ice_Temperature + - Snow_Temperature + obs bias: + input file: cycle_dir/amsua_metop-c.20231009T210000Z.satbias.nc4 + output file: cycle_dir/amsua_metop-c.20231009T210000Z.satbias.nc4 + static bc: + predictors: + - name: constant + - name: cloudWaterContent + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapseRate + order: 2 + tlapse: cycle_dir/amsua_metop-c.20231009T210000Z.tlapse.txt + - name: lapseRate + tlapse: cycle_dir/amsua_metop-c.20231009T210000Z.tlapse.txt + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: + - 1e-06 + - 10.0 + step size: 0.0001 + largest analysis variance: 10000.0 + prior: + input file: cycle_dir/amsua_metop-c.20231009T210000Z.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: cycle_dir/amsua_metop-c.20231009T210000Z.satbias_cov.nc4 + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: None + action: + name: assign error + error parameter vector: &id009 + - 2.5 + - 2.0 + - 2.0 + - 0.55 + - 0.3 + - 0.23 + - 0.23 + - 0.25 + - 0.25 + - 0.35 + - 0.4 + - 0.55 + - 0.8 + - 5.0 + - 2.5 + obs post filters: + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + minvalue: 100.0 + maxvalue: 500.0 + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: None + options: + sensor: amsua_metop-c + channels: None + test_biaspredictor: cloudWaterContentPredictor + maxvalue: 0.0 + action: + name: reject + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + sensor: amsua_metop-c + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: None + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: None + options: + channels: None + sensor: amsua_metop-c + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: + - 0.01 + - 0.02 + - 0.015 + - 0.02 + - 0.2 + obserr_dtempf: + - 0.5 + - 2.0 + - 1.0 + - 2.0 + - 4.5 + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: None + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: None + options: + sensor: amsua_metop-c + channels: None + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: + - 25.0 + - 0.25 + - 0.04 + - 3.0 + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: None + options: + channels: None + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: None + options: + channels: None + sensor: amsua_metop-c + error parameter vector: *id009 + obserr_bound_max: + - 4.5 + - 4.5 + - 4.5 + - 2.5 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.0 + - 2.5 + - 3.5 + - 4.5 + - 4.5 + - 4.5 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: None + options: + channels: None + use passive_bc: true + sensor: amsua_metop-c + use_flag: None + maxvalue: 1e-12 + action: + name: reject + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: None + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: None + options: + channels: None + use_flag: None + minvalue: 1e-12 + action: + name: reject + observation_name: amsua_metop-c + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 1500000.0 + max nobs: 10000 +local ensemble DA: + solver: Deterministic GETKF + use linear observer: true + vertical localization: + fraction of retained variance: 0.5 + lengthscale: 1.5 + lengthscale units: logp + inflation: + rtps: 0.5 + rtpp: 0.6 + mult: 1.1 +driver: + save posterior mean: true + save posterior ensemble: false + save posterior mean increment: true + save posterior ensemble increments: false +output: + filetype: auxgrid + gridtype: latlon + filename: cycle_dir/geos.analysis.mean. + field io names: + eastward_wind: ua + northward_wind: va + air_temperature: t + air_pressure_levels: pe + water_vapor_mixing_ratio_wrt_moist_air: q + cloud_liquid_ice: qi + cloud_liquid_water: ql + mole_fraction_of_ozone_in_air: o3ppmv +output increment: + filetype: auxgrid + gridtype: latlon + filename: cycle_dir/geos.mean-inc. + field io names: + eastward_wind: ua + northward_wind: va + air_temperature: t + air_pressure_at_surface: ps + air_pressure_levels: pe + water_vapor_mixing_ratio_wrt_moist_air: q + cloud_liquid_ice: qi + cloud_liquid_water: ql + mole_fraction_of_ozone_in_air: o3ppmv diff --git a/src/swell/utilities/mock_jedi_config.py b/src/swell/utilities/mock_jedi_config.py index 480514a72..6fa7bdae1 100644 --- a/src/swell/utilities/mock_jedi_config.py +++ b/src/swell/utilities/mock_jedi_config.py @@ -16,20 +16,35 @@ # -------------------------------------------------------------------------------------------------- + def mock_jedi_config(suite: str, model: str, datetime: str, executable_type: str, - copy_to_wd: bool = False) -> str: - + copy_dir: str | None = None) -> str: + + '''Generate a mock jedi config using the settings for a particular suite. Configs are generated + in a 'dry-run' mode, without checking for existence of observations. Filepaths are replaced + with placeholders + + Parameters: + suite: String of suite within swell + model: Name of model component for the suite to run (e.g. geos_marine) + datetime: Cycle that is config is generated for (e.g. 20210701T120000Z) + executable_type: JEDI executable type (e.g. variational, hofx, fgat) + copy_dir: Directory to copy rendered file to + + Rendered file is placed under a temporary experiment directory + ''' + tempdir = tempfile.mkdtemp() override_dict = {'models': {}} override_dict['experiment_root'] = tempdir override_dict['generate_yaml_and_exit'] = True - override_dict['mock_experiment_directory'] = True + override_dict['mock_experiment'] = True override_dict['models'][model] = {'check_for_obs': False} - + create_experiment_directory(suite, method='defaults', platform='nccs_discover_sles15', override=override_dict, advanced=False, slurm=None, skip_r2d2=True) @@ -39,7 +54,12 @@ def mock_jedi_config(suite: str, task_wrapper('RenderJediObservations', experiment_yaml, datetime, model, ensemblePacket=None) - task_wrapper(f'RunJedi{executable_type.capitalize()}Executable', experiment_yaml, datetime, + if executable_type == 'localensembleda': + executable_name = 'LocalEnsembleDa' + else: + executable_name = executable_type.capitalize() + + task_wrapper(f'RunJedi{executable_name}Executable', experiment_yaml, datetime, model, ensemblePacket=None) cycle_dir = os.path.join(tempdir, f'swell-{suite}', 'run', datetime, model) @@ -47,8 +67,8 @@ def mock_jedi_config(suite: str, filename = f'jedi_{executable_type}_config.yaml' config_file = os.path.join(cycle_dir, filename) - if copy_to_wd: - new_path = os.path.join(os.getcwd(), f'jedi_{suite}_config.yaml') + if copy_dir is not None: + new_path = os.path.join(copy_dir, f'jedi_{suite}_config.yaml') shutil.copy(config_file, new_path) config_file = new_path diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 46419a783..946cdc41b 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -135,9 +135,6 @@ class mock_experiment(SuiteQuestion): default_value: bool = False question_name: str = "mock_experiment" ask_question: bool = False - models: List[str] = mutable_field([ - "all_models" - ]) prompt: str = "Dry-run option for comparing configs." widget_type: WType = WType.BOOLEAN diff --git a/src/swell/utilities/scripts/create_mock_files.py b/src/swell/utilities/scripts/create_mock_files.py deleted file mode 100644 index f19342ac2..000000000 --- a/src/swell/utilities/scripts/create_mock_files.py +++ /dev/null @@ -1,60 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - -from swell.utilities.mock_jedi_config import mock_jedi_config - -# -------------------------------------------------------------------------------------------------- - -defaults_dict = {} - -marine_default_datetime = '20210701T120000Z' -atmosphere_default_datetime = '20231010T000000Z' -compo_default_datetime = '20230805T1800Z' - -defaults_dict['3dvar_marine'] = {'datetime': marine_default_datetime, - 'model': 'geos_marine', - 'executable_type': 'variational'} - -defaults_dict['3dvar_marine_cycle'] = defaults_dict['3dvar_marine'].copy() - -defaults_dict['3dfgat_marine_cycle'] = {'datetime': marine_default_datetime, - 'model': 'geos_marine', - 'executable_type': 'fgat'} - -defaults_dict['3dvar_atmos'] = {'datetime': atmosphere_default_datetime, - 'model': 'geos_atmosphere', - 'executable_type': 'variational'} - -defaults_dict['3dfgat_atmos'] = {'datetime': atmosphere_default_datetime, - 'model': 'geos_atmosphere', - 'executable_type': 'variational'} - -defaults_dict['hofx'] = {'datetime': atmosphere_default_datetime, - 'model': 'geos_atmosphere', - 'executable_type': 'hofx'} - -defaults_dict['hofx_cf'] = {'datetime': compo_default_datetime, - 'model': 'geos_cf', - 'executable_type': 'hofx'} - -defaults_dict['localensembleda'] = {'datetime': atmosphere_default_datetime, - 'model': 'geos_atmosphere', - 'executable_type': 'localensembleda'} -# -------------------------------------------------------------------------------------------------- - -def create_mock_files() -> None: - for suite, defaults in defaults_dict.items(): - model = defaults['model'] - datetime = defaults['datetime'] - executable_type = defaults['executable_type'] - - mock_jedi_config(suite, model, datetime, executable_type, True) - - -# -------------------------------------------------------------------------------------------------- From 7a20636e8da061f3d550cbca55d154f8cb529d52 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 26 Mar 2026 09:18:39 -0400 Subject: [PATCH 252/299] Add script --- .../utilities/scripts/create_mock_configs.py | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/swell/utilities/scripts/create_mock_configs.py diff --git a/src/swell/utilities/scripts/create_mock_configs.py b/src/swell/utilities/scripts/create_mock_configs.py new file mode 100644 index 000000000..04d05340b --- /dev/null +++ b/src/swell/utilities/scripts/create_mock_configs.py @@ -0,0 +1,71 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os + +from swell.swell_path import get_swell_path +from swell.utilities.mock_jedi_config import mock_jedi_config + +# -------------------------------------------------------------------------------------------------- + +defaults_dict = {} + +marine_default_datetime = '20210701T120000Z' +atmosphere_default_datetime = '20231010T000000Z' +compo_default_datetime = '20230805T1800Z' + +defaults_dict['3dvar_marine'] = {'datetime': marine_default_datetime, + 'model': 'geos_marine', + 'executable_type': 'variational'} + +defaults_dict['3dvar_marine_cycle'] = defaults_dict['3dvar_marine'].copy() + +defaults_dict['3dfgat_marine_cycle'] = {'datetime': marine_default_datetime, + 'model': 'geos_marine', + 'executable_type': 'fgat'} + +defaults_dict['3dvar_atmos'] = {'datetime': atmosphere_default_datetime, + 'model': 'geos_atmosphere', + 'executable_type': 'variational'} + +defaults_dict['3dfgat_atmos'] = {'datetime': atmosphere_default_datetime, + 'model': 'geos_atmosphere', + 'executable_type': 'variational'} + +defaults_dict['hofx'] = {'datetime': atmosphere_default_datetime, + 'model': 'geos_atmosphere', + 'executable_type': 'hofx'} + +defaults_dict['hofx_cf'] = {'datetime': '20230805T180000Z', + 'model': 'geos_cf', + 'executable_type': 'hofx'} + +defaults_dict['3dvar_cf'] = {'datetime': '20230805T180000Z', + 'model': 'geos_cf', + 'executable_type': 'variational'} + +defaults_dict['localensembleda'] = {'datetime': atmosphere_default_datetime, + 'model': 'geos_atmosphere', + 'executable_type': 'localensembleda'} + +# -------------------------------------------------------------------------------------------------- + + +def main() -> None: + for suite, defaults in defaults_dict.items(): + model = defaults['model'] + datetime = defaults['datetime'] + executable_type = defaults['executable_type'] + + copy_dir = os.path.join(get_swell_path(), 'test', 'jedi_configs') + + mock_jedi_config(suite, model, datetime, executable_type, copy_dir) + + +# -------------------------------------------------------------------------------------------------- From e6a8e05d03e9e86f8a045314798a4b94c4376ef3 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 26 Mar 2026 10:15:47 -0400 Subject: [PATCH 253/299] add to package data --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index a2c9b528a..415e71a29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ swell = [ 'deployment/platforms/*/*.yaml', 'suites/**', 'test/suite_tests/*.yaml', + 'test/jedi_configs/*.yaml', 'configuration/**', 'utilities/pinned_versions/*.yaml' ] From 9edbcffe6eb276085bff53a8b5d6f9b16f2e55ff Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 26 Mar 2026 14:15:34 -0400 Subject: [PATCH 254/299] Add xarray to requirements --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 415e71a29..759952ddc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,8 @@ dependencies = [ "h5py>=3.7.0", "flake8==6.1.0", "netCDF4", - "ruamel.yaml==0.17.16" + "ruamel.yaml==0.17.16", + "xarray==2024.7.0" ] [project.optional-dependencies] @@ -58,7 +59,7 @@ github = [ "isodate==0.6.1", "f90nml==1.4.4", "questionary==1.10.0", - "netCDF4==1.6.5", + "netCDF4==1.6.5" ] [project.urls] From 5b7bed361d091a0005f58b6989e08329cd0cc552 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 26 Mar 2026 15:50:45 -0400 Subject: [PATCH 255/299] Update mock configs --- .../jedi_configs/jedi_localensembleda_config.yaml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/swell/test/jedi_configs/jedi_localensembleda_config.yaml b/src/swell/test/jedi_configs/jedi_localensembleda_config.yaml index 92964d5b5..f68ae102a 100644 --- a/src/swell/test/jedi_configs/jedi_localensembleda_config.yaml +++ b/src/swell/test/jedi_configs/jedi_localensembleda_config.yaml @@ -9473,10 +9473,11 @@ output: filetype: auxgrid gridtype: latlon filename: cycle_dir/geos.analysis.mean. - field io names: + field io names: &id010 eastward_wind: ua northward_wind: va air_temperature: t + air_pressure_at_surface: ps air_pressure_levels: pe water_vapor_mixing_ratio_wrt_moist_air: q cloud_liquid_ice: qi @@ -9486,13 +9487,4 @@ output increment: filetype: auxgrid gridtype: latlon filename: cycle_dir/geos.mean-inc. - field io names: - eastward_wind: ua - northward_wind: va - air_temperature: t - air_pressure_at_surface: ps - air_pressure_levels: pe - water_vapor_mixing_ratio_wrt_moist_air: q - cloud_liquid_ice: qi - cloud_liquid_water: ql - mole_fraction_of_ozone_in_air: o3ppmv + field io names: *id010 From 6ffd59be038c5bcaea5bba5e4d929cbc74803295 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 30 Mar 2026 14:28:55 -0400 Subject: [PATCH 256/299] Remove print statement --- src/swell/tasks/run_jedi_variational_executable.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index ba0c1c42d..862cf5af5 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -99,7 +99,6 @@ def execute(self) -> None: # Add placeholder names if mock experiment # ---------------------------------------- if self.config.mock_experiment(False): - print('MOCK EXPERIMENT') self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') From 63dac865822b930b245b304aab1d13cdfd975b6c Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 30 Mar 2026 15:05:07 -0400 Subject: [PATCH 257/299] Move override file read --- src/swell/deployment/create_experiment.py | 28 +++-------------------- src/swell/swell.py | 7 +++++- src/swell/utilities/suite_utils.py | 12 ++++++++++ 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index c50e3451e..f89fd3649 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -31,19 +31,6 @@ # -------------------------------------------------------------------------------------------------- -def read_override_file(override_path: str | None) -> dict: - - yaml = YAML(typ='safe') - - if override_path is None: - return {} - else: - with open(override_path, 'r') as f: - return yaml.load(f) - -# -------------------------------------------------------------------------------------------------- - - def clone_config( configuration: str, experiment_id: str, @@ -233,7 +220,7 @@ def create_experiment_directory( suite_config: str, method: str, platform: str, - override: str | dict, + override: dict, advanced: bool, slurm: str | None, skip_r2d2: bool @@ -247,26 +234,17 @@ def create_experiment_directory( # --------------- logger = get_logger('SwellCreateExperiment') - # Read override file - # ------------------ - if isinstance(override, str): - override_dict = read_override_file(override) - elif override is None: - override_dict = {} - else: - override_dict = override - # Specify whether to skip registering and storing in R2D2 # ------------------------------------------------------- if skip_r2d2: # Only override this if it is true, otherwise let the suite decide - override_dict['skip_r2d2'] = skip_r2d2 + override['skip_r2d2'] = skip_r2d2 # Call the experiment config and suite generation # ------------------------------------------------ experiment_dict_str = prepare_config(suite, suite_config, method, platform, - override_dict, advanced, slurm) + override, advanced, slurm) # Load the string using yaml # -------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index 89f55a3ad..025327a8c 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -20,6 +20,7 @@ from swell.suites.all_suites import AllSuites from swell.utilities.welcome_message import write_welcome_message from swell.utilities.scripts.utility_driver import get_utilities, utility_wrapper +from swell.utilities.suite_utils import read_override_file # -------------------------------------------------------------------------------------------------- @@ -117,8 +118,12 @@ def create( """ + # Read override file + override_dict = read_override_file(override) + # Create the experiment directory - create_experiment_directory(suite, input_method, platform, override, advanced, slurm, skip_r2d2) + create_experiment_directory(suite, input_method, platform, override_dict, + advanced, slurm, skip_r2d2) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/suite_utils.py b/src/swell/utilities/suite_utils.py index 4e9c8a144..1060a8b32 100644 --- a/src/swell/utilities/suite_utils.py +++ b/src/swell/utilities/suite_utils.py @@ -11,6 +11,7 @@ import glob import os import importlib +from ruamel.yaml import YAML from swell.swell_path import get_swell_path @@ -77,3 +78,14 @@ def get_suite_tests() -> list: # -------------------------------------------------------------------------------------------------- + +def read_override_file(override_path: str | None) -> dict: + + if override_path is None: + return {} + else: + yaml = YAML(typ='safe') + with open(override_path, 'r') as f: + return yaml.load(f) + +# -------------------------------------------------------------------------------------------------- From 9a5b9959432ff9c84c99a13d33b644b50b80e612 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 31 Mar 2026 14:40:42 -0400 Subject: [PATCH 258/299] PR review --- docs/code_tests/code_tests.md | 2 +- src/swell/test/code_tests/code_tests.py | 2 +- src/swell/test/code_tests/jedi_config_test.py | 7 +++++-- src/swell/utilities/mock_jedi_config.py | 6 +++++- src/swell/utilities/scripts/create_mock_configs.py | 4 +++- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/code_tests/code_tests.md b/docs/code_tests/code_tests.md index 2dbe279a4..43e904bb2 100644 --- a/docs/code_tests/code_tests.md +++ b/docs/code_tests/code_tests.md @@ -24,4 +24,4 @@ test_cache_location: /discover/nobackup//swell-test-cache The suite creation test attempts to construct experiments for all suites within swell in a temporary directory. If one fails, try creating the suite on its own to make sure it is configured properly. Ensure all values are valid and are not filled by the templates `defer_to_model` or `defer_to_platform`. ### JEDI Config test -The JEDI config test generates mock configs for jedi executables in a dry-run mode, where obs will not be checked and placeholders will be used for experiment filepaths. These configs are compared against reference files located in `src/swell/test/jedi_configs/`, and named `jedi__config.yaml`. Any difference in values in these yamls will cause this test to fail, so ensure any differences created are intentional, then run `swell utility CreateMockConfigs` to automatically generate new reference files for all suites. These new files are placed in the `jedi_config` location in the source code. +The JEDI config test generates mock configs for jedi executables in a dry-run mode, where obs will not be checked and placeholders will be used for experiment filepaths (this can result in situations where altering some parameters will not affect the output yaml as they would in a normal experiment, such as changes to cycle times). These configs are compared against reference files located in `src/swell/test/jedi_configs/`, and named `jedi__config.yaml`. Any difference in values in these yamls will cause this test to fail, so ensure any differences created are intentional, then run `swell utility CreateMockConfigs` to automatically generate new reference files for all suites. These new files are placed in the `jedi_config` location in the source code. diff --git a/src/swell/test/code_tests/code_tests.py b/src/swell/test/code_tests/code_tests.py index 1b7b790f1..d8539004d 100644 --- a/src/swell/test/code_tests/code_tests.py +++ b/src/swell/test/code_tests/code_tests.py @@ -57,7 +57,7 @@ def code_tests() -> None: # Load Suite Creation Test test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(SuiteCreationTest)) - # Load Suite Creation Test + # Load JEDI config test test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(JEDIConfigTest)) # Create a test runner diff --git a/src/swell/test/code_tests/jedi_config_test.py b/src/swell/test/code_tests/jedi_config_test.py index adb8f5dba..0678163b4 100644 --- a/src/swell/test/code_tests/jedi_config_test.py +++ b/src/swell/test/code_tests/jedi_config_test.py @@ -14,6 +14,7 @@ from swell.swell_path import get_swell_path from swell.utilities.mock_jedi_config import mock_jedi_config from swell.utilities.scripts.create_mock_configs import defaults_dict +from swell.utilities.test_cache import get_test_cache # -------------------------------------------------------------------------------------------------- @@ -40,8 +41,10 @@ def runTest(self) -> None: datetime = defaults['datetime'] executable_type = defaults['executable_type'] + cache_location = get_test_cache() + # Create the mock jedi config - config_file = mock_jedi_config(suite, model, datetime, executable_type) + config_file = mock_jedi_config(suite, model, datetime, executable_type, cache_location) # Read the file yaml = YAML(typ='safe') @@ -60,6 +63,6 @@ def runTest(self) -> None: f'did not match comparison version {comparison_config}. ' 'Please check the file diffs. If these differences ' 'are intentional, new mock test files can be generated using ' - '`swell utility MockJediConfigs`') + '`swell utility CreateMockConfigs`') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/mock_jedi_config.py b/src/swell/utilities/mock_jedi_config.py index 6fa7bdae1..cf849a485 100644 --- a/src/swell/utilities/mock_jedi_config.py +++ b/src/swell/utilities/mock_jedi_config.py @@ -21,6 +21,7 @@ def mock_jedi_config(suite: str, model: str, datetime: str, executable_type: str, + work_dir: str | None = None, copy_dir: str | None = None) -> str: '''Generate a mock jedi config using the settings for a particular suite. Configs are generated @@ -37,7 +38,10 @@ def mock_jedi_config(suite: str, Rendered file is placed under a temporary experiment directory ''' - tempdir = tempfile.mkdtemp() + if work_dir is None: + tempdir = tempfile.mkdtemp() + else: + tempdir = work_dir override_dict = {'models': {}} override_dict['experiment_root'] = tempdir diff --git a/src/swell/utilities/scripts/create_mock_configs.py b/src/swell/utilities/scripts/create_mock_configs.py index 04d05340b..7faad4d3b 100644 --- a/src/swell/utilities/scripts/create_mock_configs.py +++ b/src/swell/utilities/scripts/create_mock_configs.py @@ -11,6 +11,7 @@ from swell.swell_path import get_swell_path from swell.utilities.mock_jedi_config import mock_jedi_config +from swell.utilities.test_cache import get_test_cache # -------------------------------------------------------------------------------------------------- @@ -63,9 +64,10 @@ def main() -> None: datetime = defaults['datetime'] executable_type = defaults['executable_type'] + work_dir = get_test_cache() copy_dir = os.path.join(get_swell_path(), 'test', 'jedi_configs') - mock_jedi_config(suite, model, datetime, executable_type, copy_dir) + mock_jedi_config(suite, model, datetime, executable_type, work_dir, copy_dir) # -------------------------------------------------------------------------------------------------- From 80531c667364e2fb11c0ccf4ba86c465476b7bc1 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 3 Apr 2026 12:08:25 -0400 Subject: [PATCH 259/299] use native type hinting --- .../interfaces/geos_cf/model/getvalues.py | 3 +- .../interfaces/geos_marine/model/getvalues.py | 3 +- src/swell/deployment/create_experiment.py | 3 +- .../prepare_config_and_suite.py | 5 +- .../question_and_answer_cli.py | 3 +- .../question_and_answer_defaults.py | 5 +- src/swell/swell.py | 14 +- src/swell/tasks/base/task_base.py | 21 +- src/swell/tasks/get_observations.py | 4 +- src/swell/tasks/link_coupled_geos_output.py | 3 +- src/swell/tasks/link_geos_output.py | 3 +- src/swell/tasks/prepare_analysis.py | 3 +- src/swell/tasks/run_jedi_hofx_executable.py | 3 +- .../tasks/run_jedi_obsfilters_executable.py | 3 +- src/swell/utilities/build.py | 3 +- src/swell/utilities/check_da_params.py | 7 +- .../data_assimilation_window_params.py | 5 +- src/swell/utilities/dataclass_utils.py | 4 +- src/swell/utilities/dictionary.py | 3 +- src/swell/utilities/filehandler.py | 9 +- src/swell/utilities/geos.py | 5 +- src/swell/utilities/get_channels.py | 5 +- src/swell/utilities/jinja2.py | 3 +- src/swell/utilities/logger.py | 3 +- src/swell/utilities/netcdf_files.py | 4 +- .../utilities/observing_system_records.py | 3 +- src/swell/utilities/question_defaults.py | 235 +++++++++--------- .../utilities/render_jedi_interface_files.py | 14 +- src/swell/utilities/run_jedi_executables.py | 9 +- .../utilities/scripts/compare_questions.py | 5 +- src/swell/utilities/shell_commands.py | 14 +- src/swell/utilities/swell_questions.py | 10 +- src/swell/utilities/test_cache.py | 4 +- 33 files changed, 195 insertions(+), 226 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_cf/model/getvalues.py b/src/swell/configuration/jedi/interfaces/geos_cf/model/getvalues.py index 671d3eee1..db8f1d1f2 100644 --- a/src/swell/configuration/jedi/interfaces/geos_cf/model/getvalues.py +++ b/src/swell/configuration/jedi/interfaces/geos_cf/model/getvalues.py @@ -6,13 +6,12 @@ # -------------------------------------------------------------------------------------------------- -from typing import Optional from collections.abc import Mapping # -------------------------------------------------------------------------------------------------- -def getvalues(template_dict: Mapping) -> Optional[Mapping]: +def getvalues(template_dict: Mapping) -> Mapping | None: getvalues = None return getvalues diff --git a/src/swell/configuration/jedi/interfaces/geos_marine/model/getvalues.py b/src/swell/configuration/jedi/interfaces/geos_marine/model/getvalues.py index 671d3eee1..db8f1d1f2 100644 --- a/src/swell/configuration/jedi/interfaces/geos_marine/model/getvalues.py +++ b/src/swell/configuration/jedi/interfaces/geos_marine/model/getvalues.py @@ -6,13 +6,12 @@ # -------------------------------------------------------------------------------------------------- -from typing import Optional from collections.abc import Mapping # -------------------------------------------------------------------------------------------------- -def getvalues(template_dict: Mapping) -> Optional[Mapping]: +def getvalues(template_dict: Mapping) -> Mapping | None: getvalues = None return getvalues diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 0b7fa81e3..44cf43803 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -15,7 +15,6 @@ import shutil import sys from ruamel.yaml import YAML -from typing import Optional from swell.suites.all_suites import AllSuites from swell.deployment.prepare_config_and_suite.prepare_config_and_suite import \ @@ -355,7 +354,7 @@ def copy_eva_files( def copy_platform_files( logger: Logger, exp_suite_path: str, - platform: Optional[str] = None + platform: str | None = None ) -> None: # Copy platform related files to the suite directory diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 69987bf5f..a327d1e4d 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -12,7 +12,6 @@ import os from ruamel.yaml import YAML from collections.abc import Mapping -from typing import Tuple, Optional from swell.swell_path import get_swell_path from swell.deployment.prepare_config_and_suite.question_and_answer_cli import GetAnswerCli @@ -374,7 +373,7 @@ def override_with_external(self) -> None: # ---------------------------------------------------------------------------------------------- - def ask_questions_and_configure_suite(self) -> Tuple[dict, dict]: + def ask_questions_and_configure_suite(self) -> tuple[dict, dict]: """ This is where we ask all the questions and as we go configure the suite file. The process @@ -530,7 +529,7 @@ def ask_a_question( self, full_question_dictionary: dict, question_key: str, - model: Optional[str] = None + model: str | None = None ) -> None: # Set flag for whether the question should be asked diff --git a/src/swell/deployment/prepare_config_and_suite/question_and_answer_cli.py b/src/swell/deployment/prepare_config_and_suite/question_and_answer_cli.py index bc6075020..240907037 100644 --- a/src/swell/deployment/prepare_config_and_suite/question_and_answer_cli.py +++ b/src/swell/deployment/prepare_config_and_suite/question_and_answer_cli.py @@ -10,7 +10,6 @@ import questionary from questionary import Choice -from typing import Optional from swell.utilities.logger import Logger from swell.utilities.swell_questions import WidgetType @@ -21,7 +20,7 @@ class GetAnswerCli: - def get_answer(self, logger: Logger, key: str, val: dict, model: Optional[str] = None): + def get_answer(self, logger: Logger, key: str, val: dict, model: str | None = None): prompt = val['prompt'] default = val['default_value'] widget_type = val['widget_type'] diff --git a/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py b/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py index f600c4198..9bacbe8c8 100644 --- a/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py +++ b/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py @@ -7,16 +7,13 @@ # -------------------------------------------------------------------------------------------------- - -from typing import Union, Optional - from swell.utilities.logger import Logger class GetAnswerDefaults: def get_answer(self, logger: Logger, key: str, val: dict, - model: Optional[str] = None) -> Union[int, float, str]: + model: str | None = None) -> int | float | str: default = val['default_value'] widget_type = val['widget_type'] diff --git a/src/swell/swell.py b/src/swell/swell.py index 89f55a3ad..513aeb524 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -9,7 +9,7 @@ import click -from typing import Union, Optional, Literal +from typing import Literal from swell.deployment.platforms.platforms import get_platforms from swell.deployment.create_experiment import clone_config, create_experiment_directory @@ -102,7 +102,7 @@ def create( suite: str, input_method: str, platform: str, - override: Union[dict, str, None], + override: dict | str | None, advanced: bool, slurm: str, skip_r2d2: bool @@ -192,9 +192,9 @@ def launch( def task( task: str, config: str, - datetime: Optional[str], - model: Optional[str], - ensemblePacket: Optional[str] + datetime: str | None, + model: str | None, + ensemblePacket: str | None ) -> None: """ Run a workflow task @@ -255,7 +255,7 @@ def test(test: str) -> None: "localensembleda", "3dvar_cycle"))) def t1test( suite: Literal["hofx", "3dvar_marine", "3dvar_atmos", "localensembleda", "3dvar_cycle"], - platform: Optional[str] = "nccs_discover_sles15" + platform: str | None = "nccs_discover_sles15" ) -> None: """ Run a particular swell suite from the tier 1 tests. @@ -277,7 +277,7 @@ def t1test( def t2test( suite: Literal["hofx", "3dvar_marine", "ufo_testing", "convert_ncdiags", "3dfgat_atmos", "build_jedi"], - platform: Optional[str] = "nccs_discover_sles15" + platform: str = "nccs_discover_sles15" ) -> None: """ Run a particular swell suite from the tier 2 tests. diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 109b3af74..90ada6a4e 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -16,7 +16,6 @@ import os import time from datetime import datetime as dt -from typing import Union, Optional # swell imports from swell.swell_path import get_swell_path @@ -38,9 +37,9 @@ class taskBase(ABC): def __init__( self, config_input: str, - datetime_input: Optional[str], + datetime_input: str | None, model: str, - ensemblePacket: Optional[str], + ensemblePacket: str | None, task_name: str ) -> None: @@ -163,7 +162,7 @@ def experiment_config_path(self) -> str: # ---------------------------------------------------------------------------------------------- - def get_ensemble_packet(self) -> Optional[str]: + def get_ensemble_packet(self) -> str | None: return self.__ensemble_packet__ # ---------------------------------------------------------------------------------------------- @@ -173,7 +172,7 @@ def get_model(self) -> str: # ---------------------------------------------------------------------------------------------- - def get_model_components(self) -> Union[str, list]: + def get_model_components(self) -> str | list: return self.__model_components__ # ---------------------------------------------------------------------------------------------- @@ -201,7 +200,7 @@ def cycle_dir(self) -> str: # ---------------------------------------------------------------------------------------------- - def forecast_dir(self, paths: Union[str, list[str]] = []) -> Optional[str]: + def forecast_dir(self, paths: str | list[str] = []) -> str | None: ''' Method to provide "forecast" directory to geos class @@ -272,9 +271,9 @@ def create_task( self, task: str, config: str, - datetime: Union[str, dt, None], + datetime: str | dt | None, model: str, - ensemblePacket: Optional[str] + ensemblePacket: str | None ) -> taskBase: # Convert camel case string to snake case @@ -332,9 +331,9 @@ def get_tasks() -> list: def task_wrapper( task: str, config: str, - datetime: Union[str, dt, None], - model: Optional[str], - ensemblePacket: Optional[str] + datetime: str | dt | None, + model: str | None, + ensemblePacket: str | None ) -> None: # Create the object diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index b48a1e570..c864b86b4 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -13,7 +13,7 @@ import os import r2d2 import shutil -from typing import Union +from collections.abc import Iterator from datetime import timedelta, datetime as dt from swell.tasks.base.task_base import taskBase @@ -316,7 +316,7 @@ def execute(self) -> None: # ---------------------------------------------------------------------------------------------- - def get_tlapse_files(self, observation_dict: dict) -> Union[None, int]: + def get_tlapse_files(self, observation_dict: dict) -> Iterator[int | None]: # Function to locate instances of tlapse in the obs operator config diff --git a/src/swell/tasks/link_coupled_geos_output.py b/src/swell/tasks/link_coupled_geos_output.py index fb566c6c1..dcc1c13c1 100644 --- a/src/swell/tasks/link_coupled_geos_output.py +++ b/src/swell/tasks/link_coupled_geos_output.py @@ -13,7 +13,6 @@ from netCDF4 import Dataset import numpy as np import xarray as xr -from typing import Tuple from swell.utilities.datetime_util import datetime_formats from swell.tasks.base.task_base import taskBase @@ -202,7 +201,7 @@ def prepare_cice6_history(self, # ---------------------------------------------------------------------------------------------- - def prepare_cice6_restart(self) -> Tuple[str, str]: + def prepare_cice6_restart(self) -> tuple[str, str]: # CICE6 input in SOCA requires aggregation of multiple variables and # time dimension added to the dataset. # SOCA needs icea area (aicen), ice volume (vicen), and snow area (vsnon) diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index f2853728a..7cd55058d 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -13,7 +13,6 @@ from netCDF4 import Dataset import numpy as np import xarray as xr -from typing import Tuple from swell.utilities.datetime_util import datetime_formats from swell.tasks.base.task_base import taskBase @@ -201,7 +200,7 @@ def prepare_cice6_history(self, # ---------------------------------------------------------------------------------------------- - def prepare_cice6_restart(self) -> Tuple[str, str]: + def prepare_cice6_restart(self) -> tuple[str, str]: # CICE6 input in SOCA requires aggregation of multiple variables and # time dimension added to the dataset. # SOCA needs icea area (aicen), ice volume (vicen), and snow area (vsnon) diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index 50738c963..da62587d6 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -12,7 +12,6 @@ import netCDF4 as nc import os import shutil -from typing import Union from swell.utilities.shell_commands import run_subprocess from swell.tasks.base.task_base import taskBase @@ -100,7 +99,7 @@ def execute(self) -> None: # ---------------------------------------------------------------------------------------- - def at_cycledir(self, paths: Union[list, str] = []) -> str: + def at_cycledir(self, paths: list | str = []) -> str: """ Get the absolute path to the model cycle directory for the given relative paths. diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 6d63cf5fc..2a57c08c0 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -11,7 +11,6 @@ import glob import os from ruamel.yaml import YAML -from typing import Optional from swell.tasks.base.task_base import taskBase from swell.utilities.netcdf_files import combine_files_without_groups @@ -252,7 +251,7 @@ def append_gomsaver( observations: list, jedi_config_dict: dict, window_begin: str, - mem: Optional[str] = None + mem: str | None = None ) -> None: # We may need to save the GeoVaLs for ensemble members. This will diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index 7af6e4867..9e79db8a8 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -10,7 +10,6 @@ import os import shutil from ruamel.yaml import YAML -from typing import Optional import random from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -22,7 +21,7 @@ class RunJediObsfiltersExecutable(taskBase): # ---------------------------------------------------------------------------------------------- - def execute(self, ensemble_members: Optional[list] = None) -> None: + def execute(self, ensemble_members: list | None = None) -> None: # Jedi application name # --------------------- diff --git a/src/swell/utilities/build.py b/src/swell/utilities/build.py index f5f2eff16..27a2e81c3 100644 --- a/src/swell/utilities/build.py +++ b/src/swell/utilities/build.py @@ -9,7 +9,6 @@ import os import shutil -from typing import Tuple from swell.utilities.logger import get_logger from jedi_bundle.utils.yaml import load_yaml @@ -21,7 +20,7 @@ # -------------------------------------------------------------------------------------------------- -def build_and_source_dirs(package_path: str) -> Tuple[str, str]: +def build_and_source_dirs(package_path: str) -> tuple[str, str]: # Make package directory # ---------------------- diff --git a/src/swell/utilities/check_da_params.py b/src/swell/utilities/check_da_params.py index 6cbfb8da9..9e635fd39 100644 --- a/src/swell/utilities/check_da_params.py +++ b/src/swell/utilities/check_da_params.py @@ -8,7 +8,6 @@ from ruamel.yaml import YAML import os -from typing import Optional from datetime import datetime from swell.utilities.logger import get_logger @@ -19,9 +18,9 @@ def check_da_params(config_list: list, model_component: str, - start_cycle_point_in: Optional[str], - final_cycle_point_in: Optional[str], - cycle_times_in: Optional[str]) -> None: + start_cycle_point_in: str | None, + final_cycle_point_in: str | None, + cycle_times_in: str | None) -> None: # From two or more experiments, check that the window parameters are the same, and gather the # common cycle times between the two. Returns times between the start and final cycle points, diff --git a/src/swell/utilities/data_assimilation_window_params.py b/src/swell/utilities/data_assimilation_window_params.py index 454cbefd2..0b1f12579 100644 --- a/src/swell/utilities/data_assimilation_window_params.py +++ b/src/swell/utilities/data_assimilation_window_params.py @@ -9,7 +9,6 @@ import datetime import isodate -from typing import Union, Tuple from swell.utilities.datetime_util import datetime_formats from swell.utilities.logger import Logger @@ -142,7 +141,7 @@ def local_background_time(self, window_length, window_type, dto=False - ) -> Union[str, Tuple[str, datetime.datetime]]: + ) -> str | tuple[str, datetime.datetime]: local_background_time = self.__get_local_background_time__(window_type, window_length) @@ -155,7 +154,7 @@ def local_background_time(self, # ---------------------------------------------------------------------------------------------- - def window_begin(self, window_length: str, dto: bool = False) -> Union[str, datetime.datetime]: + def window_begin(self, window_length: str, dto: bool = False) -> str | datetime.datetime: window_begin_dto = self.__get_window_begin_dto__(window_length) diff --git a/src/swell/utilities/dataclass_utils.py b/src/swell/utilities/dataclass_utils.py index 472e81965..6a81591d9 100644 --- a/src/swell/utilities/dataclass_utils.py +++ b/src/swell/utilities/dataclass_utils.py @@ -6,14 +6,12 @@ # -------------------------------------------------------------------------------------------------- -from typing import Union - from dataclasses import field # -------------------------------------------------------------------------------------------------- -def mutable_field(list_dict: Union[list, dict]): +def mutable_field(list_dict: list | dict): # Need to construct field using lambda because by default dataclass fields # cannot be initialized to mutable types return field(default_factory=lambda: list_dict) diff --git a/src/swell/utilities/dictionary.py b/src/swell/utilities/dictionary.py index 3facefdd6..e33439ad4 100644 --- a/src/swell/utilities/dictionary.py +++ b/src/swell/utilities/dictionary.py @@ -10,7 +10,6 @@ import io from ruamel.yaml import YAML from collections.abc import Hashable -from typing import Union from swell.utilities.logger import Logger @@ -40,7 +39,7 @@ def dict_get( # -------------------------------------------------------------------------------------------------- -def remove_matching_keys(d: Union[dict, list], key: str) -> None: +def remove_matching_keys(d: dict | list, key: str) -> None: """ Recursively locates and removes all dictionary items matching the supplied key. Parameters diff --git a/src/swell/utilities/filehandler.py b/src/swell/utilities/filehandler.py index d37796a31..78e9d04df 100755 --- a/src/swell/utilities/filehandler.py +++ b/src/swell/utilities/filehandler.py @@ -63,10 +63,9 @@ import copy import datetime as dt from shutil import copyfile -from typing import Union, Optional, Any -def get_file_handler(config: list, **kwargs) -> Union[StageFileHandler, GetDataFileHandler]: +def get_file_handler(config: list, **kwargs) -> StageFileHandler | GetDataFileHandler: """Factory for determining the file handler type for retrieving data. This method uses a heuristic algorithm to determine the staging @@ -114,7 +113,7 @@ def __init__(self, config: list, **kwargs) -> None: # ------------------------------------------------------------------------------ - def is_ready(self, fc: Optional[FileCollection] = None) -> bool: + def is_ready(self, fc: FileCollection | None = None) -> bool: """Determines if the file collection meets the criteria for readiness (e.g. minimum file count etc.) @@ -156,7 +155,7 @@ def is_ready(self, fc: Optional[FileCollection] = None) -> bool: # ------------------------------------------------------------------------ - def get(self, fc: Optional[FileCollection] = None) -> None: + def get(self, fc: FileCollection | None = None) -> None: """Retrieves the files in the specified file collection. Parameters @@ -374,7 +373,7 @@ def list(self, force: bool = False) -> list: class FileCollection(object): - def __init__(self, config: dict[Any, Any]) -> None: + def __init__(self, config: dict) -> None: self.config = copy.deepcopy(config) diff --git a/src/swell/utilities/geos.py b/src/swell/utilities/geos.py index 129f846e5..7adfe18bd 100644 --- a/src/swell/utilities/geos.py +++ b/src/swell/utilities/geos.py @@ -12,7 +12,6 @@ import glob import isodate import os -from typing import Tuple, Optional from swell.utilities.datetime_util import datetime_formats from swell.utilities.logger import Logger @@ -32,7 +31,7 @@ class Geos(): def __init__( self, logger: Logger, - forecast_dir: Optional[str], + forecast_dir: str | None, ) -> None: """ Initializes the Geos class. The intention is to share methods between forecast-only @@ -53,7 +52,7 @@ def iso_to_time_str( iso_duration: str, half: bool = False, agcm: bool = False, - ) -> Tuple[str, int, datetime.timedelta]: + ) -> tuple[str, int, datetime.timedelta]: """ Converts an ISO 8601 duration string to various time representations. diff --git a/src/swell/utilities/get_channels.py b/src/swell/utilities/get_channels.py index 7582d0a7d..5b29e9d45 100644 --- a/src/swell/utilities/get_channels.py +++ b/src/swell/utilities/get_channels.py @@ -11,7 +11,6 @@ import os from datetime import datetime as dt from itertools import groupby -from typing import Tuple, Optional from swell.utilities.exceptions import SwellError from swell.utilities.logger import Logger @@ -77,7 +76,7 @@ def get_channels( observation: str, dt_cycle_time: dt, logger: Logger -) -> Tuple[Optional[str], Optional[list[int]]]: +) -> tuple[str | None, list[int] | None]: ''' Comparing available channels and active channels from the observing @@ -123,7 +122,7 @@ def num_active_channels( path_to_observing_sys_yamls: str, observation: str, dt_cycle_time: dt -) -> Optional[int]: +) -> int | None: # Retrieve available and active channels from records yaml path_to_observing_sys_config = path_to_observing_sys_yamls + '/' + \ diff --git a/src/swell/utilities/jinja2.py b/src/swell/utilities/jinja2.py index b45c20735..bc0716f5a 100644 --- a/src/swell/utilities/jinja2.py +++ b/src/swell/utilities/jinja2.py @@ -7,7 +7,6 @@ # -------------------------------------------------------------------------------------------------- from __future__ import annotations -from typing import Union import jinja2 as j2 @@ -34,7 +33,7 @@ def __getattr__(self, name: str) -> SilentUndefined: # Return a new SilentUndefined instance but append the attribute access to the name. return SilentUndefined(name=f"{self._undefined_name}.{name}") - def __getitem__(self, key: Union[str, int]) -> SilentUndefined: + def __getitem__(self, key: str | int) -> SilentUndefined: # Similar to __getattr__, return a new instance with the key access incorporated. if isinstance(key, str): return SilentUndefined(name=f"{self._undefined_name}['{key}']") diff --git a/src/swell/utilities/logger.py b/src/swell/utilities/logger.py index ee73d2206..abda4119f 100644 --- a/src/swell/utilities/logger.py +++ b/src/swell/utilities/logger.py @@ -9,7 +9,6 @@ import os import logging -from typing import Optional # -------------------------------------------------------------------------------------------------- @@ -38,7 +37,7 @@ def assert_abort(self, condition: bool, msg: str) -> None: # -------------------------------------------------------------------------------------------------- -def get_logger(name: Optional[str] = None) -> Logger: +def get_logger(name: str | None = None) -> Logger: ''' Get a logger with custom message formatting for swell-related tasks. diff --git a/src/swell/utilities/netcdf_files.py b/src/swell/utilities/netcdf_files.py index 8b26c7a2f..0c82a98ae 100644 --- a/src/swell/utilities/netcdf_files.py +++ b/src/swell/utilities/netcdf_files.py @@ -10,7 +10,7 @@ import os import xarray as xr -from typing import Hashable, Union +from typing import Hashable from swell.utilities.logger import Logger @@ -21,7 +21,7 @@ def combine_files_without_groups( logger: Logger, list_of_input_files: list, output_file: str, - concat_dim: Union[Hashable, xr.Variable, xr.DataArray], + concat_dim: Hashable | xr.Variable | xr.DataArray, delete_input: bool = False ) -> None: diff --git a/src/swell/utilities/observing_system_records.py b/src/swell/utilities/observing_system_records.py index 47a72e7ad..bcf55b1b0 100644 --- a/src/swell/utilities/observing_system_records.py +++ b/src/swell/utilities/observing_system_records.py @@ -3,7 +3,6 @@ import pandas as pd import numpy as np import datetime as dt -from typing import Optional from swell.utilities.logger import get_logger from swell.utilities.gsi_record_parser import GSIRecordParser @@ -151,7 +150,7 @@ def parse_records(self, path_to_sat_db: str) -> None: def save_yamls( self, output_dir: str, - observation_list: Optional[list] = None + observation_list: list | None = None ) -> None: ''' diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index fe59001b9..99322d87b 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -9,7 +9,6 @@ from dataclasses import dataclass -from typing import List, Dict from swell.utilities.swell_questions import SuiteQuestion, TaskQuestion from swell.utilities.swell_questions import WidgetType as WType @@ -40,7 +39,7 @@ class cycle_times(SuiteQuestion): question_name: str = "cycle_times" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Enter the cycle times for this model." @@ -53,7 +52,7 @@ class cycling_varbc(SuiteQuestion): default_value: str = "defer_to_model" question_name: str = "cycling_varbc" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Do you want to use cycling VarBC option?" @@ -65,7 +64,7 @@ class cycling_varbc(SuiteQuestion): class ensemble_hofx_packets(SuiteQuestion): default_value: str = "defer_to_model" question_name: str = "ensemble_hofx_packets" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Enter the number of ensemble packets." @@ -77,7 +76,7 @@ class ensemble_hofx_packets(SuiteQuestion): class ensemble_hofx_strategy(SuiteQuestion): default_value: str = "defer_to_model" question_name: str = "ensemble_hofx_strategy" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Enter the ensemble hofx strategy." @@ -122,7 +121,7 @@ class marine_models(SuiteQuestion): question_name: str = "marine_models" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_marine" ]) prompt: str = "Select the active SOCA models for this model." @@ -147,7 +146,7 @@ class parser_options(SuiteQuestion): question_name: str = "parser_options" ask_question: bool = True options: list = mutable_field(['fgrep_residual_norm']) - prompt: str = "List the test types to run on the JEDI oops log." + prompt: str = "list the test types to run on the JEDI oops log." widget_type: WType = WType.STRING_DROP_LIST # -------------------------------------------------------------------------------------------------- @@ -177,7 +176,7 @@ class runahead_limit(SuiteQuestion): class skip_ensemble_hofx(SuiteQuestion): default_value: str = "defer_to_model" question_name: str = "skip_ensemble_hofx" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Enter if skip ensemble hofx." @@ -208,11 +207,11 @@ class start_cycle_point(SuiteQuestion): class window_type(SuiteQuestion): default_value: str = "defer_to_model" question_name: str = "window_type" - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "3D", "4D" ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Enter the window type for this model." @@ -227,7 +226,7 @@ class analysis_variables(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "analysis_variables" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What are the analysis variables?" @@ -240,7 +239,7 @@ class background_error_model(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "background_error_model" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Which background error model do you want to use?" @@ -253,7 +252,7 @@ class background_experiment(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "background_experiment" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the name of the name of the experiment providing the backgrounds?" @@ -265,10 +264,10 @@ class background_experiment(TaskQuestion): class background_frequency(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "background_frequency" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "window_type": "4D" }) prompt: str = "What is the frequency of the background files?" @@ -280,7 +279,7 @@ class background_frequency(TaskQuestion): class background_time_offset(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "background_time_offset" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = ("How long before the middle of the analysis window did" @@ -294,7 +293,7 @@ class bufr_obs_classes(TaskQuestion): question_name: str = "bufr_obs_classes" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What BUFR observation classes will be used?" @@ -304,7 +303,7 @@ class bufr_obs_classes(TaskQuestion): @dataclass class bundles(TaskQuestion): - default_value: List[str] = mutable_field([ + default_value: list[str] = mutable_field([ "fv3-jedi", "soca", "iodaconv", @@ -312,7 +311,7 @@ class bundles(TaskQuestion): ]) question_name: str = "bundles" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "fv3-jedi", "soca", "iodaconv", @@ -321,7 +320,7 @@ class bundles(TaskQuestion): "oops", "saber" ]) - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "jedi_build_method": "create" }) prompt: str = "Which JEDI bundles do you wish to build?" @@ -333,8 +332,8 @@ class bundles(TaskQuestion): class check_for_obs(TaskQuestion): default_value: bool = True question_name: str = "check_for_obs" - options: List[bool] = mutable_field([True, False]) - models: List[str] = mutable_field([ + options: list[bool] = mutable_field([True, False]) + models: list[str] = mutable_field([ 'all_models' ]) prompt: str = "Perform check for observations? Set to false for debugging purposes." @@ -347,7 +346,7 @@ class clean_patterns(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "clean_patterns" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Provide a list of patterns that you wish to remove from the cycle directory." @@ -359,11 +358,11 @@ class clean_patterns(TaskQuestion): class comparison_log_type(TaskQuestion): default_value: str = "variational" question_name: str = "comparison_log_type" - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ 'variational', 'fgat', ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Provide the log naming convention (e.g. 'variational', 'fgat')." @@ -375,7 +374,7 @@ class comparison_log_type(TaskQuestion): class crtm_coeff_dir(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "crtm_coeff_dir" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the path to the CRTM coefficient files?" @@ -389,7 +388,7 @@ class ensemble_hofx_packets(TaskQuestion): question_name: str = "ensemble_hofx_packets" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Enter number of packets in which ensemble observers should be computed." @@ -403,7 +402,7 @@ class ensemble_hofx_strategy(TaskQuestion): question_name: str = "ensemble_hofx_strategy" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Enter hofx strategy." @@ -416,7 +415,7 @@ class ensemble_num_members(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "ensemble_num_members" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "How many members comprise the ensemble?" @@ -428,11 +427,11 @@ class ensemble_num_members(TaskQuestion): class ensmean_only(TaskQuestion): default_value: bool = False question_name: str = "ensmean_only" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Calculate ensemble mean only?" @@ -444,11 +443,11 @@ class ensmean_only(TaskQuestion): class ensmeanvariance_only(TaskQuestion): default_value: bool = False question_name: str = "ensmeanvariance_only" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Calculate ensemble mean and variance only?" @@ -461,7 +460,7 @@ class existing_geos_gcm_build_path(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "existing_geos_gcm_build_path" ask_question: bool = True - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "geos_build_method": "use_existing" }) prompt: str = "What is the path to the existing GEOS build directory?" @@ -474,7 +473,7 @@ class existing_geos_gcm_source_path(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "existing_geos_gcm_source_path" ask_question: bool = True - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "geos_build_method": "use_existing" }) prompt: str = "What is the path to the existing GEOS source code directory?" @@ -487,7 +486,7 @@ class existing_jedi_build_directory(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "existing_jedi_build_directory" ask_question: bool = True - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "jedi_build_method": "use_existing" }) prompt: str = "What is the path to the existing JEDI build directory?" @@ -500,7 +499,7 @@ class existing_jedi_build_directory_pinned(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "existing_jedi_build_directory_pinned" ask_question: bool = True - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "jedi_build_method": "use_pinned_existing" }) prompt: str = "What is the path to the existing pinned JEDI build directory?" @@ -513,7 +512,7 @@ class existing_jedi_source_directory(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "existing_jedi_source_directory" ask_question: bool = True - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "jedi_build_method": "use_existing" }) prompt: str = "What is the path to the existing JEDI source code directory?" @@ -526,7 +525,7 @@ class existing_jedi_source_directory_pinned(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "existing_jedi_source_directory_pinned" ask_question: bool = True - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "jedi_build_method": "use_pinned_existing" }) prompt: str = "What is the path to the existing pinned JEDI source code directory?" @@ -576,7 +575,7 @@ class geos_build_method(TaskQuestion): default_value: str = "create" question_name: str = "geos_build_method" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "use_existing", "create" ]) @@ -601,7 +600,7 @@ class geos_expdir_different(TaskQuestion): default_value: str = False question_name: str = "geos_expdir_different" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) @@ -615,7 +614,7 @@ class geos_expdir_different(TaskQuestion): class geos_expdir(TaskQuestion): default_value: str = "/dev/null/" question_name: str = "geos_expdir" - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "geos_expdir_different": True }) prompt: str = ("What is the location for the EXPERIMENT Directory (to contain model " @@ -629,7 +628,7 @@ class geos_expdir(TaskQuestion): class geos_gcm_tag(TaskQuestion): default_value: str = "v11.6.0" question_name: str = "geos_gcm_tag" - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "geos_build_method": "create" }) prompt: str = "Which GEOS tag do you wish to clone?" @@ -642,11 +641,11 @@ class geos_x_background_directory(TaskQuestion): default_value: str = "/dev/null/" question_name: str = "geos_x_background_directory" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "/dev/null/", "/discover/nobackup/projects/gmao/dadev/rtodling/archive/Restarts/JEDI/541x" ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the path to the GEOS X-backgrounds directory?" @@ -659,11 +658,11 @@ class geos_x_ensemble_directory(TaskQuestion): default_value: str = "/dev/null/" question_name: str = "geos_x_ensemble_directory" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "/dev/null/", "/gpfsm/dnb05/projects/p139/rtodling/archive/" ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the path to the GEOS X-backgrounds directory?" @@ -676,7 +675,7 @@ class geovals_experiment(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "geovals_experiment" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the name of the R2D2 experiment providing the GeoVaLs?" @@ -688,7 +687,7 @@ class geovals_experiment(TaskQuestion): class geovals_provider(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "geovals_provider" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the name of the R2D2 database providing the GeoVaLs?" @@ -700,7 +699,7 @@ class geovals_provider(TaskQuestion): class gradient_norm_reduction(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "gradient_norm_reduction" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What value of gradient norm reduction for convergence?" @@ -712,7 +711,7 @@ class gradient_norm_reduction(TaskQuestion): class gsibec_configuration(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "gsibec_configuration" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Which GSIBEC climatological or hybrid?" @@ -724,7 +723,7 @@ class gsibec_configuration(TaskQuestion): class gsibec_nlats(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "gsibec_nlats" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "How many number of latutides in GSIBEC grid?" @@ -736,7 +735,7 @@ class gsibec_nlats(TaskQuestion): class gsibec_nlons(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "gsibec_nlons" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "How many number of longitudes in GSIBEC grid?" @@ -748,7 +747,7 @@ class gsibec_nlons(TaskQuestion): class horizontal_localization_lengthscale(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "horizontal_localization_lengthscale" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the length scale for horizontal covariance localization?" @@ -760,7 +759,7 @@ class horizontal_localization_lengthscale(TaskQuestion): class horizontal_localization_max_nobs(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "horizontal_localization_max_nobs" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ("What is the maximum number of observations to consider" @@ -774,7 +773,7 @@ class horizontal_localization_method(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "horizontal_localization_method" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Which localization scheme should be applied in the horizontal?" @@ -788,7 +787,7 @@ class horizontal_resolution(TaskQuestion): question_name: str = "horizontal_resolution" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the horizontal resolution for the forecast model and backgrounds?" @@ -801,7 +800,7 @@ class dry_run(TaskQuestion): default_value: bool = True question_name: str = "dry_run" ask_question: bool = False - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Dry-run mode: preview what would be ingested before storing to R2D2" @@ -815,7 +814,7 @@ class obs_to_ingest(TaskQuestion): question_name: str = "obs_to_ingest" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Which observations do you want to ingest to R2D2?" @@ -827,7 +826,7 @@ class initial_restarts_method(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "initial_restarts_method" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "geos_expdir", "r2d2", "hotstart", @@ -842,7 +841,7 @@ class ioda_locations_not_in_r2d2(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "ioda_locations_not_in_r2d2" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ( @@ -856,7 +855,7 @@ class jedi_build_method(TaskQuestion): default_value: str = "create" question_name: str = "jedi_build_method" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "use_existing", "use_pinned_existing", "create", @@ -873,10 +872,10 @@ class jedi_forecast_model(TaskQuestion): question_name: str = "jedi_forecast_model" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "window_type": "4D" }) prompt: str = "What forecast model should be used within JEDI for 4D window propagation?" @@ -888,7 +887,7 @@ class jedi_forecast_model(TaskQuestion): class local_ensemble_inflation_mult(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "local_ensemble_inflation_mult" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Specify the multiplicative prior inflation coefficient (0 inf]." @@ -900,7 +899,7 @@ class local_ensemble_inflation_mult(TaskQuestion): class local_ensemble_inflation_rtpp(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "local_ensemble_inflation_rtpp" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Specify the Relaxation To Prior Perturbation (RTPP) coefficient (0 1]." @@ -912,7 +911,7 @@ class local_ensemble_inflation_rtpp(TaskQuestion): class local_ensemble_inflation_rtps(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "local_ensemble_inflation_rtps" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Specify the Relaxation To Prior Spread (RTPS) coefficient (0 1]." @@ -924,11 +923,11 @@ class local_ensemble_inflation_rtps(TaskQuestion): class local_ensemble_save_posterior_ensemble(TaskQuestion): default_value: bool = False question_name: str = "local_ensemble_save_posterior_ensemble" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Save the posterior ensemble members?" @@ -941,11 +940,11 @@ class local_ensemble_save_posterior_ensemble_increments(TaskQuestion): default_value: bool = False question_name: str = "local_ensemble_save_posterior_ensemble_increments" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Save the posterior ensemble member increments?" @@ -958,11 +957,11 @@ class local_ensemble_save_posterior_mean(TaskQuestion): default_value: bool = False question_name: str = "local_ensemble_save_posterior_mean" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Save the posterior ensemble mean?" @@ -975,11 +974,11 @@ class local_ensemble_save_posterior_mean_increment(TaskQuestion): default_value: bool = True question_name: str = "local_ensemble_save_posterior_mean_increment" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Save the posterior ensemble mean increment?" @@ -993,7 +992,7 @@ class local_ensemble_solver(TaskQuestion): question_name: str = "local_ensemble_solver" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Which local ensemble solver type should be implemented?" @@ -1007,7 +1006,7 @@ class local_ensemble_use_linear_observer(TaskQuestion): question_name: str = "local_ensemble_use_linear_observer" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Which local ensemble solver type should be implemented?" @@ -1020,7 +1019,7 @@ class minimizer(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "minimizer" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Which data assimilation minimizer do you wish to use?" @@ -1032,11 +1031,11 @@ class minimizer(TaskQuestion): class mom6_iau(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "mom6_iau" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_marine", ]) prompt: str = "Do you wish to use IAU for MOM6?" @@ -1048,12 +1047,12 @@ class mom6_iau(TaskQuestion): class mom6_iau_nhours(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "mom6_iau_nhours" - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ 'PT3H', 'PT12H' ]) depends: dict = mutable_field({'mom6_iau': True}) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_marine", ]) prompt: str = "What is the IAU length (ODA_INCUPD_NHOURS) for MOM6?" @@ -1065,8 +1064,8 @@ class mom6_iau_nhours(TaskQuestion): class ncdiag_experiments(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "ncdiag_experiments" - options: List[str] = "defer_to_model" - models: List[str] = mutable_field([ + options: list[str] = "defer_to_model" + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Which previously run experiments do you wish to use for the NCdiag?" @@ -1079,7 +1078,7 @@ class npx_proc(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "npx_proc" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere", "geos_cf" ]) @@ -1093,7 +1092,7 @@ class npy_proc(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "npy_proc" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere", "geos_cf" ]) @@ -1107,7 +1106,7 @@ class npx(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "npx" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "What is the number of grid points in the x-direction on each cube face?" @@ -1120,7 +1119,7 @@ class npy(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "npy" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "What is the number of grid points in the y-direction on each cube face?" @@ -1132,7 +1131,7 @@ class npy(TaskQuestion): class number_of_iterations(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "number_of_iterations" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = ( @@ -1147,7 +1146,7 @@ class obs_experiment(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "obs_experiment" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the database providing the observations?" @@ -1159,7 +1158,7 @@ class obs_experiment(TaskQuestion): class obs_thinning_rej_fraction(TaskQuestion): default_value: float = 0.75 question_name: str = "obs_thinning_rej_fraction" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the rejection fraction for obs thinning?" @@ -1173,7 +1172,7 @@ class observations(TaskQuestion): question_name: str = "observations" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Which observations do you want to include?" @@ -1185,7 +1184,7 @@ class observations(TaskQuestion): class observing_system_records_mksi_path(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "observing_system_records_mksi_path" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the path to the GSI formatted observing system records?" @@ -1197,7 +1196,7 @@ class observing_system_records_mksi_path(TaskQuestion): class observing_system_records_mksi_path_tag(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "observing_system_records_mksi_path_tag" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the GSI formatted observing system records tag?" @@ -1209,7 +1208,7 @@ class observing_system_records_mksi_path_tag(TaskQuestion): class observing_system_records_path(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "observing_system_records_path" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the path to the Swell formatted observing system records?" @@ -1222,7 +1221,7 @@ class path_to_ensemble(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "path_to_ensemble" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere", "geos_marine" ]) @@ -1236,7 +1235,7 @@ class path_to_geos_adas_background(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "path_to_geos_adas_background" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ( @@ -1250,7 +1249,7 @@ class path_to_gsi_bc_coefficients(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "path_to_gsi_bc_coefficients" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the location where GSI bias correction files can be found?" @@ -1263,7 +1262,7 @@ class path_to_gsi_nc_diags(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "path_to_gsi_nc_diags" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the path to where the GSI ncdiags are stored?" @@ -1276,11 +1275,11 @@ class perhost(TaskQuestion): default_value: str = None question_name: str = "perhost" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the number of processors per host?" @@ -1293,11 +1292,11 @@ class produce_geovals(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "produce_geovals" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ("When running the ncdiag to ioda converted do you " @@ -1310,7 +1309,7 @@ class produce_geovals(TaskQuestion): class save_geovals(TaskQuestion): default_value: bool = False question_name: str = "save_geovals" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) @@ -1323,11 +1322,11 @@ class save_geovals(TaskQuestion): class single_observations(TaskQuestion): default_value: bool = False question_name: str = "single_observations" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Is it a single-observation test?" @@ -1358,7 +1357,7 @@ class total_processors(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "total_processors" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_marine", ]) prompt: str = "What is the number of processors for JEDI?" @@ -1370,11 +1369,11 @@ class total_processors(TaskQuestion): class vertical_localization_apply_log_transform(TaskQuestion): default_value: bool = True question_name: str = "vertical_localization_apply_log_transform" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ("Should a log (base 10) transformation be applied " @@ -1389,7 +1388,7 @@ class vertical_localization_function(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "vertical_localization_function" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Which localization scheme should be applied in the vertical?" @@ -1402,7 +1401,7 @@ class vertical_localization_ioda_vertical_coord(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "vertical_localization_ioda_vertical_coord" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Which coordinate should be used in constructing vertical localization?" @@ -1415,7 +1414,7 @@ class vertical_localization_ioda_vertical_coord_group(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "vertical_localization_ioda_vertical_coord_group" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ("Which vertical coordinate group should be used " @@ -1428,7 +1427,7 @@ class vertical_localization_ioda_vertical_coord_group(TaskQuestion): class vertical_localization_lengthscale(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "vertical_localization_lengthscale" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the length scale for vertical covariance localization?" @@ -1441,7 +1440,7 @@ class vertical_localization_method(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "vertical_localization_method" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ("What localization scheme should be applied in " @@ -1456,7 +1455,7 @@ class vertical_resolution(TaskQuestion): question_name: str = "vertical_resolution" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the vertical resolution for the forecast model and background?" @@ -1468,7 +1467,7 @@ class vertical_resolution(TaskQuestion): class window_length(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "window_length" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the duration for the data assimilation window?" @@ -1481,11 +1480,11 @@ class window_type(TaskQuestion): question_name: str = "window_type" default_value: str = "defer_to_model" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "3D", "4D" ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Do you want to use a 3D or 4D (including FGAT) window?" diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 779a73fb1..275d57964 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -9,9 +9,9 @@ import os from ruamel.yaml import YAML -from typing import Union, Optional, Any from importlib import import_module from collections.abc import Mapping +from typing import Any from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.get_channels import get_channels @@ -28,9 +28,9 @@ def __init__( logger: Logger, experiment_root: str, experiment_id: str, - cycle_dir: Optional[str], - cycle_time: Optional[Datetime], - jedi_interface: Optional[str] = None + cycle_dir: str | None, + cycle_time: Datetime | None, + jedi_interface: str | None = None ) -> None: # Keep a copy of the logger @@ -194,8 +194,8 @@ def __open_file_render_to_dict__(self, config_file: str) -> dict[Any, Any]: # Prepare path to oops file and call rendering def render_oops_file(self, config_name: str, - window_type: Optional[str] = None, - jedi_forecast_model: Optional[str] = None) -> dict: + window_type: str | None = None, + jedi_forecast_model: str | None = None) -> dict: # Import the module module = import_module(f'swell.configuration.jedi.oops.{config_name}') @@ -286,7 +286,7 @@ def render_interface_observations(self, config_name: str) -> dict: # Prepare path to interface metadata file and call rendering - def render_interface_meta(self, model_component_in: Union[str, dict, None] = None) -> dict: + def render_interface_meta(self, model_component_in: str | dict | None = None) -> dict: # Optionally open a different model interface model_component = self.jedi_interface diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index c4457a9c7..fcfe2ac16 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -10,7 +10,6 @@ import os import netCDF4 as nc -from typing import Optional from swell.utilities.shell_commands import run_track_log_subprocess from swell.utilities.logger import Logger @@ -19,11 +18,11 @@ def check_obs( - path_to_observing_sys_yamls: Optional[str], + path_to_observing_sys_yamls: str | None, observation: str, obs_dict: dict, - cycle_time: Optional[str], - input_and_output: Optional[bool] = False + cycle_time: str | None, + input_and_output: bool | None = False ) -> bool: use_observation = False @@ -69,7 +68,7 @@ def run_executable( jedi_executable_path: str, jedi_config_file: str, output_log: str, - perhost: Optional[int] = None + perhost: int | None = None ) -> None: # Run the JEDI executable diff --git a/src/swell/utilities/scripts/compare_questions.py b/src/swell/utilities/scripts/compare_questions.py index b04dcc6c8..6efc97723 100644 --- a/src/swell/utilities/scripts/compare_questions.py +++ b/src/swell/utilities/scripts/compare_questions.py @@ -8,7 +8,6 @@ import os -from typing import Optional, Tuple import importlib import re from enum import StrEnum, auto @@ -63,7 +62,7 @@ def get_all_tasks(suite: str) -> list: # -------------------------------------------------------------------------------------------------- -def get_question_names(config: QuestionList, model: Optional[str] = None) -> list: +def get_question_names(config: QuestionList, model: str | None = None) -> list: """ Get a list of question names from a QuestionList object. """ return [q['question_name'] for q in config.expand_question_list(model)] @@ -101,7 +100,7 @@ def questions_in_cylc(suite: str) -> list: # -------------------------------------------------------------------------------------------------- -def compare_used_and_set_questions() -> Tuple[dict, dict]: +def compare_used_and_set_questions() -> tuple[dict, dict]: """ Finds the questions which are set in the suite/task configuration, and those that are actually used by the suite. diff --git a/src/swell/utilities/shell_commands.py b/src/swell/utilities/shell_commands.py index 20ab6cd65..cd7d2c5a0 100644 --- a/src/swell/utilities/shell_commands.py +++ b/src/swell/utilities/shell_commands.py @@ -10,7 +10,7 @@ import os import stat import subprocess -from typing import Any, Optional, IO, Union +from typing import Any, IO from swell.utilities.logger import Logger @@ -20,8 +20,8 @@ def run_track_log_subprocess( logger: Logger, - command: Union[list[str], str], - output_log: Optional[str] = None, + command: list[str] | str | None, + output_log: str | None = None, **kwargs ) -> None: @@ -65,7 +65,7 @@ def run_track_log_subprocess( def run_subprocess_dev_null( logger: Logger, - command: Union[list[str], str], + command: list[str] | str | None, **kwargs ) -> None: @@ -77,9 +77,9 @@ def run_subprocess_dev_null( def run_subprocess( logger: Logger, - command: Union[list[str], str], - stdout: Union[int, IO[Any], None] = None, - stderr: Union[int, IO[Any], None] = None, + command: list[str] | str, + stdout: int | IO[Any] | None = None, + stderr: int | IO[Any] | None = None, **kwargs ) -> None: diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index ec1b5728a..db2745b30 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -10,7 +10,7 @@ import os from dataclasses import dataclass, asdict, field -from typing import List, Optional, Self, Union, Literal +from typing import Literal, Self from enum import Enum from isodate import parse_datetime, parse_duration, ISO8601Error @@ -102,7 +102,7 @@ class SwellQuestion: prompt: str question_type: str = None ask_question: bool = False - options: Optional[str] = None + options: str | None = None # -------------------------------------------------------------------------------------------------- @@ -125,7 +125,7 @@ def get_all(cls): class QuestionList: """Basic dataclass containing a list of questions for each model, suite, task""" list_name: str - questions: List[Union[SwellQuestion, Self]] + questions: list[SwellQuestion | Self] geos_atmosphere: list = field(default_factory=lambda: []) geos_cf: list = field(default_factory=lambda: []) @@ -133,7 +133,7 @@ class QuestionList: # -------------------------------------------------------------------------------------------------- - def get_all_question_names(self, suite_task: Optional[Literal['suite', 'task']] = None) -> None: + def get_all_question_names(self, suite_task: Literal['suite', 'task'] | None = None) -> None: question_list = [] for model in [None] + os.listdir(os.path.join(get_swell_path(), 'configuration', 'jedi', 'interfaces')): @@ -149,7 +149,7 @@ def get_all_question_names(self, suite_task: Optional[Literal['suite', 'task']] # -------------------------------------------------------------------------------------------------- - def expand_question_list(self, model: Optional[str] = None): + def expand_question_list(self, model: str | None = None): question_list = [] # Loop through the items in the questions list diff --git a/src/swell/utilities/test_cache.py b/src/swell/utilities/test_cache.py index 6b8988af5..5f03a9a41 100644 --- a/src/swell/utilities/test_cache.py +++ b/src/swell/utilities/test_cache.py @@ -7,15 +7,13 @@ # -------------------------------------------------------------------------------------------------- -from typing import Optional - import os from ruamel.yaml import YAML # -------------------------------------------------------------------------------------------------- -def get_test_cache() -> Optional[str]: +def get_test_cache() -> str | None: test_settings_file = os.path.expanduser('~/.swell/swell-test.yaml') if os.path.exists(test_settings_file): yaml = YAML(typ='safe') From 67ddb330f190e5b1ffaedbb19ebb2c1144010bc2 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 3 Apr 2026 15:36:15 -0400 Subject: [PATCH 260/299] Fixes for swell.py --- src/swell/deployment/create_experiment.py | 2 +- src/swell/swell.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 44cf43803..e85011797 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -232,7 +232,7 @@ def create_experiment_directory( suite_config: str, method: str, platform: str, - override: str, + override: str | dict | None, advanced: bool, slurm: str | None, skip_r2d2: bool diff --git a/src/swell/swell.py b/src/swell/swell.py index 513aeb524..83199c3e1 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -10,6 +10,7 @@ import click from typing import Literal +from ruamel.yaml import YAML from swell.deployment.platforms.platforms import get_platforms from swell.deployment.create_experiment import clone_config, create_experiment_directory @@ -151,10 +152,14 @@ def clone( # Create experiment configuration by cloning from existing experiment experiment_dict_str = clone_config(configuration, experiment_id, input_method, platform, advanced) + + yaml = YAML(typ='safe') + experiment_override = yaml.load(experiment_dict_str) + suite = experiment_override['suite_to_run'] # Create the experiment directory - create_experiment_directory(experiment_dict_str) - + create_experiment_directory(suite, method=input_method, platform=platform, + override=experiment_override, advanced=advanced) # -------------------------------------------------------------------------------------------------- From 48353f059eb997af7ad2b955ba663995421e3a4e Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 3 Apr 2026 15:39:09 -0400 Subject: [PATCH 261/299] fix --- src/swell/swell.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/swell/swell.py b/src/swell/swell.py index 83199c3e1..348c05524 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -132,12 +132,16 @@ def create( type=click.Choice(['defaults', 'cli']), help=input_method_help) @click.option('-p', '--platform', 'platform', default=None, help=platform_help) @click.option('-a', '--advanced', 'advanced', default=False, help=advanced_help) +@click.option('-s', '--slurm', 'slurm', default=None, help=slurm_help) +@click.option('-k', '--skip-r2d2', 'skip_r2d2', is_flag=True, default=False, help=skip_r2d2_help) def clone( configuration: str, experiment_id: str, input_method: str, platform: str, - advanced: bool + advanced: bool, + slurm: str, + skip_r2d2: bool ) -> None: """ Clone an existing experiment @@ -159,7 +163,8 @@ def clone( # Create the experiment directory create_experiment_directory(suite, method=input_method, platform=platform, - override=experiment_override, advanced=advanced) + override=experiment_override, advanced=advanced, slurm=slurm, + skip_r2d2=skip_r2d2) # -------------------------------------------------------------------------------------------------- From cdbd315d8bb52faceeb6e0a5e15a8dab8e70b164 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 3 Apr 2026 16:42:11 -0400 Subject: [PATCH 262/299] Type hinting and fixes --- .../geos_atmosphere/model/stage_cycle.py | 2 +- .../jedi/interfaces/geos_cf/model/stage_cycle.py | 2 +- .../interfaces/geos_marine/model/stage_cycle.py | 2 +- .../jedi/interfaces/geos_marine/model/states.py | 2 +- src/swell/cylc_swell.py | 3 ++- src/swell/deployment/create_experiment.py | 16 +++++++++++++--- .../prepare_config_and_suite.py | 6 +++--- src/swell/suites/3dfgat_atmos/__init__.py | 0 src/swell/suites/3dfgat_cycle/__init__.py | 0 src/swell/suites/3dfgat_marine_cycle/__init__.py | 0 src/swell/suites/3dvar/__init__.py | 0 src/swell/suites/3dvar_atmos/__init__.py | 0 src/swell/suites/3dvar_cf/__init__.py | 0 src/swell/suites/3dvar_cycle/__init__.py | 0 src/swell/suites/3dvar_marine/__init__.py | 0 src/swell/suites/3dvar_marine_cycle/__init__.py | 0 src/swell/suites/base/__init__.py | 0 src/swell/suites/build_geos/__init__.py | 0 src/swell/suites/build_jedi/__init__.py | 0 src/swell/suites/compare/__init__.py | 0 src/swell/suites/convert_bufr/__init__.py | 0 src/swell/suites/convert_ncdiags/__init__.py | 0 src/swell/suites/eva_capabilities/__init__.py | 0 .../suites/forecast_coupled_geos/__init__.py | 0 src/swell/suites/forecast_geos/__init__.py | 0 src/swell/suites/geosadas/__init__.py | 0 src/swell/suites/hofx/__init__.py | 0 src/swell/suites/hofx_cf/__init__.py | 0 src/swell/suites/ingest_obs/__init__.py | 0 src/swell/suites/localensembleda/__init__.py | 0 src/swell/suites/ufo_testing/__init__.py | 0 src/swell/swell.py | 2 +- src/swell/utilities/check_da_params.py | 2 +- src/swell/utilities/dictionary.py | 3 ++- 34 files changed, 26 insertions(+), 14 deletions(-) create mode 100644 src/swell/suites/3dfgat_atmos/__init__.py create mode 100644 src/swell/suites/3dfgat_cycle/__init__.py create mode 100644 src/swell/suites/3dfgat_marine_cycle/__init__.py create mode 100644 src/swell/suites/3dvar/__init__.py create mode 100644 src/swell/suites/3dvar_atmos/__init__.py create mode 100644 src/swell/suites/3dvar_cf/__init__.py create mode 100644 src/swell/suites/3dvar_cycle/__init__.py create mode 100644 src/swell/suites/3dvar_marine/__init__.py create mode 100644 src/swell/suites/3dvar_marine_cycle/__init__.py create mode 100644 src/swell/suites/base/__init__.py create mode 100644 src/swell/suites/build_geos/__init__.py create mode 100644 src/swell/suites/build_jedi/__init__.py create mode 100644 src/swell/suites/compare/__init__.py create mode 100644 src/swell/suites/convert_bufr/__init__.py create mode 100644 src/swell/suites/convert_ncdiags/__init__.py create mode 100644 src/swell/suites/eva_capabilities/__init__.py create mode 100644 src/swell/suites/forecast_coupled_geos/__init__.py create mode 100644 src/swell/suites/forecast_geos/__init__.py create mode 100644 src/swell/suites/geosadas/__init__.py create mode 100644 src/swell/suites/hofx/__init__.py create mode 100644 src/swell/suites/hofx_cf/__init__.py create mode 100644 src/swell/suites/ingest_obs/__init__.py create mode 100644 src/swell/suites/localensembleda/__init__.py create mode 100644 src/swell/suites/ufo_testing/__init__.py diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.py b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.py index 8f90e1a71..be9377b3d 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.py +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.py @@ -11,7 +11,7 @@ # -------------------------------------------------------------------------------------------------- -def stage_cycle(template_dict: Mapping) -> Mapping: +def stage_cycle(template_dict: Mapping) -> Mapping | list: cycle_dir = template_dict['cycle_dir'] swell_static_files = template_dict['swell_static_files'] diff --git a/src/swell/configuration/jedi/interfaces/geos_cf/model/stage_cycle.py b/src/swell/configuration/jedi/interfaces/geos_cf/model/stage_cycle.py index c6d50408c..b297d47bd 100644 --- a/src/swell/configuration/jedi/interfaces/geos_cf/model/stage_cycle.py +++ b/src/swell/configuration/jedi/interfaces/geos_cf/model/stage_cycle.py @@ -11,7 +11,7 @@ # -------------------------------------------------------------------------------------------------- -def stage_cycle(template_dict: Mapping) -> Mapping: +def stage_cycle(template_dict: Mapping) -> Mapping | list: cycle_dir = template_dict['cycle_dir'] swell_static_files = template_dict['swell_static_files'] diff --git a/src/swell/configuration/jedi/interfaces/geos_marine/model/stage_cycle.py b/src/swell/configuration/jedi/interfaces/geos_marine/model/stage_cycle.py index d388cb046..fa4bbb3d1 100644 --- a/src/swell/configuration/jedi/interfaces/geos_marine/model/stage_cycle.py +++ b/src/swell/configuration/jedi/interfaces/geos_marine/model/stage_cycle.py @@ -11,7 +11,7 @@ # -------------------------------------------------------------------------------------------------- -def stage_cycle(template_dict: Mapping) -> Mapping: +def stage_cycle(template_dict: Mapping) -> Mapping | list: swell_static_files = template_dict['swell_static_files'] cycle_dir = template_dict['cycle_dir'] horizontal_resolution = template_dict['horizontal_resolution'] diff --git a/src/swell/configuration/jedi/interfaces/geos_marine/model/states.py b/src/swell/configuration/jedi/interfaces/geos_marine/model/states.py index 0fd02f537..e432315a4 100644 --- a/src/swell/configuration/jedi/interfaces/geos_marine/model/states.py +++ b/src/swell/configuration/jedi/interfaces/geos_marine/model/states.py @@ -11,7 +11,7 @@ # -------------------------------------------------------------------------------------------------- -def states(template_dict: Mapping) -> Mapping: +def states(template_dict: Mapping) -> Mapping | list: experiment_id = template_dict['experiment_id'] analysis_time_iso = template_dict['analysis_time_iso'] diff --git a/src/swell/cylc_swell.py b/src/swell/cylc_swell.py index 8fb79916f..3091bcfab 100644 --- a/src/swell/cylc_swell.py +++ b/src/swell/cylc_swell.py @@ -11,6 +11,7 @@ import subprocess import os import sys +from collections.abc import Mapping from swell.deployment.platforms.platforms import SwellPlatform from swell.utilities.logger import Logger @@ -18,7 +19,7 @@ # -------------------------------------------------------------------------------------------------- -def configure_cylc_environment(append_dict: dict = {}) -> dict: +def configure_cylc_environment(append_dict: dict = {}) -> Mapping: ''' Unset the path containing Swell's cylc entry point, and set the environment according to the specified dictionary ''' diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index e85011797..829a85d75 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -62,6 +62,8 @@ def clone_config( yaml = YAML() override_dict = yaml.load(f) + suite_config = override_dict['suite_to_run'] + # Check that override_dict has a suite key and get the suite name if 'suite_to_run' not in override_dict: logger.abort('The provided configuration file does not have a \'suite_to_run\' key') @@ -75,7 +77,13 @@ def clone_config( override_dict['experiment_id'] = experiment_id # First create the configuration for the experiment. - return prepare_config(suite, method, override_dict['platform'], override_dict, advanced) + return prepare_config(suite, + suite_config=suite_config, + method=method, + platform=override_dict['platform'], + override=override_dict, + advanced=advanced, + slurm=None) # -------------------------------------------------------------------------------------------------- @@ -88,7 +96,7 @@ def prepare_config( platform: str, override: dict, advanced: bool, - slurm: str + slurm: str | None ) -> str: # Create a logger @@ -392,6 +400,8 @@ def template_modules_file( # Swell bin path # -------------- swell_bin_path = shutil.which("swell") + if swell_bin_path is None: + raise ModuleNotFoundError(f'Could not find swell executable') swell_bin_path = os.path.split(swell_bin_path)[0] # Swell lib path @@ -517,7 +527,7 @@ def prepare_cylc_suite_jinja2( # Since cycle times are used, the render_dictionary will need to include cycle_times # If there are different model components then process each to gather cycle times if len(model_components) > 0: - cycle_times = [] + cycle_times : list = [] for model_component in model_components: cycle_times_mc = experiment_dict['models'][model_component]['cycle_times'] cycle_times = list(set(cycle_times + cycle_times_mc)) diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index a327d1e4d..8f88c3106 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -75,8 +75,8 @@ def __init__( # Big dictionary that contains all user responses as well a dictionary containing the # questions that were asked - self.experiment_dict = {} - self.questions_dict = {} + self.experiment_dict: dict = {} + self.questions_dict: dict = {} # Get list of all possible models self.possible_model_components = os.listdir(os.path.join(get_swell_path(), 'configuration', @@ -579,7 +579,7 @@ def ask_a_question( # ---------------------------------------------------------------------------------------------- - def question_not_been_asked(self, question_key: str, model: str) -> bool: + def question_not_been_asked(self, question_key: str, model: str | None) -> bool: # See if a question has been answered in the experiment dict # Check model independent keys diff --git a/src/swell/suites/3dfgat_atmos/__init__.py b/src/swell/suites/3dfgat_atmos/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/3dfgat_cycle/__init__.py b/src/swell/suites/3dfgat_cycle/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/3dfgat_marine_cycle/__init__.py b/src/swell/suites/3dfgat_marine_cycle/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/3dvar/__init__.py b/src/swell/suites/3dvar/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/3dvar_atmos/__init__.py b/src/swell/suites/3dvar_atmos/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/3dvar_cf/__init__.py b/src/swell/suites/3dvar_cf/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/3dvar_cycle/__init__.py b/src/swell/suites/3dvar_cycle/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/3dvar_marine/__init__.py b/src/swell/suites/3dvar_marine/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/3dvar_marine_cycle/__init__.py b/src/swell/suites/3dvar_marine_cycle/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/base/__init__.py b/src/swell/suites/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/build_geos/__init__.py b/src/swell/suites/build_geos/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/build_jedi/__init__.py b/src/swell/suites/build_jedi/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/compare/__init__.py b/src/swell/suites/compare/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/convert_bufr/__init__.py b/src/swell/suites/convert_bufr/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/convert_ncdiags/__init__.py b/src/swell/suites/convert_ncdiags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/eva_capabilities/__init__.py b/src/swell/suites/eva_capabilities/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/forecast_coupled_geos/__init__.py b/src/swell/suites/forecast_coupled_geos/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/forecast_geos/__init__.py b/src/swell/suites/forecast_geos/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/geosadas/__init__.py b/src/swell/suites/geosadas/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/hofx/__init__.py b/src/swell/suites/hofx/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/hofx_cf/__init__.py b/src/swell/suites/hofx_cf/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/ingest_obs/__init__.py b/src/swell/suites/ingest_obs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/localensembleda/__init__.py b/src/swell/suites/localensembleda/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/ufo_testing/__init__.py b/src/swell/suites/ufo_testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/swell.py b/src/swell/swell.py index 348c05524..61a38b1a9 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -265,7 +265,7 @@ def test(test: str) -> None: "localensembleda", "3dvar_cycle"))) def t1test( suite: Literal["hofx", "3dvar_marine", "3dvar_atmos", "localensembleda", "3dvar_cycle"], - platform: str | None = "nccs_discover_sles15" + platform: str = "nccs_discover_sles15" ) -> None: """ Run a particular swell suite from the tier 1 tests. diff --git a/src/swell/utilities/check_da_params.py b/src/swell/utilities/check_da_params.py index 9e635fd39..d04fcd589 100644 --- a/src/swell/utilities/check_da_params.py +++ b/src/swell/utilities/check_da_params.py @@ -20,7 +20,7 @@ def check_da_params(config_list: list, model_component: str, start_cycle_point_in: str | None, final_cycle_point_in: str | None, - cycle_times_in: str | None) -> None: + cycle_times_in: str | None) -> tuple[list, list, list]: # From two or more experiments, check that the window parameters are the same, and gather the # common cycle times between the two. Returns times between the start and final cycle points, diff --git a/src/swell/utilities/dictionary.py b/src/swell/utilities/dictionary.py index e33439ad4..8c42e57b0 100644 --- a/src/swell/utilities/dictionary.py +++ b/src/swell/utilities/dictionary.py @@ -10,6 +10,7 @@ import io from ruamel.yaml import YAML from collections.abc import Hashable +from typing import Any from swell.utilities.logger import Logger @@ -20,7 +21,7 @@ def dict_get( logger: Logger, dictionary: dict, key: str, - default: str = 'NODEFAULT' + default: Any = 'NODEFAULT' ) -> str: if key in dictionary.keys(): From 0884c44a5b9e19f096a04e2a1edd8a8937814051 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 3 Apr 2026 16:52:15 -0400 Subject: [PATCH 263/299] Refactor r2d2_utils --- src/swell/deployment/create_experiment.py | 2 +- src/swell/tasks/get_background.py | 2 +- src/swell/tasks/get_geovals.py | 2 +- src/swell/tasks/get_ncdiags.py | 2 +- src/swell/tasks/get_observations.py | 4 ++-- src/swell/tasks/ingest_obs.py | 2 +- src/swell/tasks/save_forecast.py | 2 +- src/swell/tasks/save_obs_diags.py | 2 +- src/swell/utilities/{r2d2.py => r2d2_utils.py} | 4 ++-- src/swell/utilities/scripts/delete_obs_in_range.py | 4 ++-- src/swell/utilities/scripts/ingest_files.py | 8 ++++---- src/swell/utilities/scripts/search_ingested.py | 4 ++-- 12 files changed, 19 insertions(+), 19 deletions(-) rename src/swell/utilities/{r2d2.py => r2d2_utils.py} (98%) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 829a85d75..7454797ca 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -188,7 +188,7 @@ def prepare_config( if 'r2d2_experiment_id' in experiment_dict and 'skip_r2d2' in experiment_dict \ and not experiment_dict['skip_r2d2']: - from swell.utilities.r2d2 import load_r2d2_credentials, load_r2d2_module, unique_r2d2_id + from swell.utilities.r2d2_utils import load_r2d2_credentials, load_r2d2_module, unique_r2d2_id load_r2d2_module(logger, platform) load_r2d2_credentials(logger, platform) diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 855d8d204..2cc772fba 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -9,7 +9,7 @@ from swell.tasks.base.task_base import taskBase -from swell.utilities.r2d2 import load_r2d2_credentials, get_r2d2_model_name +from swell.utilities.r2d2_utils import load_r2d2_credentials, get_r2d2_model_name import isodate import os diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index 501d6c4e6..36960d780 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -11,7 +11,7 @@ import os from swell.tasks.base.task_base import taskBase -from swell.utilities.r2d2 import load_r2d2_credentials +from swell.utilities.r2d2_utils import load_r2d2_credentials from r2d2 import fetch diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index 2c5362e64..2ca504a33 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -10,7 +10,7 @@ import os from swell.tasks.base.task_base import taskBase from r2d2 import fetch -from swell.utilities.r2d2 import load_r2d2_credentials +from swell.utilities.r2d2_utils import load_r2d2_credentials # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index c864b86b4..15c1528df 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -17,10 +17,10 @@ from datetime import timedelta, datetime as dt from swell.tasks.base.task_base import taskBase -from swell.utilities.r2d2 import load_r2d2_credentials +from swell.utilities.r2d2_utils import load_r2d2_credentials from swell.utilities.datetime_util import datetime_formats from swell.utilities.observations import get_ioda_names_list, get_provider_for_observation -from swell.utilities.r2d2 import get_r2d2_model_name +from swell.utilities.r2d2_utils import get_r2d2_model_name # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/ingest_obs.py b/src/swell/tasks/ingest_obs.py index 2f5cad8f3..043129c32 100644 --- a/src/swell/tasks/ingest_obs.py +++ b/src/swell/tasks/ingest_obs.py @@ -18,7 +18,7 @@ import requests from swell.tasks.base.task_base import taskBase -from swell.utilities.r2d2 import load_r2d2_credentials +from swell.utilities.r2d2_utils import load_r2d2_credentials from swell.utilities.observations import get_ioda_names_list, get_provider_for_observation import r2d2 diff --git a/src/swell/tasks/save_forecast.py b/src/swell/tasks/save_forecast.py index 969b167f9..ce42939dd 100644 --- a/src/swell/tasks/save_forecast.py +++ b/src/swell/tasks/save_forecast.py @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats -from swell.utilities.r2d2 import load_r2d2_credentials +from swell.utilities.r2d2_utils import load_r2d2_credentials # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index de96a7273..6c8559e43 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -9,7 +9,7 @@ import r2d2 from swell.tasks.base.task_base import taskBase -from swell.utilities.r2d2 import load_r2d2_credentials +from swell.utilities.r2d2_utils import load_r2d2_credentials from swell.utilities.run_jedi_executables import check_obs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/r2d2.py b/src/swell/utilities/r2d2_utils.py similarity index 98% rename from src/swell/utilities/r2d2.py rename to src/swell/utilities/r2d2_utils.py index ae5df50a0..620b66775 100644 --- a/src/swell/utilities/r2d2.py +++ b/src/swell/utilities/r2d2_utils.py @@ -213,10 +213,10 @@ def random_hex_id(swell_id: str, length: int = 8): def experiment_exists(r2d2_id: str): - import r2d2 + import swell.utilities.r2d2_utils as r2d2_utils try: - r2d2.get(item='experiment', name=r2d2_id) + r2d2_utils.get(item='experiment', name=r2d2_id) except Exception as e: if '400 Client Error' in str(e): return False diff --git a/src/swell/utilities/scripts/delete_obs_in_range.py b/src/swell/utilities/scripts/delete_obs_in_range.py index d078a037e..195284771 100644 --- a/src/swell/utilities/scripts/delete_obs_in_range.py +++ b/src/swell/utilities/scripts/delete_obs_in_range.py @@ -1,4 +1,4 @@ -import r2d2 +import swell.utilities.r2d2_utils as r2d2_utils from datetime import datetime, timedelta @@ -34,7 +34,7 @@ def deregister_observations_in_range( print(f"[DRY RUN] Would delete: {observation_type} at {window_start}") else: try: - r2d2.delete( + r2d2_utils.delete( item='observation', observation_type=observation_type, provider=provider, diff --git a/src/swell/utilities/scripts/ingest_files.py b/src/swell/utilities/scripts/ingest_files.py index 8c70023cb..17e988a1c 100755 --- a/src/swell/utilities/scripts/ingest_files.py +++ b/src/swell/utilities/scripts/ingest_files.py @@ -15,7 +15,7 @@ RESET = "\033[0m" try: - import r2d2 + import swell.utilities.r2d2_utils as r2d2_utils except ImportError as e: raise ImportError( @@ -74,7 +74,7 @@ def ingest_observation(filename, file_path, parts, dry_run=True): return True try: - r2d2.store( + r2d2_utils.store( item='observation', provider=provider, observation_type=obs_type, @@ -131,7 +131,7 @@ def ingest_background(filename, file_path, parts, dry_run=True): return True try: - r2d2.store( + r2d2_utils.store( item='forecast', model='mom6', # model, experiment='s2s', # Use this for testing @@ -213,7 +213,7 @@ def ingest_bias_correction(filename, file_path, parts, dry_run=True): return True try: - r2d2.store( + r2d2_utils.store( item='bias_correction', source_file=file_path, model=model, diff --git a/src/swell/utilities/scripts/search_ingested.py b/src/swell/utilities/scripts/search_ingested.py index cabcd5843..88de9468d 100644 --- a/src/swell/utilities/scripts/search_ingested.py +++ b/src/swell/utilities/scripts/search_ingested.py @@ -1,8 +1,8 @@ -import r2d2 +import swell.utilities.r2d2_utils as r2d2_utils # providers = ['gdas'] # Search for all observations -results = r2d2.search( +results = r2d2_utils.search( item='observation', provider='odas', # gdas # or None to see all providers observation_type='adt_cryosat2n' # comment out to search only based on provider From 11d796b9baa428541932072cc792ead63b31b163 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 6 Apr 2026 14:30:30 -0400 Subject: [PATCH 264/299] more fixes --- .../geos_atmosphere/model/stage_cycle.py | 2 +- src/swell/suites/3dvar_marine/flow.cylc | 3 +- src/swell/tasks/base/task_base.py | 31 +++++++++---- src/swell/tasks/run_jedi_hofx_executable.py | 2 +- .../platform_tests/check_hashes_discover.py | 2 +- src/swell/test/suite_tests/suite_tests.py | 46 +++++++------------ src/swell/utilities/run_jedi_executables.py | 2 +- 7 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.py b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.py index be9377b3d..d95b3fb25 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.py +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage_cycle.py @@ -11,7 +11,7 @@ # -------------------------------------------------------------------------------------------------- -def stage_cycle(template_dict: Mapping) -> Mapping | list: +def stage_cycle(template_dict: Mapping) -> list: cycle_dir = template_dict['cycle_dir'] swell_static_files = template_dict['swell_static_files'] diff --git a/src/swell/suites/3dvar_marine/flow.cylc b/src/swell/suites/3dvar_marine/flow.cylc index 7afdb11af..9255b4d35 100644 --- a/src/swell/suites/3dvar_marine/flow.cylc +++ b/src/swell/suites/3dvar_marine/flow.cylc @@ -80,8 +80,7 @@ # Clean up large files EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} + EvaObservations-{{model_component}} => CleanCycle-{{model_component}} {% endif %} {% endfor %} diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 90ada6a4e..c61de8082 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -187,13 +187,9 @@ def is_datetime_dependent(self) -> bool: def cycle_dir(self) -> str: - # Check that model is set - self.logger.assert_abort(self.__model__ is not None, 'In get_cycle_dir but this ' + - 'should not be called if the task does not receive model.') - # Combine datetime string (directory format) with the model cycle_dir = os.path.join(self.experiment_path(), 'run', - self.__datetime__.string_directory(), self.__model__) + self.__dto__().string_directory(), self.__model_str__()) # Return return cycle_dir @@ -207,6 +203,9 @@ def forecast_dir(self, paths: str | list[str] = []) -> str | None: If paths are provided, it is combined with the forecast directory and returned ''' + if self.str_forecast_dir is None: + raise ValueError('str_forecast_dir is None') + # Make sure forecast directory exists # ----------------------------------- os.makedirs(self.str_forecast_dir, 0o755, exist_ok=True) @@ -222,15 +221,31 @@ def forecast_dir(self, paths: str | list[str] = []) -> str | None: # ---------------------------------------------------------------------------------------------- + def __dto__(self) -> Datetime: + if self.__datetime__ is None: + raise ValueError('Trying to call cycle datetime, but task was called without cyle time') + + return self.__datetime__ + + # ---------------------------------------------------------------------------------------------- + + def __model_str__(self) -> str: + if self.__model__ is None: + raise ValueError('Trying to call the model component, but task was not called with one') + + return self.__model__ + + # ---------------------------------------------------------------------------------------------- + def cycle_time_dto(self) -> dt: - return self.__datetime__.dto() + return self.__dto__().dto() # ---------------------------------------------------------------------------------------------- def cycle_time(self) -> str: - return self.__datetime__.string_iso() + return self.__dto__().string_iso() # ---------------------------------------------------------------------------------------------- @@ -272,7 +287,7 @@ def create_task( task: str, config: str, datetime: str | dt | None, - model: str, + model: str | None, ensemblePacket: str | None ) -> taskBase: diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 2a57c08c0..52b53aba2 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -24,7 +24,7 @@ class RunJediHofxExecutable(taskBase): # ---------------------------------------------------------------------------------------------- - def execute(self, ensemble_members: Optional[list] = None) -> None: + def execute(self, ensemble_members: list | None = None) -> None: # Jedi application name # --------------------- diff --git a/src/swell/test/platform_tests/check_hashes_discover.py b/src/swell/test/platform_tests/check_hashes_discover.py index 80a3db081..f48fde20c 100644 --- a/src/swell/test/platform_tests/check_hashes_discover.py +++ b/src/swell/test/platform_tests/check_hashes_discover.py @@ -1,5 +1,5 @@ from swell.utilities.logger import get_logger -import swell.utilities.pinned_versions.check_hashes import check_hashes +from swell.utilities.pinned_versions.check_hashes import check_hashes logger = get_logger("CheckHashesTest") bundle = "/discover/nobackup/projects/gmao/advda/jedi_bundles/current_pinned_jedi_bundle/source/" diff --git a/src/swell/test/suite_tests/suite_tests.py b/src/swell/test/suite_tests/suite_tests.py index 5b171bbe3..e8878ac94 100644 --- a/src/swell/test/suite_tests/suite_tests.py +++ b/src/swell/test/suite_tests/suite_tests.py @@ -1,6 +1,7 @@ import tempfile from ruamel.yaml import YAML import random +import os from pathlib import Path from datetime import datetime @@ -38,20 +39,20 @@ def build_jedi_for_tier2(test_dir: str, experiment_id_root: str, platform: str, if "override" in test_config: override = update_dict(override, test_config['override']) - experiment_dir = test_dir / experiment_id - experiment_dir.mkdir(parents=True, exist_ok=True) - override_yml = experiment_dir / "override.yaml" + experiment_dir = os.path.join(test_dir, experiment_id) + os.makedirs(experiment_dir, exist_ok=True) + override_yml = os.path.join(experiment_dir, "override.yaml") with open(override_yml, "w") as f: yaml.dump(override, f) create_experiment_directory( - "build_jedi", None, "defaults", platform, - str(override_yml), False, None + "build_jedi", "defaults", platform, + str(override_yml), False, None, False ) - suite_path = str(experiment_dir / f"{experiment_id}-suite") - log_path = str(experiment_dir / "log") + suite_path = os.path.join(experiment_dir, f"{experiment_id}-suite") + log_path = os.path.join(experiment_dir, "log") launch_experiment(suite_path, True, log_path) @@ -67,8 +68,9 @@ def run_suite(suite: str, platform: str, test_tier: TestSuite): experiment_id = f"{experiment_id_root}{suite}" # Get test directory from `~/.swell/swell-test.yaml` + testdir = Path(tempfile.TemporaryDirectory().name).expanduser() test_config = { - "test_root": Path(tempfile.TemporaryDirectory().name) + "test_root": testdir.name } yaml = YAML(typ='safe') yamlfile = Path("~/.swell/swell-test.yaml").expanduser() @@ -82,11 +84,10 @@ def run_suite(suite: str, platform: str, test_tier: TestSuite): except Exception as err: raise err - testdir = Path(test_config["test_root"]).expanduser() - testdir.mkdir(exist_ok=True, parents=True) + testdir.mkdir(exist_ok=True) print(f"Testing suite: {suite}") - print(f"Test directory: {testdir}") + print(f"Test directory: {testdir.name}") print(f"Experiment ID: {experiment_id}") override = { @@ -118,7 +119,8 @@ def run_suite(suite: str, platform: str, test_tier: TestSuite): and test_config["jedi_build_method"] == "use_existing" and 'existing_jedi_source_directory' in test_config and 'existing_jedi_build_directory' in test_config): - jedi_dir = build_jedi_for_tier2(testdir, experiment_id_root, platform, test_config) + jedi_dir = build_jedi_for_tier2(str(testdir), experiment_id_root, + platform, test_config) tier2_override = {"jedi_build_method": "use_existing", "existing_jedi_source_directory": f"{jedi_dir}/jedi_bundle/source", @@ -129,7 +131,7 @@ def run_suite(suite: str, platform: str, test_tier: TestSuite): if suite == "build_jedi": return None - override_yml = experiment_dir / "override.yaml" + override_yml = os.path.join(experiment_dir, "override.yaml") with open(override_yml, "w") as f: yaml.dump(override, f) @@ -139,7 +141,7 @@ def run_suite(suite: str, platform: str, test_tier: TestSuite): create_experiment_directory( suite_config, "defaults", platform, - str(override_yml), False, None + str(override_yml), False, None, True ) # TODO: Check some stuff about the experiment directory @@ -151,19 +153,3 @@ def run_suite(suite: str, platform: str, test_tier: TestSuite): # TODO: Check the outputs - -if __name__ == "__main__": - from argparse import ArgumentParser - - parser = ArgumentParser(description="Swell suite tests") - parser.add_argument("suites", nargs="+", - help="Suite(s) to run (or `all` to run all suites)") - - args = parser.parse_args() - if args.suites == ["all"]: - suites = ("3dvar_marine", "hofx", "3dvar_atmos") - else: - suites = args.suites - - for suite in suites: - run_suite(suite) diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index fcfe2ac16..489c58fce 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -21,7 +21,7 @@ def check_obs( path_to_observing_sys_yamls: str | None, observation: str, obs_dict: dict, - cycle_time: str | None, + cycle_time: str | datetime | None, input_and_output: bool | None = False ) -> bool: From d0cb405f1249f9d5f7604b7d65d136074cd36a55 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 8 Apr 2026 09:21:46 -0400 Subject: [PATCH 265/299] type hint list --- src/swell/tasks/base/task_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index c61de8082..0c29bb03f 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -196,7 +196,7 @@ def cycle_dir(self) -> str: # ---------------------------------------------------------------------------------------------- - def forecast_dir(self, paths: str | list[str] = []) -> str | None: + def forecast_dir(self, paths: str | list[str] = []) -> str | list: ''' Method to provide "forecast" directory to geos class From 083920d647b1190d885ca8ad24bd99e6545e0ac2 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 8 Apr 2026 11:48:13 -0400 Subject: [PATCH 266/299] Fix type hinting and redefs --- .../geos_atmosphere/suite_questions.yaml | 6 ---- src/swell/tasks/base/task_base.py | 2 +- src/swell/tasks/run_jedi_hofx_executable.py | 1 - src/swell/utilities/filehandler.py | 8 ++++- src/swell/utilities/logger.py | 2 +- src/swell/utilities/question_defaults.py | 32 +++---------------- src/swell/utilities/suite_utils.py | 7 ++-- src/swell/utilities/swell_questions.py | 12 ++++--- 8 files changed, 25 insertions(+), 45 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/suite_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/suite_questions.yaml index 3fcd0cf65..5e9daa266 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/suite_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/suite_questions.yaml @@ -2,11 +2,5 @@ cycle_times: default_value: ['T00', 'T06', 'T12', 'T18'] options: ['T00', 'T06', 'T12', 'T18'] -ensemble_hofx_strategy: - default_value: 'serial' - -ensemble_hofx_packets: - default_value: 1 - skip_ensemble_hofx: default_value: true diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 0c29bb03f..625ab8165 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -196,7 +196,7 @@ def cycle_dir(self) -> str: # ---------------------------------------------------------------------------------------------- - def forecast_dir(self, paths: str | list[str] = []) -> str | list: + def forecast_dir(self, paths: str | list[str] = []) -> list: ''' Method to provide "forecast" directory to geos class diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 52b53aba2..352e49dfc 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -204,7 +204,6 @@ def execute(self, ensemble_members: list | None = None) -> None: jedi_config_dict = \ self.jedi_rendering.render_oops_file(f'{jedi_application}{window_type}', window_type, - observations, jedi_forecast_model) # Continue with the yaml edits below some of which need to be diff --git a/src/swell/utilities/filehandler.py b/src/swell/utilities/filehandler.py index 78e9d04df..cbc62adc8 100755 --- a/src/swell/utilities/filehandler.py +++ b/src/swell/utilities/filehandler.py @@ -63,6 +63,7 @@ import copy import datetime as dt from shutil import copyfile +from abc import ABC, abstractmethod def get_file_handler(config: list, **kwargs) -> StageFileHandler | GetDataFileHandler: @@ -103,7 +104,7 @@ def get_file_handler(config: list, **kwargs) -> StageFileHandler | GetDataFileHa # ------------------------------------------------------------------------------ -class FileHandler(object): +class FileHandler(ABC): def __init__(self, config: list, **kwargs) -> None: @@ -226,6 +227,11 @@ def link(self, src: str, dst: str) -> None: # --------------------------------------------------------------------------- + @abstractmethod + def list(self, force: bool = False) -> list: + return [] + +# --------------------------------------------------------------------------- class StageFileHandler(FileHandler): diff --git a/src/swell/utilities/logger.py b/src/swell/utilities/logger.py index abda4119f..65a2fba03 100644 --- a/src/swell/utilities/logger.py +++ b/src/swell/utilities/logger.py @@ -18,7 +18,7 @@ class Logger(logging.Logger): # -------------------------------------------------------------------------------------------------- def abort(self, msg: str, - exception: Exception = Exception, *args, **kwargs) -> None: + exception: type[Exception] = Exception, *args, **kwargs) -> None: formatted_msg = ' Swell called ABORT: ' + msg diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 99322d87b..86240e4a6 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -59,31 +59,7 @@ class cycling_varbc(SuiteQuestion): widget_type: WType = WType.BOOLEAN # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_packets(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_packets" - models: list[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the number of ensemble packets." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_strategy(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_strategy" - models: list[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the ensemble hofx strategy." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - + @dataclass class experiment_id(SuiteQuestion): default_value: str = "defer_to_code" @@ -597,7 +573,7 @@ class geos_homdir(TaskQuestion): @dataclass class geos_expdir_different(TaskQuestion): - default_value: str = False + default_value: bool = False question_name: str = "geos_expdir_different" ask_question: bool = True options: list[bool] = mutable_field([ @@ -1064,7 +1040,7 @@ class mom6_iau_nhours(TaskQuestion): class ncdiag_experiments(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "ncdiag_experiments" - options: list[str] = "defer_to_model" + options: list[str] | str = "defer_to_model" models: list[str] = mutable_field([ "all_models" ]) @@ -1272,7 +1248,7 @@ class path_to_gsi_nc_diags(TaskQuestion): @dataclass class perhost(TaskQuestion): - default_value: str = None + default_value: str | None = None question_name: str = "perhost" ask_question: bool = True options: list[bool] = mutable_field([ diff --git a/src/swell/utilities/suite_utils.py b/src/swell/utilities/suite_utils.py index 4e9c8a144..2f5aceb6e 100644 --- a/src/swell/utilities/suite_utils.py +++ b/src/swell/utilities/suite_utils.py @@ -44,8 +44,11 @@ def get_suite_configs() -> list: suite_module = importlib.import_module(f'swell.suites.{suite}.suite_config') suite_configs = getattr(suite_module, 'SuiteConfig') - [suite_sub_list.append(suite_config[1:] if suite_config[0] == '_' else suite_config) - for suite_config in suite_configs.get_all()] + for suite_config in suite_configs.get_all(): + if suite_config[0] == '_': + suite_sub_list.append(suite_config[1:]) + else: + suite_sub_list.append(suite_config) suite_config_list.extend(sorted(suite_sub_list)) diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index db2745b30..88733b297 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -10,7 +10,7 @@ import os from dataclasses import dataclass, asdict, field -from typing import Literal, Self +from typing import Literal, Self, Any from enum import Enum from isodate import parse_datetime, parse_duration, ISO8601Error @@ -53,6 +53,8 @@ def base_type(self) -> type: if 'iso-' in self.value: return str + raise TypeError('Could not deduce widget type') + def validate_value(self, value) -> bool: """ Validate that the value matches the type and format of the widget type. """ base_type = self.base_type() @@ -96,13 +98,13 @@ def validate_value(self, value) -> bool: @dataclass class SwellQuestion: """Basic dataclass for defining Swell questions for suites and tasks""" - default_value: str + default_value: Any question_name: str widget_type: WidgetType prompt: str - question_type: str = None + question_type: str | None = None ask_question: bool = False - options: str | None = None + options: str | list | None = None # -------------------------------------------------------------------------------------------------- @@ -133,7 +135,7 @@ class QuestionList: # -------------------------------------------------------------------------------------------------- - def get_all_question_names(self, suite_task: Literal['suite', 'task'] | None = None) -> None: + def get_all_question_names(self, suite_task: Literal['suite', 'task'] | None = None) -> list: question_list = [] for model in [None] + os.listdir(os.path.join(get_swell_path(), 'configuration', 'jedi', 'interfaces')): From f7fd27b320ad3cf37ef802856c8f92daae191b15 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 8 Apr 2026 11:49:40 -0400 Subject: [PATCH 267/299] Remove redefined questions --- src/swell/utilities/question_defaults.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 86240e4a6..09d9e8b6e 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -177,22 +177,6 @@ class start_cycle_point(SuiteQuestion): prompt: str = "What is the time of the first cycle (middle of the window)?" widget_type: WType = WType.ISO_DATETIME - # -------------------------------------------------------------------------------------------------- - - @dataclass - class window_type(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "window_type" - options: list[str] = mutable_field([ - "3D", - "4D" - ]) - models: list[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the window type for this model." - widget_type: WType = WType.STRING_DROP_LIST - # -------------------------------------------------------------------------------------------------- # Task question defaults go here # -------------------------------------------------------------------------------------------------- From e0b298149ca0c12e38e2689d715fee2c2a3de721 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 8 Apr 2026 15:04:33 -0400 Subject: [PATCH 268/299] Final touches --- src/swell/deployment/create_experiment.py | 5 +++-- src/swell/swell.py | 2 +- src/swell/tasks/base/task_base.py | 2 +- src/swell/tasks/eva_comparison_increment.py | 4 ++-- src/swell/tasks/eva_comparison_jedi_log.py | 4 ++-- src/swell/tasks/eva_increment.py | 4 ++-- src/swell/tasks/link_coupled_geos_output.py | 5 ++--- src/swell/tasks/link_geos_output.py | 5 ++--- src/swell/test/suite_tests/suite_tests.py | 1 - .../data_assimilation_window_params.py | 21 ++++++++++++------- src/swell/utilities/filehandler.py | 7 ++++--- src/swell/utilities/geos.py | 18 +++++++++++----- src/swell/utilities/gsi_record_parser.py | 8 +++---- src/swell/utilities/question_defaults.py | 2 +- src/swell/utilities/shell_commands.py | 2 +- 15 files changed, 52 insertions(+), 38 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 7454797ca..43a04c074 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -188,7 +188,8 @@ def prepare_config( if 'r2d2_experiment_id' in experiment_dict and 'skip_r2d2' in experiment_dict \ and not experiment_dict['skip_r2d2']: - from swell.utilities.r2d2_utils import load_r2d2_credentials, load_r2d2_module, unique_r2d2_id + from swell.utilities.r2d2_utils import load_r2d2_credentials, load_r2d2_module, \ + unique_r2d2_id load_r2d2_module(logger, platform) load_r2d2_credentials(logger, platform) @@ -527,7 +528,7 @@ def prepare_cylc_suite_jinja2( # Since cycle times are used, the render_dictionary will need to include cycle_times # If there are different model components then process each to gather cycle times if len(model_components) > 0: - cycle_times : list = [] + cycle_times: list = [] for model_component in model_components: cycle_times_mc = experiment_dict['models'][model_component]['cycle_times'] cycle_times = list(set(cycle_times + cycle_times_mc)) diff --git a/src/swell/swell.py b/src/swell/swell.py index 61a38b1a9..8370a817a 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -156,7 +156,7 @@ def clone( # Create experiment configuration by cloning from existing experiment experiment_dict_str = clone_config(configuration, experiment_id, input_method, platform, advanced) - + yaml = YAML(typ='safe') experiment_override = yaml.load(experiment_dict_str) suite = experiment_override['suite_to_run'] diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 625ab8165..46710f9e5 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -196,7 +196,7 @@ def cycle_dir(self) -> str: # ---------------------------------------------------------------------------------------------- - def forecast_dir(self, paths: str | list[str] = []) -> list: + def forecast_dir(self, paths: str | list[str] = []) -> str: ''' Method to provide "forecast" directory to geos class diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index 8cfc16918..c0b4bbfb1 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -110,9 +110,9 @@ def execute(self) -> None: incr_file_2 = f'ocn.*.incr.{ocn_cycle_time}.nc' cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', - self.__datetime__.string_directory(), self.get_model()) + self.__dto__().string_directory(), self.get_model()) cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', - self.__datetime__.string_directory(), self.get_model()) + self.__dto__().string_directory(), self.get_model()) # Files to fill the template in the config file increment_file_path_1 = glob.glob(os.path.join(cycle_dir_1, incr_file_1))[0] diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 7c96a4155..2553564cb 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -63,9 +63,9 @@ def execute(self) -> None: experiment_id_2 = experiment_dict_2['experiment_id'] cycle_dir_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'run', - self.__datetime__.string_directory(), self.get_model()) + self.__dto__().string_directory(), self.get_model()) cycle_dir_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'run', - self.__datetime__.string_directory(), self.get_model()) + self.__dto__().string_directory(), self.get_model()) # Info to task log info_string = 'Running Eva to plot from the jedi_log file' diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index ae798fd3d..f2ac3acf9 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -50,8 +50,8 @@ def execute(self) -> None: dto=True) window_begin = window_begin_dto.strftime('%Y%m%d_%H%M%Sz') - local_bkg_dir, local_bkg_dto = self.da_window_params.local_background_time( - self.config.window_length(), self.config.window_type(), dto=True) + local_bkg_dir, local_bkg_dto = self.da_window_params.local_background_time_dto( + self.config.window_length(), self.config.window_type()) local_bkg_time = local_bkg_dto.strftime('%Y%m%d_%H%M%Sz') # Define the increment filename and path diff --git a/src/swell/tasks/link_coupled_geos_output.py b/src/swell/tasks/link_coupled_geos_output.py index dcc1c13c1..00f89be75 100644 --- a/src/swell/tasks/link_coupled_geos_output.py +++ b/src/swell/tasks/link_coupled_geos_output.py @@ -43,10 +43,9 @@ def execute(self) -> None: if self.window_type == '4D' or 'fgat' in self.suite_name(): self.background_frequency = self.config.background_frequency() - self.bkgr_time_iso, self.bkgr_time_dto = self.da_window_params.local_background_time( + self.bkgr_time_iso, self.bkgr_time_dto = self.da_window_params.local_background_time_dto( self.window_length, - self.window_type, - dto=True) + self.window_type) # Create source and destination files for linking model output to cycle directories # ----------------------------------------------------------------------------------- diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index 7cd55058d..98bffed14 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -42,10 +42,9 @@ def execute(self) -> None: if self.window_type == '4D' or 'fgat' in self.suite_name(): self.background_frequency = self.config.background_frequency() - self.bkgr_time_iso, self.bkgr_time_dto = self.da_window_params.local_background_time( + self.bkgr_time_iso, self.bkgr_time_dto = self.da_window_params.local_background_time_dto( self.window_length, - self.window_type, - dto=True) + self.window_type) # Create source and destination files for linking model output to cycle directories # ----------------------------------------------------------------------------------- diff --git a/src/swell/test/suite_tests/suite_tests.py b/src/swell/test/suite_tests/suite_tests.py index e8878ac94..64c44d41b 100644 --- a/src/swell/test/suite_tests/suite_tests.py +++ b/src/swell/test/suite_tests/suite_tests.py @@ -152,4 +152,3 @@ def run_suite(suite: str, platform: str, test_tier: TestSuite): launch_experiment(suite_path, True, log_path) # TODO: Check the outputs - diff --git a/src/swell/utilities/data_assimilation_window_params.py b/src/swell/utilities/data_assimilation_window_params.py index 0b1f12579..f765c186e 100644 --- a/src/swell/utilities/data_assimilation_window_params.py +++ b/src/swell/utilities/data_assimilation_window_params.py @@ -42,7 +42,7 @@ def __get_window_offset_dur__(self, window_length: str) -> datetime.datetime: # ---------------------------------------------------------------------------------------------- - def window_offset(self, window_length: str, dto: bool = False) -> str: + def window_offset(self, window_length: str, dto: bool = False) -> str | datetime.datetime: window_offset_dur = self.__get_window_offset_dur__(window_length) if dto: @@ -141,19 +141,26 @@ def local_background_time(self, window_length, window_type, dto=False - ) -> str | tuple[str, datetime.datetime]: + ) -> str: local_background_time = self.__get_local_background_time__(window_type, window_length) - # Return datetime object if asked - if dto: - return local_background_time.strftime(datetime_formats['directory_format']), \ - local_background_time - return local_background_time.strftime(datetime_formats['directory_format']) # ---------------------------------------------------------------------------------------------- + def local_background_time_dto(self, + window_length: str, + window_type: str) -> tuple[str, datetime.datetime]: + + local_background_time = self.__get_local_background_time__(window_type, window_length) + + # Return datetime object if asked + return local_background_time.strftime(datetime_formats['directory_format']), \ + local_background_time + + # ---------------------------------------------------------------------------------------------- + def window_begin(self, window_length: str, dto: bool = False) -> str | datetime.datetime: window_begin_dto = self.__get_window_begin_dto__(window_length) diff --git a/src/swell/utilities/filehandler.py b/src/swell/utilities/filehandler.py index cbc62adc8..1359e972f 100755 --- a/src/swell/utilities/filehandler.py +++ b/src/swell/utilities/filehandler.py @@ -108,7 +108,7 @@ class FileHandler(ABC): def __init__(self, config: list, **kwargs) -> None: - self.listing = [] + self.listing: list = [] self.config = copy.deepcopy(config) self.strict = kwargs.get('strict', True) @@ -233,6 +233,7 @@ def list(self, force: bool = False) -> list: # --------------------------------------------------------------------------- + class StageFileHandler(FileHandler): def list(self, force: bool = False) -> list: @@ -346,7 +347,7 @@ def list(self, force: bool = False) -> list: srcfile = args[0] filelist = glob.glob(srcfile) - found = found or filelist + found = found or len(filelist) > 0 for srcfile in filelist: @@ -383,7 +384,7 @@ def __init__(self, config: dict) -> None: self.config = copy.deepcopy(config) - self.listing = [] + self.listing: list = [] self.link = config.get('link', False) self.min_count = config.get('min_count', 1) self.min_age = config.get('min_age', 0) diff --git a/src/swell/utilities/geos.py b/src/swell/utilities/geos.py index 7adfe18bd..b20c1a9a2 100644 --- a/src/swell/utilities/geos.py +++ b/src/swell/utilities/geos.py @@ -47,6 +47,14 @@ def __init__( # ---------------------------------------------------------------------------------------------- + def get_forecast_dir(self) -> str: + if self.forecast_dir is None: + raise ValueError('Trying to call forecast dir but it has not been set') + + return self.forecast_dir + + # ---------------------------------------------------------------------------------------------- + def iso_to_time_str( self, iso_duration: str, @@ -108,7 +116,7 @@ def linker( self, src: str, dst: str, - dst_dir: str = None + dst_dir: str | None = None ) -> None: """ Creates a symbolic link from a source file to a destination. @@ -123,7 +131,7 @@ def linker( # Link files from BC directories # ------------------------------ if dst_dir is None: - dst_dir = self.forecast_dir + dst_dir = self.get_forecast_dir() dst = os.path.basename(dst) @@ -324,7 +332,7 @@ def process_nml( """ # Make sure input.nml is set up properly for hot/cold restart - nml_comb = f90nml.read(os.path.join(self.forecast_dir, 'input.nml')) + nml_comb = f90nml.read(os.path.join(self.get_forecast_dir(), 'input.nml')) if not cold_restart: self.logger.info('Hot start, Swell will expect rst/checkpoint files') @@ -335,10 +343,10 @@ def process_nml( if combine_fvcore: self.logger.info('Combining fvcore with input.nml') - nml2 = f90nml.read(os.path.join(self.forecast_dir, 'fvcore_layout.rc')) + nml2 = f90nml.read(os.path.join(self.get_forecast_dir(), 'fvcore_layout.rc')) nml_comb.update(nml2) - with open(os.path.join(self.forecast_dir, 'input.nml'), 'w') as f: + with open(os.path.join(self.get_forecast_dir(), 'input.nml'), 'w') as f: f90nml.write(nml_comb, f, sort=False) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/gsi_record_parser.py b/src/swell/utilities/gsi_record_parser.py index d11dabbc9..6b9fe1eee 100644 --- a/src/swell/utilities/gsi_record_parser.py +++ b/src/swell/utilities/gsi_record_parser.py @@ -18,10 +18,10 @@ def check_end_time(end_time: str) -> str: class GSIRecordParser: def __init__(self) -> None: - self.instr_df = None - self.return_df = None - self.sat = None - self.instr = None + self.instr_df = pd.DataFrame() + self.return_df = pd.DataFrame() + self.sat = '' + self.instr = '' def get_channel_list(self, start: int) -> list: channel_list = [] diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 09d9e8b6e..c04a43c86 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -59,7 +59,7 @@ class cycling_varbc(SuiteQuestion): widget_type: WType = WType.BOOLEAN # -------------------------------------------------------------------------------------------------- - + @dataclass class experiment_id(SuiteQuestion): default_value: str = "defer_to_code" diff --git a/src/swell/utilities/shell_commands.py b/src/swell/utilities/shell_commands.py index cd7d2c5a0..bd23a4392 100644 --- a/src/swell/utilities/shell_commands.py +++ b/src/swell/utilities/shell_commands.py @@ -20,7 +20,7 @@ def run_track_log_subprocess( logger: Logger, - command: list[str] | str | None, + command: list[str] | str, output_log: str | None = None, **kwargs ) -> None: From 3a5e3adbe1bb8ae4343efb00f179e34168c50361 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 8 Apr 2026 15:35:25 -0400 Subject: [PATCH 269/299] Code test fixes --- src/swell/utilities/r2d2_utils.py | 4 ++-- src/swell/utilities/run_jedi_executables.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/swell/utilities/r2d2_utils.py b/src/swell/utilities/r2d2_utils.py index 620b66775..ae5df50a0 100644 --- a/src/swell/utilities/r2d2_utils.py +++ b/src/swell/utilities/r2d2_utils.py @@ -213,10 +213,10 @@ def random_hex_id(swell_id: str, length: int = 8): def experiment_exists(r2d2_id: str): - import swell.utilities.r2d2_utils as r2d2_utils + import r2d2 try: - r2d2_utils.get(item='experiment', name=r2d2_id) + r2d2.get(item='experiment', name=r2d2_id) except Exception as e: if '400 Client Error' in str(e): return False diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index 489c58fce..aaec5e995 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -10,6 +10,7 @@ import os import netCDF4 as nc +import datetime from swell.utilities.shell_commands import run_track_log_subprocess from swell.utilities.logger import Logger @@ -21,7 +22,7 @@ def check_obs( path_to_observing_sys_yamls: str | None, observation: str, obs_dict: dict, - cycle_time: str | datetime | None, + cycle_time: str | datetime.datetime | None, input_and_output: bool | None = False ) -> bool: From 93390acc1b4975f7807443f817b0f9b495deffc9 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 9 Apr 2026 15:33:28 -0400 Subject: [PATCH 270/299] Update test cache location --- src/swell/test/code_tests/suite_creation_test.py | 9 +++++---- src/swell/utilities/mock_jedi_config.py | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/swell/test/code_tests/suite_creation_test.py b/src/swell/test/code_tests/suite_creation_test.py index 846e4b80c..f2e335b65 100644 --- a/src/swell/test/code_tests/suite_creation_test.py +++ b/src/swell/test/code_tests/suite_creation_test.py @@ -7,13 +7,13 @@ # -------------------------------------------------------------------------------------------------- -import tempfile import os import unittest from swell.suites.all_suites import get_suites from swell.deployment.create_experiment import create_experiment_directory from swell.utilities.logger import get_logger +from swell.utilities.test_cache import get_test_cache # -------------------------------------------------------------------------------------------------- @@ -40,17 +40,18 @@ def runTest(self) -> None: def suite_creation_test(self, suite: str) -> None: - tempdir = tempfile.mkdtemp() + cache_location = get_test_cache() override_dict = {} - override_dict['experiment_root'] = tempdir + override_dict['experiment_id'] = experiment_id = f'{suite}-creation' + override_dict['experiment_root'] = cache_location override_dict['skip_r2d2'] = True create_experiment_directory(suite, 'defaults', 'nccs_discover_sles15', override_dict, advanced=False, slurm=None, skip_r2d2=True) - experiment_yaml = os.path.join(tempdir, f'swell-{suite}', f'swell-{suite}-suite', + experiment_yaml = os.path.join(cache_location, experiment_id, f'{experiment_id}-suite', 'experiment.yaml') with open(experiment_yaml, 'r') as f: diff --git a/src/swell/utilities/mock_jedi_config.py b/src/swell/utilities/mock_jedi_config.py index cf849a485..c851c4513 100644 --- a/src/swell/utilities/mock_jedi_config.py +++ b/src/swell/utilities/mock_jedi_config.py @@ -44,6 +44,7 @@ def mock_jedi_config(suite: str, tempdir = work_dir override_dict = {'models': {}} + override_dict['experiment_id'] = experiment_id = f'{suite}-config' override_dict['experiment_root'] = tempdir override_dict['generate_yaml_and_exit'] = True override_dict['mock_experiment'] = True @@ -52,8 +53,8 @@ def mock_jedi_config(suite: str, create_experiment_directory(suite, method='defaults', platform='nccs_discover_sles15', override=override_dict, advanced=False, slurm=None, skip_r2d2=True) - experiment_yaml = os.path.join(tempdir, f'swell-{suite}', - f'swell-{suite}-suite', 'experiment.yaml') + experiment_yaml = os.path.join(tempdir, experiment_id, + f'{experiment_id}-suite', 'experiment.yaml') task_wrapper('RenderJediObservations', experiment_yaml, datetime, model, ensemblePacket=None) @@ -66,7 +67,7 @@ def mock_jedi_config(suite: str, task_wrapper(f'RunJedi{executable_name}Executable', experiment_yaml, datetime, model, ensemblePacket=None) - cycle_dir = os.path.join(tempdir, f'swell-{suite}', 'run', datetime, model) + cycle_dir = os.path.join(tempdir, experiment_id, 'run', datetime, model) filename = f'jedi_{executable_type}_config.yaml' config_file = os.path.join(cycle_dir, filename) From b9e77f822c113070f949881a28f58c7edaeea907 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 9 Apr 2026 15:43:24 -0400 Subject: [PATCH 271/299] code test fix --- src/swell/test/code_tests/suite_creation_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/swell/test/code_tests/suite_creation_test.py b/src/swell/test/code_tests/suite_creation_test.py index f2e335b65..be11389a2 100644 --- a/src/swell/test/code_tests/suite_creation_test.py +++ b/src/swell/test/code_tests/suite_creation_test.py @@ -9,6 +9,7 @@ import os import unittest +import tempfile from swell.suites.all_suites import get_suites from swell.deployment.create_experiment import create_experiment_directory @@ -41,6 +42,8 @@ def runTest(self) -> None: def suite_creation_test(self, suite: str) -> None: cache_location = get_test_cache() + if cache_location is None: + cache_location = tempfile.mkdtemp() override_dict = {} From 2f098a0d9388764e9aa44bdfb647d65745fff908 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 13 Apr 2026 13:57:20 -0400 Subject: [PATCH 272/299] Update 3dvar_cf config --- .../jedi_configs/jedi_3dvar_cf_config.yaml | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/swell/test/jedi_configs/jedi_3dvar_cf_config.yaml b/src/swell/test/jedi_configs/jedi_3dvar_cf_config.yaml index a319e2227..03887ec4e 100644 --- a/src/swell/test/jedi_configs/jedi_3dvar_cf_config.yaml +++ b/src/swell/test/jedi_configs/jedi_3dvar_cf_config.yaml @@ -45,7 +45,29 @@ cost function: background error: covariance model: SABER saber central block: - saber block name: ID + saber block name: BUMP_NICAS + read: + io: + data directory: cycle_dir/ + files prefix: geos_cf + alias: + - in code: volume_mixing_ratio_of_no2 + in file: fixedlevel_H_500_300_V_00_03 + drivers: + multivariate strategy: univariate + read local nicas: true + saber outer blocks: + - saber block name: StdDev + stddev scale factor: '0.25' + read: + model file: + datetime: '2023-08-05T18:00:00Z' + set datetime on read: true + filetype: cube sphere history + max allowable geometry difference: 0.1 + datapath: cycle_dir + filename: bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4 + field io names: *id001 observations: get values: observers: From ff23a32c2c3591c3b94c4f2b568f7de5f6ca9ade Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 13 Apr 2026 14:22:36 -0400 Subject: [PATCH 273/299] Add saber to other models --- .../jedi/interfaces/geos_atmosphere/task_questions.yaml | 6 ++++++ .../jedi/interfaces/geos_marine/task_questions.yaml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index 636e49e1c..efbf44f47 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -330,6 +330,12 @@ produce_geovals: - true - false +saber_central_block: + default_value: None + +saber_outer_block: + default_value: None + skip_ensemble_hofx: default_value: true options: diff --git a/src/swell/configuration/jedi/interfaces/geos_marine/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_marine/task_questions.yaml index 6f87658a1..69b858563 100644 --- a/src/swell/configuration/jedi/interfaces/geos_marine/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_marine/task_questions.yaml @@ -118,6 +118,12 @@ observations: path_to_ensemble: default_value: test +saber_central_block: + default_value: None + +saber_outer_block: + default_value: None + total_processors: default_value: 24 From 8f0369969de003e0f0b1b2394e8ad183416d6f1b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 15 Apr 2026 10:32:42 -0400 Subject: [PATCH 274/299] pycodestyle fix --- src/swell/tasks/get_observations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 2eb19fd66..42c7024df 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -81,6 +81,7 @@ def run_r2d2_fetch(r2d2_dict: dict) -> None: # -------------------------------------------------------------------------------------------------- + class GetObservations(taskBase): def execute(self) -> None: From 83c9b4ca24c5a009d96758a6cc484f61aefb0617 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 15 Apr 2026 12:01:02 -0400 Subject: [PATCH 275/299] Code test fixes --- src/swell/tasks/ingest_obs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/ingest_obs.py b/src/swell/tasks/ingest_obs.py index 07930c39b..2d9fa1abc 100644 --- a/src/swell/tasks/ingest_obs.py +++ b/src/swell/tasks/ingest_obs.py @@ -23,7 +23,6 @@ import swell.configuration.question_defaults as qd from swell.utilities.r2d2 import create_r2d2_config from swell.utilities.observations import get_ioda_names_list, get_provider_for_observation -import r2d2 # -------------------------------------------------------------------------------------------------- @@ -175,6 +174,9 @@ def process_obs_config( window_length: str, dry_run: bool, ) -> tuple[list[str], list[tuple[str, str]]]: + + import r2d2 + """Process a single observation configuration file.""" ingested = [] failed = [] From 156fb28933bc1415ca422c78aa08cb178bc302c3 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 16 Apr 2026 09:56:42 -0400 Subject: [PATCH 276/299] Fix suite configs --- .../3dfgat_marine_cycle/suite_config.py | 277 +++++++++--------- src/swell/suites/3dvar_cf/suite_config.py | 91 +++--- src/swell/suites/3dvar_marine/suite_config.py | 50 +--- src/swell/tasks/get_geovals.py | 1 - src/swell/tasks/get_ncdiags.py | 1 - src/swell/tasks/save_forecast.py | 1 - src/swell/tasks/save_restart.py | 2 - 7 files changed, 184 insertions(+), 239 deletions(-) diff --git a/src/swell/suites/3dfgat_marine_cycle/suite_config.py b/src/swell/suites/3dfgat_marine_cycle/suite_config.py index da954da5c..d75d87138 100644 --- a/src/swell/suites/3dfgat_marine_cycle/suite_config.py +++ b/src/swell/suites/3dfgat_marine_cycle/suite_config.py @@ -8,154 +8,153 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.utilities.swell_questions import QuestionList +from swell.configuration import question_defaults as qd +from swell.suites.base.suite_questions import marine +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): +suite_name = '3dfgat_marine_cycle' - # -------------------------------------------------------------------------------------------------- +_3dfgat_marine_cycle = QuestionList( + questions=[ + marine, + qd.start_cycle_point("2021-07-02T06:00:00Z"), + qd.final_cycle_point("2021-07-02T12:00:00Z"), + qd.runahead_limit("P2"), + qd.jedi_build_method("use_existing"), + qd.geos_build_method("use_existing"), + qd.model_components(['geos_marine']), + qd.comparison_log_type('fgat'), + ], + geos_marine=[ + qd.cycle_times([ + "T00", + "T06", + "T12", + "T18" + ]), + qd.analysis_variables([ + "sea_water_salinity", + "sea_water_potential_temperature", + "sea_surface_height_above_geoid", + "sea_water_cell_thickness", + "sea_ice_area_fraction", + "sea_ice_thickness", + "sea_ice_snow_thickness" + ]), + qd.window_length("PT6H"), + qd.window_type("4D"), + qd.horizontal_resolution("72x36"), + qd.vertical_resolution("50"), + qd.total_processors(6), + qd.observations([ + "adt_cryosat2n", + "adt_jason3", + "adt_saral", + "adt_sentinel3a", + "adt_sentinel3b", + "insitu_profile_argo", + "icec_amsr2_north", + "icec_amsr2_south", + "icec_nsidc_nh", + "icec_nsidc_sh", + "sst_ostia", + "sss_smos", + "sss_smapv5", + "sst_abi_g16_l3c", + "sst_gmi_l3u", + "sst_viirs_n20_l3u", + "temp_profile_xbt" + ]), + qd.number_of_iterations([10]), + qd.mom6_iau(True), + qd.background_time_offset("PT9H"), + qd.clean_patterns([ + "*.nc4", + "*.txt", + "*.rc", + "*.bin" + ]), + ] +) - _3dfgat_marine_cycle = QuestionList( - list_name="3dfgat_marine_cycle", - questions=[ - sq.marine, - qd.start_cycle_point("2021-07-02T06:00:00Z"), - qd.final_cycle_point("2021-07-02T12:00:00Z"), - qd.runahead_limit("P2"), - qd.jedi_build_method("use_existing"), - qd.geos_build_method("use_existing"), - qd.model_components(['geos_marine']), - qd.comparison_log_type('fgat'), - ], - geos_marine=[ - qd.cycle_times([ - "T00", - "T06", - "T12", - "T18" - ]), - qd.analysis_variables([ - "sea_water_salinity", - "sea_water_potential_temperature", - "sea_surface_height_above_geoid", - "sea_water_cell_thickness", - "sea_ice_area_fraction", - "sea_ice_thickness", - "sea_ice_snow_thickness" - ]), - qd.window_length("PT6H"), - qd.window_type("4D"), - qd.horizontal_resolution("72x36"), - qd.vertical_resolution("50"), - qd.total_processors(6), - qd.observations([ - "adt_cryosat2n", - "adt_jason3", - "adt_saral", - "adt_sentinel3a", - "adt_sentinel3b", - "insitu_profile_argo", - "icec_amsr2_north", - "icec_amsr2_south", - "icec_nsidc_nh", - "icec_nsidc_sh", - "sst_ostia", - "sss_smos", - "sss_smapv5", - "sst_abi_g16_l3c", - "sst_gmi_l3u", - "sst_viirs_n20_l3u", - "temp_profile_xbt" - ]), - qd.number_of_iterations([10]), - qd.mom6_iau(True), - qd.background_time_offset("PT9H"), - qd.clean_patterns([ - "*.nc4", - "*.txt", - "*.rc", - "*.bin" - ]), - ] - ) +suite_configs.register(suite_name, '3dfgat_marine_cycle', _3dfgat_marine_cycle) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- - _3dfgat_marine_cycle_tier1 = QuestionList( - list_name="3dfgat_marine_cycle_tier1", - questions=[ - _3dfgat_marine_cycle - ] - ) +_3dfgat_marine_cycle_tier1 = QuestionList( + questions=[ + _3dfgat_marine_cycle + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, '3dfgat_marine_cycle_tier1', _3dfgat_marine_cycle_tier1) - _3dfgat_marine_cycle_tier2 = QuestionList( - list_name="3dfgat_marine_cycle_tier2", +# -------------------------------------------------------------------------------------------------- - questions=[ - _3dfgat_marine_cycle, - qd.start_cycle_point("2023-07-02T12:00:00Z"), - qd.final_cycle_point("2023-07-03T12:00:00Z"), - qd.forecast_duration("P2D"), - qd.geos_homdir("/discover/nobackup/projects/gmao/advda/SwellStaticFiles/geos/homdirs/" - "dataatm_025") - ], - geos_marine=[ - qd.cycle_times([ - "T12", - ]), - qd.analysis_variables([ - "sea_water_salinity", - "sea_water_potential_temperature", - "sea_surface_height_above_geoid", - "sea_water_cell_thickness", - "sea_ice_area_fraction", - "sea_ice_thickness", - "sea_ice_snow_thickness" - ]), - qd.window_length("P1D"), - qd.mom6_iau_nhours("PT18H"), - qd.horizontal_resolution("1440x1080"), - qd.vertical_resolution("75"), - qd.total_processors(720), - qd.observations([ - "adt_cryosat2n", - "adt_jason3", - "adt_jason3n", - "adt_saral", - "adt_sentinel3a", - "adt_sentinel3b", - "adt_sentinel6a", - "adt_swot_nadir", - "insitu_profile_argo", - "insitu_profile_ctd", - "insitu_profile_pirata", - "insitu_profile_rama", - "insitu_profile_tao", - "icec_amsr2_north", - "icec_amsr2_south", - "icec_nsidc_nh", - "icec_nsidc_sh", - "sst_ostia", - "sss_smos", - "sss_smapv5", - "sst_abi_g16_l3c", - "sst_avhrrf_mb_l3u", - "sst_avhrrf_mc_l3u", - "sst_viirs_n20_l3u", - "sst_viirs_npp_l3u", - "temp_profile_xbt" - ]), - qd.number_of_iterations([50]), - qd.background_time_offset("P1DT12H"), - ] - ) +_3dfgat_marine_cycle_tier2 = QuestionList( + questions=[ + _3dfgat_marine_cycle, + qd.start_cycle_point("2023-07-02T12:00:00Z"), + qd.final_cycle_point("2023-07-03T12:00:00Z"), + qd.forecast_duration("P2D"), + qd.geos_homdir("/discover/nobackup/projects/gmao/advda/SwellStaticFiles/geos/homdirs/" + "dataatm_025") + ], + geos_marine=[ + qd.cycle_times([ + "T12", + ]), + qd.analysis_variables([ + "sea_water_salinity", + "sea_water_potential_temperature", + "sea_surface_height_above_geoid", + "sea_water_cell_thickness", + "sea_ice_area_fraction", + "sea_ice_thickness", + "sea_ice_snow_thickness" + ]), + qd.window_length("P1D"), + qd.mom6_iau_nhours("PT18H"), + qd.horizontal_resolution("1440x1080"), + qd.vertical_resolution("75"), + qd.total_processors(720), + qd.observations([ + "adt_cryosat2n", + "adt_jason3", + "adt_jason3n", + "adt_saral", + "adt_sentinel3a", + "adt_sentinel3b", + "adt_sentinel6a", + "adt_swot_nadir", + "insitu_profile_argo", + "insitu_profile_ctd", + "insitu_profile_pirata", + "insitu_profile_rama", + "insitu_profile_tao", + "icec_amsr2_north", + "icec_amsr2_south", + "icec_nsidc_nh", + "icec_nsidc_sh", + "sst_ostia", + "sss_smos", + "sss_smapv5", + "sst_abi_g16_l3c", + "sst_avhrrf_mb_l3u", + "sst_avhrrf_mc_l3u", + "sst_viirs_n20_l3u", + "sst_viirs_npp_l3u", + "temp_profile_xbt" + ]), + qd.number_of_iterations([50]), + qd.background_time_offset("P1DT12H"), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, '3dfgat_marine_cycle_tier2', _3dfgat_marine_cycle_tier2) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_cf/suite_config.py b/src/swell/suites/3dvar_cf/suite_config.py index 607c3cfa7..663c3d2de 100644 --- a/src/swell/suites/3dvar_cf/suite_config.py +++ b/src/swell/suites/3dvar_cf/suite_config.py @@ -8,58 +8,57 @@ # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd -from swell.suites.suite_questions import SuiteQuestions as sq - -from enum import Enum +from swell.utilities.swell_questions import QuestionList +from swell.configuration import question_defaults as qd +from swell.suites.base.suite_questions import common +from swell.suites.base.suite_attributes import suite_configs # -------------------------------------------------------------------------------------------------- -class SuiteConfig(QuestionContainer, Enum): +suite_name = '3dvar_cf' + +_3dvar_cf_tier1 = QuestionList( + questions=[ + common, + qd.swell_static_files("/discover/nobackup/projects/gmao/geos_cf_dev/SwellStaticFiles"), + qd.start_cycle_point("2023-08-05T18:00:00Z"), + qd.final_cycle_point("2023-08-05T18:00:00Z"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_cf']), + qd.check_for_obs(False) + ], + geos_cf=[ + qd.window_length("PT6H"), + qd.window_type("3D"), + qd.horizontal_resolution("c90"), + qd.npx(91), + qd.npy(91), + qd.npx_proc(2), + qd.npy_proc(2), + qd.vertical_resolution(72), + qd.saber_central_block('bump_nicas'), + qd.saber_outer_block('stddev_bkg_scaled'), + qd.analysis_variables(["volume_mixing_ratio_of_no2"]), + qd.background_experiment("swell_test"), + qd.observations([ + "tempo_no2_tropo", + "tropomi_s5p_no2_tropo", + ]), + qd.clean_patterns(['*.nc4', '*.txt', 'logfile.*.out']), + ] +) - # -------------------------------------------------------------------------------------------------- +suite_configs.register(suite_name, '3dvar_cf_tier1', _3dvar_cf_tier1) - _3dvar_cf_tier1 = QuestionList( - list_name="3dvar_cf", - questions=[ - sq.common, - qd.swell_static_files("/discover/nobackup/projects/gmao/geos_cf_dev/SwellStaticFiles"), - qd.start_cycle_point("2023-08-05T18:00:00Z"), - qd.final_cycle_point("2023-08-05T18:00:00Z"), - qd.jedi_build_method("use_existing"), - qd.model_components(['geos_cf']), - qd.check_for_obs(False) - ], - geos_cf=[ - qd.window_length("PT6H"), - qd.window_type("3D"), - qd.horizontal_resolution("c90"), - qd.npx(91), - qd.npy(91), - qd.npx_proc(2), - qd.npy_proc(2), - qd.vertical_resolution(72), - qd.saber_central_block('bump_nicas'), - qd.saber_outer_block('stddev_bkg_scaled'), - qd.analysis_variables(["volume_mixing_ratio_of_no2"]), - qd.background_experiment("swell_test"), - qd.observations([ - "tempo_no2_tropo", - "tropomi_s5p_no2_tropo", - ]), - qd.clean_patterns(['*.nc4', '*.txt', 'logfile.*.out']), - ] - ) +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +_3dvar_cf = QuestionList( + questions=[ + _3dvar_cf_tier1 + ] +) - _3dvar_cf = QuestionList( - list_name="3dvar_cf", - questions=[ - _3dvar_cf_tier1 - ] - ) +suite_configs.register(suite_name, '3dvar_cf', _3dvar_cf) - # -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_marine/suite_config.py b/src/swell/suites/3dvar_marine/suite_config.py index f3988571e..79caa9c67 100644 --- a/src/swell/suites/3dvar_marine/suite_config.py +++ b/src/swell/suites/3dvar_marine/suite_config.py @@ -15,7 +15,7 @@ # -------------------------------------------------------------------------------------------------- -suite_name = '3dvar' +suite_name = '3dvar_marine' _3dvar_tier1 = QuestionList( questions=[ @@ -55,63 +55,15 @@ ] ) -<<<<<<< HEAD:src/swell/suites/3dvar/suite_config.py suite_configs.register(suite_name, '3dvar_tier1', _3dvar_tier1) -======= - _3dvar_marine = QuestionList( - list_name="3dvar_marine", - questions=[ - sq.marine, - qd.start_cycle_point("2021-07-01T12:00:00Z"), - qd.final_cycle_point("2021-07-01T12:00:00Z"), - qd.jedi_build_method("use_existing"), - qd.model_components(['geos_marine']), - ], - geos_marine=[ - qd.cycle_times(['T12']), - qd.marine_models(['mom6']), - qd.window_length("P1D"), - qd.horizontal_resolution("72x36"), - qd.vertical_resolution("50"), - qd.total_processors(6), - qd.obs_experiment("s2s_v1"), - qd.observations([ - "adt_cryosat2n", - "adt_jason3", - "adt_saral", - "adt_sentinel3a", - "adt_sentinel3b", - "insitu_profile_argo", - "sst_ostia", - "sss_smos", - "sss_smapv5", - "sst_abi_g16_l3c", - "sst_gmi_l3u", - "sst_viirs_n20_l3u", - "temp_profile_xbt" - ]), - qd.background_time_offset("PT18H"), - qd.clean_patterns(['*.nc4', '*.txt']), - ] - ) ->>>>>>> develop:src/swell/suites/3dvar_marine/suite_config.py # -------------------------------------------------------------------------------------------------- -<<<<<<< HEAD:src/swell/suites/3dvar/suite_config.py _3dvar = QuestionList( questions=[ _3dvar_tier1 ] ) -======= - _3dvar_marine_tier1 = QuestionList( - list_name="3dvar_marine_tier1", - questions=[ - _3dvar_marine - ] - ) ->>>>>>> develop:src/swell/suites/3dvar_marine/suite_config.py suite_configs.register(suite_name, '3dvar', _3dvar) diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index f19fbe083..bea6f1b82 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -14,7 +14,6 @@ from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes import swell.configuration.question_defaults as qd -from swell.utilities.r2d2 import create_r2d2_config from swell.utilities.r2d2 import load_r2d2_credentials diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index 232b8c839..0d1e30c21 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -51,7 +51,6 @@ def execute(self) -> None: # Import modules # -------------- from r2d2 import fetch - from swell.utilities.r2d2 import create_r2d2_config # Load R2D2 credentials # --------------------- diff --git a/src/swell/tasks/save_forecast.py b/src/swell/tasks/save_forecast.py index ce7992080..63e5650dc 100644 --- a/src/swell/tasks/save_forecast.py +++ b/src/swell/tasks/save_forecast.py @@ -56,7 +56,6 @@ def execute(self) -> None: """ from r2d2 import store - from swell.utilities.r2d2 import create_r2d2_config # Load R2D2 credentials # --------------------- diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index 3b504d453..80cbc6858 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -52,8 +52,6 @@ def execute(self): Does not handle 4d backgrounds properly """ - - from swell.utilities.r2d2 import create_r2d2_config from r2d2 import store self.logger.info('Skipping this task as R2D2v3 restart storage is not implemented ' + From 13534090a26256f2c0bdb266d3326b704edef04c Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 16 Apr 2026 11:06:14 -0400 Subject: [PATCH 277/299] Fix workflows --- .../interfaces/geos_cf/model/increment_cs.py | 19 -- src/swell/suites/3dvar_marine/flow.cylc | 151 ---------- src/swell/suites/3dvar_marine_cycle/flow.cylc | 258 ------------------ .../suites/3dvar_marine_cycle/suite_config.py | 208 +++++--------- .../suites/3dvar_marine_cycle/workflow.py | 3 +- 5 files changed, 69 insertions(+), 570 deletions(-) delete mode 100644 src/swell/suites/3dvar_marine/flow.cylc delete mode 100644 src/swell/suites/3dvar_marine_cycle/flow.cylc diff --git a/src/swell/configuration/jedi/interfaces/geos_cf/model/increment_cs.py b/src/swell/configuration/jedi/interfaces/geos_cf/model/increment_cs.py index de0f0ac93..6ab2ffcf7 100644 --- a/src/swell/configuration/jedi/interfaces/geos_cf/model/increment_cs.py +++ b/src/swell/configuration/jedi/interfaces/geos_cf/model/increment_cs.py @@ -6,26 +6,7 @@ # -------------------------------------------------------------------------------------------------- -<<<<<<< HEAD:src/swell/tasks/remove_forecast_dir.py -import shutil - -from swell.tasks.base.task_base import taskBase -from swell.tasks.base.task_setup import TaskSetup -from swell.tasks.base.task_attributes import task_attributes - -# -------------------------------------------------------------------------------------------------- - -task_name = 'RemoveForecastDir' - - -@task_attributes.register(task_name) -class Setup(TaskSetup): - def set_defaults(self): - self.base_name = task_name - self.is_cycling = True -======= from collections.abc import Mapping ->>>>>>> develop:src/swell/configuration/jedi/interfaces/geos_cf/model/increment_cs.py # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_marine/flow.cylc b/src/swell/suites/3dvar_marine/flow.cylc deleted file mode 100644 index b30b45392..000000000 --- a/src/swell/suites/3dvar_marine/flow.cylc +++ /dev/null @@ -1,151 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - -from swell.utilities.jinja2 import template_string_jinja2 -from swell.suites.base.cylc_workflow import CylcWorkflow -from swell.tasks.base.task_attributes import task_attributes as ta -from swell.suites.base.suite_attributes import workflows - -# -------------------------------------------------------------------------------------------------- - -template_str = ''' -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing JEDI-based non-cycling variational data assimilation - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - runahead limit = {{runahead_limit}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone JEDI source code - CloneJedi - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - {% for model_component in model_components %} - # Stage JEDI static files - CloneJedi => StageJedi-{{model_component}} - {% endfor %} - """ - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - {% for model_component in model_components %} - {% if cycle_time[model_component] %} - - # Task triggers for: {{model_component}} - # ------------------ - # GenerateBClimatology, for ocean it is cycle dependent - GetBackground-{{model_component}} => GenerateBClimatology-{{model_component}} - - # Perform staging that is cycle dependent - StageJediCycle-{{model_component}} - - # Run Jedi variational executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} - StageJedi-{{model_component}}[^] => RunJediVariationalExecutable-{{model_component}} - StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - - GenerateBClimatology-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} - RenderJediObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - - # EvaObservations - RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} - - # EvaJediLog - RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} - - # EvaIncrement - RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - - {% if not skip_r2d2 %} - # Save observations - RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} - SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} - {% endif %} - - # Clean up large files - EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & - EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => - CleanCycle-{{model_component}} - {% endif %} - {% endfor %} - """ - {% endfor %} - -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - -''' # noqa - -# -------------------------------------------------------------------------------------------------- - - -@workflows.register('3dvar') -class Workflow_3dvar(CylcWorkflow): - - def get_workflow_string(self): - workflow_str = self.default_header() - workflow_str += template_string_jinja2(logger=self.logger, - templated_string=template_str, - dictionary_of_templates=self.experiment_dict, - allow_unresolved=True) - - for task in self.tasks: - workflow_str += task.runtime_string(self.experiment_dict, - self.slurm_external) - - return workflow_str - - def set_tasks(self) -> list: - - self.tasks.append(ta.root()) - self.tasks.append(ta.CloneJedi()) - self.tasks.append(ta.BuildJediByLinking()) - self.tasks.append(ta.BuildJedi()) - - for model in self.experiment_dict['model_components']: - self.tasks.append(ta.StageJedi(model=model)) - self.tasks.append(ta.GetObservations(model=model)) - self.tasks.append(ta.GenerateBClimatology(model=model)) - self.tasks.append(ta.StageJediCycle(model=model)) - self.tasks.append(ta.GetBackground(model=model)) - self.tasks.append(ta.RenderJediObservations(model=model)) - self.tasks.append(ta.RunJediVariationalExecutable(model=model)) - self.tasks.append(ta.EvaObservations(model=model)) - self.tasks.append(ta.EvaJediLog(model=model)) - self.tasks.append(ta.EvaIncrement(model=model)) - self.tasks.append(ta.SaveObsDiags(model=model)) - self.tasks.append(ta.CleanCycle(model=model)) - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_marine_cycle/flow.cylc b/src/swell/suites/3dvar_marine_cycle/flow.cylc deleted file mode 100644 index 8b76edf85..000000000 --- a/src/swell/suites/3dvar_marine_cycle/flow.cylc +++ /dev/null @@ -1,258 +0,0 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -# -------------------------------------------------------------------------------------------------- - -# Cylc suite for executing Geos forecast - -# -------------------------------------------------------------------------------------------------- - -[scheduler] - UTC mode = True - allow implicit tasks = False - -# -------------------------------------------------------------------------------------------------- - -[scheduling] - - initial cycle point = {{start_cycle_point}} - final cycle point = {{final_cycle_point}} - - [[graph]] - R1 = """ - # Triggers for non cycle time dependent tasks - # ------------------------------------------- - # Clone Geos source code - CloneGeos - - # Clone JEDI source code - CloneJedi - - # Build Geos source code by linking - CloneGeos => BuildGeosByLinking? - - # Build JEDI source code by linking - CloneJedi => BuildJediByLinking? - - # If not able to link to build create the build - BuildGeosByLinking:fail? => BuildGeos - - # If not able to link to build create the build - BuildJediByLinking:fail? => BuildJedi - - # Need first set of restarts to run model - GetCoupledGeosRestart => PrepCoupledGeosRunDir - - # Model cannot run without code - BuildGeosByLinking? | BuildGeos => RunGeos - - {% for model_component in model_components %} - - # JEDI cannot run without code - BuildJediByLinking? | BuildJedi => RunJediVariationalExecutable-{{model_component}} - - # Stage JEDI static files - CloneJedi => StageJedi-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - - {% endfor %} - """ - - {% for cycle_time in cycle_times %} - {{cycle_time.cycle_time}} = """ - {% for model_component in model_components %} - - # Model preperation - # Run the forecast through two windows (need to output restarts at the end of the - # first window and backgrounds for the second window) - MoveDaRestart-{{model_component}}[-{{models[model_component]["window_length"]}}] => PrepCoupledGeosRunDir - PrepCoupledGeosRunDir => RunGeos - - # Run the analysis - RunGeos => LinkCoupledGeosOutput-{{model_component}} - LinkCoupledGeosOutput-{{model_component}} => GenerateBClimatology-{{model_component}} - - # Data assimilation preperation - StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - - GenerateBClimatology-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} - RenderJediObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} - - # Run analysis diagnostics - RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} - RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} - RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} - - # Prepare analysis for next forecast - EvaIncrement-{{model_component}} => PrepareAnalysis-{{model_component}} - {% if 'cice6' in models[model_component]["marine_models"] %} - PrepareAnalysis-{{model_component}} => RunJediConvertStateSoca2ciceExecutable-{{model_component}} - # RunJediConvertStateSoca2ciceExecutable-{{model_component}} => SaveRestart-{{model_component}} - RunJediConvertStateSoca2ciceExecutable-{{model_component}} => MoveDaRestart-{{model_component}} - RunJediConvertStateSoca2ciceExecutable-{{model_component}} => CleanCycle-{{model_component}} - {% else %} - # PrepareAnalysis-{{model_component}} => SaveRestart-{{model_component}} - PrepareAnalysis-{{model_component}} => MoveDaRestart-{{model_component}} - {% endif %} - - # Move restart to next cycle and then erase current forecast folder - SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} => CleanCycle-{{model_component}} - - {% if not skip_r2d2 %} - # Save analysis output - # RunJediVariationalExecutable-{{model_component}} => SaveAnalysis-{{model_component}} - RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} - {% endif %} - - # Save model output - # MoveBackground-{{model_component}} => StoreBackground-{{model_component}} - - # Clean up large files - EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} => - CleanCycle-{{model_component}} - {% endfor %} - """ - {% endfor %} -# -------------------------------------------------------------------------------------------------- - -[runtime] - - # Task defaults - # ------------- - [[root]] - pre-script = "source $CYLC_SUITE_DEF_PATH/modules" - - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneGeos]] - script = "swell task CloneGeos $config" - - [[BuildGeosByLinking]] - script = "swell task BuildGeosByLinking $config" - - [[BuildGeos]] - script = "swell task BuildGeos $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildGeos"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[RunGeos]] - script = "{{experiment_path}}/GEOSgcm/forecast/gcm_run.j" - platform = {{platform}} - [[[directives]]] - {%- for key, value in scheduling["RunGeos"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[PrepCoupledGeosRunDir]] - script = "swell task PrepCoupledGeosRunDir $config -d $datetime" - - [[GetCoupledGeosRestart]] - script = "swell task GetCoupledGeosRestart $config -d $datetime" - - {% for model_component in model_components %} - - [[LinkCoupledGeosOutput-{{model_component}}]] - script = "swell task LinkCoupledGeosOutput $config -d $datetime -m {{model_component}}" - - [[MoveDaRestart-{{model_component}}]] - script = "swell task MoveDaRestart $config -d $datetime -m {{model_component}}" - - [[StageJedi-{{model_component}}]] - script = "swell task StageJedi $config -m {{model_component}}" - - [[StageJediCycle-{{model_component}}]] - script = "swell task StageJedi $config -d $datetime -m {{model_component}}" - - [[GetObservations-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - - [[GenerateBClimatology-{{model_component}}]] - script = "swell task GenerateBClimatology $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["GenerateBClimatology"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["GenerateBClimatology"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% if 'cice6' in models["geos_marine"]["marine_models"] %} - - [[RunJediConvertStateSoca2ciceExecutable-{{model_component}}]] - script = "swell task RunJediConvertStateSoca2ciceExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediConvertStateSoca2ciceExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediConvertStateSoca2ciceExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% endif %} - - [[RenderJediObservations-{{model_component}}]] - script = "swell task RenderJediObservations $config -d $datetime -m {{model_component}}" - - [[RunJediVariationalExecutable-{{model_component}}]] - script = "swell task RunJediVariationalExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution retry delays = 1*PT5M - execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediVariationalExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[EvaJediLog-{{model_component}}]] - script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" - - [[EvaIncrement-{{model_component}}]] - script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" - - [[EvaObservations-{{model_component}}]] - script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[SaveRestart-{{model_component}}]] - script = "swell task SaveRestart $config -d $datetime -m {{model_component}}" - - [[SaveObsDiags-{{model_component}}]] - script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" - - [[PrepareAnalysis-{{model_component}}]] - script = "swell task PrepareAnalysis $config -d $datetime -m {{model_component}}" - - [[CleanCycle-{{model_component}}]] - script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" - {% endfor %} - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_marine_cycle/suite_config.py b/src/swell/suites/3dvar_marine_cycle/suite_config.py index eae1e667a..097043610 100644 --- a/src/swell/suites/3dvar_marine_cycle/suite_config.py +++ b/src/swell/suites/3dvar_marine_cycle/suite_config.py @@ -13,40 +13,29 @@ from swell.suites.base.suite_questions import marine from swell.suites.base.suite_attributes import suite_configs -suite_name = '3dfgat_cycle' +suite_name = '3dvar_marine_cycle' # -------------------------------------------------------------------------------------------------- -_3dfgat_cycle_tier1 = QuestionList( +_3dvar_marine_cycle = QuestionList( questions=[ marine, - qd.cycling_varbc(), qd.start_cycle_point("2021-07-02T06:00:00Z"), qd.final_cycle_point("2021-07-02T12:00:00Z"), qd.runahead_limit("P2"), qd.jedi_build_method("use_existing"), qd.geos_build_method("use_existing"), qd.model_components(['geos_marine']), - qd.comparison_log_type('fgat'), + qd.comparison_log_type('variational'), ], geos_marine=[ qd.cycle_times([ "T00", "T06", "T12", - "T18" - ]), - qd.analysis_variables([ - "sea_water_salinity", - "sea_water_potential_temperature", - "sea_surface_height_above_geoid", - "sea_water_cell_thickness", - "sea_ice_area_fraction", - "sea_ice_thickness", - "sea_ice_snow_thickness" + "T18", ]), qd.window_length("PT6H"), - qd.window_type("4D"), qd.horizontal_resolution("72x36"), qd.vertical_resolution("50"), qd.total_processors(6), @@ -57,10 +46,6 @@ "adt_sentinel3a", "adt_sentinel3b", "insitu_profile_argo", - "icec_amsr2_north", - "icec_amsr2_south", - "icec_nsidc_nh", - "icec_nsidc_sh", "sst_ostia", "sss_smos", "sss_smapv5", @@ -71,8 +56,10 @@ ]), qd.number_of_iterations([10]), qd.mom6_iau(True), + qd.marine_models(['mom6']), qd.background_time_offset("PT9H"), qd.clean_patterns([ + "*.nc4", "*.txt", "*.rc", "*.bin" @@ -80,136 +67,75 @@ ] ) -suite_configs.register(suite_name, '3dfgat_cycle_tier1', _3dfgat_cycle_tier1) +suite_configs.register(suite_name, '3dvar_marine_cycle', _3dvar_marine_cycle) -<<<<<<< HEAD:src/swell/suites/3dfgat_cycle/suite_config.py # -------------------------------------------------------------------------------------------------- -======= - _3dvar_marine_cycle = QuestionList( - list_name="3dvar_marine_cycle", - questions=[ - sq.marine, - qd.start_cycle_point("2021-07-02T06:00:00Z"), - qd.final_cycle_point("2021-07-02T12:00:00Z"), - qd.runahead_limit("P2"), - qd.jedi_build_method("use_existing"), - qd.geos_build_method("use_existing"), - qd.model_components(['geos_marine']), - qd.comparison_log_type('variational'), - ], - geos_marine=[ - qd.cycle_times([ - "T00", - "T06", - "T12", - "T18", - ]), - qd.window_length("PT6H"), - qd.horizontal_resolution("72x36"), - qd.vertical_resolution("50"), - qd.total_processors(6), - qd.observations([ - "adt_cryosat2n", - "adt_jason3", - "adt_saral", - "adt_sentinel3a", - "adt_sentinel3b", - "insitu_profile_argo", - "sst_ostia", - "sss_smos", - "sss_smapv5", - "sst_abi_g16_l3c", - "sst_gmi_l3u", - "sst_viirs_n20_l3u", - "temp_profile_xbt" - ]), - qd.number_of_iterations([10]), - qd.mom6_iau(True), - qd.marine_models(['mom6']), - qd.background_time_offset("PT9H"), - qd.clean_patterns([ - "*.nc4", - "*.txt", - "*.rc", - "*.bin" - ]), - ] - ) ->>>>>>> develop:src/swell/suites/3dvar_marine_cycle/suite_config.py -_3dfgat_cycle = QuestionList( +_3dvar_marine_cycle_tier1 = QuestionList( questions=[ - _3dfgat_cycle_tier1 + _3dvar_marine_cycle ] ) -<<<<<<< HEAD:src/swell/suites/3dfgat_cycle/suite_config.py -suite_configs.register(suite_name, '3dfgat_cycle', _3dfgat_cycle) -======= - _3dvar_marine_cycle_tier1 = QuestionList( - list_name="3dvar_marine_cycle_tier1", - questions=[ - _3dvar_marine_cycle - ] - ) +suite_configs.register(suite_name, '3dvar_marine_cycle_tier1', _3dvar_marine_cycle_tier1) + +# -------------------------------------------------------------------------------------------------- - # -------------------------------------------------------------------------------------------------- +_3dvar_marine_cycle_tier2 = QuestionList( + questions=[ + _3dvar_marine_cycle, + qd.start_cycle_point("2023-07-02T12:00:00Z"), + qd.final_cycle_point("2023-07-02T18:00:00Z"), + qd.forecast_duration("PT12H"), + qd.geos_homdir("/discover/nobackup/projects/gmao/advda/SwellStaticFiles/geos/homdirs/" + "dataatm_025") + ], + geos_marine=[ + qd.cycle_times([ + "T00", + "T06", + "T12", + "T18", + ]), + qd.analysis_variables([ + "sea_water_salinity", + "sea_water_potential_temperature", + "sea_surface_height_above_geoid", + "sea_water_cell_thickness", + ]), + qd.window_length("PT6H"), + qd.horizontal_resolution("1440x1080"), + qd.vertical_resolution("75"), + qd.total_processors(720), + qd.observations([ + "adt_cryosat2n", + "adt_jason3", + "adt_jason3n", + "adt_saral", + "adt_sentinel3a", + "adt_sentinel3b", + "adt_sentinel6a", + "adt_swot_nadir", + "insitu_profile_argo", + "insitu_profile_ctd", + "insitu_profile_pirata", + "insitu_profile_rama", + "insitu_profile_tao", + "sst_ostia", + "sss_smos", + "sss_smapv5", + "sst_abi_g16_l3c", + "sst_avhrrf_mb_l3u", + "sst_avhrrf_mc_l3u", + "sst_viirs_n20_l3u", + "sst_viirs_npp_l3u", + "temp_profile_xbt" + ]), + qd.number_of_iterations([50]), + qd.background_time_offset("PT9H"), + ] +) - _3dvar_marine_cycle_tier2 = QuestionList( - list_name="3dvar_marine_cycle_tier2", - questions=[ - _3dvar_marine_cycle, - qd.start_cycle_point("2023-07-02T12:00:00Z"), - qd.final_cycle_point("2023-07-02T18:00:00Z"), - qd.forecast_duration("PT12H"), - qd.geos_homdir("/discover/nobackup/projects/gmao/advda/SwellStaticFiles/geos/homdirs/" - "dataatm_025") - ], - geos_marine=[ - qd.cycle_times([ - "T00", - "T06", - "T12", - "T18", - ]), - qd.analysis_variables([ - "sea_water_salinity", - "sea_water_potential_temperature", - "sea_surface_height_above_geoid", - "sea_water_cell_thickness", - ]), - qd.window_length("PT6H"), - qd.horizontal_resolution("1440x1080"), - qd.vertical_resolution("75"), - qd.total_processors(720), - qd.observations([ - "adt_cryosat2n", - "adt_jason3", - "adt_jason3n", - "adt_saral", - "adt_sentinel3a", - "adt_sentinel3b", - "adt_sentinel6a", - "adt_swot_nadir", - "insitu_profile_argo", - "insitu_profile_ctd", - "insitu_profile_pirata", - "insitu_profile_rama", - "insitu_profile_tao", - "sst_ostia", - "sss_smos", - "sss_smapv5", - "sst_abi_g16_l3c", - "sst_avhrrf_mb_l3u", - "sst_avhrrf_mc_l3u", - "sst_viirs_n20_l3u", - "sst_viirs_npp_l3u", - "temp_profile_xbt" - ]), - qd.number_of_iterations([50]), - qd.background_time_offset("PT9H"), - ] - ) ->>>>>>> develop:src/swell/suites/3dvar_marine_cycle/suite_config.py +suite_configs.register(suite_name, '3dvar_marine_cycle_tier2', _3dvar_marine_cycle_tier2) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_marine_cycle/workflow.py b/src/swell/suites/3dvar_marine_cycle/workflow.py index 0e7429a45..7f9991200 100644 --- a/src/swell/suites/3dvar_marine_cycle/workflow.py +++ b/src/swell/suites/3dvar_marine_cycle/workflow.py @@ -11,6 +11,7 @@ from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta from swell.suites.base.suite_attributes import workflows +from swell.suites.forecast_coupled_geos.workflow import RunGeos # -------------------------------------------------------------------------------------------------- @@ -170,7 +171,7 @@ def set_tasks(self) -> list: self.tasks.append(ta.BuildGeos()) self.tasks.append(ta.GetGeosRestart()) self.tasks.append(ta.PrepGeosRunDir()) - self.tasks.append(ta.RunGeosExecutable()) + self.tasks.append(ta.RunGeos()) for model in self.experiment_dict['model_components']: self.tasks.append(ta.StageJedi(model=model)) From 4a8219c7368dbc680d6ebbd308bc05b23f88e181 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 16 Apr 2026 11:15:00 -0400 Subject: [PATCH 278/299] Fix flows --- .../suites/3dfgat_marine_cycle/workflow.py | 135 +----------------- .../forecast_coupled_geos/suite_config.py | 6 +- .../suites/forecast_coupled_geos/workflow.py | 9 +- 3 files changed, 11 insertions(+), 139 deletions(-) diff --git a/src/swell/suites/3dfgat_marine_cycle/workflow.py b/src/swell/suites/3dfgat_marine_cycle/workflow.py index 0f59ea534..dc4cecd7d 100644 --- a/src/swell/suites/3dfgat_marine_cycle/workflow.py +++ b/src/swell/suites/3dfgat_marine_cycle/workflow.py @@ -11,6 +11,7 @@ from swell.suites.base.cylc_workflow import CylcWorkflow from swell.tasks.base.task_attributes import task_attributes as ta from swell.suites.base.suite_attributes import workflows +from swell.suites.forecast_coupled_geos.workflow import RunGeos # -------------------------------------------------------------------------------------------------- @@ -136,7 +137,6 @@ # Task defaults # ------------- -<<<<<<<< HEAD:src/swell/suites/3dfgat_marine_cycle/workflow.py ''' # noqa # -------------------------------------------------------------------------------------------------- @@ -170,7 +170,7 @@ def set_tasks(self) -> None: self.tasks.append(ta.GetGeosRestart()) self.tasks.append(ta.PrepGeosRunDir()) - self.tasks.append(ta.RunGeosExecutable()) + self.tasks.append(ta.RunGeos()) for model in self.experiment_dict['model_components']: self.tasks.append(ta.RunJediFgatExecutable(model=model)) @@ -191,136 +191,5 @@ def set_tasks(self) -> None: self.tasks.append(ta.PrepareAnalysis(model=model)) self.tasks.append(ta.RemoveForecastDir(model=model)) self.tasks.append(ta.SaveObsDiags(model=model)) -======== - [[[environment]]] - datetime = $CYLC_TASK_CYCLE_POINT - config = $CYLC_SUITE_DEF_PATH/experiment.yaml - - # Tasks - # ----- - [[CloneGeos]] - script = "swell task CloneGeos $config" - - [[BuildGeosByLinking]] - script = "swell task BuildGeosByLinking $config" - - [[BuildGeos]] - script = "swell task BuildGeos $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildGeos"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildGeos"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[CloneJedi]] - script = "swell task CloneJedi $config" - - [[BuildJediByLinking]] - script = "swell task BuildJediByLinking $config" - - [[BuildJedi]] - script = "swell task BuildJedi $config" - platform = {{platform}} - execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[RunGeos]] - script = "{{experiment_path}}/GEOSgcm/forecast/gcm_run.j" - platform = {{platform}} - [[[directives]]] - {%- for key, value in scheduling["RunGeos"]["directives"]["all"].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[PrepCoupledGeosRunDir]] - script = "swell task PrepCoupledGeosRunDir $config -d $datetime" - - [[GetCoupledGeosRestart]] - script = "swell task GetCoupledGeosRestart $config -d $datetime" - - {% for model_component in model_components %} - - [[LinkCoupledGeosOutput-{{model_component}}]] - script = "swell task LinkCoupledGeosOutput $config -d $datetime -m {{model_component}}" - - [[MoveDaRestart-{{model_component}}]] - script = "swell task MoveDaRestart $config -d $datetime -m {{model_component}}" - - [[StageJedi-{{model_component}}]] - script = "swell task StageJedi $config -m {{model_component}}" - - [[StageJediCycle-{{model_component}}]] - script = "swell task StageJedi $config -d $datetime -m {{model_component}}" - - [[GetObservations-{{model_component}}]] - script = "swell task GetObservations $config -d $datetime -m {{model_component}}" - - [[GenerateBClimatology-{{model_component}}]] - script = "swell task GenerateBClimatology $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["GenerateBClimatology"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["GenerateBClimatology"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% if 'cice6' in models["geos_marine"]["marine_models"] %} - - [[RunJediConvertStateSoca2ciceExecutable-{{model_component}}]] - script = "swell task RunJediConvertStateSoca2ciceExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediConvertStateSoca2ciceExecutable"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["RunJediConvertStateSoca2ciceExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - {% endif %} - - [[RenderJediObservations-{{model_component}}]] - script = "swell task RenderJediObservations $config -d $datetime -m {{model_component}}" - - [[RunJediFgatExecutable-{{model_component}}]] - script = "swell task RunJediFgatExecutable $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["RunJediFgatExecutable"]["execution_time_limit"]}} - execution retry delays = 1*PT10M - [[[directives]]] - {%- for key, value in scheduling["RunJediFgatExecutable"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[EvaJediLog-{{model_component}}]] - script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" - - [[EvaIncrement-{{model_component}}]] - script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" - - [[EvaObservations-{{model_component}}]] - script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" - platform = {{platform}} - execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} - [[[directives]]] - {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} - --{{key}} = {{value}} - {%- endfor %} - - [[SaveRestart-{{model_component}}]] - script = "swell task SaveRestart $config -d $datetime -m {{model_component}}" - - [[SaveObsDiags-{{model_component}}]] - script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" - - [[PrepareAnalysis-{{model_component}}]] - script = "swell task PrepareAnalysis $config -d $datetime -m {{model_component}}" - - [[CleanCycle-{{model_component}}]] - script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" - {% endfor %} ->>>>>>>> develop:src/swell/suites/3dfgat_marine_cycle/flow.cylc # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_coupled_geos/suite_config.py b/src/swell/suites/forecast_coupled_geos/suite_config.py index 635fc974a..3dcf4742a 100644 --- a/src/swell/suites/forecast_coupled_geos/suite_config.py +++ b/src/swell/suites/forecast_coupled_geos/suite_config.py @@ -15,7 +15,7 @@ # -------------------------------------------------------------------------------------------------- -suite_name = 'forecast_geos' +suite_name = 'forecast_coupled_geos' forecast_geos_tier1 = QuestionList( questions=[ @@ -36,7 +36,7 @@ ], ) -suite_configs.register(suite_name, 'forecast_geos_tier1', forecast_geos_tier1) +suite_configs.register(suite_name, 'forecast_coupled_geos_tier1', forecast_geos_tier1) # -------------------------------------------------------------------------------------------------- @@ -46,6 +46,6 @@ ] ) -suite_configs.register(suite_name, 'forecast_geos', forecast_geos) +suite_configs.register(suite_name, 'forecast_coupled_geos', forecast_geos) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_coupled_geos/workflow.py b/src/swell/suites/forecast_coupled_geos/workflow.py index 76366c634..5ec9d1047 100644 --- a/src/swell/suites/forecast_coupled_geos/workflow.py +++ b/src/swell/suites/forecast_coupled_geos/workflow.py @@ -75,9 +75,12 @@ ''' -RunGeos = TaskSetup(base_name='RunGeos', is_cycling=True, - script='{{experiment_path}}/GEOSgcm/forecast/gcm_run.j', - slurm={}) +class RunGeos(TaskSetup): + def set_attributes(self): + self.base_name = 'RunGeos' + self.is_cycling = True + self.script = '{{experiment_path}}/GEOSgcm/forecast/gcm_run.j' + self.slurm = {} # -------------------------------------------------------------------------------------------------- From f85cdabaf00b17352775f8a7ea6a55e32a147dc2 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 17 Apr 2026 15:28:14 -0400 Subject: [PATCH 279/299] Fixes --- src/swell/configuration/question_defaults.py | 65 +++++++++---------- src/swell/deployment/create_experiment.py | 4 +- .../prepare_config_and_suite.py | 64 +++++++----------- .../suites/3dfgat_marine_cycle/workflow.py | 2 +- src/swell/suites/3dvar_marine/suite_config.py | 4 +- src/swell/suites/3dvar_marine/workflow.py | 2 +- .../suites/3dvar_marine_cycle/workflow.py | 2 +- src/swell/tasks/get_background.py | 2 +- src/swell/tasks/get_observations.py | 7 +- src/swell/tasks/render_jedi_observations.py | 1 - src/swell/tasks/save_obs_diags.py | 1 - 11 files changed, 61 insertions(+), 93 deletions(-) diff --git a/src/swell/configuration/question_defaults.py b/src/swell/configuration/question_defaults.py index 6c7b9b093..85e520bd1 100644 --- a/src/swell/configuration/question_defaults.py +++ b/src/swell/configuration/question_defaults.py @@ -168,6 +168,15 @@ class parser_options(SuiteQuestion): # -------------------------------------------------------------------------------------------------- +@dataclass +class r2d2_experiment_id(SuiteQuestion): + default_value: str = "defer_to_code" + question_name: str = "r2d2_experiment_id" + prompt: str = "What experiment_id should r2d2 reference for experiment?" + widget_type: WType = WType.STRING + +# ------------------------------------------------------------------------------------------------- + @dataclass class runahead_limit(SuiteQuestion): @@ -178,18 +187,9 @@ class runahead_limit(SuiteQuestion): "that may be active ahead of the current cycle " "(e.g. P1: up to 1 cycle ahead, P3: up to 3 cycles ahead, default P4).") widget_type: WType = WType.STRING - + # -------------------------------------------------------------------------------------------------- -@dataclass -class r2d2_experiment_id(SuiteQuestion): - default_value: str = "defer_to_code" - question_name: str = "r2d2_experiment_id" - prompt: str = "What experiment_id should r2d2 reference for experiment?" - widget_type: WType = WType.STRING - -# ------------------------------------------------------------------------------------------------- - @dataclass class skip_ensemble_hofx(SuiteQuestion): default_value: str = "defer_to_model" @@ -222,15 +222,6 @@ class start_cycle_point(SuiteQuestion): # -------------------------------------------------------------------------------------------------- -@dataclass -class skip_r2d2(SuiteQuestion): - default_value: bool = False - question_name: str = "skip_r2d2" - prompt: str = "Skip registering and storing results of this experiment in R2D2?" - widget_type: WType = WType.BOOLEAN - -# -------------------------------------------------------------------------------------------------- - @dataclass class use_cycle_dir(SuiteQuestion): default_value: bool = True @@ -669,15 +660,20 @@ class geos_build_method(TaskQuestion): @dataclass -class geos_experiment_directory(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "geos_experiment_directory" - ask_question: bool = True - prompt: str = "What is the path to the GEOS restarts directory?" +class geos_expdir(TaskQuestion): + default_value: str = "/dev/null/" + question_name: str = "geos_expdir" + depends: Dict = mutable_field({ + "geos_expdir_different": True + }) + prompt: str = ("What is the location for the EXPERIMENT Directory (to contain model " + "output and restart files), if it is different than your GEOS HOME " + "Directory?") widget_type: WType = WType.STRING - + # -------------------------------------------------------------------------------------------------- + @dataclass class geos_expdir_different(TaskQuestion): default_value: str = False @@ -693,19 +689,16 @@ class geos_expdir_different(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass -class geos_expdir(TaskQuestion): - default_value: str = "/dev/null/" - question_name: str = "geos_expdir" - depends: Dict = mutable_field({ - "geos_expdir_different": True - }) - prompt: str = ("What is the location for the EXPERIMENT Directory (to contain model " - "output and restart files), if it is different than your GEOS HOME " - "Directory?") +class geos_experiment_directory(TaskQuestion): + default_value: str = "defer_to_platform" + question_name: str = "geos_experiment_directory" + ask_question: bool = True + prompt: str = "What is the path to the GEOS restarts directory?" widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- + +# -------------------------------------------------------------------------------------------------- @dataclass class geos_gcm_tag(TaskQuestion): diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 002001063..fc7b5d0f1 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -293,7 +293,7 @@ def create_experiment_directory( # ------------------------------------------------ experiment_dict_str, workflow_str = prepare_config(suite, suite_config, method, platform, - override, advanced, slurm) + override_dict, advanced, slurm) # Load the string using yaml # -------------------------- @@ -507,4 +507,4 @@ def create_modules_csh( modules_file_open.write(modules_file_line) -# -------------------------------------------------------------------------------------------------- \ No newline at end of file +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index b394b3a99..0fbaa9dd7 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -20,7 +20,7 @@ from swell.deployment.prepare_config_and_suite.question_and_answer_defaults import GetAnswerDefaults from swell.utilities.dictionary import dict_get from swell.utilities.logger import Logger -from swell.utilities.dictionary import update_dict, add_dict +from swell.utilities.dictionary import add_dict from swell.suites.base.suite_attributes import suite_configs from swell.utilities.swell_questions import QuestionType @@ -323,12 +323,12 @@ def override_with_defaults(self, suite_task: QuestionType) -> None: 'default_value'] == 'defer_to_code': question['default_value'] = f'swell-{self.suite}' - if key == 'r2d2_experiment_id' and val['default_value'] == 'defer_to_code': - swell_id = self.question_dictionary_model_ind['experiment_id']['default_value'] - if swell_id == 'defer_to_code': - swell_id = f'swell-{self.suite}' - self.question_dictionary_model_ind['experiment_id']['default_value'] = swell_id - val['default_value'] = swell_id + if question_name == 'r2d2_experiment_id' and question['default_value'] == 'defer_to_code': + swell_id = self.question_dictionary_model_ind['experiment_id']['default_value'] + if swell_id == 'defer_to_code': + swell_id = f'swell-{self.suite}' + self.question_dictionary_model_ind['experiment_id']['default_value'] = swell_id + question['default_value'] = swell_id # ---------------------------------------------------------------------------------------------- @@ -340,41 +340,21 @@ def override_with_external(self, suite_task: QuestionType) -> None: # Iterate over the model_ind dictionary and override # -------------------------------------------------- - for key, val in self.question_dictionary_model_ind.items(): - if key in self.override: - val['default_value'] = self.override[key] - - if isinstance(self.override, Mapping): - override_dict = update_dict(override_dict, self.override) - - elif isinstance(self.override, str): - yaml = YAML(typ='safe') - with open(self.override, 'r') as ymlfile: - override_dict = update_dict(override_dict, yaml.load(ymlfile)) - else: - self.logger.abort(f'Override must be a dictionary or a path to a yaml file.') - - # In this case the user is sending in a dictionary that looks like the experiment - # dictionary that they will ultimately be looking at. This means the dictionary does - # not contain default_value or options and the override cannot be performed. - - # Iterate over the model_ind dictionary and override - # -------------------------------------------------- - for question_name, question in self.question_dictionary_model_ind.items(): - if question['question_type'] == suite_task: - if question_name in override_dict: - question['default_value'] = override_dict[question_name] - - # Iterate over the model_dep dictionary and override - # -------------------------------------------------- - if self.suite_needs_model_components and 'models' in override_dict.keys(): - for model, model_dict in self.question_dictionary_model_dep.items(): - for question_name, question in model_dict.items(): - if question['question_type'] == suite_task: - if model in override_dict['models']: - if question_name in override_dict['models'][model]: - question['default_value'] = override_dict[ - 'models'][model][question_name] + for question_name, question in self.question_dictionary_model_ind.items(): + if question['question_type'] == suite_task: + if question_name in self.override: + question['default_value'] = self.override[question_name] + + # Iterate over the model_dep dictionary and override + # -------------------------------------------------- + if self.suite_needs_model_components and 'models' in self.override.keys(): + for model, model_dict in self.question_dictionary_model_dep.items(): + for question_name, question in model_dict.items(): + if question['question_type'] == suite_task: + if model in self.override['models']: + if question_name in self.override['models'][model]: + question['default_value'] = self.override[ + 'models'][model][question_name] # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dfgat_marine_cycle/workflow.py b/src/swell/suites/3dfgat_marine_cycle/workflow.py index dc4cecd7d..4bb154bf0 100644 --- a/src/swell/suites/3dfgat_marine_cycle/workflow.py +++ b/src/swell/suites/3dfgat_marine_cycle/workflow.py @@ -170,7 +170,7 @@ def set_tasks(self) -> None: self.tasks.append(ta.GetGeosRestart()) self.tasks.append(ta.PrepGeosRunDir()) - self.tasks.append(ta.RunGeos()) + self.tasks.append(RunGeos()) for model in self.experiment_dict['model_components']: self.tasks.append(ta.RunJediFgatExecutable(model=model)) diff --git a/src/swell/suites/3dvar_marine/suite_config.py b/src/swell/suites/3dvar_marine/suite_config.py index 79caa9c67..28f20731f 100644 --- a/src/swell/suites/3dvar_marine/suite_config.py +++ b/src/swell/suites/3dvar_marine/suite_config.py @@ -55,7 +55,7 @@ ] ) -suite_configs.register(suite_name, '3dvar_tier1', _3dvar_tier1) +suite_configs.register(suite_name, '3dvar_marine_tier1', _3dvar_tier1) # -------------------------------------------------------------------------------------------------- @@ -65,6 +65,6 @@ ] ) -suite_configs.register(suite_name, '3dvar', _3dvar) +suite_configs.register(suite_name, '3dvar_marine', _3dvar) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_marine/workflow.py b/src/swell/suites/3dvar_marine/workflow.py index b30b45392..48922faf4 100644 --- a/src/swell/suites/3dvar_marine/workflow.py +++ b/src/swell/suites/3dvar_marine/workflow.py @@ -111,7 +111,7 @@ # -------------------------------------------------------------------------------------------------- -@workflows.register('3dvar') +@workflows.register('3dvar_marine') class Workflow_3dvar(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/3dvar_marine_cycle/workflow.py b/src/swell/suites/3dvar_marine_cycle/workflow.py index 7f9991200..dbee129f0 100644 --- a/src/swell/suites/3dvar_marine_cycle/workflow.py +++ b/src/swell/suites/3dvar_marine_cycle/workflow.py @@ -171,7 +171,7 @@ def set_tasks(self) -> list: self.tasks.append(ta.BuildGeos()) self.tasks.append(ta.GetGeosRestart()) self.tasks.append(ta.PrepGeosRunDir()) - self.tasks.append(ta.RunGeos()) + self.tasks.append(RunGeos()) for model in self.experiment_dict['model_components']: self.tasks.append(ta.StageJedi(model=model)) diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 3e98d2b7e..eb36d7173 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -39,7 +39,6 @@ def set_defaults(self): qd.background_frequency(), qd.horizontal_resolution(), qd.marine_models(), - qd.r2d2_local_path(), ] # -------------------------------------------------------------------------------------------------- @@ -59,6 +58,7 @@ def execute(self) -> None: # Load R2D2 credentials # --------------------- + import r2d2 load_r2d2_credentials(self.logger, self.platform()) # Get duration into forecast for first background file diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index b889dfa15..392574820 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -43,7 +43,6 @@ def set_defaults(self): qd.cycling_varbc(), qd.obs_experiment(), qd.observing_system_records_path(), - qd.r2d2_local_path(), qd.window_length(), ] @@ -62,6 +61,8 @@ def run_r2d2_fetch(r2d2_dict: dict) -> None: These values will be popped from the dictionary before running the fetch command """ + import r2d2 + fetch_empty_obs = r2d2_dict.pop('fetch_empty', False) cycle_dir = r2d2_dict.pop('cycle_dir') logger = r2d2_dict.pop('logger') @@ -173,10 +174,6 @@ def execute(self) -> None: "tlapse" files need to be fetched. """ - # # Local import because module is not loaded until experiment launch - # -------------- - import r2d2 - # Load R2D2 credentials # --------------------- load_r2d2_credentials(self.logger, self.platform()) diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index 639884191..2b6512b96 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -34,7 +34,6 @@ def set_defaults(self): qd.background_time_offset(), qd.observing_system_records_path(), qd.observations(), - qd.set_obs_as_local(), qd.window_length() ] diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 9ea212bc3..385145191 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -32,7 +32,6 @@ def set_defaults(self): qd.crtm_coeff_dir(), qd.observations(), qd.observing_system_records_path(), - qd.r2d2_local_path(), qd.window_length(), qd.marine_models() ] From 5da504d80d9a2bee5e4232eacbb57beadeac961a Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 17 Apr 2026 15:28:43 -0400 Subject: [PATCH 280/299] Remove 3dvar_cycle --- src/swell/suites/3dvar_cycle/suite_config.py | 82 -------------------- 1 file changed, 82 deletions(-) delete mode 100644 src/swell/suites/3dvar_cycle/suite_config.py diff --git a/src/swell/suites/3dvar_cycle/suite_config.py b/src/swell/suites/3dvar_cycle/suite_config.py deleted file mode 100644 index 89e6bfaa6..000000000 --- a/src/swell/suites/3dvar_cycle/suite_config.py +++ /dev/null @@ -1,82 +0,0 @@ -# -------------------------------------------------------------------------------------------------- -# @package configuration -# -# Class containing the configuration. This is a dictionary that is converted from -# an input yaml configuration file. Various function are included for interacting with the -# dictionary. -# -# -------------------------------------------------------------------------------------------------- - - -from swell.utilities.swell_questions import QuestionList -import swell.configuration.question_defaults as qd -from swell.suites.base.suite_questions import marine -from swell.suites.base.suite_attributes import suite_configs - -suite_name = '3dvar_cycle' - -# -------------------------------------------------------------------------------------------------- - -_3dvar_cycle_tier1 = QuestionList( - questions=[ - marine, - qd.cycling_varbc(), - qd.start_cycle_point("2021-07-02T06:00:00Z"), - qd.final_cycle_point("2021-07-02T12:00:00Z"), - qd.runahead_limit("P2"), - qd.jedi_build_method("use_existing"), - qd.geos_build_method("use_existing"), - qd.model_components(['geos_marine']), - ], - geos_marine=[ - qd.cycle_times([ - "T00", - "T06", - "T12", - "T18", - ]), - qd.window_length("PT6H"), - qd.horizontal_resolution("72x36"), - qd.vertical_resolution("50"), - qd.total_processors(6), - qd.observations([ - "adt_cryosat2n", - "adt_jason3", - "adt_saral", - "adt_sentinel3a", - "adt_sentinel3b", - "insitu_profile_argo", - "sst_ostia", - "sss_smos", - "sss_smapv5", - "sst_abi_g16_l3c", - "sst_gmi_l3u", - "sst_viirs_n20_l3u", - "temp_profile_xbt" - ]), - qd.number_of_iterations([10]), - qd.mom6_iau(True), - qd.marine_models(['mom6']), - qd.background_time_offset("PT9H"), - qd.clean_patterns([ - "*.nc4", - "*.txt", - "*.rc", - "*.bin" - ]), - ] -) - -suite_configs.register(suite_name, '3dvar_cycle_tier1', _3dvar_cycle_tier1) - -# -------------------------------------------------------------------------------------------------- - -_3dvar_cycle = QuestionList( - questions=[ - _3dvar_cycle_tier1 - ] -) - -suite_configs.register(suite_name, '3dvar_cycle', _3dvar_cycle) - -# -------------------------------------------------------------------------------------------------- From fa620b7fa61daca869d54e704940efa6c787ec74 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 17 Apr 2026 15:32:53 -0400 Subject: [PATCH 281/299] Fix workflow names --- src/swell/suites/3dfgat_marine_cycle/workflow.py | 2 +- src/swell/suites/3dvar_marine_cycle/workflow.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/suites/3dfgat_marine_cycle/workflow.py b/src/swell/suites/3dfgat_marine_cycle/workflow.py index 4bb154bf0..18c206ada 100644 --- a/src/swell/suites/3dfgat_marine_cycle/workflow.py +++ b/src/swell/suites/3dfgat_marine_cycle/workflow.py @@ -142,7 +142,7 @@ # -------------------------------------------------------------------------------------------------- -@workflows.register('3dfgat_cycle') +@workflows.register('3dfgat_marine_cycle') class Workflow_3dfgat_cycle(CylcWorkflow): def get_workflow_string(self): diff --git a/src/swell/suites/3dvar_marine_cycle/workflow.py b/src/swell/suites/3dvar_marine_cycle/workflow.py index dbee129f0..931f28e3f 100644 --- a/src/swell/suites/3dvar_marine_cycle/workflow.py +++ b/src/swell/suites/3dvar_marine_cycle/workflow.py @@ -144,7 +144,7 @@ # -------------------------------------------------------------------------------------------------- -@workflows.register('3dvar_cycle') +@workflows.register('3dvar_marine_cycle') class Workflow_3dvar_cycle(CylcWorkflow): def get_workflow_string(self): From 6b0a48e63c1f36e61a8c5f966a7170216521aa81 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 20 Apr 2026 16:28:44 -0400 Subject: [PATCH 282/299] More fixes --- src/swell/deployment/create_experiment.py | 1 - .../suites/3dfgat_marine_cycle/workflow.py | 11 ++--- .../suites/3dvar_marine_cycle/workflow.py | 44 +++++++++---------- .../suites/forecast_coupled_geos/workflow.py | 4 +- src/swell/tasks/get_geovals.py | 1 - src/swell/tasks/get_ncdiags.py | 1 - src/swell/tasks/ingest_obs.py | 1 - src/swell/tasks/save_restart.py | 1 - 8 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index fc7b5d0f1..07a52185a 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -193,7 +193,6 @@ def prepare_config( # Ask the task questions # ---------------------- experiment_dict, comment_dict = prepare_config_and_suite.configure_and_ask_task_questions() - if 'start_cycle_point' in suite_dict: experiment_dict['start_cycle_point'] = suite_dict['start_cycle_point'] experiment_dict['final_cycle_point'] = suite_dict['final_cycle_point'] diff --git a/src/swell/suites/3dfgat_marine_cycle/workflow.py b/src/swell/suites/3dfgat_marine_cycle/workflow.py index 18c206ada..37546c62c 100644 --- a/src/swell/suites/3dfgat_marine_cycle/workflow.py +++ b/src/swell/suites/3dfgat_marine_cycle/workflow.py @@ -156,6 +156,9 @@ def get_workflow_string(self): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) + workflow_str = template_string_jinja2(self.logger, workflow_str, self.experiment_dict, + allow_unresolved=False) + return workflow_str def set_tasks(self) -> None: @@ -167,9 +170,8 @@ def set_tasks(self) -> None: self.tasks.append(ta.BuildJedi()) self.tasks.append(ta.BuildGeos()) self.tasks.append(ta.BuildGeosByLinking()) - - self.tasks.append(ta.GetGeosRestart()) - self.tasks.append(ta.PrepGeosRunDir()) + self.tasks.append(ta.GetCoupledGeosRestart()) + self.tasks.append(ta.PrepCoupledGeosRunDir()) self.tasks.append(RunGeos()) for model in self.experiment_dict['model_components']: @@ -177,7 +179,7 @@ def set_tasks(self) -> None: self.tasks.append(ta.StageJedi(model=model)) self.tasks.append(ta.StageJediCycle(model=model)) self.tasks.append(ta.MoveDaRestart(model=model)) - self.tasks.append(ta.LinkGeosOutput(model=model)) + self.tasks.append(ta.LinkCoupledGeosOutput(model=model)) self.tasks.append(ta.GenerateBClimatology(model=model)) self.tasks.append(ta.GetObservations(model=model)) self.tasks.append(ta.EvaObservations(model=model)) @@ -189,7 +191,6 @@ def set_tasks(self) -> None: self.tasks.append(ta.SaveRestart(model=model)) self.tasks.append(ta.CleanCycle(model=model)) self.tasks.append(ta.PrepareAnalysis(model=model)) - self.tasks.append(ta.RemoveForecastDir(model=model)) self.tasks.append(ta.SaveObsDiags(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/3dvar_marine_cycle/workflow.py b/src/swell/suites/3dvar_marine_cycle/workflow.py index 931f28e3f..6e593a0d1 100644 --- a/src/swell/suites/3dvar_marine_cycle/workflow.py +++ b/src/swell/suites/3dvar_marine_cycle/workflow.py @@ -56,10 +56,10 @@ BuildJediByLinking:fail? => BuildJedi # Need first set of restarts to run model - GetGeosRestart => PrepGeosRunDir + GetCoupledGeosRestart => PrepCoupledGeosRunDir # Model cannot run without code - BuildGeosByLinking? | BuildGeos => RunGeosExecutable + BuildGeosByLinking? | BuildGeos => RunGeos {% for model_component in model_components %} @@ -76,18 +76,17 @@ {{cycle_time.cycle_time}} = """ {% for model_component in model_components %} - # Model things + # Model preperation # Run the forecast through two windows (need to output restarts at the end of the # first window and backgrounds for the second window) - MoveDaRestart-{{model_component}}[-{{models[model_component]["window_length"]}}] => PrepGeosRunDir - PrepGeosRunDir => RunGeosExecutable + MoveDaRestart-{{model_component}}[-{{models[model_component]["window_length"]}}] => PrepCoupledGeosRunDir + PrepCoupledGeosRunDir => RunGeos # Run the analysis - # RunGeosExecutable => StageJediCycle-{{model_component}} - RunGeosExecutable => LinkGeosOutput-{{model_component}} - LinkGeosOutput-{{model_component}} => GenerateBClimatology-{{model_component}} + RunGeos => LinkCoupledGeosOutput-{{model_component}} + LinkCoupledGeosOutput-{{model_component}} => GenerateBClimatology-{{model_component}} - # Data assimilation things + # Data assimilation preperation StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} GenerateBClimatology-{{model_component}} => RunJediVariationalExecutable-{{model_component}} @@ -111,27 +110,25 @@ PrepareAnalysis-{{model_component}} => MoveDaRestart-{{model_component}} {% endif %} - # Move restart to next cycle - # SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} + # Move restart to next cycle and then erase current forecast folder + SaveRestart-{{model_component}} => MoveDaRestart-{{model_component}} => CleanCycle-{{model_component}} + {% if not skip_r2d2 %} # Save analysis output # RunJediVariationalExecutable-{{model_component}} => SaveAnalysis-{{model_component}} - RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} + {% endif %} # Save model output # MoveBackground-{{model_component}} => StoreBackground-{{model_component}} - # Remove Run Directory - # MoveDaRestart-{{model_component}} & MoveBackground-{{model_component}} => RemoveForecastDir - MoveDaRestart-{{model_component}} => RemoveForecastDir - # Clean up large files - # EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & SaveObsDiags-{{model_component}} & RemoveForecastDir => - EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} & SaveObsDiags-{{model_component}} => + EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} => CleanCycle-{{model_component}} {% endfor %} """ {% endfor %} + # -------------------------------------------------------------------------------------------------- [runtime] @@ -158,6 +155,9 @@ def get_workflow_string(self): workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) + workflow_str = template_string_jinja2(self.logger, workflow_str, self.experiment_dict, + allow_unresolved=False) + return workflow_str def set_tasks(self) -> list: @@ -169,26 +169,26 @@ def set_tasks(self) -> list: self.tasks.append(ta.BuildGeosByLinking()) self.tasks.append(ta.BuildJedi()) self.tasks.append(ta.BuildGeos()) - self.tasks.append(ta.GetGeosRestart()) - self.tasks.append(ta.PrepGeosRunDir()) + self.tasks.append(ta.GetCoupledGeosRestart()) + self.tasks.append(ta.PrepCoupledGeosRunDir()) self.tasks.append(RunGeos()) for model in self.experiment_dict['model_components']: self.tasks.append(ta.StageJedi(model=model)) self.tasks.append(ta.StageJediCycle(model=model)) self.tasks.append(ta.RunJediVariationalExecutable(model=model)) - self.tasks.append(ta.LinkGeosOutput(model=model)) + self.tasks.append(ta.LinkCoupledGeosOutput(model=model)) self.tasks.append(ta.GenerateBClimatology(model=model)) self.tasks.append(ta.GetObservations(model=model)) self.tasks.append(ta.PrepareAnalysis(model=model)) self.tasks.append(ta.RenderJediObservations(model=model)) self.tasks.append(ta.RunJediConvertStateSoca2ciceExecutable(model=model)) self.tasks.append(ta.MoveDaRestart(model=model)) - self.tasks.append(ta.RemoveForecastDir(model=model)) self.tasks.append(ta.EvaObservations(model=model)) self.tasks.append(ta.EvaJediLog(model=model)) self.tasks.append(ta.EvaIncrement(model=model)) self.tasks.append(ta.SaveObsDiags(model=model)) self.tasks.append(ta.CleanCycle(model=model)) + self.tasks.append(ta.SaveRestart(model=model)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/forecast_coupled_geos/workflow.py b/src/swell/suites/forecast_coupled_geos/workflow.py index 5ec9d1047..1aabc1938 100644 --- a/src/swell/suites/forecast_coupled_geos/workflow.py +++ b/src/swell/suites/forecast_coupled_geos/workflow.py @@ -76,10 +76,10 @@ ''' class RunGeos(TaskSetup): - def set_attributes(self): + def set_defaults(self): self.base_name = 'RunGeos' self.is_cycling = True - self.script = '{{experiment_path}}/GEOSgcm/forecast/gcm_run.j' + self.script = '{{experiment_root}}/{{experiment_id}}/GEOSgcm/forecast/gcm_run.j' self.slurm = {} # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index bea6f1b82..7f74e7055 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -35,7 +35,6 @@ def set_defaults(self): qd.observing_system_records_path(), qd.geovals_experiment(), qd.geovals_provider(), - qd.r2d2_local_path(), qd.window_length(), ] diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index 0d1e30c21..2cbc7c660 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -33,7 +33,6 @@ def set_defaults(self): qd.observing_system_records_path(), qd.ncdiag_experiments(), qd.marine_models(), - qd.r2d2_local_path(), qd.window_length(), ] diff --git a/src/swell/tasks/ingest_obs.py b/src/swell/tasks/ingest_obs.py index e74d81d89..43442b359 100644 --- a/src/swell/tasks/ingest_obs.py +++ b/src/swell/tasks/ingest_obs.py @@ -39,7 +39,6 @@ def set_defaults(self): self.questions = [ qd.dry_run(), qd.obs_to_ingest(), - qd.r2d2_local_path(), qd.window_length(), # qd.window_offset(), ] diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index 80cbc6858..1b1475183 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -36,7 +36,6 @@ def set_defaults(self): qd.forecast_duration(), qd.horizontal_resolution(), qd.marine_models(), - qd.r2d2_local_path() ] # -------------------------------------------------------------------------------------------------- From ac8d0107d5ada6053adb66fa77750e132ab36756 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 20 Apr 2026 17:33:53 -0400 Subject: [PATCH 283/299] variational fixes --- src/swell/tasks/run_jedi_variational_executable.py | 2 ++ src/swell/tasks/stage_jedi.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index d8e6f8257..4e12acd43 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -56,6 +56,8 @@ def set_defaults(self): qd.total_processors(), qd.perhost(), qd.comparison_log_type('variational'), + qd.saber_central_block(), + qd.saber_outer_block(), ] # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 09db32074..fa1967ab6 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -37,7 +37,9 @@ def set_defaults(self): qd.gsibec_nlats(), qd.gsibec_nlons(), qd.horizontal_resolution(), - qd.vertical_resolution() + qd.vertical_resolution(), + qd.saber_central_block(), + qd.saber_outer_block(), ] From 4ba400fbe426ce954cb06f85a9e8b63d73df7c80 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 21 Apr 2026 09:26:54 -0400 Subject: [PATCH 284/299] cf static files --- src/swell/suites/hofx_cf/suite_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/swell/suites/hofx_cf/suite_config.py b/src/swell/suites/hofx_cf/suite_config.py index 15fa7a5f2..8c98bf8e3 100644 --- a/src/swell/suites/hofx_cf/suite_config.py +++ b/src/swell/suites/hofx_cf/suite_config.py @@ -21,6 +21,7 @@ hofx_cf = QuestionList( questions=[ common, + qd.swell_static_files("/discover/nobackup/projects/gmao/geos_cf_dev/SwellStaticFiles"), qd.start_cycle_point("2023-08-05T18:00:00Z"), qd.final_cycle_point("2023-08-05T18:00:00Z"), qd.jedi_build_method("use_existing"), From 3d048df6bdd08192067c3405ec9c11e84d4063cd Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 21 Apr 2026 10:13:00 -0400 Subject: [PATCH 285/299] add 3dvar_cf workflow --- src/swell/suites/3dvar_cf/__init__.py | 0 src/swell/suites/3dvar_cf/workflow.py | 140 ++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 src/swell/suites/3dvar_cf/__init__.py create mode 100644 src/swell/suites/3dvar_cf/workflow.py diff --git a/src/swell/suites/3dvar_cf/__init__.py b/src/swell/suites/3dvar_cf/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/swell/suites/3dvar_cf/workflow.py b/src/swell/suites/3dvar_cf/workflow.py new file mode 100644 index 000000000..bd473ce50 --- /dev/null +++ b/src/swell/suites/3dvar_cf/workflow.py @@ -0,0 +1,140 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.jinja2 import template_string_jinja2 +from swell.suites.base.cylc_workflow import CylcWorkflow +from swell.tasks.base.task_attributes import task_attributes as ta +from swell.suites.base.suite_attributes import workflows + +# -------------------------------------------------------------------------------------------------- + +template_str = ''' +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing JEDI-based non-cycling variational data assimilation + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + # Task triggers for: {{model_component}} + # ------------------ + # Generate background error covariance + GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + + # Perform staging that is cycle dependent + StageJediCycle-{{model_component}} + + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediVariationalExecutable-{{model_component}} + StageJediCycle-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + GetBackground-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + + + GetObservations-{{model_component}} => RenderJediObservations-{{model_component}} + RenderJediObservations-{{model_component}} => RunJediVariationalExecutable-{{model_component}} + + # EvaObservations + RunJediVariationalExecutable-{{model_component}} => EvaObservations-{{model_component}} + + # EvaJediLog + RunJediVariationalExecutable-{{model_component}} => EvaJediLog-{{model_component}} + + # EvaIncrement + RunJediVariationalExecutable-{{model_component}} => EvaIncrement-{{model_component}} + + {% if not skip_r2d2 %} + # Save observations + RunJediVariationalExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + SaveObsDiags-{{model_component}} => CleanCycle-{{model_component}} + {% endif %} + + # Clean up large files + EvaObservations-{{model_component}} & EvaJediLog-{{model_component}} & EvaIncrement-{{model_component}} => + CleanCycle-{{model_component}} + + {% endif %} + {% endfor %} + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + +''' # noqa + +# -------------------------------------------------------------------------------------------------- + + +@workflows.register('3dvar_cf') +class Workflow_hofx_cf(CylcWorkflow): + + def get_workflow_string(self): + workflow_str = self.default_header() + workflow_str += template_string_jinja2(logger=self.logger, + templated_string=template_str, + dictionary_of_templates=self.experiment_dict, + allow_unresolved=True) + + for task in self.tasks: + workflow_str += task.runtime_string(self.experiment_dict, + self.slurm_external) + + return workflow_str + + def set_tasks(self) -> list: + + self.tasks.append(ta.root()) + self.tasks.append(ta.CloneJedi()) + self.tasks.append(ta.BuildJediByLinking()) + self.tasks.append(ta.BuildJedi()) + + for model in self.experiment_dict['model_components']: + self.tasks.append(ta.StageJediCycle(model=model)) + self.tasks.append(ta.GetBackground(model=model)) + self.tasks.append(ta.GetObservations(model=model)) + self.tasks.append(ta.RenderJediObservations(model=model)) + self.tasks.append(ta.RunJediVariationalExecutable(model=model)) + self.tasks.append(ta.EvaObservations(model=model)) + self.tasks.append(ta.SaveObsDiags(model=model)) + self.tasks.append(ta.CleanCycle(model=model)) + +# -------------------------------------------------------------------------------------------------- From c2f111e828ee245823a4036f416a8a6540c3b761 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 21 Apr 2026 16:08:03 -0400 Subject: [PATCH 286/299] Fix stage_jedi --- src/swell/tasks/stage_jedi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index fa1967ab6..d63f3782a 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -33,6 +33,8 @@ def set_defaults(self): self.questions = [ qd.swell_static_files(), qd.swell_static_files_user(), + qd.npx_proc(), + qd.npy_proc(), qd.gsibec_configuration(), qd.gsibec_nlats(), qd.gsibec_nlons(), From 5833f552384e33b6e59767a95f93092420603b26 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 21 Apr 2026 17:10:46 -0400 Subject: [PATCH 287/299] Fix implicit tasks --- src/swell/suites/3dvar_cf/workflow.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/swell/suites/3dvar_cf/workflow.py b/src/swell/suites/3dvar_cf/workflow.py index bd473ce50..0efeac1eb 100644 --- a/src/swell/suites/3dvar_cf/workflow.py +++ b/src/swell/suites/3dvar_cf/workflow.py @@ -133,6 +133,8 @@ def set_tasks(self) -> list: self.tasks.append(ta.GetObservations(model=model)) self.tasks.append(ta.RenderJediObservations(model=model)) self.tasks.append(ta.RunJediVariationalExecutable(model=model)) + self.tasks.append(ta.EvaIncrement(model=model)) + self.tasks.append(ta.EvaJediLog(model=model)) self.tasks.append(ta.EvaObservations(model=model)) self.tasks.append(ta.SaveObsDiags(model=model)) self.tasks.append(ta.CleanCycle(model=model)) From fca7d8a3121e7c585ccb8ef302ce4c11226b73cb Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 16 Apr 2026 16:13:16 -0400 Subject: [PATCH 288/299] Add comparisons for 3dvar_cf (#750) * Add eva yamls and suite config for cf * Fixes * Fix overlapping text * Remove increment file from clean patterns --- src/swell/suites/3dvar_cf/suite_config.py | 2 +- .../eva/comparison_increment-geos_cf.yaml | 104 ++++ .../eva/comparison_jedi_log-geos_cf.yaml | 110 ++++ ...parison_observations-3dvar_cf-geos_cf.yaml | 528 ++++++++++++++++++ src/swell/suites/compare/suite_config.py | 11 + 5 files changed, 754 insertions(+), 1 deletion(-) create mode 100644 src/swell/suites/compare/eva/comparison_increment-geos_cf.yaml create mode 100644 src/swell/suites/compare/eva/comparison_jedi_log-geos_cf.yaml create mode 100644 src/swell/suites/compare/eva/comparison_observations-3dvar_cf-geos_cf.yaml diff --git a/src/swell/suites/3dvar_cf/suite_config.py b/src/swell/suites/3dvar_cf/suite_config.py index 607c3cfa7..93dd6118c 100644 --- a/src/swell/suites/3dvar_cf/suite_config.py +++ b/src/swell/suites/3dvar_cf/suite_config.py @@ -49,7 +49,7 @@ class SuiteConfig(QuestionContainer, Enum): "tempo_no2_tropo", "tropomi_s5p_no2_tropo", ]), - qd.clean_patterns(['*.nc4', '*.txt', 'logfile.*.out']), + qd.clean_patterns(['*.txt', 'logfile.*.out']), ] ) diff --git a/src/swell/suites/compare/eva/comparison_increment-geos_cf.yaml b/src/swell/suites/compare/eva/comparison_increment-geos_cf.yaml new file mode 100644 index 000000000..b69ac2dbd --- /dev/null +++ b/src/swell/suites/compare/eva/comparison_increment-geos_cf.yaml @@ -0,0 +1,104 @@ +datasets: + +- group: increment + type: LatLon + filename: {{increment_file_path_1}} + name: experiment_increment_1 + variables: [NO2, lat, lon] + +- group: increment + type: LatLon + filename: {{increment_file_path_2}} + name: experiment_increment_2 + variables: [NO2, lat, lon] + +graphics: + + plotting_backend: Emcpy + figure_list: + + #map plot for NO2 increment (near surface) + - batch figure: + variables: [NO2] + figure: + figure size: [60,30] + layout: [3,1] + title: 'NO2 Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_surface.png' + tight_layout: true + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: NO2 Increment + add_grid: + add_title: '{{experiment_tag_1}}' + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::NO2 + slices: '[71,...]' + label: "{{experiment_tag_1}} NO2 increment (surface level)" + colorbar: true + cmap: 'bwr' + vmin: -1e-10 + vmax: 1e-10 + + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: NO2 Increment + add_grid: + add_title: '{{experiment_tag_2}}' + layers: + - type: MapGridded + longitude: + variable: experiment_increment_2::increment::lon + latitude: + variable: experiment_increment_2::increment::lat + data: + variable: experiment_increment_2::increment::NO2 + slices: '[71,...]' + label: "{{experiment_tag_2}} NO2 increment (surface level)" + colorbar: true + cmap: 'bwr' + vmin: -1e-10 + vmax: 1e-10 + + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: NO2 Increment + add_grid: + add_title: '{{experiment_tag_1}} - {{experiment_tag_2}}' + layers: + - type: MapGridded + longitude: + variable: experiment_increment_1::increment::lon + latitude: + variable: experiment_increment_1::increment::lat + data: + variable: experiment_increment_1::increment::NO2_diff + slices: '[71,...]' + label: "{{experiment_tag_1}} NO2 increment diff (surface level)" + colorbar: true + cmap: 'bwr' + vmin: -1e-11 + vmax: 1e-11 + +transforms: +- equals: experiment_increment_1::increment::NO2-experiment_increment_2::increment::NO2 + for: + variable: NO2 + new name: experiment_increment_1::increment::NO2_diff + transform: arithmetic diff --git a/src/swell/suites/compare/eva/comparison_jedi_log-geos_cf.yaml b/src/swell/suites/compare/eva/comparison_jedi_log-geos_cf.yaml new file mode 100644 index 000000000..6cb16062a --- /dev/null +++ b/src/swell/suites/compare/eva/comparison_jedi_log-geos_cf.yaml @@ -0,0 +1,110 @@ +datasets: + +- type: JediLog + collection_name: JediLogTest_1 + jedi_log_to_parse: '{{cycle_dir_1}}/jedi_variational_log.log' + data_to_parse: + convergence: true + +- type: JediLog + collection_name: JediLogTest_2 + jedi_log_to_parse: '{{cycle_dir_2}}/jedi_variational_log.log' + data_to_parse: + convergence: true + +transforms: +- transform: arithmetic + new name: JediLogTest_1::convergence::${variable}_log + equals: log(JediLogTest_1::convergence::${variable}) + for: + variable: [residual_norm, norm_reduction] + +- transform: arithmetic + new name: JediLogTest_2::convergence::${variable}_log + equals: log(JediLogTest_2::convergence::${variable}) + for: + variable: [residual_norm, norm_reduction] + +graphics: + + plotting_backend: Emcpy + figure_list: + + - figure: + layout: [3,1] + figure size: [12,10] + title: 'Residual Norm and Norm Reduction Plots' + output name: '{{cycle_dir}}/eva/jedi_log/convergence/residual_norm_reduction.png' + plots: + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Residual norm' + add_legend: + layers: + - type: LinePlot + x: + variable: JediLogTest_1::convergence::total_iteration + y: + variable: JediLogTest_1::convergence::residual_norm + color: 'black' + label: '{{experiment_tag_1}} Residual norm' + - type: LinePlot + x: + variable: JediLogTest_2::convergence::total_iteration + y: + variable: JediLogTest_2::convergence::residual_norm + color: 'red' + label: '{{experiment_tag_2}} Residual norm' + + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Log(norm reduction)' + add_legend: + layers: + - type: LinePlot + x: + variable: JediLogTest_1::convergence::total_iteration + y: + variable: JediLogTest_1::convergence::norm_reduction + color: 'blue' + label: '{{experiment_tag_1}} Norm reduction' + + - type: LinePlot + x: + variable: JediLogTest_2::convergence::total_iteration + y: + variable: JediLogTest_2::convergence::norm_reduction + color: 'green' + label: '{{experiment_tag_2}} Norm reduction' + + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Log(reduction)' + add_legend: + layers: + - type: LinePlot + x: + variable: JediLogTest_1::convergence::total_iteration + y: + variable: JediLogTest_1::convergence::residual_norm_log + color: 'black' + label: '{{experiment_tag_1}} Log(residual norm)' + - type: LinePlot + x: + variable: JediLogTest_2::convergence::total_iteration + y: + variable: JediLogTest_2::convergence::residual_norm_log + color: 'red' + label: '{{experiment_tag_2}} Log(residual norm)' + + - type: LinePlot + x: + variable: JediLogTest_1::convergence::total_iteration + y: + variable: JediLogTest_1::convergence::norm_reduction_log + color: 'blue' + label: '{{experiment_tag_1}} Log norm reduction' + - type: LinePlot + x: + variable: JediLogTest_2::convergence::total_iteration + y: + variable: JediLogTest_2::convergence::norm_reduction_log + color: 'green' + label: '{{experiment_tag_2}} Log norm reduction' diff --git a/src/swell/suites/compare/eva/comparison_observations-3dvar_cf-geos_cf.yaml b/src/swell/suites/compare/eva/comparison_observations-3dvar_cf-geos_cf.yaml new file mode 100644 index 000000000..2e0ec0df8 --- /dev/null +++ b/src/swell/suites/compare/eva/comparison_observations-3dvar_cf-geos_cf.yaml @@ -0,0 +1,528 @@ +datasets: + +- name: experiment_1 + type: IodaObsSpace + filenames: + - {{obs_path_file_1}} + groups: + - name: ObsValue + variables: &variables {{simulated_variables}} + - name: hofx0 + - name: hofx1 + - name: ombg + - name: oman + - name: MetaData + - name: EffectiveQC0 + - name: EffectiveQC1 + +- name: experiment_2 + type: IodaObsSpace + filenames: + - {{obs_path_file_2}} + groups: + - name: ObsValue + variables: *variables + - name: hofx0 + - name: hofx1 + - name: ombg + - name: oman + - name: MetaData + - name: EffectiveQC0 + - name: EffectiveQC1 + +transforms: + +# Generate Increment for JEDI +- transform: arithmetic + new name: experiment_1::increment::${variable} + equals: experiment_1::ombg::${variable}-experiment_1::oman::${variable} + for: + variable: *variables + +- transform: arithmetic + new name: experiment_2::increment::${variable} + equals: experiment_2::ombg::${variable}-experiment_2::oman::${variable} + for: + variable: *variables + +# Generate hofx0 that passed QC for JEDI +- transform: accept where + new name: experiment_1::hofx0PassedQc::${variable} + starting field: experiment_1::hofx0::${variable} + where: + - experiment_1::EffectiveQC0::${variable} == 0 + for: + variable: *variables +- transform: accept where + new name: experiment_2::hofx0PassedQc::${variable} + starting field: experiment_2::hofx0::${variable} + where: + - experiment_2::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +# Generate hofx1 that passed QC for JEDI +- transform: accept where + new name: experiment_1::hofx1PassedQc::${variable} + starting field: experiment_1::hofx1::${variable} + where: + - experiment_1::EffectiveQC1::${variable} == 0 + for: + variable: *variables +- transform: accept where + new name: experiment_2::hofx1PassedQc::${variable} + starting field: experiment_2::hofx1::${variable} + where: + - experiment_2::EffectiveQC1::${variable} == 0 + for: + variable: *variables + +# Generate ombg that passed QC for JEDI +- transform: accept where + new name: experiment_1::ombgPassedQc::${variable} + starting field: experiment_1::ombg::${variable} + where: + - experiment_1::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +- transform: accept where + new name: experiment_2::ombgPassedQc::${variable} + starting field: experiment_2::ombg::${variable} + where: + - experiment_2::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +# Generate oman that passed QC for JEDI +- transform: accept where + new name: experiment_1::omanPassedQc::${variable} + starting field: experiment_1::oman::${variable} + where: + - experiment_1::EffectiveQC1::${variable} == 0 + for: + variable: *variables + +- transform: accept where + new name: experiment_2::omanPassedQc::${variable} + starting field: experiment_2::oman::${variable} + where: + - experiment_2::EffectiveQC1::${variable} == 0 + for: + variable: *variables + +# Generate obs that passed QC for JEDI +- transform: accept where + new name: experiment_1::ObsValuePassedQc::${variable} + starting field: experiment_1::ObsValue::${variable} + where: + - experiment_1::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +- transform: accept where + new name: experiment_2::ObsValuePassedQc::${variable} + starting field: experiment_2::ObsValue::${variable} + where: + - experiment_2::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +graphics: + + plotting_backend: Emcpy + figure_list: + + # Correlation scatter plots + # ------------------------- + + # JEDI h(x) vs Observations + - batch figure: + variables: *variables + figure: + layout: [2,1] + figure size: [60,30] + title: 'Observations vs. JEDI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}/jedi_hofx_vs_obs_{{instrument}}_${variable}.png' + tight_layout: + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'JEDI h(x)' + add_grid: + add_title: '{{experiment_tag_1}}' + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment_1::ObsValue::${variable} + y: + variable: experiment_1::hofx0PassedQc::${variable} + markersize: 5 + color: 'black' + label: 'JEDI h(x)_0 versus obs (passed QC in JEDI)' + - type: Scatter + x: + variable: experiment_1::ObsValue::${variable} + y: + variable: experiment_1::hofx1PassedQc::${variable} + markersize: 5 + color: 'red' + label: 'JEDI h(x)_1 versus obs (passed QC in JEDI)' + + - add_xlabel: 'Observation Value' + add_ylabel: 'JEDI h(x)' + add_grid: + add_title: '{{experiment_tag_2}}' + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment_2::ObsValue::${variable} + y: + variable: experiment_2::hofx0PassedQc::${variable} + markersize: 5 + color: 'black' + label: 'JEDI h(x)_0 versus obs (passed QC in JEDI)' + - type: Scatter + x: + variable: experiment_2::ObsValue::${variable} + y: + variable: experiment_2::hofx1PassedQc::${variable} + markersize: 5 + color: 'red' + label: 'JEDI h(x)_1 versus obs (passed QC in JEDI)' + + + # Histogram plots + # --------------- + + # JEDI h(x) vs Observations + - batch figure: + variables: *variables + dynamic options: + - type: histogram_bins + data variable: experiment_1::omanPassedQc::${variable} + number of bins rule: 'rice' + figure: + layout: [2,1] + figure size: [60,30] + title: 'Observations vs. JEDI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/histogram/${variable}/ombg_oman_{{instrument}}_${variable}.png' + tight_layout: + plots: + - add_xlabel: 'Difference' + add_ylabel: 'Count' + set_xlim: [-5e-4, 5e-4] + add_title: '{{experiment_tag_1}}' + add_legend: + loc: 'upper left' + statistics: + fields: + - field_name: experiment_1::ombgPassedQc::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + color: 'black' + fontsize: 8 + fontfamily: monospace + - field_name: experiment_1::omanPassedQc::${variable} + xloc: 0.5 + yloc: -0.13 + kwargs: + color: 'red' + fontsize: 8 + fontfamily: monospace + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Histogram + data: + variable: experiment_1::ombgPassedQc::${variable} + color: 'red' + label: 'observations minus background ' + bins: ${dynamic_bins} + alpha: 0.5 + density: true + - type: Histogram + data: + variable: experiment_1::omanPassedQc::${variable} + color: 'blue' + label: 'observations minus analysis' + bins: ${dynamic_bins} + alpha: 0.5 + density: true + + - add_xlabel: 'Difference' + add_ylabel: 'Count' + set_xlim: [-5e-4, 5e-4] + add_title: '{{experiment_tag_2}}' + add_legend: + loc: 'upper left' + statistics: + fields: + - field_name: experiment_2::ombgPassedQc::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + color: 'black' + fontsize: 8 + fontfamily: monospace + - field_name: experiment_2::omanPassedQc::${variable} + xloc: 0.5 + yloc: -0.13 + kwargs: + color: 'red' + fontsize: 8 + fontfamily: monospace + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Histogram + data: + variable: experiment_2::ombgPassedQc::${variable} + color: 'red' + label: 'observations minus background ' + bins: ${dynamic_bins} + alpha: 0.5 + density: true + - type: Histogram + data: + variable: experiment_2::omanPassedQc::${variable} + color: 'blue' + label: 'observations minus analysis' + bins: ${dynamic_bins} + alpha: 0.5 + density: true + + # Map plots + # --------- + # Increment + - batch figure: + variables: *variables + dynamic options: + - type: vminvmaxcmap + data variable: experiment_1::ombgPassedQc::${variable} + figure: + figure size: [60,30] + layout: [4,1] + title: '{{instrument_title}} | Passed QC' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}/ombg_oman_{{instrument}}_${variable}.png' + tight_layout: + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_title: '{{experiment_tag_1}}' + add_colorbar: + label: ObsValue + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment_1::MetaData::longitude + latitude: + variable: experiment_1::MetaData::latitude + data: + variable: experiment_1::ombgPassedQc::${variable} + markersize: 1 + label: OmAn + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_title: '{{experiment_tag_2}}' + add_colorbar: + label: ObsValue + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment_2::MetaData::longitude + latitude: + variable: experiment_2::MetaData::latitude + data: + variable: experiment_2::ombgPassedQc::${variable} + markersize: 1 + label: OmAn + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_title: '{{experiment_tag_1}}' + add_colorbar: + label: ObsValue + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment_1::MetaData::longitude + latitude: + variable: experiment_1::MetaData::latitude + data: + variable: experiment_1::omanPassedQc::${variable} + markersize: 1 + label: OmBg + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_title: '{{experiment_tag_2}}' + add_colorbar: + label: ObsValue + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment_2::MetaData::longitude + latitude: + variable: experiment_2::MetaData::latitude + data: + variable: experiment_2::omanPassedQc::${variable} + markersize: 1 + label: OmBg + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + - batch figure: + variables: *variables + dynamic options: + - type: vminvmaxcmap + data variable: experiment_1::EffectiveQC1::${variable} + figure: + figure size: [40,20] + layout: [2,1] + title: '{{instrument_title}} | Passed QC' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}/effectiveQC_{{instrument}}_${variable}.png' + tight_layout: + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_title: '{{experiment_tag_1}}' + add_colorbar: + label: EffectiveQC1 + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment_1::MetaData::longitude + latitude: + variable: experiment_1::MetaData::latitude + data: + variable: experiment_1::EffectiveQC1::${variable} + markersize: 2 + label: OmAn + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_title: '{{experiment_tag_2}}' + add_colorbar: + label: EffectiveQC1 + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment_2::MetaData::longitude + latitude: + variable: experiment_2::MetaData::latitude + data: + variable: experiment_2::EffectiveQC1::${variable} + markersize: 2 + label: OmAn + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + - batch figure: + variables: *variables + dynamic options: + - type: vminvmaxcmap + data variable: experiment_1::increment::${variable} + figure: + figure size: [40,20] + layout: [2,1] + title: '{{instrument_title}} | Passed QC' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}/increment_{{instrument}}_${variable}.png' + tight_layout: + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_title: '{{experiment_tag_1}}' + add_colorbar: + label: Increment (OmBg - OmAn) + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment_1::MetaData::longitude + latitude: + variable: experiment_1::MetaData::latitude + data: + variable: experiment_1::increment::${variable} + markersize: 2 + label: OmAn + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_title: '{{experiment_tag_2}}' + add_colorbar: + label: Increment (OmBg - OmAn) + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment_2::MetaData::longitude + latitude: + variable: experiment_2::MetaData::latitude + data: + variable: experiment_2::increment::${variable} + markersize: 2 + label: OmAn + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} diff --git a/src/swell/suites/compare/suite_config.py b/src/swell/suites/compare/suite_config.py index 9f8c4807e..7847c366a 100644 --- a/src/swell/suites/compare/suite_config.py +++ b/src/swell/suites/compare/suite_config.py @@ -57,6 +57,17 @@ class SuiteConfig(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + compare_variational_cf = QuestionList( + list_name="compare_variational_cf", + questions=[ + compare, + qd.comparison_log_type('variational'), + qd.model_components(['geos_cf']), + ] + ) + + # -------------------------------------------------------------------------------------------------- + compare_fgat_marine = QuestionList( list_name="compare_fgat_marine", questions=[ From dd017a6f1add1d536d0ee35d66605bc3057d27df Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 1 May 2026 15:02:16 -0400 Subject: [PATCH 289/299] move requests import --- src/swell/tasks/ingest_obs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/swell/tasks/ingest_obs.py b/src/swell/tasks/ingest_obs.py index 43442b359..a3ff73387 100644 --- a/src/swell/tasks/ingest_obs.py +++ b/src/swell/tasks/ingest_obs.py @@ -15,8 +15,6 @@ import yaml from datetime import datetime -import requests - from swell.tasks.base.task_base import taskBase from swell.tasks.base.task_setup import TaskSetup from swell.tasks.base.task_attributes import task_attributes @@ -171,6 +169,7 @@ def process_obs_config( ) -> tuple[list[str], list[tuple[str, str]]]: import r2d2 + import requests """Process a single observation configuration file.""" ingested = [] From cf0c8753b80d2382709fca127ba1da71840e6849 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 1 May 2026 15:11:35 -0400 Subject: [PATCH 290/299] Code test fixes --- src/swell/utilities/slurm.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index d9853a0e1..41a7cfe51 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -140,9 +140,11 @@ def slurm_global_defaults( with open(yaml_path, "r") as yaml_file: user_globals['slurm_directives_global'] = yaml.safe_load(yaml_file) ''' - yaml = YAML(typ='safe') - with open(yaml_path, 'r') as yaml_file: - user_globals = yaml.load(yaml_file) + user_globals = {} + if os.path.exists(yaml_path): + yaml = YAML(typ='safe') + with open(yaml_path, 'r') as yaml_file: + user_globals = yaml.load(yaml_file) return user_globals # -------------------------------------------------------------------------------------------------- From 644bc6e8632488f8117cd4a93fbedcd4fd07772d Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 1 May 2026 15:24:48 -0400 Subject: [PATCH 291/299] Fix templating --- src/swell/suites/3dfgat_marine_cycle/workflow.py | 2 +- src/swell/suites/3dvar_marine_cycle/workflow.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/suites/3dfgat_marine_cycle/workflow.py b/src/swell/suites/3dfgat_marine_cycle/workflow.py index a0891c227..6497f8894 100644 --- a/src/swell/suites/3dfgat_marine_cycle/workflow.py +++ b/src/swell/suites/3dfgat_marine_cycle/workflow.py @@ -147,7 +147,7 @@ class Workflow_3dfgat_marine_cycle(CylcWorkflow): def get_workflow_string(self): workflow_str = self.default_header() - + workflow_str += template_str for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) diff --git a/src/swell/suites/3dvar_marine_cycle/workflow.py b/src/swell/suites/3dvar_marine_cycle/workflow.py index 86269da5d..a70490510 100644 --- a/src/swell/suites/3dvar_marine_cycle/workflow.py +++ b/src/swell/suites/3dvar_marine_cycle/workflow.py @@ -146,7 +146,7 @@ class Workflow_3dvar_marine_cycle(CylcWorkflow): def get_workflow_string(self): workflow_str = self.default_header() - + workflow_str += template_str for task in self.tasks: workflow_str += task.runtime_string(self.experiment_dict, self.slurm_external) From 63f905587e73deccc0f694754c467737333811a1 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 12 May 2026 11:16:36 -0400 Subject: [PATCH 292/299] Code test fix --- src/swell/utilities/question_defaults.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index db830bacd..8c2f2cd82 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -825,7 +825,7 @@ class obs_to_download(TaskQuestion): default_value: list = mutable_field([]) question_name: str = "obs_to_download" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Which observations do you want to download from remote servers?" @@ -838,7 +838,7 @@ class converter_path(TaskQuestion): default_value: str = "" question_name: str = "converter_path" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = ("Path to directory containing ioda-converter scripts" From 19d4776ecce34fe36e003aa84f91be7ac2117665 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 18 May 2026 16:55:17 -0400 Subject: [PATCH 293/299] change config name --- src/swell/suites/3dvar_marine/workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/suites/3dvar_marine/workflow.py b/src/swell/suites/3dvar_marine/workflow.py index 48922faf4..96d6f266c 100644 --- a/src/swell/suites/3dvar_marine/workflow.py +++ b/src/swell/suites/3dvar_marine/workflow.py @@ -112,7 +112,7 @@ @workflows.register('3dvar_marine') -class Workflow_3dvar(CylcWorkflow): +class Workflow_3dvar_marine(CylcWorkflow): def get_workflow_string(self): workflow_str = self.default_header() From 9cb27425017c2059a6be7fe42b22426e15d3b6ac Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 19 May 2026 16:43:17 -0400 Subject: [PATCH 294/299] test fixes --- pyproject.toml | 3 ++- src/swell/configuration/question_defaults.py | 2 +- src/swell/suites/ingest_obs/suite_config.py | 1 - src/swell/tasks/convert_obs_to_ioda.py | 1 + src/swell/tasks/download_obs.py | 1 + src/swell/tasks/publish_comparisons.py | 2 ++ src/swell/utilities/suite_utils.py | 1 + 7 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 759952ddc..bae3766bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,8 @@ dependencies = [ "flake8==6.1.0", "netCDF4", "ruamel.yaml==0.17.16", - "xarray==2024.7.0" + "xarray==2024.7.0", + "requests>=2.23.0" ] [project.optional-dependencies] diff --git a/src/swell/configuration/question_defaults.py b/src/swell/configuration/question_defaults.py index 0ffdf732d..ec1a19665 100644 --- a/src/swell/configuration/question_defaults.py +++ b/src/swell/configuration/question_defaults.py @@ -69,7 +69,7 @@ class download_convert_pipeline(SuiteQuestion): "(DownloadObs -> ConvertObsToIoda) -> IngestObs to R2D2") widget_type: WType = WType.BOOLEAN -# -------------------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------------------- @dataclass diff --git a/src/swell/suites/ingest_obs/suite_config.py b/src/swell/suites/ingest_obs/suite_config.py index 08eee3614..cdeab38c8 100644 --- a/src/swell/suites/ingest_obs/suite_config.py +++ b/src/swell/suites/ingest_obs/suite_config.py @@ -76,4 +76,3 @@ suite_configs.register(suite_name, 'ingest_obs_cf', ingest_obs_cf) # -------------------------------------------------------------------------------------------------- - diff --git a/src/swell/tasks/convert_obs_to_ioda.py b/src/swell/tasks/convert_obs_to_ioda.py index 09ee5e7d0..045fba9d2 100644 --- a/src/swell/tasks/convert_obs_to_ioda.py +++ b/src/swell/tasks/convert_obs_to_ioda.py @@ -42,6 +42,7 @@ def set_defaults(self): # -------------------------------------------------------------------------------------------------- + class ConvertObsToIoda(taskBase): """Convert downloaded raw observation files to IODA format. diff --git a/src/swell/tasks/download_obs.py b/src/swell/tasks/download_obs.py index 28cacd368..2507123c6 100644 --- a/src/swell/tasks/download_obs.py +++ b/src/swell/tasks/download_obs.py @@ -40,6 +40,7 @@ def set_defaults(self): # -------------------------------------------------------------------------------------------------- + class DownloadObs(taskBase): """Download raw observation files from a remote HTTPS server. diff --git a/src/swell/tasks/publish_comparisons.py b/src/swell/tasks/publish_comparisons.py index adae35255..94a5c843c 100644 --- a/src/swell/tasks/publish_comparisons.py +++ b/src/swell/tasks/publish_comparisons.py @@ -18,6 +18,8 @@ # -------------------------------------------------------------------------------------------------- task_name = 'PublishComparisons' + + @task_attributes.register(task_name) class Setup(TaskSetup): def set_defaults(self): diff --git a/src/swell/utilities/suite_utils.py b/src/swell/utilities/suite_utils.py index afd1681a7..c2acd812c 100644 --- a/src/swell/utilities/suite_utils.py +++ b/src/swell/utilities/suite_utils.py @@ -39,6 +39,7 @@ def read_override_file(override_path: str | None) -> dict: # -------------------------------------------------------------------------------------------------- + def read_override_file(override_path: str | None) -> dict: if override_path is None: From 41adba4af59d4f550cbadb10084c08baa202b07d Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 9 Jun 2026 17:02:07 -0400 Subject: [PATCH 295/299] Code test fix --- src/swell/tasks/get_restart_cf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/get_restart_cf.py b/src/swell/tasks/get_restart_cf.py index 37d717e89..8ee079c7f 100644 --- a/src/swell/tasks/get_restart_cf.py +++ b/src/swell/tasks/get_restart_cf.py @@ -16,7 +16,6 @@ import isodate import os -import r2d2 # ---------------------------------------------------------------------------------------------- @@ -46,6 +45,8 @@ def execute(self) -> None: """ + import r2d2 + # Load R2D2 credentials # --------------------- load_r2d2_credentials(self.logger, self.platform()) From 9bb8a919456adb91de78e9fefa14a5ba95f88cd7 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 9 Jun 2026 17:11:17 -0400 Subject: [PATCH 296/299] Remove r2d2 imports --- src/swell/tasks/save_forecast_cf.py | 3 ++- src/swell/tasks/save_restart_cf.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/swell/tasks/save_forecast_cf.py b/src/swell/tasks/save_forecast_cf.py index db26478ee..702605fd5 100644 --- a/src/swell/tasks/save_forecast_cf.py +++ b/src/swell/tasks/save_forecast_cf.py @@ -10,7 +10,6 @@ import isodate import os -from r2d2 import store import swell.configuration.question_defaults as qd from swell.configuration.jedi.interfaces.geos_cf.model.r2d2 import forecast_filename, r2d2 @@ -53,6 +52,8 @@ def execute(self) -> None: See the taskBase constructor for more information. """ + from r2d2 import store + # Load R2D2 credentials # --------------------- load_r2d2_credentials(self.logger, self.platform()) diff --git a/src/swell/tasks/save_restart_cf.py b/src/swell/tasks/save_restart_cf.py index 26edf8166..077651415 100644 --- a/src/swell/tasks/save_restart_cf.py +++ b/src/swell/tasks/save_restart_cf.py @@ -9,7 +9,6 @@ import isodate import os -from r2d2 import store import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase @@ -48,6 +47,8 @@ def execute(self): can be retrieved by GetRestartCf in the subsequent cycle. """ + from r2d2 import store + model = self.__model__ # Load R2D2 credentials From 33f4f509347f317c3e745f40089e3842693307f5 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Thu, 11 Jun 2026 10:02:56 -0400 Subject: [PATCH 297/299] Use native type hinting --- src/swell/configuration/question_defaults.py | 270 +++++++++---------- 1 file changed, 135 insertions(+), 135 deletions(-) diff --git a/src/swell/configuration/question_defaults.py b/src/swell/configuration/question_defaults.py index f03e62e56..d890274e5 100644 --- a/src/swell/configuration/question_defaults.py +++ b/src/swell/configuration/question_defaults.py @@ -9,7 +9,7 @@ from dataclasses import dataclass -from typing import List, Dict, Union, Literal +from typing import Literal from swell.utilities.swell_questions import SuiteQuestion, TaskQuestion from swell.utilities.swell_questions import WidgetType as WType @@ -37,7 +37,7 @@ class cycle_times(SuiteQuestion): question_name: str = "cycle_times" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Enter the cycle times for this model." @@ -51,7 +51,7 @@ class cycling_varbc(SuiteQuestion): default_value: str = "defer_to_model" question_name: str = "cycling_varbc" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Do you want to use cycling VarBC option?" @@ -86,7 +86,7 @@ class email_address(SuiteQuestion): class ensemble_hofx_packets(SuiteQuestion): default_value: str = "defer_to_model" question_name: str = "ensemble_hofx_packets" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Enter the number of ensemble packets." @@ -99,7 +99,7 @@ class ensemble_hofx_packets(SuiteQuestion): class ensemble_hofx_strategy(SuiteQuestion): default_value: str = "defer_to_model" question_name: str = "ensemble_hofx_strategy" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Enter the ensemble hofx strategy." @@ -148,7 +148,7 @@ class marine_models(SuiteQuestion): question_name: str = "marine_models" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_marine" ]) prompt: str = "Select the active SOCA models for this model." @@ -186,7 +186,7 @@ class parser_options(SuiteQuestion): question_name: str = "parser_options" ask_question: bool = True options: list = mutable_field(['fgrep_residual_norm']) - prompt: str = "List the test types to run on the JEDI oops log." + prompt: str = "list the test types to run on the JEDI oops log." widget_type: WType = WType.STRING_CHECK_LIST # -------------------------------------------------------------------------------------------------- @@ -280,7 +280,7 @@ class saber_outer_block(SuiteQuestion): class skip_ensemble_hofx(SuiteQuestion): default_value: str = "defer_to_model" question_name: str = "skip_ensemble_hofx" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Enter if skip ensemble hofx." @@ -326,11 +326,11 @@ class use_cycle_dir(SuiteQuestion): class window_type(SuiteQuestion): default_value: str = "defer_to_model" question_name: str = "window_type" - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "3D", "4D" ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Enter the window type for this model." @@ -346,7 +346,7 @@ class analysis_variables(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "analysis_variables" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What are the analysis variables?" @@ -360,7 +360,7 @@ class background_error_model(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "background_error_model" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Which background error model do you want to use?" @@ -374,7 +374,7 @@ class background_experiment(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "background_experiment" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the name of the name of the experiment providing the backgrounds?" @@ -387,10 +387,10 @@ class background_experiment(TaskQuestion): class background_frequency(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "background_frequency" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "window_type": "4D" }) prompt: str = "What is the frequency of the background files?" @@ -403,7 +403,7 @@ class background_frequency(TaskQuestion): class background_time_offset(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "background_time_offset" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = ("How long before the middle of the analysis window did" @@ -419,7 +419,7 @@ class bufr_obs_classes(TaskQuestion): question_name: str = "bufr_obs_classes" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What BUFR observation classes will be used?" @@ -430,7 +430,7 @@ class bufr_obs_classes(TaskQuestion): @dataclass class bundles(TaskQuestion): - default_value: List[str] = mutable_field([ + default_value: list[str] = mutable_field([ "fv3-jedi", "soca", "iodaconv", @@ -438,7 +438,7 @@ class bundles(TaskQuestion): ]) question_name: str = "bundles" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "fv3-jedi", "soca", "iodaconv", @@ -447,7 +447,7 @@ class bundles(TaskQuestion): "oops", "saber" ]) - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "jedi_build_method": "create" }) prompt: str = "Which JEDI bundles do you wish to build?" @@ -459,7 +459,7 @@ class bundles(TaskQuestion): class cache_fetch(TaskQuestion): default_value: bool = True question_name: str = "cache_fetch" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) @@ -473,8 +473,8 @@ class cache_fetch(TaskQuestion): class check_for_obs(TaskQuestion): default_value: bool = True question_name: str = "check_for_obs" - options: List[bool] = mutable_field([True, False]) - models: List[str] = mutable_field([ + options: list[bool] = mutable_field([True, False]) + models: list[str] = mutable_field([ 'all_models' ]) prompt: str = "Perform check for observations? Set to false for debugging purposes." @@ -488,7 +488,7 @@ class clean_patterns(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "clean_patterns" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Provide a list of patterns that you wish to remove from the cycle directory." @@ -501,11 +501,11 @@ class clean_patterns(TaskQuestion): class comparison_log_type(TaskQuestion): default_value: str = "variational" question_name: str = "comparison_log_type" - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ 'variational', 'fgat', ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Provide the log naming convention (e.g. 'variational', 'fgat')." @@ -519,7 +519,7 @@ class converter_path(TaskQuestion): default_value: str = "" question_name: str = "converter_path" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = ("Path to directory containing ioda-converter scripts" @@ -533,7 +533,7 @@ class converter_path(TaskQuestion): class crtm_coeff_dir(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "crtm_coeff_dir" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the path to the CRTM coefficient files?" @@ -547,7 +547,7 @@ class dry_run(TaskQuestion): default_value: bool = True question_name: str = "dry_run" ask_question: bool = False - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Dry-run mode: preview what would be ingested before storing to R2D2" @@ -562,7 +562,7 @@ class ensemble_hofx_packets(TaskQuestion): question_name: str = "ensemble_hofx_packets" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Enter number of packets in which ensemble observers should be computed." @@ -577,7 +577,7 @@ class ensemble_hofx_strategy(TaskQuestion): question_name: str = "ensemble_hofx_strategy" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Enter hofx strategy." @@ -591,7 +591,7 @@ class ensemble_num_members(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "ensemble_num_members" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "How many members comprise the ensemble?" @@ -604,11 +604,11 @@ class ensemble_num_members(TaskQuestion): class ensmean_only(TaskQuestion): default_value: bool = False question_name: str = "ensmean_only" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Calculate ensemble mean only?" @@ -621,11 +621,11 @@ class ensmean_only(TaskQuestion): class ensmeanvariance_only(TaskQuestion): default_value: bool = False question_name: str = "ensmeanvariance_only" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Calculate ensemble mean and variance only?" @@ -639,7 +639,7 @@ class existing_geos_gcm_build_path(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "existing_geos_gcm_build_path" ask_question: bool = True - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "geos_build_method": "use_existing" }) prompt: str = "What is the path to the existing GEOS build directory?" @@ -653,7 +653,7 @@ class existing_geos_gcm_source_path(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "existing_geos_gcm_source_path" ask_question: bool = True - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "geos_build_method": "use_existing" }) prompt: str = "What is the path to the existing GEOS source code directory?" @@ -667,7 +667,7 @@ class existing_jedi_build_directory(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "existing_jedi_build_directory" ask_question: bool = True - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "jedi_build_method": "use_existing" }) prompt: str = "What is the path to the existing JEDI build directory?" @@ -681,7 +681,7 @@ class existing_jedi_build_directory_pinned(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "existing_jedi_build_directory_pinned" ask_question: bool = True - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "jedi_build_method": "use_pinned_existing" }) prompt: str = "What is the path to the existing pinned JEDI build directory?" @@ -695,7 +695,7 @@ class existing_jedi_source_directory(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "existing_jedi_source_directory" ask_question: bool = True - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "jedi_build_method": "use_existing" }) prompt: str = "What is the path to the existing JEDI source code directory?" @@ -709,7 +709,7 @@ class existing_jedi_source_directory_pinned(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "existing_jedi_source_directory_pinned" ask_question: bool = True - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "jedi_build_method": "use_pinned_existing" }) prompt: str = "What is the path to the existing pinned JEDI source code directory?" @@ -744,7 +744,7 @@ class forecast_length(TaskQuestion): default_value: str = "PT12H" question_name: str = "forecast_length" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "Duration of the GEOS-CF forecast (ISO 8601 duration, e.g. PT12H)" @@ -758,7 +758,7 @@ class forecast_output_frequency(TaskQuestion): default_value: str = "PT1H" question_name: str = "forecast_output_frequency" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "Frequency of forecast output files (ISO 8601 duration, e.g. PT1H)" @@ -782,7 +782,7 @@ class geos_build_method(TaskQuestion): default_value: str = "create" question_name: str = "geos_build_method" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "use_existing", "create" ]) @@ -797,7 +797,7 @@ class geos_cf_install_dir(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "geos_cf_install_dir" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "What is the path to the GEOS-CF install directory?" @@ -811,7 +811,7 @@ class geos_cf_run_dir(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "geos_cf_run_dir" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "What is the path to the GEOS-CF model run directory?" @@ -824,7 +824,7 @@ class geos_cf_run_dir(TaskQuestion): class geos_expdir(TaskQuestion): default_value: str = "/dev/null/" question_name: str = "geos_expdir" - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "geos_expdir_different": True }) prompt: str = ("What is the location for the EXPERIMENT Directory (to contain model " @@ -840,7 +840,7 @@ class geos_expdir_different(TaskQuestion): default_value: str = False question_name: str = "geos_expdir_different" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) @@ -867,7 +867,7 @@ class geos_experiment_directory(TaskQuestion): class geos_gcm_tag(TaskQuestion): default_value: str = "v11.6.0" question_name: str = "geos_gcm_tag" - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "geos_build_method": "create" }) prompt: str = "Which GEOS tag do you wish to clone?" @@ -904,11 +904,11 @@ class geos_x_background_directory(TaskQuestion): default_value: str = "/dev/null/" question_name: str = "geos_x_background_directory" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "/dev/null/", "/discover/nobackup/projects/gmao/dadev/rtodling/archive/Restarts/JEDI/541x" ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the path to the GEOS X-backgrounds directory?" @@ -922,11 +922,11 @@ class geos_x_ensemble_directory(TaskQuestion): default_value: str = "/dev/null/" question_name: str = "geos_x_ensemble_directory" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "/dev/null/", "/gpfsm/dnb05/projects/p139/rtodling/archive/" ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the path to the GEOS X-backgrounds directory?" @@ -940,7 +940,7 @@ class geosfp_exp(TaskQuestion): default_value: str = "f5295_fp" question_name: str = "geosfp_exp" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "What is the GEOS FP experiment ID used for IAU analysis files?" @@ -954,7 +954,7 @@ class geosfp_path(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "geosfp_path" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "What is the path to the GEOS FP archive?" @@ -968,7 +968,7 @@ class geovals_experiment(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "geovals_experiment" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the name of the R2D2 experiment providing the GeoVaLs?" @@ -981,7 +981,7 @@ class geovals_experiment(TaskQuestion): class geovals_provider(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "geovals_provider" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the name of the R2D2 database providing the GeoVaLs?" @@ -1004,7 +1004,7 @@ class gmao_perllib_tag(TaskQuestion): class gradient_norm_reduction(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "gradient_norm_reduction" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What value of gradient norm reduction for convergence?" @@ -1017,7 +1017,7 @@ class gradient_norm_reduction(TaskQuestion): class gsibec_configuration(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "gsibec_configuration" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Which GSIBEC climatological or hybrid?" @@ -1030,7 +1030,7 @@ class gsibec_configuration(TaskQuestion): class gsibec_nlats(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "gsibec_nlats" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "How many number of latutides in GSIBEC grid?" @@ -1043,7 +1043,7 @@ class gsibec_nlats(TaskQuestion): class gsibec_nlons(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "gsibec_nlons" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "How many number of longitudes in GSIBEC grid?" @@ -1056,7 +1056,7 @@ class gsibec_nlons(TaskQuestion): class horizontal_localization_lengthscale(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "horizontal_localization_lengthscale" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the length scale for horizontal covariance localization?" @@ -1069,7 +1069,7 @@ class horizontal_localization_lengthscale(TaskQuestion): class horizontal_localization_max_nobs(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "horizontal_localization_max_nobs" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ("What is the maximum number of observations to consider" @@ -1084,7 +1084,7 @@ class horizontal_localization_method(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "horizontal_localization_method" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Which localization scheme should be applied in the horizontal?" @@ -1099,7 +1099,7 @@ class horizontal_resolution(TaskQuestion): question_name: str = "horizontal_resolution" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the horizontal resolution for the forecast model and backgrounds?" @@ -1113,11 +1113,11 @@ class iau(TaskQuestion): default_value: bool = True question_name: str = "iau" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "Use Incremental Analysis Update (IAU) in the GEOS-CF forecast?" @@ -1131,7 +1131,7 @@ class inc_template(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "inc_template" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "What is the path to the GEOS-CF increment template NetCDF file?" @@ -1145,7 +1145,7 @@ class initial_restarts_method(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "initial_restarts_method" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "geos_expdir", "r2d2", "hotstart", @@ -1161,7 +1161,7 @@ class ioda_locations_not_in_r2d2(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "ioda_locations_not_in_r2d2" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ( @@ -1176,7 +1176,7 @@ class jedi_build_method(TaskQuestion): default_value: str = "create" question_name: str = "jedi_build_method" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "use_existing", "use_pinned_existing", "create", @@ -1194,10 +1194,10 @@ class jedi_forecast_model(TaskQuestion): question_name: str = "jedi_forecast_model" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) - depends: Dict = mutable_field({ + depends: dict = mutable_field({ "window_type": "4D" }) prompt: str = "What forecast model should be used within JEDI for 4D window propagation?" @@ -1210,7 +1210,7 @@ class jedi_forecast_model(TaskQuestion): class local_ensemble_inflation_mult(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "local_ensemble_inflation_mult" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Specify the multiplicative prior inflation coefficient (0 inf]." @@ -1223,7 +1223,7 @@ class local_ensemble_inflation_mult(TaskQuestion): class local_ensemble_inflation_rtpp(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "local_ensemble_inflation_rtpp" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Specify the Relaxation To Prior Perturbation (RTPP) coefficient (0 1]." @@ -1236,7 +1236,7 @@ class local_ensemble_inflation_rtpp(TaskQuestion): class local_ensemble_inflation_rtps(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "local_ensemble_inflation_rtps" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Specify the Relaxation To Prior Spread (RTPS) coefficient (0 1]." @@ -1249,11 +1249,11 @@ class local_ensemble_inflation_rtps(TaskQuestion): class local_ensemble_save_posterior_ensemble(TaskQuestion): default_value: bool = False question_name: str = "local_ensemble_save_posterior_ensemble" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Save the posterior ensemble members?" @@ -1267,11 +1267,11 @@ class local_ensemble_save_posterior_ensemble_increments(TaskQuestion): default_value: bool = False question_name: str = "local_ensemble_save_posterior_ensemble_increments" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Save the posterior ensemble member increments?" @@ -1285,11 +1285,11 @@ class local_ensemble_save_posterior_mean(TaskQuestion): default_value: bool = False question_name: str = "local_ensemble_save_posterior_mean" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Save the posterior ensemble mean?" @@ -1303,11 +1303,11 @@ class local_ensemble_save_posterior_mean_increment(TaskQuestion): default_value: bool = True question_name: str = "local_ensemble_save_posterior_mean_increment" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Save the posterior ensemble mean increment?" @@ -1322,7 +1322,7 @@ class local_ensemble_solver(TaskQuestion): question_name: str = "local_ensemble_solver" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Which local ensemble solver type should be implemented?" @@ -1337,7 +1337,7 @@ class local_ensemble_use_linear_observer(TaskQuestion): question_name: str = "local_ensemble_use_linear_observer" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Which local ensemble solver type should be implemented?" @@ -1351,7 +1351,7 @@ class minimizer(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "minimizer" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Which data assimilation minimizer do you wish to use?" @@ -1364,11 +1364,11 @@ class minimizer(TaskQuestion): class mom6_iau(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "mom6_iau" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_marine", ]) prompt: str = "Do you wish to use IAU for MOM6?" @@ -1381,12 +1381,12 @@ class mom6_iau(TaskQuestion): class mom6_iau_nhours(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "mom6_iau_nhours" - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ 'PT3H', 'PT12H' ]) depends: dict = mutable_field({'mom6_iau': True}) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_marine", ]) prompt: str = "What is the IAU length (ODA_INCUPD_NHOURS) for MOM6?" @@ -1399,8 +1399,8 @@ class mom6_iau_nhours(TaskQuestion): class ncdiag_experiments(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "ncdiag_experiments" - options: List[str] = "defer_to_model" - models: List[str] = mutable_field([ + options: list[str] = "defer_to_model" + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Which previously run experiments do you wish to use for the NCdiag?" @@ -1414,7 +1414,7 @@ class npx(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "npx" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "What is the number of grid points in the x-direction on each cube face?" @@ -1428,7 +1428,7 @@ class npx_proc(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "npx_proc" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere", "geos_cf" ]) @@ -1443,7 +1443,7 @@ class npy(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "npy" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "What is the number of grid points in the y-direction on each cube face?" @@ -1457,7 +1457,7 @@ class npy_proc(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "npy_proc" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere", "geos_cf" ]) @@ -1471,7 +1471,7 @@ class npy_proc(TaskQuestion): class number_of_iterations(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "number_of_iterations" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = ( @@ -1487,7 +1487,7 @@ class obs_experiment(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "obs_experiment" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the database providing the observations?" @@ -1500,7 +1500,7 @@ class obs_experiment(TaskQuestion): class obs_thinning_rej_fraction(TaskQuestion): default_value: float = 0.75 question_name: str = "obs_thinning_rej_fraction" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the rejection fraction for obs thinning?" @@ -1514,7 +1514,7 @@ class obs_to_download(TaskQuestion): default_value: list = mutable_field([]) question_name: str = "obs_to_download" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Which observations do you want to download from remote servers?" @@ -1529,7 +1529,7 @@ class obs_to_ingest(TaskQuestion): question_name: str = "obs_to_ingest" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Which observations do you want to ingest to R2D2?" @@ -1544,7 +1544,7 @@ class observations(TaskQuestion): question_name: str = "observations" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Which observations do you want to include?" @@ -1557,7 +1557,7 @@ class observations(TaskQuestion): class observing_system_records_mksi_path(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "observing_system_records_mksi_path" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the path to the GSI formatted observing system records?" @@ -1570,7 +1570,7 @@ class observing_system_records_mksi_path(TaskQuestion): class observing_system_records_mksi_path_tag(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "observing_system_records_mksi_path_tag" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the GSI formatted observing system records tag?" @@ -1583,7 +1583,7 @@ class observing_system_records_mksi_path_tag(TaskQuestion): class observing_system_records_path(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "observing_system_records_path" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the path to the Swell formatted observing system records?" @@ -1597,7 +1597,7 @@ class path_to_ensemble(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "path_to_ensemble" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere", "geos_marine" ]) @@ -1612,7 +1612,7 @@ class path_to_geos_adas_background(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "path_to_geos_adas_background" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ( @@ -1627,7 +1627,7 @@ class path_to_gsi_bc_coefficients(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "path_to_gsi_bc_coefficients" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the location where GSI bias correction files can be found?" @@ -1641,7 +1641,7 @@ class path_to_gsi_nc_diags(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "path_to_gsi_nc_diags" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the path to where the GSI ncdiags are stored?" @@ -1667,11 +1667,11 @@ class perhost(TaskQuestion): default_value: str = None question_name: str = "perhost" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the number of processors per host?" @@ -1685,11 +1685,11 @@ class produce_geovals(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "produce_geovals" ask_question: bool = True - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ("When running the ncdiag to ioda converted do you " @@ -1704,7 +1704,7 @@ class rst_experiment(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "rst_experiment" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "What is the name of the experiment providing the restart files in R2D2?" @@ -1718,7 +1718,7 @@ class rst_file_types(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "rst_file_types" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = "What are the restart file types to fetch/store from R2D2?" @@ -1732,7 +1732,7 @@ class rst_store_interval(TaskQuestion): default_value: str = None question_name: str = "rst_store_interval" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_cf" ]) prompt: str = ("After how many cycles should restart files be stored as real files" @@ -1747,7 +1747,7 @@ class rst_store_interval(TaskQuestion): class save_geovals(TaskQuestion): default_value: bool = False question_name: str = "save_geovals" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) @@ -1761,11 +1761,11 @@ class save_geovals(TaskQuestion): class single_observations(TaskQuestion): default_value: bool = False question_name: str = "single_observations" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Is it a single-observation test?" @@ -1778,7 +1778,7 @@ class single_observations(TaskQuestion): class swell_static_files(TaskQuestion): default_value: str = "defer_to_platform" question_name: str = "swell_static_files" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the path to the Swell Static files directory?" @@ -1799,7 +1799,7 @@ class swell_static_files_user(TaskQuestion): @dataclass class task_email_parameters(TaskQuestion): - default_value: Union[Literal["auto"], dict] = "auto" + default_value: Literal["auto"] | dict = "auto" question_name: str = "task_email_parameters" prompt: str = ("Provide a dictionary mapping tasks to cylc event statuses, or 'auto' to " "automatically configure these based on the graph.") @@ -1813,7 +1813,7 @@ class total_processors(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "total_processors" ask_question: bool = True - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_marine", ]) prompt: str = "What is the number of processors for JEDI?" @@ -1826,11 +1826,11 @@ class total_processors(TaskQuestion): class vertical_localization_apply_log_transform(TaskQuestion): default_value: bool = True question_name: str = "vertical_localization_apply_log_transform" - options: List[bool] = mutable_field([ + options: list[bool] = mutable_field([ True, False ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ("Should a log (base 10) transformation be applied " @@ -1846,7 +1846,7 @@ class vertical_localization_function(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "vertical_localization_function" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Which localization scheme should be applied in the vertical?" @@ -1860,7 +1860,7 @@ class vertical_localization_ioda_vertical_coord(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "vertical_localization_ioda_vertical_coord" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "Which coordinate should be used in constructing vertical localization?" @@ -1874,7 +1874,7 @@ class vertical_localization_ioda_vertical_coord_group(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "vertical_localization_ioda_vertical_coord_group" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ("Which vertical coordinate group should be used " @@ -1888,7 +1888,7 @@ class vertical_localization_ioda_vertical_coord_group(TaskQuestion): class vertical_localization_lengthscale(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "vertical_localization_lengthscale" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the length scale for vertical covariance localization?" @@ -1902,7 +1902,7 @@ class vertical_localization_method(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "vertical_localization_method" options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = ("What localization scheme should be applied in " @@ -1918,7 +1918,7 @@ class vertical_resolution(TaskQuestion): question_name: str = "vertical_resolution" ask_question: bool = True options: str = "defer_to_model" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the vertical resolution for the forecast model and background?" @@ -1931,7 +1931,7 @@ class vertical_resolution(TaskQuestion): class window_length(TaskQuestion): default_value: str = "defer_to_model" question_name: str = "window_length" - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "What is the duration for the data assimilation window?" @@ -1945,11 +1945,11 @@ class window_type(TaskQuestion): question_name: str = "window_type" default_value: str = "defer_to_model" ask_question: bool = True - options: List[str] = mutable_field([ + options: list[str] = mutable_field([ "3D", "4D" ]) - models: List[str] = mutable_field([ + models: list[str] = mutable_field([ "all_models" ]) prompt: str = "Do you want to use a 3D or 4D (including FGAT) window?" From 72e36f4488f4cde1bfd60572dc23533e43421bab Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 15 Jun 2026 09:43:24 -0400 Subject: [PATCH 298/299] address remaining comments --- src/swell/tasks/base/task_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 12bec65d3..5127fb3b4 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -193,7 +193,7 @@ def cycle_dir(self) -> str: 'should not be called if the task does not receive model.') # Check whether to send to cycle dir - if self.config.use_cycle_dir(True): + if self.config.use_cycle_dir(): # Combine datetime string (directory format) with the model cycle_dir = os.path.join(self.experiment_path(), 'run', From 675aad18e318145579c80be13d02aa2860b43725 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 15 Jun 2026 10:00:16 -0400 Subject: [PATCH 299/299] Code test fix --- src/swell/tasks/base/task_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 5127fb3b4..e4de2f49d 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -193,7 +193,8 @@ def cycle_dir(self) -> str: 'should not be called if the task does not receive model.') # Check whether to send to cycle dir - if self.config.use_cycle_dir(): + # Set to true since not set by default + if self.config.use_cycle_dir(True): # Combine datetime string (directory format) with the model cycle_dir = os.path.join(self.experiment_path(), 'run',