Restart writing in CAM-SIMA

.gitmodules

@@ -6,8 +6,8 @@
 	fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework
 [submodule "history"]
 	path          = src/history/buffers
-	url           = https://github.com/ESMCI/history_output
-	fxtag         = history01_02
+	url           = https://github.com/peverwhee/history_output
+	fxtag         = a8e4bf4ffd69e1332aebab7f5d34ddf2ffb228fd
 	fxrequired    = AlwaysRequired
 	fxDONOTUSEurl = https://github.com/ESMCI/history_output
 [submodule "mpas"]

cime_config/buildlib

@@ -84,6 +84,7 @@ def _build_cam():
         dycore = config.get_value('dyn')
         reg_dir = config.get_value('reg_dir')
         init_dir = config.get_value('init_dir')
+        restart_dir = config.get_value('restart_dir')
         phys_dirs_str = config.get_value('phys_dirs')
 
         #Convert the phys_dirs_str into a proper list:
@@ -96,7 +97,7 @@ def _build_cam():
         filepath_src = os.path.join(caseroot, "Buildconf",
                                     "camconf", "Filepath")
         filepath_dst = os.path.join(bldroot, "Filepath")
-        paths = [source_mods_dir, reg_dir, init_dir,
+        paths = [source_mods_dir, reg_dir, init_dir, restart_dir,
                  os.path.join(atm_root, "src", "data"),
                  os.path.join(atm_root, "src", "control"),
                  os.path.join(atm_root, "src", "cpl",

cime_config/cam_autogen.py

@@ -33,6 +33,7 @@
 # Import needed registry and other src/data scripts:
 from generate_registry_data import gen_registry
 from write_init_files import write_init_files
+from write_restart_physics import write_restart_physics
 
 ###############################################################################
 
@@ -454,7 +455,7 @@ def generate_registry(data_search, build_cache, atm_root, bldroot,
                                    gen_fort_indent, source_mods_dir, atm_root,
                                    logger=_LOGGER, schema_paths=data_search,
                                    error_on_no_validate=True)
-            retcode, reg_file_list, ic_names, registry_constituents, vars_init_value = retvals
+            retcode, reg_file_list, ic_names, registry_constituents, restart_vars, vars_init_value = retvals
             # Raise error if gen_registry failed:
             if retcode != 0:
                 emsg = "ERROR:Unable to generate CAM data structures from {}, err = {}"
@@ -468,16 +469,17 @@ def generate_registry(data_search, build_cache, atm_root, bldroot,
         # Save build details in the build cache
         reg_file_paths = [x.file_path for x in reg_file_list if x.file_path]
         build_cache.update_registry(gen_reg_file, registry_files, dycore,
-                                    reg_file_paths, ic_names, registry_constituents, vars_init_value)
+                                    reg_file_paths, ic_names, registry_constituents, restart_vars, vars_init_value)
     else:
         # If we did not run the registry generator, retrieve info from cache
         reg_file_paths = build_cache.reg_file_list()
         ic_names = build_cache.ic_names()
         registry_constituents = build_cache.constituents()
+        restart_vars = build_cache.restart_vars()
         vars_init_value = build_cache.vars_init_value()
     # End if
 
-    return genreg_dir, do_gen_registry, reg_file_paths, ic_names, registry_constituents, vars_init_value
+    return genreg_dir, do_gen_registry, reg_file_paths, ic_names, registry_constituents, restart_vars, vars_init_value
 
 ###############################################################################
 def generate_physics_suites(build_cache, preproc_defs, host_name,
@@ -808,6 +810,55 @@ def generate_init_routines(build_cache, bldroot, force_ccpp, force_init,
 
     return init_dir
 
+###############################################################################
+def generate_restart_routines(build_cache, bldroot, force_ccpp, force_restart,
+                              source_mods_dir, gen_fort_indent, cap_database,
+                              ic_names, registry_constituents, restart_vars):
+###############################################################################
+    """
+    Generate the physics restart routines (restart_physics.F90) using
+    both the registry and the CCPP physics suites if required
+    (new case or changes to registry or CCPP source(s), meta-data,
+    and/or script).
+    """
+
+    #Add new directory to build path:
+    restart_dir = os.path.join(bldroot, "phys_restart")
+    # Use this for cache check
+    gen_restart_file = os.path.join(_REG_GEN_DIR, "write_restart_physics.py")
+
+    # Figure out if we need to generate new restart routines:
+    if os.path.exists(restart_dir):
+        # Check if registry and / or CCPP suites were modified:
+        if force_ccpp or force_restart:
+            do_gen_restart = True
+        else:
+            #If not, then check cache to see if actual
+            #"restart_physics.py" was modified:
+            do_gen_restart = build_cache.restart_write_mismatch(gen_restart_file)
+    else:
+        #If no directory exists, then one will need
+        # to create new routines:
+        os.mkdir(restart_dir)
+        do_gen_restart = True
+    # End if
+    if do_gen_restart:
+        source_paths = [source_mods_dir, _REG_GEN_DIR]
+        retmsg = write_restart_physics(cap_database, ic_names, registry_constituents,
+                                       restart_vars, restart_dir, _find_file, 
+                                       source_paths, gen_fort_indent, _LOGGER)
+
+        #Check that script ran properly
+        if retmsg:
+            emsg = f"ERROR: Unable to generate CAM restart source code, error message is:\n{retmsg}"
+            raise CamAutoGenError(emsg)
+        #-----
+
+        # save build details in the build cache
+        build_cache.update_restart_gen(gen_restart_file)
+
+    return restart_dir
+
 #############
 # End of file
 #############

cime_config/cam_build_cache.py

@@ -213,6 +213,7 @@ def __init__(self, build_cache):
         # Set empty values sure to trigger processing
         self.__gen_reg_file = None
         self.__gen_init_file = None
+        self.__gen_restart_file = None
         self.__registry_files = {}
         self.__dycore = None
         self.__sdfs = {}
@@ -227,6 +228,7 @@ def __init__(self, build_cache):
         self.__reg_gen_files = []
         self.__ic_names = {}
         self.__constituents = []
+        self.__restart_dict = {}
         self.__vars_init_value = []
         if os.path.exists(build_cache):
             # Initialize build cache state
@@ -240,6 +242,9 @@ def __init__(self, build_cache):
                         elif item.tag == 'generate_init_file':
                             new_entry = new_entry_from_xml(item)
                             self.__gen_init_file = new_entry
+                        elif item.tag == 'generate_restart_file':
+                            new_entry = new_entry_from_xml(item)
+                            self.__gen_restart_file = new_entry
                         elif item.tag == 'registry_file':
                             new_entry = new_entry_from_xml(item)
                             self.__registry_files[new_entry.key] = new_entry
@@ -258,6 +263,10 @@ def __init__(self, build_cache):
                             stdname = item.get('standard_name')
                             itext = clean_xml_text(item)
                             self.__constituents.append(itext)
+                        elif item.tag == 'restart_entry':
+                            stdname = item.get('standard_name')
+                            itext = clean_xml_text(item)
+                            self.__restart_dict[stdname] = itext
                         elif item.tag == 'vars_init_value_entry':
                             itext = clean_xml_text(item)
                             self.__vars_init_value.append(itext)
@@ -322,7 +331,8 @@ def __init__(self, build_cache):
         # end if
 
     def update_registry(self, gen_reg_file, registry_source_files,
-                        dycore, reg_file_list, ic_names, constituents, vars_init_value):
+                        dycore, reg_file_list, ic_names, constituents,
+                        restart_dict, vars_init_value):
         """Replace the registry cache data with input data
         """
         self.__dycore = dycore
@@ -338,6 +348,7 @@ def update_registry(self, gen_reg_file, registry_source_files,
         # and should already be of type dict:
         self.__ic_names = ic_names
         self.__constituents = constituents
+        self.__restart_dict = restart_dict
         self.__vars_init_value = vars_init_value
 
     def update_ccpp(self, suite_definition_files, scheme_files, host_files,
@@ -383,6 +394,14 @@ def update_init_gen(self, gen_init_file):
         """
         self.__gen_init_file = FileStatus(gen_init_file, 'generate_init_file')
 
+    def update_restart_gen(self, gen_restart_file):
+        """
+        Replace the restart writer
+        (write_restart_physics.py) cache
+        data with input data
+        """
+        self.__gen_restart_file = FileStatus(gen_restart_file, 'generate_restart_file')
+
     def write(self):
         """Write out the current cache state"""
         new_cache = ET.Element("CAMBuildCache")
@@ -394,6 +413,9 @@ def write(self):
         new_xml_entry(registry, 'generate_registry_file',
                       self.__gen_reg_file.file_path,
                       self.__gen_reg_file.file_hash)
+        new_xml_entry(registry, 'generate_restart_file',
+                      self.__gen_restart_file.file_path,
+                      self.__gen_restart_file.file_hash)
         for rfile in self.__registry_files.values():
             new_xml_entry(registry, 'registry_file',
                           rfile.file_path, rfile.file_hash)
@@ -415,6 +437,11 @@ def write(self):
             const_entry = ET.SubElement(registry, 'constituent_entry')
             const_entry.text = stdname
         # end for
+        for stdname, restart_name in self.__restart_dict.items():
+            restart_entry = ET.SubElement(registry, 'restart_entry')
+            restart_entry.set('standard_name', stdname)
+            restart_entry.text = restart_name
+        # end for
         for stdname in self.__vars_init_value:
             var_entry = ET.SubElement(registry, 'vars_init_value_entry')
             var_entry.text = stdname
@@ -603,6 +630,22 @@ def init_write_mismatch(self, gen_init_file):
         #Return mismatch logical:
         return mismatch
 
+    def restart_write_mismatch(self, gen_restart_file):
+        """
+        Determine if the restart_files writer (write_restart_files.py)
+            differs from the data stored in our cache. Return True
+            if the data differs.
+        """
+
+        # Initialize variable
+        mismatch = False
+
+        # Check file hash to see if mis-match exists:
+        mismatch = self.__gen_restart_file.hash_mismatch(gen_restart_file)
+
+        # Return mismatch logical:
+        return mismatch
+
     def scheme_nl_metadata(self):
         """Return the stored list of scheme namelist metadata files"""
         return self.__scheme_nl_metadata
@@ -625,6 +668,10 @@ def constituents(self):
         """Return a copy of the registry constituents list"""
         return list(self.__constituents)
 
+    def restart_vars(self):
+        """Return a copy of the registry's list of variables for the restart file"""
+        return dict(self.__restart_dict)
+
     def vars_init_value(self):
         """Return a copy of the list of variables with initial_value"""
         return list(self.__vars_init_value)

cime_config/cam_config.py

@@ -32,6 +32,7 @@
 # Import fortran auto-generation routines:
 from cam_autogen import generate_registry, generate_physics_suites
 from cam_autogen import generate_init_routines
+from cam_autogen import generate_restart_routines
 
 ###############################################################################
 #HELPER FUNCTIONS
@@ -867,7 +868,7 @@ def generate_cam_src(self, gen_fort_indent):
         retvals = generate_registry(data_search, build_cache, self.__atm_root,
                                     self.__bldroot, source_mods_dir,
                                     dyn, gen_fort_indent)
-        reg_dir, force_ccpp, reg_files, ic_names, registry_constituents, vars_init_value = retvals
+        reg_dir, force_ccpp, reg_files, ic_names, registry_constituents, restart_vars, vars_init_value = retvals
 
         #Add registry path to config object:
         reg_dir_desc = "Location of auto-generated registry code."
@@ -907,6 +908,19 @@ def generate_cam_src(self, gen_fort_indent):
         init_dir_desc = "Location of auto-generated physics initialization code."
         self.create_config("init_dir", init_dir_desc, init_dir)
 
+        #---------------------------------------------------------
+        # Create restart init/write/read routines
+        #---------------------------------------------------------
+        restart_dir = generate_restart_routines(build_cache, self.__bldroot,
+                                                force_ccpp, force_init,
+                                                source_mods_dir, gen_fort_indent,
+                                                capgen_db, ic_names,
+                                                registry_constituents, restart_vars)
+
+        #Add registry path to config object:
+        restart_dir_desc = "Location of auto-generated physics restart code."
+        self.create_config("restart_dir", restart_dir_desc, restart_dir)
+
         #--------------------------------------------------------------
         # write out the cache here as we have completed pre-processing
         #--------------------------------------------------------------

src/control/cam_comp.F90

@@ -467,51 +467,24 @@ end subroutine cam_run3
    !-----------------------------------------------------------------------
    !
 
-   subroutine cam_run4(rstwr, nlend,                         &
-        yr_spec, mon_spec, day_spec, sec_spec)
+   subroutine cam_run4(rstwr, nlend)
+      logical, intent(in)  :: rstwr    ! write restart file
+      logical, intent(in)  :: nlend    ! this is final timestep
 
       !-----------------------------------------------------------------------
       !
-      ! Purpose:  Final phase of atmosphere model run method. This consists
-      !           of all the restart output, history writes, and other
-      !           file output.
+      ! Purpose:
       !
       !-----------------------------------------------------------------------
-!      use cam_restart,  only: cam_write_restart
 !      use qneg_module,  only: qneg_print_summary
 
-      logical,         intent(in)           :: rstwr    ! write restart file
-      logical,         intent(in)           :: nlend    ! this is final timestep
-      integer,         intent(in), optional :: yr_spec  ! Simulation year
-      integer,         intent(in), optional :: mon_spec ! Simulation month
-      integer,         intent(in), optional :: day_spec ! Simulation day
-      integer,         intent(in), optional :: sec_spec ! Secs in current simulation day
-
-      !
-      ! Write restart files
-      !
-      if (rstwr) then
-         call t_startf('cam_write_restart')
-         if (present(yr_spec) .and. present(mon_spec) .and.                   &
-              present(day_spec).and.present(sec_spec)) then
-!!XXgoldyXX: v need to import this
-!            call cam_write_restart(cam_in, cam_out, dyn_out, yr_spec=yr_spec, &
-!                 mon_spec=mon_spec, day_spec=day_spec, sec_spec= sec_spec)
-!!XXgoldyXX: ^ need to import this
-         else
-!!XXgoldyXX: v need to import this
-!            call cam_write_restart(cam_in, cam_out, dyn_out)
-!!XXgoldyXX: ^ need to import this
-         end if
-         call t_stopf('cam_write_restart')
-      end if
-
    end subroutine cam_run4
 
    !
    !-----------------------------------------------------------------------
    !
-   subroutine cam_timestep_final(rstwr, nlend, do_ncdata_check, do_history_write)
+   subroutine cam_timestep_final(rstwr, nlend, do_ncdata_check, &
+                   yr_spec, mon_spec, day_spec, sec_spec, do_history_write)
       !-----------------------------------------------------------------------
       !
       ! Purpose:   Timestep final runs at the end of each timestep
@@ -521,15 +494,20 @@ subroutine cam_timestep_final(rstwr, nlend, do_ncdata_check, do_history_write)
       use phys_comp,    only: phys_timestep_final
       use cam_history,  only: history_write_files
       use cam_history,  only: history_wrap_up
+      use cam_restart,  only: cam_write_restart
       logical, intent(in)  :: rstwr    ! write restart file
       logical, intent(in)  :: nlend    ! this is final timestep
       !Flag for whether a snapshot (ncdata) check should be run or not
       ! - flag is true if this is not the first or last step
       logical, intent(in)  :: do_ncdata_check
+      integer,         intent(in), optional :: yr_spec  ! Simulation year
+      integer,         intent(in), optional :: mon_spec ! Simulation month
+      integer,         intent(in), optional :: day_spec ! Simulation day
+      integer,         intent(in), optional :: sec_spec ! Secs in current simulation day
       !Flag for whether to perform the history write
       logical, optional, intent(in) :: do_history_write
 
-      logical :: history_write_loc
+      logical                               :: history_write_loc
 
       if (present(do_history_write)) then
          history_write_loc = do_history_write
@@ -540,7 +518,20 @@ subroutine cam_timestep_final(rstwr, nlend, do_ncdata_check, do_history_write)
       if (history_write_loc) then
          call history_write_files()
       end if
-      ! peverwhee - todo: handle restarts
+      !
+      ! Write restart files
+      !
+      if (rstwr) then
+         call t_startf('cam_write_restart')
+         if (present(yr_spec) .and. present(mon_spec) .and.                   &
+              present(day_spec).and.present(sec_spec)) then
+            call cam_write_restart(dyn_out, yr_spec=yr_spec, &
+                 mon_spec=mon_spec, day_spec=day_spec, sec_spec= sec_spec)
+         else
+            call cam_write_restart(dyn_out)
+         end if
+         call t_stopf('cam_write_restart')
+      end if
       call history_wrap_up(rstwr, nlend)
 
       !