v8.4.1 merge

.github/workflows/bld_mpas_images.yaml

@@ -4,8 +4,8 @@ run-name: CI Image Build for MPAS-A
 on:
   push:
     branches:
-      # Only build containers when pushing to gsl/develop
-      - "gsl/develop"
+      # Only build containers when pushing to noaa/develop
+      - "noaa/develop"
 
 jobs:
   docker:

.github/workflows/run_mpas.yml

@@ -26,12 +26,12 @@ on: [push, pull_request, workflow_dispatch]
 #
 # Tests:
 #        Baseline Codebase Repository:Branch   Physics                       build type
-# 1/7)   MPAS-Dev:v8.3.0                       mesoscale_reference           Release/Debug
-# 2/8)   MPAS-Dev:v8.3.0                       convection_permitting         Release/Debug
-# 3/9)   MPAS-Dev:v8.3.0                       mesoscale_reference_noahmp    Release/Debug
-# 4/10)  ufs-community:gsl/develop             mesoscale_reference           Release/Debug
-# 5/11)  ufs-community:gsl/develop             convection_permitting         Release/Debug
-# 6/12)  ufs-community:gsl/develop             mesoscale_reference_noahmp    Release/Debug
+# 1/7)   MPAS-Dev:v8.4.0                       mesoscale_reference           Release/Debug
+# 2/8)   MPAS-Dev:v8.4.0                       convection_permitting         Release/Debug
+# 3/9)   MPAS-Dev:v8.4.0                       mesoscale_reference_noahmp    Release/Debug
+# 4/10)  ufs-community:noaa/develop            mesoscale_reference           Release/Debug
+# 5/11)  ufs-community:noaa/develop            convection_permitting         Release/Debug
+# 6/12)  ufs-community:noaa/develop            mesoscale_reference_noahmp    Release/Debug
 #
 #############################################################################################
 jobs:
@@ -43,13 +43,13 @@ jobs:
         f-compiler: [gfortran]#,ifx]
         physics:    [mesoscale_reference, convection_permitting, mesoscale_reference_noahmp]
         repo:       [ufs-community, MPAS-Dev]
-        branch:     [gsl/develop, v8.3.0]
+        branch:     [noaa/develop, v8.4.0]
         build-type: [Debug, Release]
         exclude:
           - repo: MPAS-Dev
-            branch: gsl/develop
+            branch: noaa/develop
           - repo: ufs-community
-            branch: v8.3.0
+            branch: v8.4.0
         include:
         # Set container images for each compiler
         - f-compiler: gfortran

.github/workflows/run_mpas_hrrr.yml

@@ -22,9 +22,9 @@ on: [push, pull_request, workflow_dispatch]
 #
 # Tests:
 #        Baseline Codebase Repository:Branch  Physics   IC source   season   build-type
-# 1/4)   ufs-community:gsl/develop            hrrrv5    gfs         winter   Release/Debug
-# 2/5)   ufs-community:gsl/develop            hrrrv5    rap         summer   Release/Debug
-# 3/6)   ufs-community:gsl/develop            hrrrv5    rap         winter   Release/Debug
+# 1/4)   ufs-community:noaa/develop           hrrrv5    gfs         winter   Release/Debug
+# 2/5)   ufs-community:noaa/develop           hrrrv5    rap         summer   Release/Debug
+# 3/6)   ufs-community:noaa/develop           hrrrv5    rap         winter   Release/Debug
 #
 #############################################################################################
 jobs:
@@ -90,15 +90,15 @@ jobs:
       if: contains(matrix.build-type, 'Debug')
       run: |
         cd ${runner_ROOT}
-        git clone --recursive --branch gsl/develop https://github.com/ufs-community/MPAS-Model.git MPAS-Model-BL
+        git clone --recursive --branch noaa/develop https://github.com/ufs-community/MPAS-Model.git MPAS-Model-BL
         cd ${mpas_bl_ROOT}
         make ${{matrix.bld_target}} CORE=atmosphere DEBUG=true
 
     - name: Checkout and build MPAS standalone for baseline generation (Release)
       if: contains(matrix.build-type, 'Release')
       run: |
         cd ${runner_ROOT}
-        git clone --recursive --branch gsl/develop https://github.com/ufs-community/MPAS-Model.git MPAS-Model-BL
+        git clone --recursive --branch noaa/develop https://github.com/ufs-community/MPAS-Model.git MPAS-Model-BL
         cd ${mpas_bl_ROOT}
         make ${{matrix.bld_target}} CORE=atmosphere
 

Makefile

@@ -16,11 +16,11 @@ gnu:   # BUILDTARGET GNU Fortran, C, and C++ compilers
 	"CC_SERIAL = gcc" \
 	"CXX_SERIAL = g++" \
 	"FFLAGS_PROMOTION = -fdefault-real-8 -fdefault-double-8" \
-	"FFLAGS_OPT = -std=f2008 -O3 -ffree-line-length-none -fconvert=big-endian -ffree-form" \
+	"FFLAGS_OPT = -std=f2008 -fimplicit-none -O3 -ffree-line-length-none -fconvert=big-endian -ffree-form" \
 	"CFLAGS_OPT = -O3" \
 	"CXXFLAGS_OPT = -O3" \
 	"LDFLAGS_OPT = -O3" \
-	"FFLAGS_DEBUG = -std=f2008 -g -ffree-line-length-none -fconvert=big-endian -ffree-form -fcheck=all -fbacktrace -ffpe-trap=invalid,zero,overflow" \
+	"FFLAGS_DEBUG = -std=f2008 -fimplicit-none -g -ffree-line-length-none -fconvert=big-endian -ffree-form -fcheck=all -fbacktrace -ffpe-trap=invalid,zero,overflow" \
 	"CFLAGS_DEBUG = -g" \
 	"CXXFLAGS_DEBUG = -g" \
 	"LDFLAGS_DEBUG = -g" \
@@ -822,11 +822,11 @@ intel:   # BUILDTARGET Intel oneAPI Fortran, C, and C++ compiler suite
 	"CC_SERIAL = icx" \
 	"CXX_SERIAL = icpx" \
 	"FFLAGS_PROMOTION = -real-size 64" \
-	"FFLAGS_OPT = -O3 -convert big_endian -free -align array64byte" \
+	"FFLAGS_OPT = -O3 -convert big_endian -free -align array64byte -Qoption,fpp,-macro_expand=vc" \
 	"CFLAGS_OPT = -O3" \
 	"CXXFLAGS_OPT = -O3" \
 	"LDFLAGS_OPT = -O3" \
-	"FFLAGS_DEBUG = -g -convert big_endian -free -check bounds,pointers,arg_temp_created,format,shape,contiguous -fpe0 -traceback" \
+	"FFLAGS_DEBUG = -g -convert big_endian -free -check bounds,pointers,arg_temp_created,format,shape,contiguous -fpe0 -traceback -Qoption,fpp,-macro_expand=vc" \
 	"CFLAGS_DEBUG = -g -traceback" \
 	"CXXFLAGS_DEBUG = -g -traceback" \
 	"LDFLAGS_DEBUG = -g -traceback" \
@@ -844,6 +844,24 @@ CPPINCLUDES =
 FCINCLUDES =
 LIBS =
 
+export MPAS_ESMF ?= embedded
+ifeq "$(MPAS_ESMF)" "external"
+  ifeq ($(wildcard $(ESMFMKFILE)), )
+    $(error ESMFMKFILE must be set if MPAS_ESMF=external)
+  endif
+  include $(ESMFMKFILE)
+  export MPAS_ESMF_INC = $(ESMF_F90COMPILEPATHS)
+  export MPAS_ESMF_LIB = $(ESMF_F90LINKPATHS) $(ESMF_F90ESMFLINKPATHS) $(ESMF_F90ESMFLINKLIBS)
+  override CPPFLAGS += -DMPAS_EXTERNAL_ESMF_LIB=true
+  ESMF_MESSAGE="MPAS was built with an external ESMF library using ESMFMKFILE"
+else ifeq "$(MPAS_ESMF)" "embedded"
+  export MPAS_ESMF_INC = -I$(PWD)/src/external/esmf_time_f90
+  export MPAS_ESMF_LIB = -L$(PWD)/src/external/esmf_time_f90 -lesmf_time
+  ESMF_MESSAGE="MPAS was built with the embedded ESMF timekeeping library."
+else
+  $(error Invalid MPAS_ESMF option: $(MPAS_ESMF) - valid options "embedded", "external")
+endif
+
 ifneq "$(PIO)" ""
 #
 # Regardless of PIO library version, look for a lib subdirectory of PIO path
@@ -918,6 +936,15 @@ endif
 	LIBS += $(NCLIB)
 endif
 
+ifneq "$(SCOTCH)" ""
+	SCOTCH_INCLUDES += -I$(SCOTCH)/include
+	SCOTCH_LIBS += -L$(SCOTCH)/lib64 -lptscotch -lscotch  -lptscotcherr -lm
+	SCOTCH_FLAGS = -DMPAS_SCOTCH
+
+	CPPINCLUDES += $(SCOTCH_INCLUDES)
+	LIBS += $(SCOTCH_LIBS)
+	override CPPFLAGS += $(SCOTCH_FLAGS)
+endif
 
 ifneq "$(PNETCDF)" ""
 ifneq ($(wildcard $(PNETCDF)/lib/libpnetcdf.*), )
@@ -1031,6 +1058,20 @@ $(if $(PRECISION),$(info NOTE: PRECISION=single is unnecessary, single is the de
 	PRECISION_MESSAGE="MPAS was built with default single-precision reals."
 endif #PRECISION IF
 
+# Optional MUSICA support for chemistry
+ifeq "$(shell echo $(MUSICA) | tr '[:upper:]' '[:lower:]')" "true"
+ifeq ($(shell pkg-config --exists musica-fortran && echo yes || echo no), no)
+$(error "musica-fortran package is not installed. Please install it to proceed.")
+endif
+	MUSICA_FCINCLUDES += $(shell pkg-config --cflags musica-fortran)
+	MUSICA_LIBS += $(shell pkg-config --libs musica-fortran)
+	MUSICA_FFLAGS = -DMPAS_USE_MUSICA
+
+	FCINCLUDES += $(MUSICA_FCINCLUDES)
+	LIBS += $(MUSICA_LIBS)
+	override CPPFLAGS += $(MUSICA_FFLAGS)
+endif
+
 ifeq "$(USE_PAPI)" "true"
 	CPPINCLUDES += -I$(PAPI)/include -D_PAPI
 	FCINCLUDES += -I$(PAPI)/include
@@ -1533,6 +1574,63 @@ mpi_f08_test:
 	$(if $(findstring 1,$(MPAS_MPI_F08)), $(eval MPI_F08_MESSAGE = "Using the mpi_f08 module."), )
 	$(if $(findstring 1,$(MPAS_MPI_F08)), $(info mpi_f08 module detected.))
 
+musica_fortran_test:
+	@#
+	@# Create a Fortran test program that will link against the MUSICA library
+	@#
+	$(info Checking for a working MUSICA-Fortran library...)
+	$(eval MUSICA_FORTRAN_TEST := $(shell $\
+		printf "program test_musica_fortran\n$\
+		&   use musica_util, only : string_t\n$\
+		&   use musica_micm, only : get_micm_version\n$\
+		&   type(string_t) :: version_string\n$\
+		&   version_string = get_micm_version()\n$\
+		&   print *, \"MUSICA support is available. MICM version: \", version_string%%value_\n$\
+		end program test_musica_fortran\n" | sed 's/&/ /' > test_musica_fortran.f90; $\
+		$\
+		$(FC) $(MUSICA_FCINCLUDES) $(MUSICA_FFLAGS) test_musica_fortran.f90 -o test_musica_fortran.x $(MUSICA_LIBS) > /dev/null 2>&1; $\
+		musica_fortran_status=$$?; $\
+		rm -f test_musica_fortran.f90 test_musica_fortran.x; $\
+		if [ $$musica_fortran_status -eq 0 ]; then $\
+			printf "1"; $\
+		else $\
+			printf "0"; $\
+		fi $\
+	))
+	$(if $(findstring 0,$(MUSICA_FORTRAN_TEST)), $(error Could not build a simple test program with MUSICA-Fortran))
+	$(eval MUSICA_FORTRAN_VERSION := $(shell pkg-config --modversion musica-fortran))
+	$(if $(findstring 1,$(MUSICA_FORTRAN_TEST)), $(info Built a simple test program with MUSICA-Fortran version $(MUSICA_FORTRAN_VERSION)), )
+
+scotch_c_test:
+	@#
+	@# Create a C test program and try to build against the PT-SCOTCH library
+	@#
+	$(info Checking for a working Scotch library...)
+	$(eval SCOTCH_C_TEST := $(shell $\
+	    printf "#include <stdio.h>\n\
+			&#include \"mpi.h\"\n\
+			&#include \"ptscotch.h\"\n\
+			&int main(){\n\
+			&    int err;\n\
+			&    SCOTCH_Dgraph *dgraph;\n\
+			&    err = SCOTCH_dgraphInit(dgraph, MPI_COMM_WORLD);\n\
+			&    SCOTCH_dgraphExit(dgraph);\n\
+			&    return err;\n\
+			&}\n" | sed 's/&/ /' > ptscotch_c_test.c; $\
+		$\
+		$(CC) $(CPPINCLUDES) $(CFLAGS) $(LDFLAGS) ptscotch_c_test.c -o ptscotch_c_test.x $(SCOTCH_LIBS) > ptscotch_c_test.log 2>&1; $\
+		scotch_c_status=$$?; $\
+		if [ $$scotch_c_status -eq 0 ]; then $\
+			printf "1"; $\
+			rm -f ptscotch_c_test.c ptscotch_c_test.x ptscotch_c_test.log; $\
+		else $\
+			printf "0"; $\
+		fi $\
+	))
+	$(if $(findstring 0,$(SCOTCH_C_TEST)), $(error Could not build a simple C program with Scotch. $\
+		Test program ptscotch_c_test.c and output ptscotch_c_test.log have been left $\
+	    in the top-level MPAS directory for further debugging ))
+	$(if $(findstring 1,$(SCOTCH_C_TEST)), $(info Built a simple C program with Scotch ))
 
 pnetcdf_test:
 	@#
@@ -1583,6 +1681,20 @@ IO_MESSAGE = "Using the SMIOL library."
 override CPPFLAGS += "-DMPAS_SMIOL_SUPPORT"
 endif
 
+ifneq "$(MUSICA_FFLAGS)" ""
+MAIN_DEPS += musica_fortran_test
+MUSICA_MESSAGE = "MPAS was linked with the MUSICA-Fortran library version $(MUSICA_FORTRAN_VERSION)."
+else
+MUSICA_MESSAGE = "MPAS was not linked with the MUSICA-Fortran library."
+endif
+
+ifneq "$(SCOTCH)" ""
+MAIN_DEPS += scotch_c_test
+SCOTCH_MESSAGE = "MPAS has been linked with the Scotch graph partitioning library."
+else
+SCOTCH_MESSAGE = "MPAS was NOT linked with the Scotch graph partitioning library."
+endif
+
 mpas_main: $(MAIN_DEPS)
 	cd src; $(MAKE) FC="$(FC)" \
                  CC="$(CC)" \
@@ -1619,13 +1731,16 @@ mpas_main: $(MAIN_DEPS)
 	@echo $(OPENMP_MESSAGE)
 	@echo $(OPENMP_OFFLOAD_MESSAGE)
 	@echo $(OPENACC_MESSAGE)
+	@echo $(MUSICA_MESSAGE)
+	@echo $(SCOTCH_MESSAGE)
 	@echo $(SHAREDLIB_MESSAGE)
 ifeq "$(AUTOCLEAN)" "true"
 	@echo $(AUTOCLEAN_MESSAGE)
 endif
 	@echo $(GEN_F90_MESSAGE)
 	@echo $(TIMER_MESSAGE)
 	@echo $(IO_MESSAGE)
+	@echo $(ESMF_MESSAGE)
 	@echo "*******************************************************************************"
 clean:
 	cd src; $(MAKE) clean RM="$(RM)" CORE="$(CORE)" AUTOCLEAN="$(AUTOCLEAN)"
@@ -1682,6 +1797,9 @@ errmsg:
 	@echo "    OPENACC=true  - builds and links with OpenACC flags. Default is to not use OpenACC."
 	@echo "    PRECISION=double - builds with default double-precision real kind. Default is to use single-precision."
 	@echo "    SHAREDLIB=true - generate position-independent code suitable for use in a shared library. Default is false."
+	@echo "    MPAS_ESMF=opt  - Selects the ESMF library to be used for MPAS. Options are:"
+	@echo "                     MPAS_ESMF=embedded - Use the embedded ESMF timekeeping library (default)"
+	@echo "                     MPAS_ESMF=external - Use an external ESMF library, determined by ESMFMKFILE"
 	@echo ""
 	@echo "Ensure that NETCDF, PNETCDF, PIO, and PAPI (if USE_PAPI=true) are environment variables"
 	@echo "that point to the absolute paths for the libraries."

README.md

@@ -1,4 +1,4 @@
-MPAS-v8.3.1-2.26
+MPAS-v8.4.0-1.0
 ====
 
 The Model for Prediction Across Scales (MPAS) is a collaborative project for

cmake/Functions/MPAS_Functions.cmake

@@ -121,6 +121,7 @@ function(mpas_fortran_target target)
         list(APPEND MPAS_FORTRAN_TARGET_COMPILE_OPTIONS_PUBLIC
                 $<$<COMPILE_LANGUAGE:Fortran>:-align array64byte>
                 $<$<COMPILE_LANGUAGE:Fortran>:-convert big_endian>
+                $<$<COMPILE_LANGUAGE:Fortran>:-Qoption,fpp,-macro_expand=vc>
         )
         if(MPAS_DOUBLE_PRECISION)
             list(APPEND MPAS_FORTRAN_TARGET_COMPILE_OPTIONS_PRIVATE

cmake/Modules/FindPnetCDF.cmake

@@ -133,27 +133,29 @@ set(_new_components)
 if(PnetCDF_Fortran_FOUND AND NOT TARGET PnetCDF::PnetCDF_Fortran)
     add_library(PnetCDF::PnetCDF_Fortran INTERFACE IMPORTED)
     set_target_properties(PnetCDF::PnetCDF_Fortran PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${PnetCDF_INCLUDE_DIR}
-                                                              INTERFACE_LINK_DIRECTORIES ${PnetCDF_LIB_DIR})
+                                                              INTERFACE_LINK_DIRECTORIES ${PnetCDF_LIB_DIR}
+                                                              INTERFACE_LINK_LIBRARIES pnetcdf)
     if(PnetCDF_MODULE_DIR AND NOT PnetCDF_MODULE_DIR STREQUAL PnetCDF_INCLUDE_DIR )
         set_property(TARGET PnetCDF::PnetCDF_Fortran APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PnetCDF_MODULE_DIR})
     endif()
     set(_new_components 1)
-    target_link_libraries(PnetCDF::PnetCDF_Fortran INTERFACE -lpnetcdf)
 endif()
 
 # PnetCDF::PnetCDF_C imported interface target
 if(PnetCDF_C_FOUND AND NOT TARGET PnetCDF::PnetCDF_C)
     add_library(PnetCDF::PnetCDF_C INTERFACE IMPORTED)
     set_target_properties(PnetCDF::PnetCDF_C PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${PnetCDF_INCLUDE_DIR}
-                                                        INTERFACE_LINK_DIRECTORIES ${PnetCDF_LIB_DIR})
+                                                        INTERFACE_LINK_DIRECTORIES ${PnetCDF_LIB_DIR}
+                                                        INTERFACE_LINK_LIBRARIES pnetcdf)
     set(_new_components 1)
 endif()
 
 # PnetCDF::PnetCDF_CXX imported interface target
 if(PnetCDF_CXX_FOUND AND NOT TARGET PnetCDF::PnetCDF_CXX)
     add_library(PnetCDF::PnetCDF_CXX INTERFACE IMPORTED)
     set_target_properties(PnetCDF::PnetCDF_CXX PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${PnetCDF_INCLUDE_DIR}
-                                                          INTERFACE_LINK_DIRECTORIES ${PnetCDF_LIB_DIR})
+                                                          INTERFACE_LINK_DIRECTORIES ${PnetCDF_LIB_DIR}
+                                                          INTERFACE_LINK_LIBRARIES pnetcdf)
     set(_new_components 1)
 endif()
 

src/Makefile

@@ -9,7 +9,7 @@ else
 all: mpas
 
 mpas: $(AUTOCLEAN_DEPS) externals frame ops dycore drver
-	$(LINKER) $(LDFLAGS) -o $(EXE_NAME) driver/*.o -L. -ldycore -lops -lframework $(LIBS) -I./external/esmf_time_f90 -L./external/esmf_time_f90 -lesmf_time
+	$(LINKER) $(LDFLAGS) -o $(EXE_NAME) driver/*.o -L. -ldycore -lops -lframework $(LIBS) $(MPAS_ESMF_INC) $(MPAS_ESMF_LIB)
 
 externals: $(AUTOCLEAN_DEPS)
 	( cd external; $(MAKE) FC="$(FC)" SFC="$(SFC)" CC="$(CC)" SCC="$(SCC)" FFLAGS="$(FFLAGS)" CFLAGS="$(CFLAGS)" CPP="$(CPP)" NETCDF="$(NETCDF)" CORE="$(CORE)" all )

src/core_atmosphere/CMakeLists.txt

@@ -151,7 +151,7 @@ if(NOT EXISTS ${ATMOSPHERE_CORE_PHYSICS_MMM_DIR})
     endif()
 
 else()
-    message(STATUS "Directory ${DIR_TO_CHECK} already exists, skipping clone")
+    message( STATUS "Finished checking out external repos via manage_externals" )
 endif()
 
 set(ATMOSPHERE_CORE_PHYSICS_MMM_SOURCES
@@ -476,6 +476,7 @@ list(TRANSFORM ATMOSPHERE_CORE_DIAGNOSTIC_SOURCES PREPEND diagnostics/)
 # dynamics/
 set(ATMOSPHERE_CORE_DYNAMICS_SOURCES
     mpas_atm_boundaries.F
+    mpas_atm_dissipation_models.F
     mpas_atm_iau.F
     mpas_atm_time_integration.F)
 list(TRANSFORM ATMOSPHERE_CORE_DYNAMICS_SOURCES PREPEND dynamics/)