UFS-dev PR#184

.github/workflows/GCC.yml

@@ -15,33 +15,34 @@ on:
     - develop
 
 jobs:
-  GCC:
+  build_spack:
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        cmake_opts: ["-D32BIT=ON", "-D32BIT=OFF"]
-        gcc_ver: ["11"]
-        mpi: ["mpich"]
+        gcc_ver: ["12"]
+        mpi: ["mpich", "openmpi"]
 
     steps:
 
-    - name: install-doxygen
-      run: |
-        sudo apt-get install doxygen graphviz
-
     - name: checkout-fv3atm
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         path: ${{ github.workspace }}/fv3atm
         submodules: recursive
 
+    - name: install-cmake
+      run: | 
+        cd ${{ github.workspace }}
+        curl -f -s -S -R -L https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-Linux-x86_64.tar.gz | tar -zx
+        echo "${{ github.workspace }}/cmake-3.29.2-linux-x86_64/bin" >> $GITHUB_PATH
+
     - name: cache-spack
       id: cache-spack
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ github.workspace }}/spack-develop
-        key: spack-${{ hashFiles('fv3atm/ci/spack.yaml') }}-gcc${{ matrix.gcc_ver }}-2-${{ matrix.cmake_opts }}-${{ matrix.mpich }}
+        key: spack-${{ hashFiles('fv3atm/ci/spack.yaml') }}-gcc${{ matrix.gcc_ver }}-${{ matrix.mpi }}
 
     # Building dependencies takes 40+ min
     - name: spack-install
@@ -58,6 +59,42 @@ jobs:
         spack config add "packages:mpi:require:'${{ matrix.mpi }}'"
         spack concretize |& tee ${SPACK_ENV}/log.concretize
         spack install -j2 --fail-fast
+        spack clean --all
+
+  build_fv3atm:
+    needs: build_spack
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        cmake_opts: ["-D32BIT=ON", "-D32BIT=OFF"]
+        gcc_ver: ["12"]
+        mpi: ["mpich", "openmpi"]
+
+    steps:
+
+    - name: install-doxygen
+      run: |
+        sudo apt-get install doxygen graphviz
+
+    - name: install-cmake
+      run: | 
+        cd ${{ github.workspace }}
+        curl -f -s -S -R -L https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-Linux-x86_64.tar.gz | tar -zx
+        echo "${{ github.workspace }}/cmake-3.29.2-linux-x86_64/bin" >> $GITHUB_PATH
+
+    - name: checkout-fv3atm
+      uses: actions/checkout@v4
+      with:
+        path: ${{ github.workspace }}/fv3atm
+        submodules: recursive
+
+    - name: cache-spack
+      id: cache-spack
+      uses: actions/cache/restore@v4
+      with:
+        path: ${{ github.workspace }}/spack-develop
+        key: spack-${{ hashFiles('fv3atm/ci/spack.yaml') }}-gcc${{ matrix.gcc_ver }}-${{ matrix.mpi }}
 
     - name: build-fv3atm
       run: |
@@ -70,20 +107,24 @@ jobs:
         mkdir ${GITHUB_WORKSPACE}/build
         sed -i 's/doc /upp_doc /' upp/docs/CMakeLists.txt
         cd ${GITHUB_WORKSPACE}/build
+        export CC=mpicc
+        export CXX=mpicxx
+        export FC=mpif90
+        cat /home/runner/work/fv3atm/fv3atm/spack-develop/opt/spack/linux-ubuntu22.04-zen2/gcc-12.3.0/fms-2023.04-*/lib/cmake/fms/fms-config.cmake
         cmake ${GITHUB_WORKSPACE}/fv3atm -DBUILD_TESTING=ON ${{ matrix.cmake_opts }} -DENABLE_DOCS=ON
         make -j2
         ls -l /home/runner/work/fv3atm/fv3atm/fv3atm/io
 
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
-        name: docs
+        name: docs-gcc${{ matrix.gcc_ver }}-${{ matrix.mpi }}-${{ matrix.cmake_opts }}
         path: |
           build/docs/html
-          
+
     - name: debug-artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: ${{ failure() }}
       with:
-        name: ccpp_prebuild_logs
+        name: ccpp_prebuild_logs-gcc${{ matrix.gcc_ver }}-${{ matrix.mpi }}-${{ matrix.cmake_opts }}
         path: ${{ github.workspace }}/build/ccpp/ccpp_prebuild.*
 

CMakeLists.txt

@@ -9,7 +9,7 @@ option(ENABLE_DOCS "Enable generation of doxygen-based documentation." OFF)
 # Determine whether or not to generate documentation.
 if(ENABLE_DOCS)
   find_package(Doxygen REQUIRED)
-  add_subdirectory(docs)  
+  add_subdirectory(docs)
 endif()
 
 # Enable CI build & unit testing:
@@ -22,6 +22,7 @@ endif()
 ### CCPP
 ###############################################################################
 
+set(MPI ON)
 add_subdirectory(ccpp)
 
 ###############################################################################
@@ -37,6 +38,7 @@ if(MOVING_NEST)
   set(MOVING_NEST ON)
 endif()
 add_subdirectory(atmos_cubed_sphere)
+target_compile_definitions(fv3 PRIVATE BYPASS_BREED_SLP_INLINE)
 
 ###############################################################################
 ### fv3atm
@@ -104,6 +106,7 @@ add_library(fv3atm
   cpl/module_block_data.F90
   cpl/module_cplfields.F90
   cpl/module_cap_cpl.F90
+  cpl/module_cplscalars.F90
   io/fv3atm_common_io.F90
   io/fv3atm_clm_lake_io.F90
   io/fv3atm_rrfs_sd_io.F90

atmos_model.F90

@@ -450,7 +450,7 @@ end subroutine update_atmos_radiation_physics
 !   variable type are allocated for the global grid (without halo regions).
 ! </INOUT>
 subroutine atmos_timestep_diagnostics(Atmos)
-  use mpi
+  use mpi_f08
   implicit none
   type (atmos_data_type), intent(in) :: Atmos
 !--- local variables---
@@ -582,7 +582,8 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
    call atmosphere_diag_axes (Atmos%axes)
    call atmosphere_etalvls (Atmos%ak, Atmos%bk, flip=flip_vc)
 
-   call atmosphere_control_data (isc, iec, jsc, jec, nlev, p_hydro, hydro, tile_num)
+   tile_num=-1
+   call atmosphere_control_data (isc, iec, jsc, jec, nlev, p_hydro, hydro, global_tile_num=tile_num)
 
    allocate (Atmos%lon(nlon,nlat), Atmos%lat(nlon,nlat))
    call atmosphere_grid_ctr (Atmos%lon, Atmos%lat)
@@ -3142,7 +3143,8 @@ subroutine setup_exportdata(rc)
 
     use ESMF
 
-    use module_cplfields, only: exportFields, chemistryFieldNames
+    use module_cplfields,  only: exportFields, chemistryFieldNames
+    use module_cplscalars, only: flds_scalar_name
 
     !--- arguments
     integer, optional, intent(out) :: rc
@@ -3192,33 +3194,36 @@ subroutine setup_exportdata(rc)
       if (isFound) then
         call ESMF_FieldGet(exportFields(n), name=fieldname, rank=rank, typekind=datatype, rc=localrc)
         if (ESMF_LogFoundError(rcToCheck=localrc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return
-        if (datatype == ESMF_TYPEKIND_R8) then
-           select case (rank)
-             case (2)
-               call ESMF_FieldGet(exportFields(n),farrayPtr=datar82d,localDE=0, rc=localrc)
-               if (ESMF_LogFoundError(rcToCheck=localrc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return
-             case (3)
-               call ESMF_FieldGet(exportFields(n),farrayPtr=datar83d,localDE=0, rc=localrc)
-               if (ESMF_LogFoundError(rcToCheck=localrc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return
-             case default
-               !--- skip field
-               isFound = .false.
-           end select
-        else if (datatype == ESMF_TYPEKIND_R4) then
-           select case (rank)
-             case (2)
-               call ESMF_FieldGet(exportFields(n),farrayPtr=datar42d,localDE=0, rc=localrc)
-               if (ESMF_LogFoundError(rcToCheck=localrc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return
-             case default
-               !--- skip field
-               isFound = .false.
-           end select
-        else
-          !--- skip field
+        if (trim(fieldname) == trim(flds_scalar_name)) then
           isFound = .false.
+        else
+          if (datatype == ESMF_TYPEKIND_R8) then
+            select case (rank)
+            case (2)
+              call ESMF_FieldGet(exportFields(n),farrayPtr=datar82d,localDE=0, rc=localrc)
+              if (ESMF_LogFoundError(rcToCheck=localrc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return
+            case (3)
+              call ESMF_FieldGet(exportFields(n),farrayPtr=datar83d,localDE=0, rc=localrc)
+              if (ESMF_LogFoundError(rcToCheck=localrc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return
+            case default
+              !--- skip field
+              isFound = .false.
+            end select
+          else if (datatype == ESMF_TYPEKIND_R4) then
+            select case (rank)
+            case (2)
+              call ESMF_FieldGet(exportFields(n),farrayPtr=datar42d,localDE=0, rc=localrc)
+              if (ESMF_LogFoundError(rcToCheck=localrc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return
+            case default
+              !--- skip field
+              isFound = .false.
+            end select
+          else
+            !--- skip field
+            isFound = .false.
+          end if
         end if
       end if
-
       !--- skip field if only required for chemistry
       if (isFound .and. GFS_control%cplchm) isFound = .not.any(trim(fieldname) == chemistryFieldNames)
 

ccpp/config/ccpp_prebuild_config.py

@@ -29,6 +29,7 @@
 TYPEDEFS_NEW_METADATA = {
     'ccpp_types' : {
         'ccpp_t' : 'cdata',
+        'MPI_Comm' : '',
         'ccpp_types' : '',
         },
     'machine' : {

ccpp/data/GFS_typedefs.F90

@@ -1,5 +1,6 @@
 module GFS_typedefs
 
+   use mpi_f08
    use machine,                  only: kind_phys, kind_dbl_prec, kind_sngl_prec
    use physcons,                 only: con_cp, con_fvirt, con_g, rholakeice,           &
                                        con_hvap, con_hfus, con_pi, con_rd, con_rv,     &
@@ -94,7 +95,7 @@ module GFS_typedefs
   type GFS_init_type
     integer :: me                                !< my MPI-rank
     integer :: master                            !< master MPI-rank
-    integer :: fcst_mpi_comm                     !< forecast tasks mpi communicator
+    type(MPI_Comm) :: fcst_mpi_comm              !< forecast tasks mpi communicator
     integer :: fcst_ntasks                       !< total number of forecast tasks
     integer :: tile_num                          !< tile number for this MPI rank
     integer :: isc                               !< starting i-index for this MPI-domain
@@ -700,7 +701,7 @@ module GFS_typedefs
 
     integer              :: me              !< MPI rank designator
     integer              :: master          !< MPI rank of master atmosphere processor
-    integer              :: communicator    !< MPI communicator
+    type(MPI_Comm)       :: communicator    !< MPI communicator
     integer              :: ntasks          !< MPI size in communicator
     integer              :: nthreads        !< OpenMP threads available for physics
     integer              :: nlunit          !< unit for namelist
@@ -1159,6 +1160,7 @@ module GFS_typedefs
     logical              :: lheatstrg       !< flag for canopy heat storage parameterization
     logical              :: lseaspray       !< flag for sea spray parameterization
     logical              :: cnvcld
+    logical              :: xr_cnvcld       !< flag for adding suspended convective clouds to Xu-Randall cloud fraction
     logical              :: random_clds     !< flag controls whether clouds are random
     logical              :: shal_cnv        !< flag for calling shallow convection
     logical              :: do_deep         !< whether to do deep convection
@@ -3306,7 +3308,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, &
     real(kind=kind_phys), dimension(:), intent(in) :: bk
     logical,                intent(in) :: restart
     logical,                intent(in) :: hydrostatic
-    integer,                intent(in) :: communicator
+    type(MPI_Comm),         intent(in) :: communicator
     integer,                intent(in) :: ntasks
     integer,                intent(in) :: nthreads
 
@@ -3316,9 +3318,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, &
     integer :: seed0
     logical :: exists
     real(kind=kind_phys) :: tem
-    real(kind=kind_phys) :: rinc(5)
-    real(kind=kind_sngl_prec) :: rinc4(5)
-    real(kind=kind_dbl_prec) :: rinc8(5)
+    real(kind=kind_dbl_prec) :: rinc(5)
     real(kind=kind_phys) :: wrk(1)
     real(kind=kind_phys), parameter :: con_hr = 3600.
 
@@ -3681,6 +3681,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, &
     logical              :: lheatstrg      = .false.                  !< flag for canopy heat storage parameterization
     logical              :: lseaspray      = .false.                  !< flag for sea spray parameterization
     logical              :: cnvcld         = .false.
+    logical              :: xr_cnvcld      = .true.                   !< flag for including suspended convective clouds in Xu-Randall cloud fraction
     logical              :: random_clds    = .false.                  !< flag controls whether clouds are random
     logical              :: shal_cnv       = .false.                  !< flag for calling shallow convection
     integer              :: imfshalcnv     =  1                       !< flag for mass-flux shallow convection scheme
@@ -3974,7 +3975,6 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, &
 
     real(kind=kind_phys) :: radar_tten_limits(2) = (/ limit_unspecified, limit_unspecified /)
     integer :: itime
-    integer :: w3kindreal,w3kindint
 
 !--- END NAMELIST VARIABLES
 
@@ -4064,8 +4064,8 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, &
                                hwrf_samfdeep, hwrf_samfshal,progsigma,betascu,betamcu,      &
                                betadcu,h2o_phys, pdfcld, shcnvcw, redrag, hybedmf, satmedmf,&
                                shinhong, do_ysu, dspheat, lheatstrg, lseaspray, cnvcld,     &
-                               random_clds, shal_cnv, imfshalcnv, imfdeepcnv, isatmedmf,    &
-                               do_deep, jcap,                                               &
+                               xr_cnvcld, random_clds, shal_cnv, imfshalcnv, imfdeepcnv,    &
+                               isatmedmf, do_deep, jcap,                                    &
                                cs_parm, flgmin, cgwf, ccwf, cdmbgwd, sup, ctei_rm, crtrh,   &
                                dlqf, rbcr, shoc_parm, psauras, prauras, wminras,            &
                                do_sppt, do_shum, do_skeb,                                   &
@@ -4894,6 +4894,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, &
     Model%lheatstrg         = lheatstrg
     Model%lseaspray         = lseaspray
     Model%cnvcld            = cnvcld
+    Model%xr_cnvcld         = xr_cnvcld
     Model%random_clds       = random_clds
     Model%shal_cnv          = shal_cnv
     Model%imfshalcnv        = imfshalcnv
@@ -5608,19 +5609,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, &
     Model%cdec             = -9999.
     Model%clstp            = -9999
     rinc(1:5)              = 0
-    call w3kind(w3kindreal,w3kindint)
-    if (w3kindreal == 8) then
-       rinc8(1:5) = 0
-       call w3difdat(jdat,idat,4,rinc8)
-       rinc = rinc8
-    else if (w3kindreal == 4) then
-       rinc4(1:5) = 0
-       call w3difdat(jdat,idat,4,rinc4)
-       rinc = rinc4
-    else
-       write(0,*)' FATAL ERROR: Invalid w3kindreal'
-       call abort
-    endif
+    call w3difdat(jdat,idat,4,rinc)
     Model%phour            = rinc(4)/con_hr
     Model%fhour            = (rinc(4) + Model%dtp)/con_hr
     Model%zhour            = mod(Model%phour,Model%fhzero)
@@ -6195,7 +6184,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, &
                  ' do_shoc=',   Model%do_shoc,   ' nshoc3d=',  Model%nshoc_3d,  &
                  ' nshoc_2d=',  Model%nshoc_2d,  ' shoc_cld=', Model%shoc_cld,  &
                  ' nkbfshoc=',  Model%nkbfshoc,  ' nahdshoc=', Model%nahdshoc,  &
-                 ' nscfshoc=',  Model%nscfshoc,                                 &
+                 ' nscfshoc=',  Model%nscfshoc,  ' xr_cnvcld=',Model%xr_cnvcld, &
                  ' uni_cld=',   Model%uni_cld,                                  &
                  ' ntot3d=',    Model%ntot3d,    ' ntot2d=',   Model%ntot2d,    &
                  ' shocaftcnv=',Model%shocaftcnv,' indcld=',   Model%indcld,    &
@@ -6418,7 +6407,7 @@ subroutine control_print(Model)
       print *, 'basic control parameters'
       print *, ' me                : ', Model%me
       print *, ' master            : ', Model%master
-      print *, ' communicator      : ', Model%communicator
+      print *, ' communicator      : ', Model%communicator%mpi_val
       print *, ' nlunit            : ', Model%nlunit
       print *, ' fn_nml            : ', trim(Model%fn_nml)
       print *, ' fhzero            : ', Model%fhzero