From bb2889b878320c84d77230df5c7c27ac4b8439b4 Mon Sep 17 00:00:00 2001 From: junmeiban Date: Tue, 17 Nov 2020 11:03:19 -0700 Subject: [PATCH] Reduce file size via 2-stream model initialization and DA cycling (#5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR consists of 4 changes: **1. Two-stream model initialization:** Currently, we use the restart file for MPAS-JEDI cycling. While we move to higher-resolution cycling experiments, we face the challenge of both memory usage and disk storage. This PR addresses the disk storage issue. The idea is: instead of using restart files, we move to use the file (init.nc type background/analysis file) that keeps only the necessary fields. Also split out static fields into a separate file. These v7-based changes are based off of a v6-based branch (https://github.com/skamaroc/MPAS-Model/commit/096b5d345ceb46351c9edcaed5238630c932784d). **2. Allow comparable two-stream 'mpasout' cycling result with that of one-stream 'restart' cycling:** This PR also fixes the large T2m/Q2m and low-level T/Q error growth issue that shows up in 120-km cycling experiments with this 2-stream workflow, which is caused by the different initialization of land use fields when config_do_restart is true and false. After this change in mpas_atmphys_landuse.F, using this two-stream code gives comparable cycling results to the original restart cycling. **3. Add 4 more variables (pressure_base, pressure_p, u/v at cell center) in core_init_atmosphere/Registry.xml to ease init.nc file to be used for the model-space verification purpose.** **4. Commented out one line in mpas_init_atm_cases.F to fix the over-specification issue of sea ice.** Files changed: M src/core_atmosphere/Registry.xml M src/core_atmosphere/mpas_atm_core.F M src/core_atmosphere/mpas_atm_core_interface.F M src/core_atmosphere/physics/mpas_atmphys_landuse.F M src/core_init_atmosphere/Registry.xml M src/core_init_atmosphere/mpas_init_atm_cases.F **Details for two-stream model initialization and DA cycling:** - static stream: Includes the mesh, some of sfc_input variables(landmask, shdmin, albedo12m, etc) and parameters for gravity wave drag over orography. - da_state stream: Fields are specified in the MPAS-Atmosphere Registry. Includes fields needed for DA purposes (either analysis variables or fixed input needed for CRTM or other obs operators) - For cold start FC, both the static stream file and the input stream file should be set to the “init.nc” file produced by the init_atmosphere core; - For cycling run (FC), the input stream file should be the new da_state stream file that was previously written by the model and modified/updated in the DA cycle. (Both static and da_state stream need to be specified in the streams.atmosphere file) - For cycling run (DA): use static.nc to read in mesh fields (specified in the streams.atmosphere file) and set: config_do_restart = false - After implementing the capability, the size of the 120-km init.nc type background/analysis file is about 430M (double precision); ~50 variables (the size of original restart file is about 2GB). Memory usage also decreased when we separated the static stream: memory_usage - If users would like to use restart files in cycling DA, the only difference from before is in prepare “1stCycle_background” step, please see example: (pay attention to streams.atmosphere, namelist.atmosphere and run_fc2018041418.csh) script: /glade/work/jban/pandac/JB01_restart_1stCycle_background/run_fc2018041418.csh results:/glade/scratch/jban/pandac/JB01_restart_1stCycle_background - The modification has no impact on restart mode run. The following figure shows 6-h forecast from restart cycling run (current code: old; this PR :new) verified against GFSANA and there is no difference between them. ![JB01_restart-expmgfs_day0p25_NXTro_surface_pressure_RMS](https://user-images.githubusercontent.com/23242703/98064199-f690eb00-1e0e-11eb-8009-14070edeae57.png) - Tested on Cheyenne. All ctests passed. --- src/core_atmosphere/Registry.xml | 152 +++++++++++++++--- src/core_atmosphere/mpas_atm_core.F | 4 +- src/core_atmosphere/mpas_atm_core_interface.F | 2 +- .../physics/mpas_atmphys_landuse.F | 6 +- src/core_init_atmosphere/Registry.xml | 4 + .../mpas_init_atm_cases.F | 2 +- 6 files changed, 143 insertions(+), 27 deletions(-) diff --git a/src/core_atmosphere/Registry.xml b/src/core_atmosphere/Registry.xml index 1b2c7e04ee..85fdd251a0 100644 --- a/src/core_atmosphere/Registry.xml +++ b/src/core_atmosphere/Registry.xml @@ -395,9 +395,10 @@ - @@ -442,6 +443,10 @@ + + + + @@ -467,29 +472,73 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + - - - - - - + + + + + + + + + + @@ -506,19 +555,11 @@ + + - - - - - - - - - - @@ -1117,6 +1158,73 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/core_atmosphere/mpas_atm_core.F b/src/core_atmosphere/mpas_atm_core.F index 5b56d653a8..84754e94f4 100644 --- a/src/core_atmosphere/mpas_atm_core.F +++ b/src/core_atmosphere/mpas_atm_core.F @@ -92,16 +92,18 @@ function atm_core_init(domain, startTimeStamp) result(ierr) if (config_do_restart) then call MPAS_stream_mgr_read(domain % streamManager, streamID='restart', ierr=ierr) else + call MPAS_stream_mgr_read(domain % streamManager, streamID='static', whence=MPAS_STREAM_NEAREST, ierr=ierr) call MPAS_stream_mgr_read(domain % streamManager, streamID='input', ierr=ierr) end if + if (ierr /= MPAS_STREAM_MGR_NOERR) then call mpas_log_write('********************************************************************************', messageType=MPAS_LOG_ERR) call mpas_log_write('Error reading initial conditions', messageType=MPAS_LOG_ERR) call mpas_log_write('********************************************************************************', messageType=MPAS_LOG_CRIT) end if + call MPAS_stream_mgr_reset_alarms(domain % streamManager, streamID='static', direction=MPAS_STREAM_INPUT, ierr=ierr) call MPAS_stream_mgr_reset_alarms(domain % streamManager, streamID='input', direction=MPAS_STREAM_INPUT, ierr=ierr) call MPAS_stream_mgr_reset_alarms(domain % streamManager, streamID='restart', direction=MPAS_STREAM_INPUT, ierr=ierr) - ! ! Read all other inputs ! For now we don't do this here to match results with previous code; to match requires diff --git a/src/core_atmosphere/mpas_atm_core_interface.F b/src/core_atmosphere/mpas_atm_core_interface.F index 437502430e..c17a272b15 100644 --- a/src/core_atmosphere/mpas_atm_core_interface.F +++ b/src/core_atmosphere/mpas_atm_core_interface.F @@ -314,7 +314,7 @@ function atm_get_mesh_stream(configs, stream) result(ierr) else if (config_do_restart) then write(stream,'(a)') 'restart' else - write(stream,'(a)') 'input' + write(stream,'(a)') 'static' end if end function atm_get_mesh_stream diff --git a/src/core_atmosphere/physics/mpas_atmphys_landuse.F b/src/core_atmosphere/physics/mpas_atmphys_landuse.F index 779b85a4f9..309e79c289 100644 --- a/src/core_atmosphere/physics/mpas_atmphys_landuse.F +++ b/src/core_atmosphere/physics/mpas_atmphys_landuse.F @@ -95,7 +95,7 @@ subroutine landuse_init_forMPAS(dminfo,julday,mesh,configs,diag_physics,sfc_inpu integer,intent(in):: julday !local pointers: - logical,pointer:: config_do_restart, & + logical,pointer:: config_do_restart, config_do_DAcycling, & config_frac_seaice, & config_sfc_albedo @@ -133,6 +133,7 @@ subroutine landuse_init_forMPAS(dminfo,julday,mesh,configs,diag_physics,sfc_inpu !call mpas_log_write('--- enter subroutine landuse_init_forMPAS:') call mpas_pool_get_config(configs,'config_do_restart' ,config_do_restart ) + call mpas_pool_get_config(configs,'config_do_DAcycling' ,config_do_DAcycling ) call mpas_pool_get_config(configs,'config_frac_seaice',config_frac_seaice) call mpas_pool_get_config(configs,'config_sfc_albedo' ,config_sfc_albedo ) @@ -252,8 +253,9 @@ subroutine landuse_init_forMPAS(dminfo,julday,mesh,configs,diag_physics,sfc_inpu !call mpas_log_write('--- isice =$i',intArgs=(/isice/)) !call mpas_log_write('--- iswater =$i',intArgs=(/iswater/)) !call mpas_log_write('--- isurban =$i',intArgs=(/isurban/)) - if(config_do_restart) then + if(config_do_restart .or. config_do_DAcycling) then call mpas_log_write('--- config_do_restart =$l', logicArgs=(/config_do_restart/)) + call mpas_log_write('--- config_do_DAcycling =$l', logicArgs=(/config_do_DAcycling/)) call mpas_log_write('--- skip the end of landuse_init_forMPAS') return endif diff --git a/src/core_init_atmosphere/Registry.xml b/src/core_init_atmosphere/Registry.xml index 515b881d01..90dad39fdc 100644 --- a/src/core_init_atmosphere/Registry.xml +++ b/src/core_init_atmosphere/Registry.xml @@ -555,6 +555,10 @@ + + + + diff --git a/src/core_init_atmosphere/mpas_init_atm_cases.F b/src/core_init_atmosphere/mpas_init_atm_cases.F index 84e056b9a0..1fb7c73045 100644 --- a/src/core_init_atmosphere/mpas_init_atm_cases.F +++ b/src/core_init_atmosphere/mpas_init_atm_cases.F @@ -4498,7 +4498,7 @@ subroutine init_atm_case_gfs(block, mesh, nCells, nEdges, nVertLevels, fg, state if (allocated(maskslab)) deallocate(maskslab) ! Freeze really cold ocean - where (sst < 271.0 .and. landmask == 0) xice = 1.0 + !where (sst < 271.0 .and. landmask == 0) xice = 1.0 ! Limit XICE to values between 0 and 1. Although the input meteorological field is between 0. ! and 1., interpolation to the MPAS grid can yield values of XiCE less than 0. and greater