Skip to content

Commit

Permalink
Merge commit '2e2434d10e35404be5284022270f474d50d2db26' into noresm
Browse files Browse the repository at this point in the history
  • Loading branch information
mvdebolskiy committed Sep 22, 2023
2 parents aeb4498 + 2e2434d commit febcc84
Show file tree
Hide file tree
Showing 49 changed files with 3,209 additions and 2,627 deletions.
6 changes: 6 additions & 0 deletions .git-blame-ignore-revs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# Ran python directory through black python formatter
d229b5c6689efc4c2a6cef077515c4ccd5c18ff6
4cd83cb3ee6d85eb909403487abf5eeaf4d98911
0aa2957c1f8603c63fa30b11295c06cfddff44a5
2cdb380febb274478e84cd90945aee93f29fa2e6
Expand All @@ -10,6 +11,11 @@ b88e1cd1b28e3609684c79a2ec0e88f26cfc362b
b771971e3299c4fa56534b93421f7a2b9c7282fd
9de88bb57ea9855da408cbec1dc8acb9079eda47
8bc4688e52ea23ef688e283698f70a44388373eb
0a5a9e803b56ec1bbd6232eff1c99dbbeef25eb7
810cb346f05ac1aabfff931ab1a2b7b584add241
5933b0018f8e29413e30dda9b906370d147bad45
# Ran SystemTests and python/ctsm through black python formatter
5364ad66eaceb55dde2d3d598fe4ce37ac83a93c
8056ae649c1b37f5e10aaaac79005d6e3a8b2380
540b256d1f3382f4619d7b0877c32d54ce5c40b6
8a168bb0895f4f2421608dd2589398e13a6663e6
2 changes: 1 addition & 1 deletion bld/CLMBuildNamelist.pm
Original file line number Diff line number Diff line change
Expand Up @@ -4413,7 +4413,7 @@ sub setup_logic_misc {
add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'for_testing_use_second_grain_pool');
add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'for_testing_use_repr_structure_pool');
add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'for_testing_no_crop_seed_replenishment');
add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'hist_master_list_file');
add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'hist_fields_list_file');
}

#-------------------------------------------------------------------------------
Expand Down
4 changes: 2 additions & 2 deletions bld/namelist_files/namelist_defaults_ctsm.xml
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ attributes from the config_cache.xml file (with keys converted to upper-case).
<for_testing_use_repr_structure_pool>.false.</for_testing_use_repr_structure_pool>
<for_testing_no_crop_seed_replenishment>.false.</for_testing_no_crop_seed_replenishment>

<!-- Set to .true. in namelist to write hist fields master list file -->
<hist_master_list_file>.false.</hist_master_list_file>
<!-- Set to .true. in namelist to write file with all history fields -->
<hist_fields_list_file>.false.</hist_fields_list_file>

<!-- In accelerated spinup mode reduce the amount of history output -->
<hist_empty_htapes clm_accelerated_spinup="on">.true.</hist_empty_htapes>
Expand Down
4 changes: 2 additions & 2 deletions bld/namelist_files/namelist_definition_ctsm.xml
Original file line number Diff line number Diff line change
Expand Up @@ -764,9 +764,9 @@ SNICAR (SNow, ICe, and Aerosol Radiative model) optical data file name
SNICAR (SNow, ICe, and Aerosol Radiative model) snow aging data file name
</entry>

<entry id="hist_master_list_file" type="logical" category="history"
<entry id="hist_fields_list_file" type="logical" category="history"
group="clm_inparm" valid_values="" value=".false.">
If TRUE, write master field list to separate file for documentation purposes
If TRUE, write list of all output fields to separate file for documentation purposes
</entry>

<entry id="hist_avgflag_pertape" type="char*10(10)" category="history"
Expand Down
29 changes: 19 additions & 10 deletions cime_config/SystemTests/fsurdatmodifyctsm.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,18 @@

import os
import re
import systemtest_utils as stu
from CIME.SystemTests.system_tests_common import SystemTestsCommon
from CIME.XML.standard_module_setup import *
from CIME.SystemTests.test_utils.user_nl_utils import append_to_user_nl_files

# For calling fsurdat_modifier
from argparse import Namespace

_CTSM_PYTHON = os.path.join(
os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "python"
)
sys.path.insert(1, _CTSM_PYTHON)

logger = logging.getLogger(__name__)


Expand Down Expand Up @@ -66,16 +73,18 @@ def _create_config_file(self):
cfg_out.write(line)

def _run_modify_fsurdat(self):
tool_path = os.path.join(self._ctsm_root, "tools/modify_input_files/fsurdat_modifier")

self._case.load_env(reset=True)
command = f"python3 {tool_path} {self._cfg_file_path}"
stu.run_python_script(
self._get_caseroot(),
"ctsm_pylib",
command,
tool_path,
fsurdat_modifier_args = Namespace(
cfg_path=self._cfg_file_path,
debug=False,
fsurdat_in="UNSET",
fsurdat_out="UNSET",
overwrite=False,
silent=False,
verbose=False,
)
from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier

fsurdat_modifier(fsurdat_modifier_args)

def _modify_user_nl(self):
append_to_user_nl_files(
Expand Down
63 changes: 53 additions & 10 deletions cime_config/SystemTests/rxcropmaturity.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,8 @@ def run_phase(self):
case_gddgen.check_all_input_data()

# Make custom version of surface file
logger.info("RXCROPMATURITY log: run make_fsurdat_all_crops_everywhere")
self._run_make_fsurdat_all_crops_everywhere()
logger.info("RXCROPMATURITY log: run fsurdat_modifier")
self._run_fsurdat_modifier()

# -------------------------------------------------------------------
# (2) Perform GDD-generating run and generate prescribed GDDs file
Expand Down Expand Up @@ -239,7 +239,7 @@ def _setup_all(self):
logger.info("RXCROPMATURITY log: _setup_all done")

# Make a surface dataset that has every crop in every gridcell
def _run_make_fsurdat_all_crops_everywhere(self):
def _run_fsurdat_modifier(self):

# fsurdat should be defined. Where is it?
self._fsurdat_in = None
Expand All @@ -255,20 +255,29 @@ def _run_make_fsurdat_all_crops_everywhere(self):
raise RuntimeError(error_message)

# Where we will save the fsurdat version for this test
self._fsurdat_out = os.path.join(self._path_gddgen, "fsurdat.nc")
path, ext = os.path.splitext(self._fsurdat_in)
dir_in, filename_in_noext = os.path.split(path)
self._fsurdat_out = os.path.join(
self._path_gddgen, f"{filename_in_noext}.all_crops_everywhere{ext}"
)

# Make fsurdat for this test, if not already done
if not os.path.exists(self._fsurdat_out):
tool_path = os.path.join(
self._ctsm_root,
"python",
"ctsm",
"crop_calendars",
"make_fsurdat_all_crops_everywhere.py",
"tools",
"modify_input_files",
"fsurdat_modifier",
)
command = (
f"python3 {tool_path} " + f"-i {self._fsurdat_in} " + f"-o {self._fsurdat_out}"

# Create configuration file for fsurdat_modifier
self._cfg_path = os.path.join(
self._path_gddgen,
"modify_fsurdat_allcropseverywhere.cfg",
)
self._create_config_file_evenlysplitcrop()

command = f"python3 {tool_path} {self._cfg_path} "
stu.run_python_script(
self._get_caseroot(),
self._this_conda_env,
Expand All @@ -287,6 +296,40 @@ def _run_make_fsurdat_all_crops_everywhere(self):
]
)

def _create_config_file_evenlysplitcrop(self):
"""
Open the new and the template .cfg files
Loop line by line through the template .cfg file
When string matches, replace that line's content
"""
cfg_template_path = os.path.join(
self._ctsm_root, "tools/modify_input_files/modify_fsurdat_template.cfg"
)

with open(self._cfg_path, "w", encoding="utf-8") as cfg_out:
# Copy template, replacing some lines
with open(cfg_template_path, "r", encoding="utf-8") as cfg_in:
for line in cfg_in:
if re.match(r" *evenly_split_cropland *=", line):
line = f"evenly_split_cropland = True"
elif re.match(r" *fsurdat_in *=", line):
line = f"fsurdat_in = {self._fsurdat_in}"
elif re.match(r" *fsurdat_out *=", line):
line = f"fsurdat_out = {self._fsurdat_out}"
elif re.match(r" *process_subgrid_section *=", line):
line = f"process_subgrid_section = True"
cfg_out.write(line)

# Add new lines
cfg_out.write("\n")
cfg_out.write("[modify_fsurdat_subgrid_fractions]\n")
cfg_out.write("PCT_CROP = 100.0\n")
cfg_out.write("PCT_NATVEG = 0.0\n")
cfg_out.write("PCT_GLACIER = 0.0\n")
cfg_out.write("PCT_WETLAND = 0.0\n")
cfg_out.write("PCT_LAKE = 0.0\n")
cfg_out.write("PCT_URBAN = 0.0 0.0 0.0\n")

def _run_check_rxboth_run(self):

output_dir = os.path.join(self._get_caseroot(), "run")
Expand Down
63 changes: 46 additions & 17 deletions cime_config/SystemTests/systemtest_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,27 +10,41 @@ def cmds_to_setup_conda(caseroot):
# Use semicolon here since it's OK to fail
#
conda_setup_commands = ". " + caseroot + "/.env_mach_specific.sh; "
# Setting CONDA_PREFIX to empty ensures that this works even if called from
# a shell with a conda environment activated
conda_setup_commands += "CONDA_PREFIX=; "
# Execute the module unload/load when "which conda" fails
# eg on cheyenne
try:
subprocess.run("which conda", shell=True, check=True)
except subprocess.CalledProcessError:
# Remove python and add conda to environment for cheyennne
conda_setup_commands += " module unload python; module load conda;"
unload_python_load_conda = "module unload python; module load conda;"
# Make sure that adding this actually loads conda
subprocess.run(unload_python_load_conda + "which conda", shell=True, check=True)
# Save
conda_setup_commands += " " + unload_python_load_conda

return conda_setup_commands


def run_python_script(caseroot, this_conda_env, command, tool_path):

def cmds_to_run_via_conda(caseroot, conda_run_call, command):
# Run in the specified conda environment
conda_setup_commands = cmds_to_setup_conda(caseroot)
conda_setup_commands += f" conda run -n {this_conda_env}"
conda_setup_commands += " " + conda_run_call

# Finish with Python script call
command = conda_setup_commands + " " + command
print(f"command: {command}")

return command


def run_python_script(caseroot, this_conda_env, command_in, tool_path):

# First, try with "conda run -n"
command = cmds_to_run_via_conda(caseroot, f"conda run -n {this_conda_env}", command_in)

# Run with logfile
tool_name = os.path.split(tool_path)[-1]
try:
Expand All @@ -39,19 +53,34 @@ def run_python_script(caseroot, this_conda_env, command, tool_path):
command, shell=True, check=True, text=True, stdout=f, stderr=subprocess.STDOUT
)
except subprocess.CalledProcessError as error:
print("ERROR while getting the conda environment and/or ")
print(f"running the {tool_name} tool: ")
print(f"(1) If your {this_conda_env} environment is out of date or you ")
print(f"have not created the {this_conda_env} environment, yet, you may ")
print("get past this error by running ./py_env_create ")
print("in your ctsm directory and trying this test again. ")
print("(2) If conda is not available, install and load conda, ")
print("run ./py_env_create, and then try this test again. ")
print("(3) If (1) and (2) are not the issue, then you may be ")
print(f"getting an error within {tool_name} itself. ")
print("Default error message: ")
print(error.output)
raise
# Retry with the original "conda activate" method
command = cmds_to_run_via_conda(
caseroot,
f"conda activate {this_conda_env} && ",
command_in,
)
try:
with open(tool_name + ".log2", "w") as f:
subprocess.run(
command, shell=True, check=True, text=True, stdout=f, stderr=subprocess.STDOUT
)
except subprocess.CalledProcessError as error:
print("ERROR while getting the conda environment and/or ")
print(f"running the {tool_name} tool: ")
print(f"(1) If your {this_conda_env} environment is out of date or you ")
print(f"have not created the {this_conda_env} environment, yet, you may ")
print("get past this error by running ./py_env_create ")
print("in your ctsm directory and trying this test again. ")
print("(2) If conda is not available, install and load conda, ")
print("run ./py_env_create, and then try this test again. ")
print("(3) If (1) and (2) are not the issue, then you may be ")
print(f"getting an error within {tool_name} itself. ")
print("Default error message: ")
print(error.output)
raise
except:
print(f"ERROR trying to run {tool_name}.")
raise
except:
print(f"ERROR trying to run {tool_name}.")
raise
4 changes: 2 additions & 2 deletions cime_config/testdefs/testlist_clm.xml
Original file line number Diff line number Diff line change
Expand Up @@ -1007,7 +1007,7 @@
</machines>
<options>
<option name="wallclock">00:20:00</option>
<option name="comment" >Among other extra outputs, ensure that writing the history field master list to a separate file does not cause failure"</option>
<option name="comment" >Among other extra outputs, ensure that writing the list of all history fields to a separate file does not cause failure"</option>
</options>
</test>
<test name="ERS_D_Ld3" grid="f10_f10_mg37" compset="I1850Clm50BgcCrop" testmods="clm/default">
Expand Down Expand Up @@ -2594,7 +2594,7 @@
</machines>
</test>

<test name="RXCROPMATURITY_Lm61" grid="f09_g17" compset="IHistClm50BgcCrop">
<test name="RXCROPMATURITY_Lm61" grid="f09_g17" compset="IHistClm50BgcCrop" testmods="clm/cropMonthOutput">
<machines>
<machine name="cheyenne" compiler="intel" category="ctsm_sci">
<options>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
use_lch4 = .false.
hist_master_list_file = .true.
hist_fields_list_file = .true.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
hist_fields_list_file = .true.
4 changes: 2 additions & 2 deletions cime_config/testdefs/testmods_dirs/clm/extra_outputs/README
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
This test mod turns on extra diagnostic fields

It also outputs an optional text file containing a table of the
history fields master list
It also outputs an optional text file containing a table of all
the possible history fields
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
calc_human_stress_indices = 'ALL'
hist_master_list_file = .true.
hist_fields_list_file = .true.

hist_fincl1 += 'GSSUN:L43200', 'GSSHA:L43200', 'FSDSND:L43200', 'FSRND:L43200', 'FSRSFND:L43200',
'SSRE_FSRND:L43200', 'FSDSVD:L43200', 'FSDSVI:L43200',
Expand Down
Loading

0 comments on commit febcc84

Please sign in to comment.