diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 5d41d0441d..a7a96805e9 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -277,6 +277,13 @@ steps: # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge false --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 21600 --dt 200secs --dt_rad 6hours --idealized_insolation true --mono_surface true --h_elem 4 --precip_model 0M --run_name target_params_in_slab1" # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab1_artifacts/total_energy*.png" + - label: "AMIP target: albedo from function" + key: "target_amip_albedo_function" + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_amip_albedo_function.yml" + artifact_paths: "experiments/AMIP/output/amip/target_amip_albedo_function_artifacts/*" + agents: + slurm_mem: 20GB + - label: "AMIP - Float64 + hourly checkpoint" key: "amip" command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64_hourly_checkpoints.yml" @@ -377,6 +384,17 @@ steps: slurm_mem: 20GB slurm_gpus: 1 + # GPU RUNS: AMIP + - label: "GPU AMIP target: albedo from function" + key: "gpu_target_amip_albedo_function" + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_target_amip_albedo_function.yml" + artifact_paths: "experiments/AMIP/output/amip/gpu_target_amip_albedo_function_artifacts/*" + agents: + slurm_mem: 20GB + slurm_gpus: 1 + + - wait + - wait # plot job performance history diff --git a/config/model_configs/default_mono.yml b/config/model_configs/default_mono.yml index b9f902fa05..a43189651c 100644 --- a/config/model_configs/default_mono.yml +++ b/config/model_configs/default_mono.yml @@ -1,6 +1,6 @@ apply_limiter: false dt: "200secs" -dt_cpl 200: +dt_cpl: 200 dt_save_to_sol: "3600secs" energy_check: true h_elem: 4 diff --git a/config/model_configs/gpu_target_amip_albedo_function.yml b/config/model_configs/gpu_target_amip_albedo_function.yml new file mode 100644 index 0000000000..55c8e23dd2 --- /dev/null +++ b/config/model_configs/gpu_target_amip_albedo_function.yml @@ -0,0 +1,23 @@ +alpha_rayleigh_uh: 0 +alpha_rayleigh_w: 10 +apply_limiter: false +dt: "150secs" +dt_cpl: 150 +dt_rad: "1hours" +dt_save_to_sol: "1days" +dz_bottom: 30 +dz_top: 3000 +h_elem: 16 +job_id: "gpu_target_amip_albedo_function" +kappa_4: 1e16 +land_albedo_type: "function" +mode_name: "amip" +moist: "equil" +precip_model: "0M" +rad: "gray" +rayleigh_sponge: true +run_name: "gpu_target_amip_albedo_function" +t_end: "300secs" +vert_diff: "true" +z_elem: 50 +z_stretch: false diff --git a/config/model_configs/target_amip_albedo_function.yml b/config/model_configs/target_amip_albedo_function.yml new file mode 100644 index 0000000000..5fc762e397 --- /dev/null +++ b/config/model_configs/target_amip_albedo_function.yml @@ -0,0 +1,23 @@ +alpha_rayleigh_uh: 0 +alpha_rayleigh_w: 10 +apply_limiter: false +dt: "150secs" +dt_cpl: 150 +dt_rad: "1hours" +dt_save_to_sol: "1days" +dz_bottom: 30 +dz_top: 3000 +h_elem: 16 +job_id: "target_amip_albedo_function" +kappa_4: 1e16 +land_albedo_type: "function" +mode_name: "amip" +moist: "equil" +precip_model: "0M" +rad: "gray" +rayleigh_sponge: true +run_name: "target_amip_albedo_function" +t_end: "300secs" +vert_diff: "true" +z_elem: 50 +z_stretch: false diff --git a/experiments/AMIP/user_io/amip_visualizer.jl b/experiments/AMIP/user_io/amip_visualizer.jl index 88166bd29d..69714df533 100644 --- a/experiments/AMIP/user_io/amip_visualizer.jl +++ b/experiments/AMIP/user_io/amip_visualizer.jl @@ -78,8 +78,9 @@ function read_latest_model_data(name::Symbol, filedir::String, root::String) varfile_root = @sprintf "%s%s" string(name) root filename = glob("*" * varfile_root * "*", filedir)[end] - comms_ctx = ClimaComms.SingletonCommsContext() - hdfreader = InputOutput.HDF5Reader(filename) + # Ensure file gets read onto CPU for postprocessing + cpu_singleton_context = ClimaComms.SingletonCommsContext(ClimaComms.CPUSingleThreaded()) + hdfreader = InputOutput.HDF5Reader(filename, cpu_singleton_context) var = InputOutput.read_field(hdfreader, string(name)) close(hdfreader) return var diff --git a/experiments/AMIP/user_io/user_diagnostics.jl b/experiments/AMIP/user_io/user_diagnostics.jl index 0407cb9c96..26b7605134 100644 --- a/experiments/AMIP/user_io/user_diagnostics.jl +++ b/experiments/AMIP/user_io/user_diagnostics.jl @@ -56,25 +56,23 @@ Top of the atmosphere radiation fluxes (W m⁻²). function get_var(cs::CoupledSimulation, ::Val{:toa_fluxes}) atmos_sim = cs.model_sims.atmos_sim face_space = axes(atmos_sim.integrator.u.f) - z = parent(Fields.coordinate_field(face_space).z) - Δz_top = round(FT(0.5) * (z[end, 1, 1, 1, 1] - z[end - 1, 1, 1, 1, 1])) - n_faces = length(z[:, 1, 1, 1, 1]) + nz_faces = length(face_space.grid.vertical_grid.topology.mesh.faces) LWd_TOA = Fields.level( CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation.radiation_model.face_lw_flux_dn), face_space), - n_faces - half, + nz_faces - half, ) LWu_TOA = Fields.level( CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation.radiation_model.face_lw_flux_up), face_space), - n_faces - half, + nz_faces - half, ) SWd_TOA = Fields.level( CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation.radiation_model.face_sw_flux_dn), face_space), - n_faces - half, + nz_faces - half, ) SWu_TOA = Fields.level( CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation.radiation_model.face_sw_flux_up), face_space), - n_faces - half, + nz_faces - half, ) radiation_sources = @. -(LWd_TOA + SWd_TOA - LWu_TOA - SWu_TOA) diff --git a/src/Regridder.jl b/src/Regridder.jl index 1cdc880a35..ce166a9323 100644 --- a/src/Regridder.jl +++ b/src/Regridder.jl @@ -163,10 +163,10 @@ function hdwrite_regridfile_rll_to_cgll( end # If doesn't make sense to regrid with GPUs/MPI processes - cpu_context = ClimaComms.SingletonCommsContext(ClimaComms.CPUSingleThreaded()) + cpu_singleton_context = ClimaComms.SingletonCommsContext(ClimaComms.CPUSingleThreaded()) topology = Topologies.Topology2D( - cpu_context, + cpu_singleton_context, Spaces.topology(space2d).mesh, Topologies.spacefillingcurve(Spaces.topology(space2d).mesh), ) @@ -245,7 +245,7 @@ function hdwrite_regridfile_rll_to_cgll( ) map( - x -> write_to_hdf5(REGRID_DIR, hd_outfile_root, times[x], offline_fields[x], varname, cpu_context), + x -> write_to_hdf5(REGRID_DIR, hd_outfile_root, times[x], offline_fields[x], varname, cpu_singleton_context), 1:length(times), ) jldsave(joinpath(REGRID_DIR, hd_outfile_root * "_times.jld2"); times = times)