From 677eaa9b296989986ea4b4a518354dd3aab1df67 Mon Sep 17 00:00:00 2001 From: Zhaoyi Shen <11598433+szy21@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:15:45 -0700 Subject: [PATCH 1/2] add radiative fluxes to leaderboard --- experiments/ClimaEarth/Artifacts.toml | 11 ++++-- experiments/ClimaEarth/run_amip.jl | 2 +- .../user_io/leaderboard/cmip_rmse.jl | 33 ++++++++++------- .../user_io/leaderboard/compare_with_obs.jl | 36 +++++++++++-------- test/Artifacts.toml | 11 ++++-- 5 files changed, 61 insertions(+), 32 deletions(-) diff --git a/experiments/ClimaEarth/Artifacts.toml b/experiments/ClimaEarth/Artifacts.toml index 385fb2673c..83bfdd3654 100644 --- a/experiments/ClimaEarth/Artifacts.toml +++ b/experiments/ClimaEarth/Artifacts.toml @@ -10,10 +10,10 @@ git-tree-sha1 = "9969543acd3c9084b8238b7858d0c76aa0bf98a7" git-tree-sha1 = "b377e7c997804b47720b8e7243361d399d108441" [cmip_model_rmse] -git-tree-sha1 = "aa2d5178a656e9ce7a37698d978d8ed92aa46d99" +git-tree-sha1 = "53a27731c0a2ec60756af24e4c60ac37da4a32bc" [[cmip_model_rmse.download]] - sha256 = "b019d40510cebc678899a74d2cf14e33aa60452805478b2bf7176b891012aa97" + sha256 = "3e8fa41df226dda9d375f8cb9dbeee3e10f9fceabcad2c7b91c9633b489b0437" url = "https://caltech.box.com/shared/static/lmzvnjkxctlb0fanieeqxll0vdrbei9e.gz" [precipitation_obs] @@ -22,3 +22,10 @@ git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af" [[precipitation_obs.download]] sha256 = "463ebc1886c4e0be9656656da699f2cbb87d7c2568d5e81277c0c8a415dc8143" url = "https://caltech.box.com/shared/static/j63uak557kw7tlzb3dgwifoaeczekzm4.gz" + +[radiation_obs] +git-tree-sha1 = "c9eef049fc3bf057f35332c316755a694b0eb5ef" + + [[radiation_obs.download]] + sha256 = "3274926803246cd3738e2d9f36c3fc2652fdb564fba80ddcb10860fd5d8cb552" + url = "https://caltech.box.com/shared/static/odvaz76hi3ujakhwgkilzbitk49ty1fw.gz" diff --git a/experiments/ClimaEarth/run_amip.jl b/experiments/ClimaEarth/run_amip.jl index e79cd3ee60..fb1422ad3b 100644 --- a/experiments/ClimaEarth/run_amip.jl +++ b/experiments/ClimaEarth/run_amip.jl @@ -938,7 +938,7 @@ if ClimaComms.iamroot(comms_ctx) @info output_dates include("user_io/leaderboard.jl") - compare_vars = ["pr"] + compare_vars = ["pr", "rsut", "rlut"] function compute_biases(dates) if isempty(dates) return map(x -> 0.0, compare_vars) diff --git a/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl b/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl index 71dc462a20..812f7e70b9 100644 --- a/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl +++ b/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl @@ -1,19 +1,26 @@ import Statistics: median -pr_file_path = joinpath(@clima_artifact("cmip_model_rmse"), "pr_rmse_amip_pr_amip_5yr.csv") +const RMSE_FILE_PATHS = Dict() -open(pr_file_path, "r") do io - # Skip the header - header = readline(io) +RMSE_FILE_PATHS["pr"] = joinpath(@clima_artifact("cmip_model_rmse"), "pr_rmse_amip_pr_amip_5yr.csv") +RMSE_FILE_PATHS["rsut"] = joinpath(@clima_artifact("cmip_model_rmse"), "rsut_rmse_amip_rsut_amip_5yr.csv") +RMSE_FILE_PATHS["rlut"] = joinpath(@clima_artifact("cmip_model_rmse"), "rlut_rmse_amip_rlut_amip_5yr.csv") - # Process each line - for line in eachline(io) - # Split the line by comma - fields = split(line, ',') - model_name = fields[1] - DJF, MAM, JJA, SON, ANN = map(x -> parse(Float64, x), fields[2:end]) +short_names = ["pr", "rsut", "rlut"] +for short_name in short_names + open(RMSE_FILE_PATHS[short_name], "r") do io + # Skip the header + header = readline(io) - push!(OTHER_MODELS_RMSEs["pr"], RMSEs(; model_name, DJF, MAM, JJA, SON, ANN)) + # Process each line + for line in eachline(io) + # Split the line by comma + fields = split(line, ',') + model_name = fields[1] + DJF, MAM, JJA, SON, ANN = map(x -> parse(Float64, x), fields[2:end]) + + push!(OTHER_MODELS_RMSEs[short_name], RMSEs(; model_name, DJF, MAM, JJA, SON, ANN)) + end end end @@ -71,4 +78,6 @@ function RSME_stats(vecRMSEs) (; best_single_model = best_single_model(vecRMSEs), median_model, worst_model, best_model) end -COMPARISON_RMSEs["pr"] = RSME_stats(OTHER_MODELS_RMSEs["pr"]) +for short_name in short_names + COMPARISON_RMSEs[short_name] = RSME_stats(OTHER_MODELS_RMSEs[short_name]) +end diff --git a/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl b/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl index a5d0e740ea..ed8843013f 100644 --- a/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl +++ b/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl @@ -8,6 +8,10 @@ function preprocess_pr_fn(data) return data .* Float32(-86400) end +function replace_nan(x; v = 0.0) + return map(x -> isnan(x) ? zero(x) : x, v) +end + Base.@kwdef struct RMSEs model_name::String ANN::Union{<:Real, ClimaAnalysis.OutputVar} = 0.0 @@ -28,26 +32,28 @@ OBS_DS["pr"] = ObsDataSource(; path = joinpath(@clima_artifact("precipitation_obs"), "gpcp.precip.mon.mean.197901-202305.nc"), var_name = "precip", ) - SIM_DS_KWARGS["pr"] = (; preprocess_data_fn = preprocess_pr_fn, new_units = "mm / day") - OTHER_MODELS_RMSEs["pr"] = [] -include("cmip_rmse.jl") +OBS_DS["rsut"] = ObsDataSource(; + path = joinpath(@clima_artifact("radiation_obs"), "CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc"), + var_name = "toa_sw_all_mon", +) +SIM_DS_KWARGS["rsut"] = (;) +OTHER_MODELS_RMSEs["rsut"] = [] -# OBS_DS["rsut"] = ObsDataSource(; -# path = "OBS/CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc", -# var_name = "toa_sw_all_mon", -# ) +OBS_DS["rlut"] = ObsDataSource(; + path = joinpath(@clima_artifact("radiation_obs"), "CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc"), + var_name = "toa_lw_all_mon", +) +SIM_DS_KWARGS["rlut"] = (;) +OTHER_MODELS_RMSEs["rlut"] = [] -# OBS_DS["rlut"] = ObsDataSource(; -# path = "OBS/CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc", -# var_name = "toa_lw_all_mon", -# ) +include("cmip_rmse.jl") function bias(output_dir::AbstractString, short_name::AbstractString, target_dates::AbstractArray{<:Dates.DateTime}) obs = OBS_DS[short_name] - sim = SimDataSource(; path = output_dir, short_name, SIM_DS_KWARGS["pr"]...) + sim = SimDataSource(; path = output_dir, short_name, SIM_DS_KWARGS[short_name]...) return bias(obs, sim, target_dates) end @@ -59,7 +65,7 @@ function plot_biases(biases; output_path) fig = CairoMakie.Figure(; size = (600, 300 * length(biases))) loc = 1 for bias_var in biases - ClimaAnalysis.Visualize.heatmap2D_on_globe!(fig, bias_var; p_loc = (1, loc)) + ClimaAnalysis.Visualize.heatmap2D_on_globe!(fig, bias_var; p_loc = (loc, 1)) loc = loc + 1 end CairoMakie.save(output_path, fig) @@ -101,10 +107,10 @@ function plot_leaderboard(rmses; output_path) short_name = rmse.ANN.attributes["var_short_name"] units = rmse.ANN.attributes["units"] ax = CairoMakie.Axis( - fig[1, loc], + fig[loc, 1], ylabel = "$short_name [$units]", xticks = (1:5, ["Ann", "DJF", "JJA", "MAM", "SON"]), - title = "Global RMSE", + title = "Global RMSE $short_name [$units]", ) # Against other models diff --git a/test/Artifacts.toml b/test/Artifacts.toml index e3b19ade8b..1ad28857ca 100644 --- a/test/Artifacts.toml +++ b/test/Artifacts.toml @@ -1,8 +1,8 @@ [cmip_model_rmse] -git-tree-sha1 = "aa2d5178a656e9ce7a37698d978d8ed92aa46d99" +git-tree-sha1 = "53a27731c0a2ec60756af24e4c60ac37da4a32bc" [[cmip_model_rmse.download]] - sha256 = "b019d40510cebc678899a74d2cf14e33aa60452805478b2bf7176b891012aa97" + sha256 = "3e8fa41df226dda9d375f8cb9dbeee3e10f9fceabcad2c7b91c9633b489b0437" url = "https://caltech.box.com/shared/static/lmzvnjkxctlb0fanieeqxll0vdrbei9e.gz" [precipitation_obs] @@ -11,3 +11,10 @@ git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af" [[precipitation_obs.download]] sha256 = "463ebc1886c4e0be9656656da699f2cbb87d7c2568d5e81277c0c8a415dc8143" url = "https://caltech.box.com/shared/static/j63uak557kw7tlzb3dgwifoaeczekzm4.gz" + +[radiation_obs] +git-tree-sha1 = "c9eef049fc3bf057f35332c316755a694b0eb5ef" + + [[radiation_obs.download]] + sha256 = "3274926803246cd3738e2d9f36c3fc2652fdb564fba80ddcb10860fd5d8cb552" + url = "https://caltech.box.com/shared/static/odvaz76hi3ujakhwgkilzbitk49ty1fw.gz" From dc7a6ba0046e2b6eb19f000b5b525c19567ab044 Mon Sep 17 00:00:00 2001 From: Zhaoyi Shen <11598433+szy21@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:51:37 -0700 Subject: [PATCH 2/2] change amip start date to 20100101 --- config/ci_configs/gpu_amip_target_topo_diagedmf_shortrun.yml | 2 +- config/longrun_configs/amip_target.yml | 2 +- config/longrun_configs/amip_target_topo.yml | 2 +- config/longrun_configs/dyamond_target.yml | 2 +- config/longrun_configs/longrun_amip_dyamond.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/ci_configs/gpu_amip_target_topo_diagedmf_shortrun.yml b/config/ci_configs/gpu_amip_target_topo_diagedmf_shortrun.yml index 434fbd0bc8..5b97c29b27 100644 --- a/config/ci_configs/gpu_amip_target_topo_diagedmf_shortrun.yml +++ b/config/ci_configs/gpu_amip_target_topo_diagedmf_shortrun.yml @@ -15,7 +15,7 @@ land_albedo_type: "map_temporal" mode_name: "amip" mono_surface: false netcdf_output_at_levels: true -start_date: "19790301" +start_date: "20100101" surface_setup: "PrescribedSurface" t_end: "200secs" topo_smoothing: true diff --git a/config/longrun_configs/amip_target.yml b/config/longrun_configs/amip_target.yml index e74afbd6f8..5a8f95faaa 100644 --- a/config/longrun_configs/amip_target.yml +++ b/config/longrun_configs/amip_target.yml @@ -14,7 +14,7 @@ land_albedo_type: "map_temporal" mode_name: "amip" mono_surface: false rad: "allskywithclear" -start_date: "19790301" +start_date: "20100101" surface_setup: "PrescribedSurface" t_end: "100days" turb_flux_partition: "CombinedStateFluxesMOST" diff --git a/config/longrun_configs/amip_target_topo.yml b/config/longrun_configs/amip_target_topo.yml index d70ff31d6f..3fc472091a 100644 --- a/config/longrun_configs/amip_target_topo.yml +++ b/config/longrun_configs/amip_target_topo.yml @@ -16,7 +16,7 @@ mode_name: "amip" mono_surface: false netcdf_interpolate_z_over_msl: true rad: "allskywithclear" -start_date: "19790301" +start_date: "20100101" surface_setup: "PrescribedSurface" t_end: "100days" topo_smoothing: true diff --git a/config/longrun_configs/dyamond_target.yml b/config/longrun_configs/dyamond_target.yml index 396720a72f..92acd9fe3e 100644 --- a/config/longrun_configs/dyamond_target.yml +++ b/config/longrun_configs/dyamond_target.yml @@ -8,6 +8,6 @@ land_albedo_type: "map_temporal" mode_name: "amip" mono_surface: false monthly_checkpoint: false -start_date: "19790301" +start_date: "20100101" t_end: "1days" turb_flux_partition: "CombinedStateFluxesMOST" diff --git a/config/longrun_configs/longrun_amip_dyamond.yml b/config/longrun_configs/longrun_amip_dyamond.yml index 59b1694573..c716714fa6 100644 --- a/config/longrun_configs/longrun_amip_dyamond.yml +++ b/config/longrun_configs/longrun_amip_dyamond.yml @@ -8,6 +8,6 @@ land_albedo_type: "map_temporal" mode_name: "amip" mono_surface: false monthly_checkpoint: false -start_date: "19790301" +start_date: "20100101" t_end: "1days" turb_flux_partition: "CombinedStateFluxesMOST"