Merge pull request #846 from CliMA/zs/leaderboard_radiation

add radiative fluxes to leaderboard

config/ci_configs/gpu_amip_target_topo_diagedmf_shortrun.yml

@@ -15,7 +15,7 @@ land_albedo_type: "map_temporal"
 mode_name: "amip"
 mono_surface: false
 netcdf_output_at_levels: true
-start_date: "19790301"
+start_date: "20100101"
 surface_setup: "PrescribedSurface"
 t_end: "200secs"
 topo_smoothing: true

config/longrun_configs/amip_target.yml

@@ -14,7 +14,7 @@ land_albedo_type: "map_temporal"
 mode_name: "amip"
 mono_surface: false
 rad: "allskywithclear"
-start_date: "19790301"
+start_date: "20100101"
 surface_setup: "PrescribedSurface"
 t_end: "100days"
 turb_flux_partition: "CombinedStateFluxesMOST"

config/longrun_configs/amip_target_topo.yml

@@ -16,7 +16,7 @@ mode_name: "amip"
 mono_surface: false
 netcdf_interpolate_z_over_msl: true
 rad: "allskywithclear"
-start_date: "19790301"
+start_date: "20100101"
 surface_setup: "PrescribedSurface"
 t_end: "100days"
 topo_smoothing: true

config/longrun_configs/dyamond_target.yml

@@ -8,6 +8,6 @@ land_albedo_type: "map_temporal"
 mode_name: "amip"
 mono_surface: false
 monthly_checkpoint: false
-start_date: "19790301"
+start_date: "20100101"
 t_end: "1days"
 turb_flux_partition: "CombinedStateFluxesMOST"

config/longrun_configs/longrun_amip_dyamond.yml

@@ -8,6 +8,6 @@ land_albedo_type: "map_temporal"
 mode_name: "amip"
 mono_surface: false
 monthly_checkpoint: false
-start_date: "19790301"
+start_date: "20100101"
 t_end: "1days"
 turb_flux_partition: "CombinedStateFluxesMOST"

experiments/ClimaEarth/Artifacts.toml

@@ -10,10 +10,10 @@ git-tree-sha1 = "9969543acd3c9084b8238b7858d0c76aa0bf98a7"
 git-tree-sha1 = "b377e7c997804b47720b8e7243361d399d108441"
 
 [cmip_model_rmse]
-git-tree-sha1 = "aa2d5178a656e9ce7a37698d978d8ed92aa46d99"
+git-tree-sha1 = "53a27731c0a2ec60756af24e4c60ac37da4a32bc"
 
     [[cmip_model_rmse.download]]
-    sha256 = "b019d40510cebc678899a74d2cf14e33aa60452805478b2bf7176b891012aa97"
+    sha256 = "3e8fa41df226dda9d375f8cb9dbeee3e10f9fceabcad2c7b91c9633b489b0437"
     url = "https://caltech.box.com/shared/static/lmzvnjkxctlb0fanieeqxll0vdrbei9e.gz"
 
 [precipitation_obs]
@@ -22,3 +22,10 @@ git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af"
     [[precipitation_obs.download]]
     sha256 = "463ebc1886c4e0be9656656da699f2cbb87d7c2568d5e81277c0c8a415dc8143"
     url = "https://caltech.box.com/shared/static/j63uak557kw7tlzb3dgwifoaeczekzm4.gz"
+
+[radiation_obs]
+git-tree-sha1 = "c9eef049fc3bf057f35332c316755a694b0eb5ef"
+
+    [[radiation_obs.download]]
+    sha256 = "3274926803246cd3738e2d9f36c3fc2652fdb564fba80ddcb10860fd5d8cb552"
+    url = "https://caltech.box.com/shared/static/odvaz76hi3ujakhwgkilzbitk49ty1fw.gz"

experiments/ClimaEarth/run_amip.jl

@@ -938,7 +938,7 @@ if ClimaComms.iamroot(comms_ctx)
             @info output_dates
 
             include("user_io/leaderboard.jl")
-            compare_vars = ["pr"]
+            compare_vars = ["pr", "rsut", "rlut"]
             function compute_biases(dates)
                 if isempty(dates)
                     return map(x -> 0.0, compare_vars)

experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl

@@ -1,19 +1,26 @@
 import Statistics: median
 
-pr_file_path = joinpath(@clima_artifact("cmip_model_rmse"), "pr_rmse_amip_pr_amip_5yr.csv")
+const RMSE_FILE_PATHS = Dict()
 
-open(pr_file_path, "r") do io
-    # Skip the header
-    header = readline(io)
+RMSE_FILE_PATHS["pr"] = joinpath(@clima_artifact("cmip_model_rmse"), "pr_rmse_amip_pr_amip_5yr.csv")
+RMSE_FILE_PATHS["rsut"] = joinpath(@clima_artifact("cmip_model_rmse"), "rsut_rmse_amip_rsut_amip_5yr.csv")
+RMSE_FILE_PATHS["rlut"] = joinpath(@clima_artifact("cmip_model_rmse"), "rlut_rmse_amip_rlut_amip_5yr.csv")
 
-    # Process each line
-    for line in eachline(io)
-        # Split the line by comma
-        fields = split(line, ',')
-        model_name = fields[1]
-        DJF, MAM, JJA, SON, ANN = map(x -> parse(Float64, x), fields[2:end])
+short_names = ["pr", "rsut", "rlut"]
+for short_name in short_names
+    open(RMSE_FILE_PATHS[short_name], "r") do io
+        # Skip the header
+        header = readline(io)
 
-        push!(OTHER_MODELS_RMSEs["pr"], RMSEs(; model_name, DJF, MAM, JJA, SON, ANN))
+        # Process each line
+        for line in eachline(io)
+            # Split the line by comma
+            fields = split(line, ',')
+            model_name = fields[1]
+            DJF, MAM, JJA, SON, ANN = map(x -> parse(Float64, x), fields[2:end])
+
+            push!(OTHER_MODELS_RMSEs[short_name], RMSEs(; model_name, DJF, MAM, JJA, SON, ANN))
+        end
     end
 end
 
@@ -71,4 +78,6 @@ function RSME_stats(vecRMSEs)
     (; best_single_model = best_single_model(vecRMSEs), median_model, worst_model, best_model)
 end
 
-COMPARISON_RMSEs["pr"] = RSME_stats(OTHER_MODELS_RMSEs["pr"])
+for short_name in short_names
+    COMPARISON_RMSEs[short_name] = RSME_stats(OTHER_MODELS_RMSEs[short_name])
+end

experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl

@@ -8,6 +8,10 @@ function preprocess_pr_fn(data)
     return data .* Float32(-86400)
 end
 
+function replace_nan(x; v = 0.0)
+    return map(x -> isnan(x) ? zero(x) : x, v)
+end
+
 Base.@kwdef struct RMSEs
     model_name::String
     ANN::Union{<:Real, ClimaAnalysis.OutputVar} = 0.0
@@ -28,26 +32,28 @@ OBS_DS["pr"] = ObsDataSource(;
     path = joinpath(@clima_artifact("precipitation_obs"), "gpcp.precip.mon.mean.197901-202305.nc"),
     var_name = "precip",
 )
-
 SIM_DS_KWARGS["pr"] = (; preprocess_data_fn = preprocess_pr_fn, new_units = "mm / day")
-
 OTHER_MODELS_RMSEs["pr"] = []
 
-include("cmip_rmse.jl")
+OBS_DS["rsut"] = ObsDataSource(;
+    path = joinpath(@clima_artifact("radiation_obs"), "CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc"),
+    var_name = "toa_sw_all_mon",
+)
+SIM_DS_KWARGS["rsut"] = (;)
+OTHER_MODELS_RMSEs["rsut"] = []
 
-# OBS_DS["rsut"] = ObsDataSource(;
-#                              path = "OBS/CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc",
-#                              var_name = "toa_sw_all_mon",
-#                              )
+OBS_DS["rlut"] = ObsDataSource(;
+    path = joinpath(@clima_artifact("radiation_obs"), "CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc"),
+    var_name = "toa_lw_all_mon",
+)
+SIM_DS_KWARGS["rlut"] = (;)
+OTHER_MODELS_RMSEs["rlut"] = []
 
-# OBS_DS["rlut"] = ObsDataSource(;
-#                              path = "OBS/CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc",
-#                              var_name = "toa_lw_all_mon",
-#                              )
+include("cmip_rmse.jl")
 
 function bias(output_dir::AbstractString, short_name::AbstractString, target_dates::AbstractArray{<:Dates.DateTime})
     obs = OBS_DS[short_name]
-    sim = SimDataSource(; path = output_dir, short_name, SIM_DS_KWARGS["pr"]...)
+    sim = SimDataSource(; path = output_dir, short_name, SIM_DS_KWARGS[short_name]...)
     return bias(obs, sim, target_dates)
 end
 
@@ -59,7 +65,7 @@ function plot_biases(biases; output_path)
     fig = CairoMakie.Figure(; size = (600, 300 * length(biases)))
     loc = 1
     for bias_var in biases
-        ClimaAnalysis.Visualize.heatmap2D_on_globe!(fig, bias_var; p_loc = (1, loc))
+        ClimaAnalysis.Visualize.heatmap2D_on_globe!(fig, bias_var; p_loc = (loc, 1))
         loc = loc + 1
     end
     CairoMakie.save(output_path, fig)
@@ -101,10 +107,10 @@ function plot_leaderboard(rmses; output_path)
         short_name = rmse.ANN.attributes["var_short_name"]
         units = rmse.ANN.attributes["units"]
         ax = CairoMakie.Axis(
-            fig[1, loc],
+            fig[loc, 1],
             ylabel = "$short_name [$units]",
             xticks = (1:5, ["Ann", "DJF", "JJA", "MAM", "SON"]),
-            title = "Global RMSE",
+            title = "Global RMSE $short_name [$units]",
         )
 
         # Against other models

test/Artifacts.toml

@@ -1,8 +1,8 @@
 [cmip_model_rmse]
-git-tree-sha1 = "aa2d5178a656e9ce7a37698d978d8ed92aa46d99"
+git-tree-sha1 = "53a27731c0a2ec60756af24e4c60ac37da4a32bc"
 
     [[cmip_model_rmse.download]]
-    sha256 = "b019d40510cebc678899a74d2cf14e33aa60452805478b2bf7176b891012aa97"
+    sha256 = "3e8fa41df226dda9d375f8cb9dbeee3e10f9fceabcad2c7b91c9633b489b0437"
     url = "https://caltech.box.com/shared/static/lmzvnjkxctlb0fanieeqxll0vdrbei9e.gz"
 
 [precipitation_obs]
@@ -11,3 +11,10 @@ git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af"
     [[precipitation_obs.download]]
     sha256 = "463ebc1886c4e0be9656656da699f2cbb87d7c2568d5e81277c0c8a415dc8143"
     url = "https://caltech.box.com/shared/static/j63uak557kw7tlzb3dgwifoaeczekzm4.gz"
+
+[radiation_obs]
+git-tree-sha1 = "c9eef049fc3bf057f35332c316755a694b0eb5ef"
+
+    [[radiation_obs.download]]
+    sha256 = "3274926803246cd3738e2d9f36c3fc2652fdb564fba80ddcb10860fd5d8cb552"
+    url = "https://caltech.box.com/shared/static/odvaz76hi3ujakhwgkilzbitk49ty1fw.gz"