add radiative fluxes to leaderboard

CliMA · Jun 11, 2024 · 22334d6 · 22334d6
1 parent c0f9d8b
commit 22334d6
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 29 deletions.
diff --git a/experiments/ClimaEarth/Artifacts.toml b/experiments/ClimaEarth/Artifacts.toml
@@ -10,10 +10,10 @@ git-tree-sha1 = "9969543acd3c9084b8238b7858d0c76aa0bf98a7"
 git-tree-sha1 = "b377e7c997804b47720b8e7243361d399d108441"
 
 [cmip_model_rmse]
-git-tree-sha1 = "aa2d5178a656e9ce7a37698d978d8ed92aa46d99"
+git-tree-sha1 = "53a27731c0a2ec60756af24e4c60ac37da4a32bc"
 
     [[cmip_model_rmse.download]]
-    sha256 = "b019d40510cebc678899a74d2cf14e33aa60452805478b2bf7176b891012aa97"
+    sha256 = "3e8fa41df226dda9d375f8cb9dbeee3e10f9fceabcad2c7b91c9633b489b0437"
     url = "https://caltech.box.com/shared/static/lmzvnjkxctlb0fanieeqxll0vdrbei9e.gz"
 
 [precipitation_obs]
@@ -22,3 +22,10 @@ git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af"
     [[precipitation_obs.download]]
     sha256 = "463ebc1886c4e0be9656656da699f2cbb87d7c2568d5e81277c0c8a415dc8143"
     url = "https://caltech.box.com/shared/static/j63uak557kw7tlzb3dgwifoaeczekzm4.gz"
+
+[radiation_obs]
+git-tree-sha1 = "c9eef049fc3bf057f35332c316755a694b0eb5ef"
+
+    [[radiation_obs.download]]
+    sha256 = "3274926803246cd3738e2d9f36c3fc2652fdb564fba80ddcb10860fd5d8cb552"
+    url = "https://caltech.box.com/shared/static/odvaz76hi3ujakhwgkilzbitk49ty1fw.gz"
diff --git a/experiments/ClimaEarth/run_amip.jl b/experiments/ClimaEarth/run_amip.jl
@@ -938,7 +938,7 @@ if ClimaComms.iamroot(comms_ctx)
             @info output_dates
 
             include("user_io/leaderboard.jl")
-            compare_vars = ["pr"]
+            compare_vars = ["pr", "rsut", "rlut"]
             function compute_biases(dates)
                 if isempty(dates)
                     return map(x -> 0.0, compare_vars)

diff --git a/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl b/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl
@@ -1,19 +1,26 @@
 import Statistics: median
 
-pr_file_path = joinpath(@clima_artifact("cmip_model_rmse"), "pr_rmse_amip_pr_amip_5yr.csv")
+const RMSE_FILE_PATHS = Dict()
 
-open(pr_file_path, "r") do io
-    # Skip the header
-    header = readline(io)
+RMSE_FILE_PATHS["pr"] = joinpath(@clima_artifact("cmip_model_rmse"), "pr_rmse_amip_pr_amip_5yr.csv")
+RMSE_FILE_PATHS["rsut"] = joinpath(@clima_artifact("cmip_model_rmse"), "rsut_rmse_amip_rsut_amip_5yr.csv")
+RMSE_FILE_PATHS["rlut"] = joinpath(@clima_artifact("cmip_model_rmse"), "rlut_rmse_amip_rlut_amip_5yr.csv")
 
-    # Process each line
-    for line in eachline(io)
-        # Split the line by comma
-        fields = split(line, ',')
-        model_name = fields[1]
-        DJF, MAM, JJA, SON, ANN = map(x -> parse(Float64, x), fields[2:end])
+short_names = ["pr", "rsut", "rlut"]
+for short_name in short_names
+    open(RMSE_FILE_PATHS[short_name], "r") do io
+        # Skip the header
+        header = readline(io)
 
-        push!(OTHER_MODELS_RMSEs["pr"], RMSEs(; model_name, DJF, MAM, JJA, SON, ANN))
+        # Process each line
+        for line in eachline(io)
+            # Split the line by comma
+            fields = split(line, ',')
+            model_name = fields[1]
+            DJF, MAM, JJA, SON, ANN = map(x -> parse(Float64, x), fields[2:end])
+
+            push!(OTHER_MODELS_RMSEs[short_name], RMSEs(; model_name, DJF, MAM, JJA, SON, ANN))
+        end
     end
 end
 
@@ -71,4 +78,6 @@ function RSME_stats(vecRMSEs)
     (; best_single_model = best_single_model(vecRMSEs), median_model, worst_model, best_model)
 end
 
-COMPARISON_RMSEs["pr"] = RSME_stats(OTHER_MODELS_RMSEs["pr"])
+for short_name in short_names
+    COMPARISON_RMSEs[short_name] = RSME_stats(OTHER_MODELS_RMSEs[short_name])
+end
diff --git a/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl b/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl
@@ -28,26 +28,28 @@ OBS_DS["pr"] = ObsDataSource(;
     path = joinpath(@clima_artifact("precipitation_obs"), "gpcp.precip.mon.mean.197901-202305.nc"),
     var_name = "precip",
 )
-
 SIM_DS_KWARGS["pr"] = (; preprocess_data_fn = preprocess_pr_fn, new_units = "mm / day")
-
 OTHER_MODELS_RMSEs["pr"] = []
 
-include("cmip_rmse.jl")
+OBS_DS["rsut"] = ObsDataSource(;
+    path = joinpath(@clima_artifact("radiation_obs"), "CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc"),
+    var_name = "toa_sw_all_mon",
+)
+SIM_DS_KWARGS["rsut"] = (;)
+OTHER_MODELS_RMSEs["rsut"] = []
 
-# OBS_DS["rsut"] = ObsDataSource(;
-#                              path = "OBS/CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc",
-#                              var_name = "toa_sw_all_mon",
-#                              )
+OBS_DS["rlut"] = ObsDataSource(;
+    path = joinpath(@clima_artifact("radiation_obs"), "CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc"),
+    var_name = "toa_lw_all_mon",
+)
+SIM_DS_KWARGS["rlut"] = (;)
+OTHER_MODELS_RMSEs["rlut"] = []
 
-# OBS_DS["rlut"] = ObsDataSource(;
-#                              path = "OBS/CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc",
-#                              var_name = "toa_lw_all_mon",
-#                              )
+include("cmip_rmse.jl")
 
 function bias(output_dir::AbstractString, short_name::AbstractString, target_dates::AbstractArray{<:Dates.DateTime})
     obs = OBS_DS[short_name]
-    sim = SimDataSource(; path = output_dir, short_name, SIM_DS_KWARGS["pr"]...)
+    sim = SimDataSource(; path = output_dir, short_name, SIM_DS_KWARGS[short_name]...)
     return bias(obs, sim, target_dates)
 end
 

diff --git a/test/Artifacts.toml b/test/Artifacts.toml
@@ -1,8 +1,8 @@
 [cmip_model_rmse]
-git-tree-sha1 = "aa2d5178a656e9ce7a37698d978d8ed92aa46d99"
+git-tree-sha1 = "53a27731c0a2ec60756af24e4c60ac37da4a32bc"
 
     [[cmip_model_rmse.download]]
-    sha256 = "b019d40510cebc678899a74d2cf14e33aa60452805478b2bf7176b891012aa97"
+    sha256 = "3e8fa41df226dda9d375f8cb9dbeee3e10f9fceabcad2c7b91c9633b489b0437"
     url = "https://caltech.box.com/shared/static/lmzvnjkxctlb0fanieeqxll0vdrbei9e.gz"
 
 [precipitation_obs]
@@ -11,3 +11,10 @@ git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af"
     [[precipitation_obs.download]]
     sha256 = "463ebc1886c4e0be9656656da699f2cbb87d7c2568d5e81277c0c8a415dc8143"
     url = "https://caltech.box.com/shared/static/j63uak557kw7tlzb3dgwifoaeczekzm4.gz"
+
+[radiation_obs]
+git-tree-sha1 = "c9eef049fc3bf057f35332c316755a694b0eb5ef"
+
+    [[radiation_obs.download]]
+    sha256 = "3274926803246cd3738e2d9f36c3fc2652fdb564fba80ddcb10860fd5d8cb552"
+    url = "https://caltech.box.com/shared/static/odvaz76hi3ujakhwgkilzbitk49ty1fw.gz"