Merge pull request #821 from CliMA/gb/leaderboard

Add other models to leaderboard

.github/workflows/ci.yml

@@ -36,8 +36,18 @@ jobs:
             ${{ runner.os }}-test-
             ${{ runner.os }}-
       - uses: julia-actions/julia-buildpkg@v1
+
+        # Needed to download its unique artifacts
+      - name: "Instantiate ClimaEarth"
+        run: julia --color=yes --project=experiments/ClimaEarth -e 'using Pkg; Pkg.instantiate()'
+
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1
         with:
           file: lcov.info
+      - name: Submit coverage
+        if: ${{ matrix.version == '1.10' }} && ${{ matrix.os == 'ubuntu-latest' }}
+        uses: codecov/[email protected]
+        with:
+          token: ${{secrets.CODECOV_TOKEN}}

Project.toml

@@ -21,10 +21,10 @@ TempestRemap_jll = "8573a8c5-1df0-515e-a024-abad257ee284"
 Thermodynamics = "b60c26fb-14c3-4610-9d3e-2d17fe7ff00c"
 
 [compat]
+CUDA = "5"
 ClimaComms = "0.5.6"
 ClimaCore = "0.14"
 ClimaCoreTempestRemap = "0.3"
-CUDA = "5"
 Dates = "1"
 DocStringExtensions = "0.8, 0.9"
 JLD2 = "0.4"

artifacts/Artifacts.toml

@@ -4,9 +4,6 @@ git-tree-sha1 = "0c6039bfcc39f6c6cb20fb331a71f6a381d93f24"
 [land_mask]
 git-tree-sha1 = "e41bc8c44124f867b64a9d70f1599515ae27f38a"
 
-[pr_obs_data]
-git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af"
-
 [sic]
 git-tree-sha1 = "139d23f6de29900917d79b41a87a8ee6483c72d0"
 

artifacts/artifact_funcs.jl

@@ -48,18 +48,6 @@ function mask_dataset_path()
     return AW.get_data_folder(mask_dataset)
 end
 
-function pr_obs_data_path()
-    pr_obs_data = AW.ArtifactWrapper(
-        @__DIR__,
-        "pr_obs_data",
-        AW.ArtifactFile[AW.ArtifactFile(
-            url = "https://caltech.box.com/shared/static/k1or5d0d9xdvyfiytl1os7z341monkzn.nc",
-            filename = "gpcp.precip.mon.mean.197901-202305.nc",
-        ),],
-    )
-    return AW.get_data_folder(pr_obs_data)
-end
-
 """
     artifact_data(datapath_full, filename)
 

artifacts/download_artifacts.jl

@@ -23,7 +23,6 @@ function trigger_download()
     @info "sic dataset path: `$(sic_dataset_path())`"
     @info "co2 dataset path: `$(co2_dataset_path())`"
     @info "mask dataset path: `$(mask_dataset_path())`"
-    @info "pr obs data path: `$(pr_obs_data_path())`"
     return nothing
 end
 trigger_download()

experiments/ClimaEarth/Artifacts.toml

@@ -1,9 +1,5 @@
 [land_mask]
 git-tree-sha1 = "e41bc8c44124f867b64a9d70f1599515ae27f38a"
-
-[pr_obs_data]
-git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af"
-
 [sic]
 git-tree-sha1 = "139d23f6de29900917d79b41a87a8ee6483c72d0"
 
@@ -12,3 +8,17 @@ git-tree-sha1 = "9969543acd3c9084b8238b7858d0c76aa0bf98a7"
 
 [test]
 git-tree-sha1 = "b377e7c997804b47720b8e7243361d399d108441"
+
+[cmip_model_rmse]
+git-tree-sha1 = "aa2d5178a656e9ce7a37698d978d8ed92aa46d99"
+
+    [[cmip_model_rmse.download]]
+    sha256 = "b019d40510cebc678899a74d2cf14e33aa60452805478b2bf7176b891012aa97"
+    url = "https://caltech.box.com/shared/static/lmzvnjkxctlb0fanieeqxll0vdrbei9e.gz"
+
+[precipitation_obs]
+git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af"
+
+    [[precipitation_obs.download]]
+    sha256 = "463ebc1886c4e0be9656656da699f2cbb87d7c2568d5e81277c0c8a415dc8143"
+    url = "https://caltech.box.com/shared/static/j63uak557kw7tlzb3dgwifoaeczekzm4.gz"

experiments/ClimaEarth/Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.3"
 manifest_format = "2.0"
-project_hash = "c2b41c62ef471acee30499943ca8e1f5eb29f796"
+project_hash = "1a2614b593c01321f60aa5713efe4d6d7b0cf5ba"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "daf26bbdec60d9ca1c0003b70f389d821ddb4224"

experiments/ClimaEarth/Project.toml

@@ -2,6 +2,7 @@
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
 ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
 ArtifactWrappers = "a14bc488-3040-4b00-9dc1-f6467924858a"
+Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 AtmosphericProfilesLibrary = "86bc3604-9858-485a-bdbe-831ec50de11d"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
@@ -16,6 +17,7 @@ ClimaCoupler = "4ade58fe-a8da-486c-bd89-46df092ec0c7"
 ClimaLand = "08f4d4ce-cf43-44bb-ad95-9d2d5f413532"
 ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c"
 ClimaTimeSteppers = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
+ClimaUtilities = "b3f4f4ca-9299-4f7f-bd9b-81e1242a7513"
 CloudMicrophysics = "6a9e3e04-43cd-43ba-94b9-e8782df3c71b"
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157"
@@ -60,6 +62,7 @@ ClimaCorePlots = "0.2"
 ClimaLand = "0.12"
 ClimaParams = "0.10"
 ClimaTimeSteppers = "0.7"
+ClimaUtilities = "0.1"
 Colors = "0.12"
 Dierckx = "0.5"
 ForwardDiff = "0.10"

experiments/ClimaEarth/run_amip.jl

@@ -971,7 +971,7 @@ if ClimaComms.iamroot(comms_ctx)
                 1:length(compare_vars),
             )
 
-            Leaderboard.plot_leaderboard(rmses; output_path = "bias_leaderboard.png")
+            Leaderboard.plot_leaderboard(rmses; output_path = joinpath(dir_paths.artifacts, "bias_leaderboard.png"))
         end
     end
 

experiments/ClimaEarth/user_io/leaderboard/Leaderboard.jl

@@ -1,5 +1,6 @@
 module Leaderboard
 
+import ClimaComms
 import ClimaAnalysis
 import Dates
 import NCDatasets
@@ -8,11 +9,10 @@ import CairoMakie
 import GeoMakie
 import ClimaCoupler
 import Statistics: mean
+import ClimaUtilities.ClimaArtifacts: @clima_artifact
 
-include(joinpath(pkgdir(ClimaCoupler), "artifacts", "artifact_funcs.jl"))
 include("data_sources.jl")
 include("utils.jl")
 include("compare_with_obs.jl")
 
-
 end

experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl

@@ -0,0 +1,74 @@
+import Statistics: median
+
+pr_file_path = joinpath(@clima_artifact("cmip_model_rmse"), "pr_rmse_amip_pr_amip_5yr.csv")
+
+open(pr_file_path, "r") do io
+    # Skip the header
+    header = readline(io)
+
+    # Process each line
+    for line in eachline(io)
+        # Split the line by comma
+        fields = split(line, ',')
+        model_name = fields[1]
+        DJF, MAM, JJA, SON, ANN = map(x -> parse(Float64, x), fields[2:end])
+
+        push!(OTHER_MODELS_RMSEs["pr"], RMSEs(; model_name, DJF, MAM, JJA, SON, ANN))
+    end
+end
+
+"""
+    best_single_model(RMSEs)
+
+Return the one model that has the overall smallest error.
+"""
+function best_single_model(RMSEs)
+    _, index = findmin(r -> abs.(values(r)), RMSEs)
+    return RMSEs[index]
+end
+
+"""
+    RSME_stats(RMSEs)
+
+Return:
+- best single model
+- "model" with all the medians
+- "model" with all the best values
+- "model" with all the worst values
+"""
+function RSME_stats(vecRMSEs)
+    # Collect into vectors that we can process independently
+    all_values = stack(values.(vecRMSEs))
+    ANN, DJF, JJA, MAM, SON = ntuple(i -> all_values[i, :], 5)
+
+    median_model = RMSEs(;
+        model_name = "Median",
+        ANN = median(ANN),
+        DJF = median(DJF),
+        JJA = median(JJA),
+        MAM = median(MAM),
+        SON = median(SON),
+    )
+
+    worst_model = RMSEs(;
+        model_name = "Worst",
+        ANN = maximum(abs.(ANN)),
+        DJF = maximum(abs.(DJF)),
+        JJA = maximum(abs.(JJA)),
+        MAM = maximum(abs.(MAM)),
+        SON = maximum(abs.(SON)),
+    )
+
+    best_model = RMSEs(;
+        model_name = "Best",
+        ANN = minimum(abs.(ANN)),
+        DJF = minimum(abs.(DJF)),
+        JJA = minimum(abs.(JJA)),
+        MAM = minimum(abs.(MAM)),
+        SON = minimum(abs.(SON)),
+    )
+
+    (; best_single_model = best_single_model(vecRMSEs), median_model, worst_model, best_model)
+end
+
+COMPARISON_RMSEs["pr"] = RSME_stats(OTHER_MODELS_RMSEs["pr"])

experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl

@@ -1,6 +1,7 @@
 const OBS_DS = Dict()
 const SIM_DS_KWARGS = Dict()
 const OTHER_MODELS_RMSEs = Dict()
+const COMPARISON_RMSEs = Dict()
 
 function preprocess_pr_fn(data)
     # -1 kg/m/s2 -> 1 mm/day
@@ -23,13 +24,16 @@ function Base.values(r::RMSEs)
     return val_or_rmse.([r.ANN, r.DJF, r.JJA, r.MAM, r.SON])
 end
 
-OBS_DS["pr"] =
-    ObsDataSource(; path = joinpath(pr_obs_data_path(), "gpcp.precip.mon.mean.197901-202305.nc"), var_name = "precip")
+OBS_DS["pr"] = ObsDataSource(;
+    path = joinpath(@clima_artifact("precipitation_obs"), "gpcp.precip.mon.mean.197901-202305.nc"),
+    var_name = "precip",
+)
 
 SIM_DS_KWARGS["pr"] = (; preprocess_data_fn = preprocess_pr_fn, new_units = "mm / day")
 
-# TODO: These numbers are eyeballed and should not be really used. Use instead real values from the various models
-OTHER_MODELS_RMSEs["pr"] = [RMSEs(; model_name = "AM4.0", ANN = 0.5, DJF = 1.0, JJA = 1.5, MAM = 0.5, SON = 1.0)]
+OTHER_MODELS_RMSEs["pr"] = []
+
+include("cmip_rmse.jl")
 
 # OBS_DS["rsut"] = ObsDataSource(;
 #                              path = "OBS/CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc",
@@ -74,12 +78,33 @@ function plot_leaderboard(rmses; output_path)
             xticks = (1:5, ["Ann", "DJF", "JJA", "MAM", "SON"]),
             title = "Global RMSE",
         )
-        CairoMakie.scatter!(ax, 1:5, values(rmse), label = rmse.model_name)
-        for other_model_rmse in OTHER_MODELS_RMSEs[short_name]
-            CairoMakie.scatter!(ax, 1:5, values(other_model_rmse), label = other_model_rmse.model_name)
-        end
+
+        # Against other models
+        (; best_single_model, median_model, worst_model, best_model) = COMPARISON_RMSEs[short_name]
+
+        CairoMakie.errorbars!(
+            ax,
+            1:5,
+            values(median_model),
+            values(best_model),
+            values(worst_model),
+            whiskerwidth = 10,
+            color = :black,
+            linewidth = 0.5,
+        )
+        CairoMakie.scatter!(
+            ax,
+            1:5,
+            values(median_model),
+            label = median_model.model_name,
+            color = :black,
+            marker = :hline,
+        )
+        CairoMakie.scatter!(ax, 1:5, values(best_single_model), label = best_single_model.model_name)
+        CairoMakie.scatter!(ax, 1:5, values(rmse), label = rmse.model_name, marker = :star5)
+
         # Add a fake extra point to center the legend a little better
-        CairoMakie.scatter!(ax, [6], [0.1], markersize = 0.01)
+        CairoMakie.scatter!(ax, [6.5], [0.1], markersize = 0.01)
         CairoMakie.axislegend()
         loc = loc + 1
     end

test/Project.toml

@@ -14,6 +14,7 @@ ClimaCoupler = "4ade58fe-a8da-486c-bd89-46df092ec0c7"
 ClimaLand = "08f4d4ce-cf43-44bb-ad95-9d2d5f413532"
 ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c"
 ClimaTimeSteppers = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
+ClimaUtilities = "b3f4f4ca-9299-4f7f-bd9b-81e1242a7513"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 GeoMakie = "db073c08-6b98-4ee5-b6a4-5efafb3259c6"
@@ -43,6 +44,7 @@ ArtifactWrappers = "0.2"
 ClimaAtmos = "0.24"
 ClimaLand = "0.12"
 ClimaParams = "0.10"
+ClimaUtilities = "0.1"
 Dates = "1"
 Downloads = "1"
 IntervalSets = "0.5, 0.6, 0.7"

test/experiment_tests/Artifacts.toml

@@ -0,0 +1,6 @@
+[precipitation_obs]
+git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af"
+
+    [[precipitation_obs.download]]
+    sha256 = "463ebc1886c4e0be9656656da699f2cbb87d7c2568d5e81277c0c8a415dc8143"
+    url = "https://caltech.box.com/shared/static/j63uak557kw7tlzb3dgwifoaeczekzm4.gz"

test/experiment_tests/leaderboard.jl

@@ -1,6 +1,8 @@
 import Test: @test, @testset, @test_throws
+import ClimaComms
 import ClimaCoupler
 import ClimaAnalysis
+import ClimaUtilities.ClimaArtifacts: @clima_artifact
 import Dates
 
 # Load file to test
@@ -67,7 +69,7 @@ end
     @test Leaderboard.data_at_date(sim_datasource, Dates.DateTime(1979, 1, 2)) == preprocess_fn(pr.data[1, :, :])
 
     obs_datasource = Leaderboard.ObsDataSource(;
-        path = joinpath(pr_obs_data_path(), "gpcp.precip.mon.mean.197901-202305.nc"),
+        path = joinpath(@clima_artifact("precipitation_obs"), "gpcp.precip.mon.mean.197901-202305.nc"),
         var_name = "precip",
         preprocess_data_fn = preprocess_fn,
     )