From 63e1f0399d6cf96e3d869d8a5aa44684fc31c348 Mon Sep 17 00:00:00 2001 From: Gabriele Bozzola Date: Mon, 10 Jun 2024 09:02:20 -0700 Subject: [PATCH 1/2] Add boxes in bias_leaderboard --- .../user_io/leaderboard/cmip_rmse.jl | 22 ++++++++++- .../user_io/leaderboard/compare_with_obs.jl | 38 +++++++++++++------ 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl b/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl index 812f7e70b9..c67dece5cc 100644 --- a/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl +++ b/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl @@ -1,4 +1,4 @@ -import Statistics: median +import Statistics: median, quantile const RMSE_FILE_PATHS = Dict() @@ -75,7 +75,25 @@ function RSME_stats(vecRMSEs) SON = minimum(abs.(SON)), ) - (; best_single_model = best_single_model(vecRMSEs), median_model, worst_model, best_model) + quantile25 = RMSEs(; + model_name = "Quantile 0.25", + ANN = quantile(ANN, 0.25), + DJF = quantile(DJF, 0.25), + JJA = quantile(JJA, 0.25), + MAM = quantile(MAM, 0.25), + SON = quantile(SON, 0.25), + ) + + quantile75 = RMSEs(; + model_name = "Quantile 0.75", + ANN = quantile(ANN, 0.75), + DJF = quantile(DJF, 0.75), + JJA = quantile(JJA, 0.75), + MAM = quantile(MAM, 0.75), + SON = quantile(SON, 0.75), + ) + + (; best_single_model = best_single_model(vecRMSEs), median_model, worst_model, best_model, quantile25, quantile75) end for short_name in short_names diff --git a/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl b/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl index 32e2646883..b58fdd59f4 100644 --- a/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl +++ b/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl @@ -119,26 +119,42 @@ function plot_leaderboard(rmses; output_path) squares[begin:NUM_BOXES, var_num] .= values(rmse) ./ values(median_model) squares[(NUM_BOXES + 1):end, var_num] .= values(best_single_model) ./ values(median_model) - CairoMakie.errorbars!( + CairoMakie.scatter!( ax, 1:5, values(median_model), - values(best_model), - values(worst_model), - whiskerwidth = 10, + label = median_model.model_name, color = :black, - linewidth = 0.5, + marker = :hline, + markersize = 15, + ) + + categories = vcat(map(_ -> collect(1:5), 1:length(OTHER_MODELS_RMSEs[short_name]))...) + + CairoMakie.boxplot!( + ax, + categories, + vcat(values.(OTHER_MODELS_RMSEs[short_name])...); + whiskerwidth = 1, + width = 0.35, + mediancolor = :black, + color = :gray, + whiskerlinewidth = 1, ) + + for model in OTHER_MODELS_RMSEs[short_name] + CairoMakie.scatter!(ax, 1:5, values(model), marker = :hline) + end + CairoMakie.scatter!( ax, 1:5, - values(median_model), - label = median_model.model_name, - color = :black, - marker = :hline, + values(rmse), + label = rmse.model_name, + marker = :star5, + markersize = 20, + color = :orange, ) - CairoMakie.scatter!(ax, 1:5, values(best_single_model), label = best_single_model.model_name) - CairoMakie.scatter!(ax, 1:5, values(rmse), label = rmse.model_name, marker = :star5) # Add a fake extra point to center the legend a little better CairoMakie.scatter!(ax, [6.5], [0.1], markersize = 0.01) From 4e8ffb6c0b89f7b3ef314c74b7263f1012516f30 Mon Sep 17 00:00:00 2001 From: Gabriele Bozzola Date: Wed, 12 Jun 2024 07:52:48 -0700 Subject: [PATCH 2/2] Ensure that leaderboard box plot has same order as RMSE --- .../user_io/leaderboard/compare_with_obs.jl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl b/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl index b58fdd59f4..e58d410304 100644 --- a/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl +++ b/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl @@ -86,7 +86,7 @@ function plot_leaderboard(rmses; output_path) loc_squares = length(rmses) + 1 ax_squares = CairoMakie.Axis( fig[loc_squares, 1], - yticks = (1:num_variables, var_names), + yticks = (1:num_variables, reverse(var_names)), xticks = ([3, NUM_BOXES + 3], ["CliMA", "Best model"]), aspect = NUM_BOXES * NUM_MODELS, ) @@ -116,8 +116,8 @@ function plot_leaderboard(rmses; output_path) # Against other models (; best_single_model, median_model, worst_model, best_model) = COMPARISON_RMSEs[short_name] - squares[begin:NUM_BOXES, var_num] .= values(rmse) ./ values(median_model) - squares[(NUM_BOXES + 1):end, var_num] .= values(best_single_model) ./ values(median_model) + squares[begin:NUM_BOXES, end - var_num + 1] .= values(rmse) ./ values(median_model) + squares[(NUM_BOXES + 1):end, end - var_num + 1] .= values(best_single_model) ./ values(median_model) CairoMakie.scatter!( ax, @@ -142,9 +142,10 @@ function plot_leaderboard(rmses; output_path) whiskerlinewidth = 1, ) - for model in OTHER_MODELS_RMSEs[short_name] - CairoMakie.scatter!(ax, 1:5, values(model), marker = :hline) - end + # If we want to plot other models + # for model in OTHER_MODELS_RMSEs[short_name] + # CairoMakie.scatter!(ax, 1:5, values(model), marker = :hline) + # end CairoMakie.scatter!( ax,