Try fixing invalidations via code loading

.buildkite/ci_deps.jl

@@ -0,0 +1,43 @@
+# This code loading order was found with
+# https://github.com/CliMA/CodeLoadingOrders.jl
+import NCDatasets
+import SciMLBase
+import CUDA
+import Krylov
+import CairoMakie
+import ClimaAnalysis
+import StatsBase
+import ClimaReproducibilityTests
+import Statistics
+import ClimaTimeSteppers
+import PrettyTables
+import PrecompileCI
+import HDF5
+import ClimaCoreTempestRemap
+import SnoopCompile
+import ClimaCore
+import MPI
+import JSON
+import SnoopCompileCore
+import YAML
+import ClimaUtilities
+import ClimaAtmos
+import NCDatasets
+import ClimaCoreSpectra
+import Random
+import ProfileCanvas
+import IntervalSets
+import NullBroadcasts
+import Test
+import ClimaComms
+import ArtifactWrappers
+import ArgParse
+import JET
+import OrderedCollections
+import BenchmarkTools
+import Interpolations
+import DiffEqBase
+import Profile
+import JLD2
+import Poppler_jll
+import ClimaDiagnostics

.buildkite/ci_driver.jl

@@ -9,6 +9,7 @@ redirect_stderr(IOContext(stderr, :stacktrace_types_limited => Ref(false)))
 # To load in the precompiled methods, run `using PrecompileCI` before loading ClimaAtmos. 
 # To see what methods are precompiled, open julia: `julia --project=.buildkite/PrecompileCI`
 # and run `using PrecompileTools; PrecompileTools.verbose[] = true; include(".buildkite/PrecompileCI/src/PrecompileCI.jl")`
+include("ci_deps.jl")
 using PrecompileCI
 import ClimaComms
 ClimaComms.@import_required_backends

perf/invalidations.jl

@@ -1,14 +1,15 @@
-import ClimaComms
-ClimaComms.@import_required_backends
-import ClimaAtmos as CA
 redirect_stderr(IOContext(stderr, :stacktrace_types_limited => Ref(false)))
 
 # From: https://timholy.github.io/SnoopCompile.jl/dev/tutorials/invalidations/#Tutorial-on-@snoop_invalidations
 using SnoopCompileCore
 invalidations = @snoop_invalidations begin
+    include(joinpath(dirname(@__DIR__), ".buildkite", "ci_deps.jl"))
+    import ClimaComms
+    ClimaComms.@import_required_backends
+    import ClimaAtmos as CA
     (; config_file, job_id) = CA.commandline_kwargs()
     config = CA.AtmosConfig(config_file; job_id)
-    include(joinpath(dirname(@__DIR__), "examples", "hybrid", "driver.jl"))
+    include(joinpath(dirname(@__DIR__), ".buildkite", "ci_driver.jl"))
     nothing
 end;