restart

CliMA · Jun 17, 2024 · 21f4936 · 21f4936
1 parent 686ba4f
commit 21f4936
Show file tree

Hide file tree

Showing 2 changed files with 67 additions and 62 deletions.
diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml
@@ -17,30 +17,8 @@ timeout_in_minutes: 1440
 
 steps:
 
-  - label: "init :computer:"
-    key: "init_cpu_env"
-    command:
-      - "echo $$JULIA_DEPOT_PATH"
-
-      - echo "--- Instantiate AMIP env"
-      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
-      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
-
-      - echo "--- Download artifacts"
-      - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'"
-      - "julia --project=artifacts -e 'using Pkg; Pkg.status()'"
-      - "julia --project=artifacts artifacts/download_artifacts.jl"
-
-    agents:
-      slurm_cpus_per_task: 8
-    env:
-      JULIA_NUM_PRECOMPILE_TASKS: 8
-      JULIA_MAX_NUM_PRECOMPILE_FILES: 50
-
-  # - label: "init clima :computer:"
-  #   key: "init_cpu_env_clima"
+  # - label: "init :computer:"
+  #   key: "init_cpu_env"
   #   command:
   #     - "echo $$JULIA_DEPOT_PATH"
 
@@ -49,61 +27,83 @@ steps:
   #     - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
   #     - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
 
-  #     # - echo "--- Instantiate test env"
-  #     # - "julia --project=test/ -e 'using Pkg; Pkg.develop(path=\".\")'"
-  #     # - "julia --project=test/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-  #     # - "julia --project=test/ -e 'using Pkg; Pkg.precompile()'"
-  #     # - "julia --project=test/ -e 'using Pkg; Pkg.status()'"
-
   #     - echo "--- Download artifacts"
   #     - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
   #     - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'"
   #     - "julia --project=artifacts -e 'using Pkg; Pkg.status()'"
   #     - "julia --project=artifacts artifacts/download_artifacts.jl"
 
   #   agents:
-  #     queue: clima
-  #     modules: common
+  #     slurm_cpus_per_task: 8
   #   env:
   #     JULIA_NUM_PRECOMPILE_TASKS: 8
   #     JULIA_MAX_NUM_PRECOMPILE_FILES: 50
 
+  - label: "init clima :computer:"
+    key: "init_cpu_env_clima"
+    command:
+      - "echo $$JULIA_DEPOT_PATH"
+
+      - echo "--- Instantiate AMIP env"
+      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
+      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
+
+      # - echo "--- Instantiate test env"
+      # - "julia --project=test/ -e 'using Pkg; Pkg.develop(path=\".\")'"
+      # - "julia --project=test/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      # - "julia --project=test/ -e 'using Pkg; Pkg.precompile()'"
+      # - "julia --project=test/ -e 'using Pkg; Pkg.status()'"
+
+      - echo "--- Download artifacts"
+      - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'"
+      - "julia --project=artifacts -e 'using Pkg; Pkg.status()'"
+      - "julia --project=artifacts artifacts/download_artifacts.jl"
+
+    agents:
+      queue: clima
+      modules: common
+    env:
+      JULIA_NUM_PRECOMPILE_TASKS: 8
+      JULIA_MAX_NUM_PRECOMPILE_FILES: 50
+
   - wait
 
   - group: "Current target tests on GPU: AMIP surface"
 
     steps:
 
-      - label: "MPI AMIP FINE: new target amip: topo + diagedmf"
-        key: "amip_target_topo_diagedmf_cpu"
-        command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf_cpu.yml --job_id amip_target_topo_diagedmf_cpu"
-        artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_cpu_artifacts/*"
-        env:
-          CLIMACORE_DISTRIBUTED: "MPI"
-          BUILD_HISTORY_HANDLE: ""
-        agents:
-          slurm_ntasks_per_node: 16
-          slurm_nodes: 8
-          slurm_mem_per_cpu: 20G
-
-      # - label: "GPU AMIP FINE: new target amip: topo + diagedmf"
-      #   key: "amip_target_topo_diagedmf_gpu"
-      #   command:
-      #     - echo "--- Run simulation"
-      #     - "srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf_gpu.yml --job_id amip_target_topo_diagedmf_gpu"
-      #   artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_gpu_artifacts/*"
-      #   timeout_in_minutes: 1440
+      # - label: "MPI AMIP FINE: new target amip: topo + diagedmf"
+      #   key: "amip_target_topo_diagedmf_cpu"
+      #   command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf_cpu.yml --job_id amip_target_topo_diagedmf_cpu"
+      #   artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_cpu_artifacts/*"
       #   env:
-      #     CLIMACOMMS_CONTEXT: "MPI"
-      #     RESTART_DIR: "/scratch/clima/slurm-buildkite/climacoupler-longruns/718/climacoupler-longruns/experiments/ClimaEarth/output/amip/gpu_amip_target_topo_diagedmf_artifacts/"
-      #     RESTART_T: 6912000
+      #     CLIMACORE_DISTRIBUTED: "MPI"
+      #     BUILD_HISTORY_HANDLE: ""
       #   agents:
-      #     queue: clima
-      #     slurm_gpus_per_task: 1
-      #     slurm_cpus_per_task: 4
-      #     slurm_ntasks: 1
-      #     slurm_mem: 30GB
-      #     modules: common
+      #     slurm_ntasks_per_node: 16
+      #     slurm_nodes: 8
+      #     slurm_mem_per_cpu: 20G
+
+      - label: "GPU AMIP FINE: new target amip: topo + diagedmf"
+        key: "amip_target_topo_diagedmf_gpu"
+        command:
+          - echo "--- Run simulation"
+          - "srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf_gpu.yml --restart_dir $$RESTART_DIR --restart_t $$RESTART_T --job_id amip_target_topo_diagedmf_gpu"
+        artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_gpu_artifacts/*"
+        timeout_in_minutes: 1440
+        env:
+          CLIMACOMMS_CONTEXT: "MPI"
+          RESTART_DIR: "/scratch/clima/slurm-buildkite/climacoupler-longruns/724/climacoupler-longruns/experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_gpu_artifacts/"
+          RESTART_T: 5184000
+        agents:
+          queue: clima
+          slurm_gpus_per_task: 1
+          slurm_cpus_per_task: 4
+          slurm_ntasks: 1
+          slurm_mem: 30GB
+          modules: common
 
       # - label: "GPU AMIP FINE: new target amip: topo + diagedmf ft64"
       #   key: "gpu_amip_target_topo_diagedmf_ft64"

diff --git a/config/longrun_configs/amip_target_topo_diagedmf_gpu.yml b/config/longrun_configs/amip_target_topo_diagedmf_gpu.yml
@@ -14,11 +14,16 @@ mode_name: "amip"
 mono_surface: false
 netcdf_output_at_levels: true
 output_default_diagnostics: true
-start_date: "20100510"
+start_date: "20100709"
 surface_setup: "PrescribedSurface"
-t_end: "180days"
+t_end: "30days"
 topo_smoothing: true
 topography: "Earth"
 coupler_toml_file: "toml/amip_target_topo_diagedmf.toml"
 turb_flux_partition: "CombinedStateFluxesMOST"
 use_reference_state: false
+diagnostics:
+  - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]
+    period: 1days
+  - short_name: [arup, waup, taup, thetaaup, haup, husup, hurup, clwup, cliup, waen, tke, lmix]
+    period: 1days