From 2015f22c9179d9da66935e7cd73d6327940ef28d Mon Sep 17 00:00:00 2001 From: Gabriele Bozzola Date: Mon, 17 Jun 2024 17:19:25 -0700 Subject: [PATCH] Sync NetCDF file at every write This change essentially disables the internal buffer in the NetCDF library. With this change, every time something has to be written to disk, it is immediately written (as opposed to buffering writes). This ensures that datasets are always written and no data is lost, but comes to a performance cost. This was already the default behavior for GPU runs, so this change only affects CPU runs --- perf/flame.jl | 4 ++-- src/callbacks/get_callbacks.jl | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/perf/flame.jl b/perf/flame.jl index 6b3a80997a..6b50840e10 100644 --- a/perf/flame.jl +++ b/perf/flame.jl @@ -38,14 +38,14 @@ ProfileCanvas.html_file(joinpath(output_dir, "flame.html"), results) ##### allocs_limit = Dict() -allocs_limit["flame_perf_target"] = 1_160_723 +allocs_limit["flame_perf_target"] = 1_298_056 allocs_limit["flame_perf_target_tracers"] = 1_378_216 allocs_limit["flame_perf_target_edmfx"] = 1_383_200 allocs_limit["flame_perf_diagnostics"] = 21_359_336 allocs_limit["flame_perf_target_diagnostic_edmfx"] = 1_936_480 allocs_limit["flame_perf_target_frierson"] = 1_378_072 allocs_limit["flame_perf_target_threaded"] = 2_298_355 -allocs_limit["flame_perf_target_callbacks"] = 1_285_460 +allocs_limit["flame_perf_target_callbacks"] = 1_446_496 allocs_limit["flame_perf_gw"] = 3_268_961_856 allocs_limit["flame_perf_target_prognostic_edmfx_aquaplanet"] = 4_000_488 allocs_limit["flame_gpu_implicit_barowave_moist"] = 336_378 diff --git a/src/callbacks/get_callbacks.jl b/src/callbacks/get_callbacks.jl index 46150c9062..f94f3e1b6d 100644 --- a/src/callbacks/get_callbacks.jl +++ b/src/callbacks/get_callbacks.jl @@ -43,6 +43,7 @@ function get_diagnostics(parsed_args, atmos_model, Y, p, t_start, dt) p.output_dir, num_points = num_netcdf_points; z_sampling_method, + sync_schedule = CAD.EveryStepSchedule(), ) writers = (hdf5_writer, netcdf_writer)