diff --git a/benchmarks/Project.toml b/benchmarks/Project.toml new file mode 100644 index 00000000..ff6d9d46 --- /dev/null +++ b/benchmarks/Project.toml @@ -0,0 +1,46 @@ +name = "benchmarks" +uuid = "b465d61d-1350-4ef5-9abc-112f9dc9b757" +version = "0.0.1" + +[deps] +ACSets = "227ef7b5-1206-438b-ac65-934d6da304b8" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +CombinatorialSpaces = "b1c52339-7909-45ad-8b6a-6e388f7c67f2" +ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Decapodes = "679ab3ea-c928-4fe6-8d59-fd451142d391" +DiagrammaticEquations = "6f00c28b-6bed-4403-80fa-30e0dc12f317" +DrWatson = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1" +GeometryBasics = "5c1252a2-5f33-56bf-86c9-59e7332b4326" +JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +MLStyle = "d8e11817-5142-5d16-987a-aa16d5891078" +OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" +PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" + +[compat] +ACSets = "0.2" +BenchmarkTools = "1.5" +CUDA = "5.4" +CombinatorialSpaces = "0.6.7" +ComponentArrays = "0.15" +DataFrames = "1.6" +Decapodes = "0.5.5" +DiagrammaticEquations = "0.1.6" +DrWatson = "2.15.0" +GeometryBasics = "0.4" +JLD2 = "0.4" +LinearAlgebra = "1.10" +MLStyle = "0.4" +OrdinaryDiffEq = "6.86" +PrettyTables = "2.3" +TOML = "1.0" +julia = "1.10.0" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..03d0baff --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,89 @@ +# Benchmarks + +## DrWatson Initialization + +This code base is using the [Julia Language](https://julialang.org/) and +[DrWatson](https://juliadynamics.github.io/DrWatson.jl/stable/) +to make a reproducible scientific project named +> benchmarks + +To (locally) reproduce this project, do the following: + +0. Download this code base. Notice that raw data are typically not included in the + git-history and may need to be downloaded independently. +1. Open a Julia console and do: + + ```julia + julia> using Pkg + julia> Pkg.add("DrWatson") # install globally, for using `quickactivate` + julia> Pkg.activate("path/to/this/project") + julia> Pkg.instantiate() + ``` + +This will install all necessary packages for you to be able to run the scripts and +everything should work out of the box, including correctly finding local paths. + +You may notice that most scripts start with the commands: + +```julia +using DrWatson +@quickactivate :benchmarks +``` + +which auto-activate the project, enable local path handling from DrWatson and provide several helper functions. Note that `@quickactivate :benchmarks` is only usable from within the `benchmarks` directory. + +## Establishing Simulation Configurations + +To establish a set of configurations for a simulation, you should create a file `src/$physics/config.toml` with the below structure. + +1. For a given physics (heat, brusselator, etc), entries are structured as `[$physics.$architecture.$tag]`, e.g. `[heat.cpu.example]`. +2. Under an entry, list all the parameters desired. This should be structured as either `$param_name = [val1, val2, ...]` or `$param_name = val`. +3. You should always include a `code_target`, which takes either `CPUTarget` or `CUDATarget` as a string. + +## Creating a Simulation File + +These benchmarks depend upon you to create the simulation files to be benchmarked. For a certain simulation named ```example```, the simulation file would be ```src/example/example.jl```. + +Always start the file with the following, as it provides you access to helper functions located in ```src/helpers```. + +```julia +using DrWatson +@quickactivate :benchmarks +``` + +Additionally, this file should contain the following functions. + +1. ```setup_config```, which will take in a ```Dict{String, Any}``` with the provided parameter names pointing to that task's provided configuration values. It's up to you to take these values, process them and then organize them to be passed along. +2. ```setup_benchmark```, which will create the Decapode and run ```eval(gensim())``` on it and return that evaluated function. +3. ```create_mesh```, which will create the mesh upon which the simulation will run and also initialize the initial conditions and any constants/parameters. Return the mesh, initial conditions and constants/parameters in that order. +4. ```create_simulate```, which will take the generated mesh and evaluated function and run the simulate function. Return the resulting function. +5. ```run_simulation```, which will take the resulting simulation function, initial conditions and constants/parameters and run the solve. Return the result of the solve. + +## Running the Benchmarks + +Use the `main_config.toml` to list out which physics configuration entries you would like to be run. These entries correspond one-to-one with the entries in the physics configurations. However, these `main_config.toml` entries can take optional arguements to customize how the simulations are run. Currently supported arguments are: + +1. `slurm_args`, passed as a vector of strings for each seperate `sbatch` argument. These arguments will be added to each job for that entry. +2. `concur_jobs`, passed as an integer that determines how many jobs can run at a time for that configuration. + +Once done, simply run `scripts/main.sh`. + +As another option, the name of a specific configuration entry in the `main_config.toml` can be passed to `main.sh` to only run that job, e.g. `main.sh heat cpu example`. + +**Warning**: The caller of this script should be in the `benchmarks` directory. + +## Data Collection and Processing + +Once a simulation is run, their outputs will be saved in `data/sims/$physics`. The benchmark JSON files will contain the full result of the benchmarking run while the stat JLD2 files will contain the solve result statistics from DEStats from OrdinaryDiffEq.jl. + +These files will then be processed and have their data stored in `data/exp_pro/$physics/$slurm_job_id/autogen`. Each result file in that directory will contain the processed results from one single task. + +These result files can then be collected with `collect_results` on the `autogen` directory and post-processed with `DataFrames.jl` functions to create the desired tables. An example of this kind of post-processing script is included in `scripts/post_processing`. + +Because these files are expected to be collected using DrWatson's `collect_results` , there is no intended correlation between a task and its result file name. + +**Warning**: Not all information from the benchmarking run is saved to the result files and any files in `data/sims/$physics`, specifically the JSON and JLD2 files mentioned above, will be deleted upon the next benchmark run for that physics. On the other hand, result files in the `autogen` directory mentioned before will never be deleted by the benchmarking. + +## Testing + +The benchmarks have a few tests that can be used to establish the robustness of the system. To run them, activate the `benchmarks` environment and then enter `test`. diff --git a/benchmarks/scripts/array.jl b/benchmarks/scripts/array.jl new file mode 100644 index 00000000..375b84e9 --- /dev/null +++ b/benchmarks/scripts/array.jl @@ -0,0 +1,59 @@ +using DrWatson +@quickactivate :benchmarks + +using BenchmarkTools +using TOML + +const task_key = ARGS[1] +const physics = ARGS[2] +const arch = ARGS[3] +const tag = ARGS[4] + +sim_namedata = SimNameData(physics, arch, tag, task_key) + +function extract_task_config(config_data) + if !haskey(config_data, task_key) + error("Warning: Task with key $(task_key) could not find config data") + end + + task_config_data = all_config_data[task_key] + @info string(task_config_data) + + task_config_data +end + +@info "Running $physics on $arch, tagged as $tag, array id is $task_key" + +# Extract data +all_config_data = TOML.parsefile(simconfig_path(sim_namedata)) +task_config_data = extract_task_config(all_config_data) + +# Grab user's physics file +include(physicsfile_path(sim_namedata)) + +sim_instance = pass_simulation_instance() + +config = sim_instance.setup_config(task_config_data) + +# Get intermediate variables to use in benchmarking +sim = sim_instance.setup_benchmark(config); +sd, u0, cnst_param = sim_instance.create_mesh(config); +fm = sim_instance.create_simulate(config, sd, sim); +result = sim_instance.run_simulation(config, fm, u0, cnst_param); + +# Save solver statistics +stats_data = tostringdict(struct2dict(result.stats)) +wsave(statsfile_path(sim_namedata), stats_data) + +# Setup and run benchmarking +simulation_suite = BenchmarkGroup() + +stages = solver_stages() +simulation_suite[task_key][stages[1]] = @benchmarkable sim_instance.setup_benchmark($config) gctrial=true +simulation_suite[task_key][stages[2]] = @benchmarkable sim_instance.create_mesh($config) gcsample=true +simulation_suite[task_key][stages[3]] = @benchmarkable sim_instance.create_simulate($config, $sd, $sim) gctrial=true +simulation_suite[task_key][stages[4]] = @benchmarkable sim_instance.run_simulation($config, $fm, $u0, $cnst_param) gcsample=true + +tune!(simulation_suite) +deca_sim_results = run(simulation_suite; verbose = true) +BenchmarkTools.save(benchfile_path(sim_namedata), deca_sim_results) diff --git a/benchmarks/scripts/array.sh b/benchmarks/scripts/array.sh new file mode 100644 index 00000000..087b3359 --- /dev/null +++ b/benchmarks/scripts/array.sh @@ -0,0 +1,17 @@ +#!/bin/bash +#SBATCH --job-name=PRINT +#SBATCH --output=printlog_%A_%a.txt +#SBATCH --mem=16GB +#SBATCH --time=01:00:00 + +pwd; hostname; date + +module load julia + +SIMNAME=$1 +ARCH=$2 +TAG=$3 + +julia array.jl $SLURM_ARRAY_TASK_ID $SIMNAME $ARCH $TAG + +date diff --git a/benchmarks/scripts/clean.jl b/benchmarks/scripts/clean.jl new file mode 100644 index 00000000..bbe8ded1 --- /dev/null +++ b/benchmarks/scripts/clean.jl @@ -0,0 +1,13 @@ +using DrWatson +@quickactivate :benchmarks + +# TODO: Can improve by seperating bench and final logs +file_regex = r"^.*log_(\d+)(?:_\d+)?\.txt$" + +file_matches = filter(!isnothing, map(x -> match(file_regex, x), readdir(scriptsdir()))) + +for file in file_matches + tgt_dir = scriptsdir("slurm_logs", "logs_"*file[1]) + mkpath(tgt_dir) + mv(scriptsdir(file.match), joinpath(tgt_dir, file.match)) +end diff --git a/benchmarks/scripts/final.jl b/benchmarks/scripts/final.jl new file mode 100644 index 00000000..2fe9a108 --- /dev/null +++ b/benchmarks/scripts/final.jl @@ -0,0 +1,13 @@ +using DrWatson +@quickactivate :benchmarks + +include(helpersdir("data_aggr_helper.jl")) + +using TOML + +const slurm_id = ARGS[1] +const physics = ARGS[2] + +aggregate_data(slurm_id, physics) + +run(`julia --threads=auto $(postprocessdir("default_out.jl")) $slurm_id $physics`) diff --git a/benchmarks/scripts/final.sh b/benchmarks/scripts/final.sh new file mode 100644 index 00000000..23395afc --- /dev/null +++ b/benchmarks/scripts/final.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH --job-name=FINAL +#SBATCH --output=finallog_%j.txt +#SBATCH --mem=1GB +#SBATCH --time=01:00:00 + +pwd; hostname; date + +module load julia + +SIMNAME=$1 + +julia final.jl $SLURM_JOB_ID $SIMNAME + +date + +julia clean.jl + diff --git a/benchmarks/scripts/main.jl b/benchmarks/scripts/main.jl new file mode 100644 index 00000000..8e376a61 --- /dev/null +++ b/benchmarks/scripts/main.jl @@ -0,0 +1,32 @@ +using DrWatson +@quickactivate :benchmarks + +@info "Precompiling Julia" +using Pkg +Pkg.instantiate() +Pkg.precompile() +@info "Finished precompiling Julia" + +using TOML + +include(helpersdir("main_source.jl")) + +if length(ARGS) == 0 + @info "Running all sims" + generate_all_configs() + run_all_physics() + +elseif length(ARGS) == 3 + @info "Running single sim" + generate_all_configs() + + const physics = ARGS[1] + const arch = ARGS[2] + const tag = ARGS[3] + + run_physics = SimNameData(physics, arch, tag) + is_valid_config_instance(run_physics) + run_single_physics(physics, [run_physics]) +else + error("Usage: ['physics' 'architecture' 'tag']") +end diff --git a/benchmarks/scripts/main.sh b/benchmarks/scripts/main.sh new file mode 100644 index 00000000..20125135 --- /dev/null +++ b/benchmarks/scripts/main.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +DIR=scripts +cd $DIR + +module load julia + +if [ $# == 3 ] +then + SIMNAME=$1 + ARCH=$2 + TAG=$3 + julia --threads=auto main.jl $SIMNAME $ARCH $TAG +elif [ $# == 0 ] +then + julia --threads=auto main.jl +else + echo "Usage: ['physics' 'architecture' 'tag']" + exit 1 +fi + diff --git a/benchmarks/scripts/post_processing/default_out.jl b/benchmarks/scripts/post_processing/default_out.jl new file mode 100644 index 00000000..70dd5b9f --- /dev/null +++ b/benchmarks/scripts/post_processing/default_out.jl @@ -0,0 +1,44 @@ +using DrWatson +@quickactivate :benchmarks + +include(helpersdir("data_aggr_helper.jl")) + +using TOML +using DataFrames +using PrettyTables + +const slurm_id = ARGS[1] +const physics = ARGS[2] + +sims_to_process = collect_mainconfig_simentries(physics) + +# TODO: Have meta config information be in a seperate toml +config_data = load_simconfig(first(sims_to_process)) +meta_config = meta_config_info(config_data) +const meta_field_names = split(meta_config["fields"], ",") + +# TODO: Meant as a basic data processing pipeline +# Can create multiple scripts to roughly process data in general ways +pretty_results = collect_results(aggdatadir(physics, slurm_id)) + +median_times = map(stage -> benchmark_headername(stage, "Median", "time"), solver_stages()) +table_header = vcat(["Task ID", "statsfile", "benchfile"], meta_field_names, median_times, ["nf"]) + +select!(pretty_results, table_header) + +for time in median_times + transform!(pretty_results, [time] => ByRow(x -> x / 1e9) => [time]) +end + +# TODO: Can choose other sorting methods +for field_name in reverse(meta_field_names) + sort!(pretty_results, Symbol(field_name)) +end + +mkpath(tablesdir(physics, slurm_id)) + +# TODO: Can change this backened to be different from markdown +open(tablesdir(physics, slurm_id, "default_output.md"), "w") do results_file + conf = set_pt_conf(tf = tf_markdown) + pretty_table_with_conf(conf, results_file, pretty_results; header = table_header) +end diff --git a/benchmarks/src/benchmarks.jl b/benchmarks/src/benchmarks.jl new file mode 100644 index 00000000..a998789c --- /dev/null +++ b/benchmarks/src/benchmarks.jl @@ -0,0 +1,9 @@ +module benchmarks + +include(joinpath("helpers", "constants.jl")) +include(joinpath("helpers", "paths.jl")) +include(joinpath("helpers", "param_parsing.jl")) +include(joinpath("helpers", "sim_interface.jl")) +include(joinpath("helpers", "main_config_helper.jl")) + +end diff --git a/benchmarks/src/helpers/constants.jl b/benchmarks/src/helpers/constants.jl new file mode 100644 index 00000000..b0c75332 --- /dev/null +++ b/benchmarks/src/helpers/constants.jl @@ -0,0 +1,11 @@ +export solver_stages, supported_arches, is_supported_arch, meta_config_id, mainsim_config_path + +const solver_stages_list = ["Setup", "Mesh", "Simulate", "Solve"] +solver_stages() = return solver_stages_list + +const supported_architectures_list = ["cpu", "cuda"] +supported_arches() = return supported_architectures_list +is_supported_arch(arch) = return arch in supported_arches() + +const meta_key = string(0) +meta_config_id() = return meta_key diff --git a/benchmarks/src/helpers/data_aggr_helper.jl b/benchmarks/src/helpers/data_aggr_helper.jl new file mode 100644 index 00000000..813983dc --- /dev/null +++ b/benchmarks/src/helpers/data_aggr_helper.jl @@ -0,0 +1,112 @@ +using DrWatson +@quickactivate :benchmarks + +using BenchmarkTools +using DataFrames +using JLD2 + +function aggregate_data(slurm_id, physics) + sims_to_process = collect_mainconfig_simentries(physics) + + for sim_namedata in sims_to_process + + # Meant to work when not all sims in main_config are run + if !all_resultfiles_exist(sim_namedata) + @info "Result file not found for $(sim_namedata), skipping" + continue + end + + aggregated_results = process_simdata(sim_namedata) + + savefile_path = aggdatadir(physics, slurm_id, "results.jld2") + safesave(savefile_path, aggregated_results) + + end +end + +function all_resultfiles_exist(sim_namedata) + return isfile(benchfile_path(sim_namedata)) && isfile(statsfile_path(sim_namedata)) +end + +# TODO: Come back and clean up this function +function collect_mainconfig_simentries(physics) + + entries = SimNameData[] + + physics_configurations = collect_simsfor_physics(physics) + for sim_namedata in physics_configurations + + if !isfile(simconfig_path(sim_namedata)) + @info "Config file for $sim_namedata not found, skipping" + continue + end + + config_entries = simconfig_size(load_simconfig(sim_namedata)) + for task_id in 1:config_entries + + physics = sim_namedata.physics + arch = sim_namedata.arch + tag = sim_namedata.tag + task_key = string(task_id) + + push!(entries, SimNameData(physics, arch, tag, task_key)) + end + end + + entries +end + +function process_simdata(snd::SimNameData) + return process_simdata(snd, load_benchfile(snd), load_statsfile(snd), load_simconfig(snd)) +end + +function process_simdata(sim_namedata::SimNameData, benchmark_data, stats_data, config_data) + data_row = Dict{String, Any}() + + add_debug_simdata!(data_row, sim_namedata) + add_config_data!(data_row, sim_namedata.task_key, config_data) + add_benchmark_data!(data_row, sim_namedata.task_key, benchmark_data) + add_solver_stats_data!(data_row, stats_data) + + return data_row +end + +function add_debug_simdata!(data_row, sim_namedata::SimNameData) + push!(data_row, "statsfile" => statsfile_name(sim_namedata)) + push!(data_row, "benchfile" => benchfile_name(sim_namedata)) + push!(data_row, "Task ID" => sim_namedata.task_key) + push!(data_row, "Tagged Name" => sim_namedata.tag) + push!(data_row, "Architecture" => sim_namedata.arch) + push!(data_row, "Simulation Name" => sim_namedata.physics) +end + +function add_config_data!(data_row, task_key::String, config_data) + merge!(data_row, config_data[task_key]) # Adds sim parameters +end + +function add_benchmark_data!(data_row, task_key::String, benchmark_data) + median_trial = median(benchmark_data[task_key]) + min_trial = minimum(benchmark_data[task_key]) + + add_trial_data!(data_row, "Median", median_trial) + add_trial_data!(data_row, "Minimum", min_trial) +end + +# TODO: Add benchmark parameters for each stage +function add_trial_data!(data_row, statistic_name::String, trial_data) + for stage in solver_stages() + stage_data = trial_data[stage] + for field in fieldnames(typeof(stage_data)) + field != :params || continue + push!(data_row, benchmark_headername(stage, statistic_name, String(field)) => getfield(stage_data, field)) + end + end +end + +function benchmark_headername(stage::String, statistic_name::String, category::String) + return "$(stage) $(statistic_name) $(category)" +end + +function add_solver_stats_data!(data_row, stats_data) + merge!(data_row, stats_data) +end diff --git a/benchmarks/src/helpers/main_config_helper.jl b/benchmarks/src/helpers/main_config_helper.jl new file mode 100644 index 00000000..94e32821 --- /dev/null +++ b/benchmarks/src/helpers/main_config_helper.jl @@ -0,0 +1,87 @@ +using DrWatson +@quickactivate :benchmarks + +using TOML + +export mainsim_config_path, load_main_config, listof_main_physics, + collect_simsfor_physics, has_config_args, config_args, config_arg + +mainsim_config_path() = srcdir("main_config.toml") + +load_main_config() = TOML.parsefile(mainsim_config_path()) + +listof_main_physics() = collect(keys(load_main_config())) + +collect_simsfor_physics(physics) = collect_simsfor_physics(load_main_config(), physics) + +function collect_simsfor_physics(main_config_info, physics) + entries = SimNameData[] + add_entriesfor_physics!(entries, main_config_info, physics) +end + +function add_entriesfor_physics!(entries, main_config_info, physics) + physics_info = main_config_physics_info(main_config_info, physics) + for arch in keys(physics_info) + add_entriesfor_physics_arch!(entries, physics_info, physics, arch) + end + return entries +end + +function add_entriesfor_physics_arch!(entries, physics_info, physics, arch) + tag_info = physics_config_arch_info(physics_info, arch) + for tag in keys(tag_info) + add_entryfor_tagged!(entries, physics, arch, tag) + end + return entries +end + +function add_entryfor_tagged!(entries, physics, arches, tag) + push!(entries, SimNameData(physics, arches, tag)) + return entries +end + +function main_config_physics_info(main_config_info, physics) + haskey(main_config_info, physics) || error("Physics $physics does not exist in the provided main configuration") + return main_config_info[physics] +end + +function physics_config_arch_info(physics_info, arch) + haskey(physics_info, arch) || error("Architecture $arch does not exist in the provided physics entry") + physics_arch_taglist = physics_info[arch] + return physics_arch_taglist +end + +function has_config_args(config_info, snd::SimNameData) + is_supported_arch(snd.arch) || return false + + haskey(config_info, snd.physics) || return false + physics_info = config_info[snd.physics] + + haskey(physics_info, snd.arch) || return false + arch_info = physics_info[snd.arch] + + haskey(arch_info, snd.tag) || return false + tag_info = arch_info[snd.tag] + + return !isempty(tag_info) +end + +function config_args(config_info, snd::SimNameData) + if !(has_config_args(config_info, snd)) + error("Arguments for $(snd) were not found in the main configuration or provided architecture $(snd.arch) is invalid") + end + return config_info[snd.physics][snd.arch][snd.tag] +end + +function config_arg(config_info, snd::SimNameData, arg) + if !has_config_args(config_info, snd) + return nothing + end + + args = config_args(config_info, snd) + if !haskey(args, arg) + return nothing + end + + return args[arg] +end diff --git a/benchmarks/src/helpers/main_source.jl b/benchmarks/src/helpers/main_source.jl new file mode 100644 index 00000000..06445599 --- /dev/null +++ b/benchmarks/src/helpers/main_source.jl @@ -0,0 +1,110 @@ +using DrWatson +@quickactivate :benchmarks + +using TOML +using MLStyle + +include(helpersdir("physics_config_helper.jl")) + +const DEFAULT_CPU_SLURMARGS = `--partition=hpg-milan` +const DEFAULT_CUDA_SLURMARGS = `--partition=gpu --gres=gpu:a100:1` +const DEFAULT_MAXJOBS = 6 + +function generate_all_configs() + for physics in listof_main_physics() + generate_configsfor_physics(physics) + end +end + +function run_all_physics() + main_config_info = load_main_config() + validate_all_physics(main_config_info) + + for physics in listof_main_physics() + physics_configs = collect_simsfor_physics(main_config_info, physics) + run_single_physics(physics, physics_configs) + end +end + +function validate_all_physics(main_config_info) + for physics in listof_main_physics() + physics_configs = collect_simsfor_physics(main_config_info, physics) + for config in physics_configs + is_valid_config_instance(config) + end + end +end + +function is_valid_config_instance(sim_namedata::SimNameData) + is_supported_arch(sim_namedata.arch) || error("Architecture $(arch) is not in list $(supported_arches())") + simfile = physicsfile_path(sim_namedata) + if !isfile(simfile) + error("Simulation file at $(simfile) was not found") + end + + config = simconfig_path(sim_namedata) + if !isfile(config) + error("Config file at $(config) was not found") + end + return true +end + +function run_single_physics(physics, physics_configs) + + if isempty(physics_configs) + return + end + + rm(resultsdir(physics), recursive=true, force=true) + mkpath(resultsdir(physics)) + + dependency_ids = [] + + for config in physics_configs + run_single_config!(dependency_ids, config) + end + + !isempty(dependency_ids) || return + + run(`sbatch --dependency=afterok:$(join(dependency_ids, ",")) $(scriptsdir("final.sh")) $physics`) +end + +function run_single_config!(dependency_ids, sim_namedata::SimNameData) + + main_config_info = load_main_config() + concurrent_jobs = get_concur_jobs(main_config_info, sim_namedata) + slurm_args = get_slurm_args(main_config_info, sim_namedata) + + physics = sim_namedata.physics + arch = sim_namedata.arch + tag = sim_namedata.tag + + count = simconfig_size(load_simconfig(sim_namedata)) + + @info "Running $count $arch tasks" + + jobid = readchomp(`sbatch --array=1-$(count)%$(concurrent_jobs) --parsable $(slurm_args) $(scriptsdir("array.sh")) $physics $arch $tag`) + + @info "Job ID for $(tagfor_run(sim_namedata)) is: $jobid" + push!(dependency_ids, jobid) + return dependency_ids +end + +function get_slurm_args(main_config_info, snd::SimNameData) + + slurm_args = config_arg(main_config_info, snd, "slurm_args") + + if isnothing(slurm_args) + return @match snd.arch begin + "cpu" => DEFAULT_CPU_SLURMARGS + "cuda" => DEFAULT_CUDA_SLURMARGS + end + end + + return Cmd(slurm_args) +end + +function get_concur_jobs(main_config_info, snd::SimNameData) + job_arg = config_arg(main_config_info, snd, "concur_jobs") + return isnothing(job_arg) ? DEFAULT_MAXJOBS : job_arg +end diff --git a/benchmarks/src/helpers/param_parsing.jl b/benchmarks/src/helpers/param_parsing.jl new file mode 100644 index 00000000..9c4e35b9 --- /dev/null +++ b/benchmarks/src/helpers/param_parsing.jl @@ -0,0 +1,31 @@ +# Collection of functions meant to convert TOML data into a sim-usable format +using MLStyle +using Decapodes + +export parse_float_type, parse_code_target, arch_to_code_target + +function parse_float_type(float_data) + @match float_data begin + "Float32" => Float32 + "Float64" => Float64 + "ComplexF32" => ComplexF32 + "ComplexF64" => ComplexF64 + _ => error("Float data $(float_data) is not valid") + end +end + +function parse_code_target(code_target_data) + @match code_target_data begin + "CPUTarget" => CPUTarget() + "CUDATarget" => CUDATarget() + _ => error("Code target data $(code_target_data) is not in list [\"CPUTarget\", \"CUDATarget\"]") + end +end + +function arch_to_code_target(architecture) + @match architecture begin + "cpu" => "CPUTarget" + "cuda" => "CUDATarget" + _ => error("Architecture $(architecture) is not in list [\"cpu\", \"cuda\"]") + end +end diff --git a/benchmarks/src/helpers/paths.jl b/benchmarks/src/helpers/paths.jl new file mode 100644 index 00000000..065529ce --- /dev/null +++ b/benchmarks/src/helpers/paths.jl @@ -0,0 +1,77 @@ +using DrWatson +@quickactivate "benchmarks" + +using TOML +using BenchmarkTools + +export physicsdir, resultsdir, tablesdir, helpersdir, aggdatadir, postprocessdir +export tagfor_run, tagfor_task +export load_simconfig, simconfig_name, simconfig_path, + physicsfile_name, physicsfile_path, + load_physicsconfig, physicsconfig_name, physicsconfig_path, + load_statsfile, statsfile_name, statsfile_path, + load_benchfile, benchfile_name, benchfile_path +export simconfig_size, meta_config_info + +export SimNameData +import Base.show + +helpersdir(args...) = srcdir("helpers", args...) + +physicsdir(args...) = srcdir("physics", args...) + +resultsdir(physics, args...) = datadir("sims", physics, args...) + +tablesdir(physics, slurm_id, args...) = datadir("exp_pro", physics, slurm_id, args...) +aggdatadir(physics, slurm_id, args...) = tablesdir(physics, slurm_id, "autogen", args...) +postprocessdir(args...) = scriptsdir("post_processing", args...) + +struct SimNameData + physics::String + arch::String + tag::String + task_key::String +end + +function SimNameData(physics::String, arch::String, tag::String) + return SimNameData(physics, arch, tag, "0") +end + +function SimNameData(physics::String, arch::String, tag::String, task_id::Int) + return SimNameData(physics, arch, tag, string(task_id)) +end + +function Base.show(io::IO, snd::SimNameData) + if snd.task_key == "0" + print(io, "$(snd.physics) on $(snd.arch) tagged as '$(snd.tag)'") + else + print(io, "$(snd.physics) on $(snd.arch) tagged as '$(snd.tag)' running for task $(snd.task_key)") + end +end + +tagfor_run(simdata::SimNameData) = return "$(simdata.physics)_$(simdata.arch)_$(simdata.tag)" +tagfor_task(simdata::SimNameData) = return "$(tagfor_run(simdata))_$(simdata.task_key)" + +load_simconfig(simdata::SimNameData) = TOML.parsefile(simconfig_path(simdata)) +simconfig_name(simdata::SimNameData) = return "$(tagfor_run(simdata)).toml" +simconfig_path(simdata::SimNameData) = physicsdir(simdata.physics, simconfig_name(simdata)) + +physicsfile_name(simdata::SimNameData) = return "$(simdata.physics).jl" +physicsfile_path(simdata::SimNameData) = physicsdir(simdata.physics, physicsfile_name(simdata)) + +load_physicsconfig(simdata::SimNameData) = TOML.parsefile(physicsconfig_path(simdata)) +physicsconfig_name() = return "config.toml" +physicsconfig_path(simdata::SimNameData) = physicsdir(simdata.physics, physicsconfig_name()) + +load_statsfile(simdata::SimNameData) = wload(statsfile_path(simdata)) +statsfile_name(simdata::SimNameData) = return "stats_$(tagfor_task(simdata)).jld2" +statsfile_path(simdata::SimNameData) = resultsdir(simdata.physics, statsfile_name(simdata)) + +load_benchfile(simdata::SimNameData) = only(BenchmarkTools.load(benchfile_path(simdata))) +benchfile_name(simdata::SimNameData) = return "benchmarks_$(tagfor_task(simdata)).json" +benchfile_path(simdata::SimNameData) = resultsdir(simdata.physics, benchfile_name(simdata)) + + +simconfig_size(config_data) = return length(keys(config_data)) - 1 # Don't include meta info + +meta_config_info(benchmark_config) = return benchmark_config[meta_config_id()] diff --git a/benchmarks/src/helpers/physics_config_helper.jl b/benchmarks/src/helpers/physics_config_helper.jl new file mode 100644 index 00000000..fb6ea0ff --- /dev/null +++ b/benchmarks/src/helpers/physics_config_helper.jl @@ -0,0 +1,49 @@ +using DrWatson +@quickactivate :benchmarks +using TOML + +function generate_configsfor_physics(physics) + simulations = collect_simsfor_physics(physics) + + for sim in simulations + if !has_physicsconfig_args(sim) + error("Simulation $(sim) does not have a valid configuration entry") + end + end + + for sim in simulations + load_save_benchmark_data(sim) + end +end + +function load_save_benchmark_data(sim) + all_parameters = physicsconfig_args(sim) + task_simconfig = process_simulation_config(all_parameters) + open(simconfig_path(sim), "w") do io + TOML.print(io, task_simconfig) + end +end + +has_physicsconfig_args(sim) = has_config_args(load_physicsconfig(sim), sim) +physicsconfig_args(sim) = config_args(load_physicsconfig(sim), sim) + +function process_simulation_config(entry) + received_params = dict_list(entry) + + params_list = Dict() + add_meta_data!(params_list, received_params) + add_task_data!(params_list, received_params) + + params_list +end + +function add_meta_data!(params_list, received_params) + meta_data = Dict("fields" => join(keys(first(received_params)), ",")) + push!(params_list, string(0) => meta_data) +end + +function add_task_data!(params_list, received_params) + for (idx, param) in enumerate(received_params) + push!(params_list, string(idx) => param) + end +end diff --git a/benchmarks/src/helpers/sim_interface.jl b/benchmarks/src/helpers/sim_interface.jl new file mode 100644 index 00000000..6a45ec0b --- /dev/null +++ b/benchmarks/src/helpers/sim_interface.jl @@ -0,0 +1,9 @@ +export SimulationInstance + +struct SimulationInstance + setup_config::Function + setup_benchmark::Function + create_mesh::Function + create_simulate::Function + run_simulation::Function +end diff --git a/benchmarks/src/main_config.toml b/benchmarks/src/main_config.toml new file mode 100644 index 00000000..54619c6b --- /dev/null +++ b/benchmarks/src/main_config.toml @@ -0,0 +1,2 @@ +[heat.cpu.test] +[heat.cuda.test] \ No newline at end of file diff --git a/benchmarks/src/physics/brussel/brussel.jl b/benchmarks/src/physics/brussel/brussel.jl new file mode 100644 index 00000000..2443a22c --- /dev/null +++ b/benchmarks/src/physics/brussel/brussel.jl @@ -0,0 +1,85 @@ +using DrWatson +@quickactivate :benchmarks + +using ACSets +using CUDA +using CUDA.CUSPARSE +using CombinatorialSpaces +using ComponentArrays +using Decapodes +using DiagrammaticEquations +using GeometryBasics: Point2, Point3 +using LinearAlgebra +using MLStyle +using OrdinaryDiffEq + +using Decapodes.Canon.Chemistry + +function pass_simulation_instance() + return SimulationInstance(setup_config, setup_benchmark, create_mesh, create_simulate, run_simulation) +end + +struct BrusselConfig + float_type + code_target + res +end + +function setup_config(task_config_data::Dict{String, Any}) + float_type = parse_float_type(task_config_data["float_type"]) + code_target = parse_code_target(task_config_data["code_target"]) + resolution = Float64(task_config_data["resolution"]) + @info "Float type: $(float_type), Code target: $(code_target), Resolution: $(resolution)" + + BrusselConfig(float_type, code_target, resolution) +end + +function setup_benchmark(config::BrusselConfig) + eval(gensim(brusselator, code_target = config.code_target, stateeltype = config.float_type)) +end + +function create_mesh(config::BrusselConfig) + s = triangulated_grid(1, 1, config.res, config.res, Point2{config.float_type}) + sd = EmbeddedDeltaDualComplex2D{Bool, config.float_type, Point2{config.float_type}}(s) + subdivide_duals!(sd, Circumcenter()) + + U = config.float_type.(map(sd[:point]) do (_,y) + 22 * (y *(1-y))^(3/2) + end) + + V = config.float_type.(map(sd[:point]) do (x,_) + 27 * (x *(1-x))^(3/2) + end) + + F₁ = config.float_type.(map(sd[:point]) do (x,y) + (x-0.3)^2 + (y-0.6)^2 ≤ (0.1)^2 ? 5.0 : 0.0 + end) + + F₂ = zeros(config.float_type, nv(sd)) + + if config.code_target isa CUDATarget + U = CuArray(U) + V = CuArray(V) + F₁ = CuArray(F₁) + F₂ = CuArray(F₂) + end + + u₀ = ComponentArray(U=U, V=V) + + constants_and_parameters = ( + α = config.float_type.(0.001), + F = t -> t ≥ config.float_type.(1.1) ? F₁ : F₂) + + (sd, u₀, constants_and_parameters) +end + +function create_simulate(config::BrusselConfig, sd, simulate) + simulate(sd, nothing, DiagonalHodge()) +end + +function run_simulation(config::BrusselConfig, fm, u0, cnst_param) + tₑ = config.float_type(11.5) + prob = ODEProblem(fm, u0, (0, tₑ), cnst_param) + + soln = solve(prob, Tsit5(), saveat=0.1) +end diff --git a/benchmarks/src/physics/brussel/config.toml b/benchmarks/src/physics/brussel/config.toml new file mode 100644 index 00000000..b3b290e7 --- /dev/null +++ b/benchmarks/src/physics/brussel/config.toml @@ -0,0 +1,9 @@ +[brussel.cpu.default] +code_target = "CPUTarget" +float_type = ["Float32", "Float64"] +resolution = [0.05, 0.02, 0.01] + +[brussel.cuda.default] +code_target = "CUDATarget" +float_type = ["Float32", "Float64"] +resolution = [0.05, 0.02, 0.01] \ No newline at end of file diff --git a/benchmarks/src/physics/cahnhilliard/cahnhilliard.jl b/benchmarks/src/physics/cahnhilliard/cahnhilliard.jl new file mode 100644 index 00000000..18a18058 --- /dev/null +++ b/benchmarks/src/physics/cahnhilliard/cahnhilliard.jl @@ -0,0 +1,71 @@ +using DrWatson +@quickactivate :benchmarks + +using ACSets +using CUDA +using CUDA.CUSPARSE +using CombinatorialSpaces +using ComponentArrays +using Decapodes +using DiagrammaticEquations +using GeometryBasics: Point2, Point3 +using LinearAlgebra +using MLStyle +using OrdinaryDiffEq + +function pass_simulation_instance() + return SimulationInstance(setup_config, setup_benchmark, create_mesh, create_simulate, run_simulation) +end + +struct CHConfig + float_type + code_target + res +end + +function setup_config(task_config_data::Dict{String, Any}) + float_type = parse_float_type(task_config_data["float_type"]) + code_target = parse_code_target(task_config_data["code_target"]) + resolution = float_type(task_config_data["resolution"]) + @info "Float type: $(float_type), Code target: $(code_target), Resolution: $(resolution)" + + CHConfig(float_type, code_target, resolution) +end + +function setup_benchmark(config::CHConfig) + CahnHilliard = @decapode begin + C::Form0 + (D, γ)::Constant + ∂ₜ(C) == D * Δ(C.^3 - C - γ * Δ(C)) + end + + eval(gensim(CahnHilliard, code_target = config.code_target, stateeltype = config.float_type)) +end + +function create_mesh(config::CHConfig) + s = triangulated_grid(100, 100, config.res, config.res, Point2{config.float_type}) + sd = EmbeddedDeltaDualComplex2D{Bool, config.float_type, Point2{config.float_type}}(s) + subdivide_duals!(sd, Circumcenter()) + + C = rand(config.float_type, nv(sd)) + + if config.code_target isa CUDATarget + C = CuArray(C) + end + u₀ = ComponentArray(C=C) + + constants = (D = config.float_type(0.5), γ = config.float_type(0.5)) + + (sd, u₀, constants) +end + +function create_simulate(config::CHConfig, sd, simulate) + simulate(sd, nothing, DiagonalHodge()) +end + +function run_simulation(config::CHConfig, fm, u0, cnst_param) + tₑ = config.float_type(200) + prob = ODEProblem(fm, u0, (0, tₑ), cnst_param) + + soln = solve(prob, Tsit5(), saveat=0.1) +end diff --git a/benchmarks/src/physics/cahnhilliard/config.toml b/benchmarks/src/physics/cahnhilliard/config.toml new file mode 100644 index 00000000..1805452d --- /dev/null +++ b/benchmarks/src/physics/cahnhilliard/config.toml @@ -0,0 +1,9 @@ +[cahnhilliard.cpu.test] +code_target = "CPUTarget" +float_type = ["Float32", "Float64"] +resolution = [1, 0.8, 0.5, 0.4] + +[cahnhilliard.cuda.test] +code_target = "CUDATarget" +float_type = ["Float32", "Float64"] +resolution = [1, 0.8, 0.5, 0.4] diff --git a/benchmarks/src/physics/heat/config.toml b/benchmarks/src/physics/heat/config.toml new file mode 100644 index 00000000..7c041459 --- /dev/null +++ b/benchmarks/src/physics/heat/config.toml @@ -0,0 +1,14 @@ +[heat.cpu.test] +code_target = "CPUTarget" +float_type = ["Float32", "Float64"] +resolution = [5, 2, 1] + +[heat.cpu.test2] +code_target = "CPUTarget" +float_type = "Float64" +resolution = 5 + +[heat.cuda.test] +code_target = "CUDATarget" +float_type = ["Float32", "Float64"] +resolution = [5, 2, 1] diff --git a/benchmarks/src/physics/heat/heat.jl b/benchmarks/src/physics/heat/heat.jl new file mode 100644 index 00000000..5dfbcdb8 --- /dev/null +++ b/benchmarks/src/physics/heat/heat.jl @@ -0,0 +1,69 @@ +using DrWatson +@quickactivate :benchmarks + +using ACSets +using CUDA +using CUDA.CUSPARSE +using CombinatorialSpaces +using ComponentArrays +using Decapodes +using DiagrammaticEquations +using GeometryBasics: Point2, Point3 +using LinearAlgebra +using MLStyle +using OrdinaryDiffEq + +function pass_simulation_instance() + return SimulationInstance(setup_config, setup_benchmark, create_mesh, create_simulate, run_simulation) +end + +struct HeatConfig + float_type + code_target + res +end + +function setup_config(task_config_data::Dict{String, Any}) + float_type = parse_float_type(task_config_data["float_type"]) + code_target = parse_code_target(task_config_data["code_target"]) + resolution = Float64(task_config_data["resolution"]) + @info "Float type: $(float_type), Code target: $(code_target), Resolution: $(resolution)" + + HeatConfig(float_type, code_target, resolution) +end + +function setup_benchmark(config::HeatConfig) + Heat = @decapode begin + U::Form0 + ∂ₜ(U) == 100 * Δ(U) + end + + eval(gensim(Heat, code_target = config.code_target, stateeltype = config.float_type)) +end + +function create_mesh(config::HeatConfig) + s = triangulated_grid(100, 100, config.res, config.res, Point2{config.float_type}) + sd = EmbeddedDeltaDualComplex2D{Bool, config.float_type, Point2{config.float_type}}(s) + subdivide_duals!(sd, Circumcenter()) + + U = map(sd[:point]) do (x,_) + return x + end + if config.code_target isa CUDATarget + U = CuArray(U) + end + u₀ = ComponentArray(U=U) + + (sd, u₀, ()) +end + +function create_simulate(config::HeatConfig, sd, simulate) + simulate(sd, nothing, DiagonalHodge()) +end + +function run_simulation(config::HeatConfig, fm, u0, cnst_param) + tₑ = config.float_type(10) + prob = ODEProblem(fm, u0, (0, tₑ), cnst_param) + + soln = solve(prob, Tsit5(), saveat=0.1) +end diff --git a/benchmarks/test/config_gen.jl b/benchmarks/test/config_gen.jl new file mode 100644 index 00000000..c3bc6fdc --- /dev/null +++ b/benchmarks/test/config_gen.jl @@ -0,0 +1,65 @@ +module ConfigGen + +using DrWatson +@quickactivate :benchmarks + +include(helpersdir("physics_config_helper.jl")) + +using Test +using TOML + +# @testset "Config file validation" begin + +# bad_toml = TOML.parse(""" +# [heat.cpu.test] +# test_val = [2] +# """) +# @test is_valid_config(bad_toml) === nothing + +# empty_sim_toml = TOML.parse("[heat.cpu]\n") +# @test_throws "defined but empty" is_valid_config(empty_sim_toml) + +# good_toml = TOML.parse("[physics.cpu]\ntest_val = [2]\n") +# @test is_valid_config(good_toml) === nothing +# end + +@testset "Config file loading" begin + temp_list = Dict() + temp_params = [Dict("test" => 1)] + add_meta_data!(temp_list, temp_params) + @test temp_list["0"]["fields"] == "test" + + temp_list = Dict() + temp_params = dict_list(Dict("test" => [1, 2])) + add_task_data!(temp_list, temp_params) + @test temp_list["1"]["test"] == 1 + @test temp_list["2"]["test"] == 2 + + init_params = Dict("full_test" => ["a", "b"]) + temp_list = process_simulation_config(init_params) + @test temp_list["0"]["fields"] == "full_test" + @test temp_list["1"]["full_test"] == "a" + @test temp_list["2"]["full_test"] == "b" + @test simconfig_size(temp_list) == 2 + + init_params = Dict("full_test" => ["a", "b", "c", "d"]) + temp_list = process_simulation_config(init_params) + @test simconfig_size(temp_list) == 4 + @test meta_config_info(temp_list)["fields"] == "full_test" +end + +@testset "Parameter parsing" begin + test_toml = TOML.parse("[physics.cpu]\ntest_val = [2, 3]\ntest_val2 = [10, 11]") + @test !isempty(test_toml["physics"]["cpu"]) + + test_data = test_toml["physics"]["cpu"] + task_data = process_simulation_config(test_data) + @test length(task_data) == 4 + 1 # Each entry plus meta + + for key in keys(task_data) + key == "0" && continue + @test keys(task_data[key]) == Set(["test_val", "test_val2"]) + end +end + +end diff --git a/benchmarks/test/data_aggr.jl b/benchmarks/test/data_aggr.jl new file mode 100644 index 00000000..e00e2153 --- /dev/null +++ b/benchmarks/test/data_aggr.jl @@ -0,0 +1,61 @@ +module DataAggr + +using DrWatson +@quickactivate :benchmarks + +using Test +using BenchmarkTools + +include(helpersdir("data_aggr_helper.jl")) + +@testset "Benchmark names" begin + @test benchmark_headername("Solve", "Maximum", "Time") == "Solve Maximum Time" +end + +@testset "Adding debug data" begin + data_row_debug = Dict{String, Any}() + physics = "heat" + arch = "cpu" + tag = "testing" + test_key = "1" + test_namedata = SimNameData(physics, arch, tag, test_key) + + debug_data = add_debug_simdata!(data_row_debug, test_namedata) + + debug_vals = values(debug_data) + @test test_key in debug_vals + @test arch in debug_vals + @test physics in debug_vals + @test statsfile_name(test_namedata) in debug_vals + @test benchfile_name(test_namedata) in debug_vals +end + +@testset "Adding benchmark data" begin + test_suite = BenchmarkGroup() + test_key = "1" + for stage in solver_stages() + test_suite[test_key][stage] = @benchmarkable rand(100) samples=10 + end + test_run = run(test_suite) + + data_row_bench = Dict{String, Any}() + median_test_run = median(test_run[test_key]) + add_trial_data!(data_row_bench, "Median", median_test_run) + + for stage in solver_stages() + for stat in ["time", "memory", "gctime", "allocs"] + @test benchmark_headername(stage, "Median", stat) in keys(data_row_bench) + end + end +end + +@testset "Add solver stats data" begin + pseudo_stats = Dict{String, Any}("steps" => 10, "evals" => 100) + data_row_stats = Dict{String, Any}() + add_solver_stats_data!(data_row_stats, pseudo_stats) + + @test data_row_stats["steps"] == 10 + @test data_row_stats["evals"] == 100 +end + +end diff --git a/benchmarks/test/main_config.jl b/benchmarks/test/main_config.jl new file mode 100644 index 00000000..c7175533 --- /dev/null +++ b/benchmarks/test/main_config.jl @@ -0,0 +1,121 @@ +module MainConfig + +using DrWatson +@quickactivate :benchmarks + +using Test +using TOML + +import benchmarks: main_config_physics_info, physics_config_arch_info + +@testset "Gather Nested Config Info" begin + main_config_info = TOML.parse(""" + [heat.cpu.test] + args = "test" + [heat.cpu.test2] + [heat.cuda.cutest] + + [not_heat.cuda.default] + arg2 = "test2" + """) + + not_heat_entry = main_config_physics_info(main_config_info, "not_heat") + @test length(keys(not_heat_entry)) == 1 + + not_heat_cuda = physics_config_arch_info(not_heat_entry, "cuda") + @test keys(not_heat_cuda) == Set(["default"]) + + + heat_entry = main_config_physics_info(main_config_info, "heat") + @test length(keys(heat_entry)) == 2 + + heat_cpu = physics_config_arch_info(heat_entry, "cpu") + @test keys(heat_cpu) == Set(["test", "test2"]) + + heat_cuda = physics_config_arch_info(heat_entry, "cuda") + @test keys(heat_cuda) == Set(["cutest"]) +end + +@testset "Collecting Config Entries" begin + simple_toml = TOML.parse(""" + [heat.cpu.test] + """) + + simple_entries = collect_simsfor_physics(simple_toml, "heat") + @test length(simple_entries) == 1 + + + multiple_physics_toml = TOML.parse(""" + [heat.cpu.test2] + + [cold.cuda.cutest] + """) + + multiphys_heat_entries = collect_simsfor_physics(multiple_physics_toml, "heat") + @test length(multiphys_heat_entries) == 1 + + multiphys_cold_entries = collect_simsfor_physics(multiple_physics_toml, "cold") + @test length(multiphys_heat_entries) == 1 + + @test multiphys_cold_entries != multiphys_heat_entries + + + multiple_arches_toml = TOML.parse(""" + [heat.cpu.test2] + [heat.cuda.cutest2] + [heat.cuda.final] + """) + + multiarches_entries = collect_simsfor_physics(multiple_arches_toml, "heat") + @test length(multiarches_entries) == 3 + @test allunique(multiarches_entries) +end + +@testset "Automatic Collection" begin + main_config_info = TOML.parse(""" + [heat.cpu.a] + args = "test" + [heat.cpu.b] + [heat.cuda.c] + + [not_heat.cuda.a] + arg2 = "test2" + """) + + hcpa = SimNameData("heat", "cpu", "a") + @test has_config_args(main_config_info, hcpa) + hcpa_args = config_args(main_config_info, hcpa) + @test hcpa_args["args"] == "test" + + @test is_supported_arch("cpu") + @test is_supported_arch("cuda") + + @test !is_supported_arch("wrongarch") + + no_entries = SimNameData("heat", "cuda", "c") + @test has_config_args(main_config_info, no_entries) == false + @test_throws "not found in the main config" config_args(main_config_info, no_entries) + + no_tag = SimNameData("heat", "cpu", "badtag") + @test has_config_args(main_config_info, no_tag) == false + @test_throws "not found in the main config" config_args(main_config_info, no_tag) + + no_arch = SimNameData("not_heat", "cpu", "c") + @test has_config_args(main_config_info, no_arch) == false + @test_throws "not found in the main config" config_args(main_config_info, no_arch) + + bad_arch = SimNameData("not_heat", "fake", "c") + @test has_config_args(main_config_info, bad_arch) == false + @test_throws "not found in the main config" config_args(main_config_info, bad_arch) + + no_physics = SimNameData("brussel", "cuda", "c") + @test has_config_args(main_config_info, no_physics) == false + @test_throws "not found in the main config" config_args(main_config_info, no_physics) + + empty_toml = TOML.parse("") + empty_snd = SimNameData("", "", "") + @test has_config_args(empty_toml, empty_snd) == false + @test_throws "not found in the main configuration" config_args(empty_toml, empty_snd) +end + +end diff --git a/benchmarks/test/pathfiles.jl b/benchmarks/test/pathfiles.jl new file mode 100644 index 00000000..44b65f7c --- /dev/null +++ b/benchmarks/test/pathfiles.jl @@ -0,0 +1,24 @@ +module PathFiles + +using DrWatson +@quickactivate :benchmarks + +using Test + +@testset "File names" begin + @test "heat_cpu_test.toml" == simconfig_name(SimNameData("heat", "cpu", "test")) + + @test "stats_heat_cpu_test_1.jld2" == statsfile_name(SimNameData("heat", "cpu", "test", "1")) + @test "benchmarks_heat_cuda_test_3.json" == benchfile_name(SimNameData("heat", "cuda", "test", "3")) + + @test occursin("stats_heat_cuda_test_5.jld2", statsfile_path(SimNameData("heat", "cuda", "test", "5"))) + @test occursin("benchmarks_heat_cpu_test2_7.json", benchfile_path(SimNameData("heat", "cpu", "test2", "7"))) + + @test "heat_cpu_default.toml" == simconfig_name(SimNameData("heat", "cpu", "default")) + @test "test_cuda_default.toml" == simconfig_name(SimNameData("test", "cuda", "default")) + + expected_config_dir = joinpath("heat", simconfig_name(SimNameData("heat", "cuda", "test"))) + @test occursin(expected_config_dir, simconfig_path(SimNameData("heat", "cuda", "test"))) +end + +end diff --git a/benchmarks/test/runtests.jl b/benchmarks/test/runtests.jl new file mode 100644 index 00000000..5d0223d8 --- /dev/null +++ b/benchmarks/test/runtests.jl @@ -0,0 +1,17 @@ +using Test + +@testset "Paths and Files" begin + include("pathfiles.jl") +end + +@testset "Main Config Collection" begin + include("main_config.jl") +end + +@testset "Config Generation" begin + include("config_gen.jl") +end + +@testset "Data Aggregation" begin + include("data_aggr.jl") +end