From f9ed3f1a54d005f4b9b68ddc0b519d21c51ef21f Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Tue, 22 Aug 2023 20:29:58 -0500 Subject: [PATCH 01/22] remove Pkg.add from Benchmark --- benchmark/run.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmark/run.jl b/benchmark/run.jl index ba2290db7..63145da11 100644 --- a/benchmark/run.jl +++ b/benchmark/run.jl @@ -2,8 +2,7 @@ using Pkg Pkg.develop(PackageSpec(path=dirname(@__DIR__))) Pkg.instantiate() using PkgBenchmark, MixedModels -# explicit `Pkg.add` is a crutch until we've got a good base on main # Pkg.update() allows us to benchmark even when dependencies/compat requirements change -juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update(); Pkg.add([\"BenchmarkTools\", \"StatsModels\"])"` +juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"` config = BenchmarkConfig(; id="origin/HEAD", juliacmd) export_markdown("benchmark.md", judge(MixedModels, config; verbose=true)) From 74e7c2126dcdf596c58b0f2ff309474ec8a16a8f Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Tue, 22 Aug 2023 22:05:59 -0500 Subject: [PATCH 02/22] update --- benchmark/run.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/run.jl b/benchmark/run.jl index 63145da11..b91e3ac39 100644 --- a/benchmark/run.jl +++ b/benchmark/run.jl @@ -1,6 +1,6 @@ using Pkg Pkg.develop(PackageSpec(path=dirname(@__DIR__))) -Pkg.instantiate() +Pkg.update() using PkgBenchmark, MixedModels # Pkg.update() allows us to benchmark even when dependencies/compat requirements change juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"` From 23a2d24720921182f2d32085fde3f3468f085481 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Tue, 22 Aug 2023 22:25:50 -0500 Subject: [PATCH 03/22] use median --- benchmark/Project.toml | 1 + benchmark/benchmarks.jl | 2 +- benchmark/run.jl | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/benchmark/Project.toml b/benchmark/Project.toml index e4293dceb..cfb2253a8 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -2,6 +2,7 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" MixedModels = "ff71e718-51f3-5ec2-a782-8ffcbfa3c316" PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" StatsModels = "3eaba693-59b7-5ba5-a881-562e759f1c8d" [compat] diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 626146397..4b53e6bae 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -85,7 +85,7 @@ function fitbobyqa(dsnm::Symbol, i::Integer) return fit(MixedModel, fms[dsnm][i], dataset(dsnm); contrasts, progress=false) end -# these tests are so fast that they can be very noisy because the denominator is so small, +# these tests are so fast that they can be very noisy because the denominator is so small, # so we disable them by default for auto-benchmarking # SUITE["simplescalar"] = BenchmarkGroup(["single", "simple", "scalar"]) # for (ds, i) in [ diff --git a/benchmark/run.jl b/benchmark/run.jl index b91e3ac39..0a7b99e62 100644 --- a/benchmark/run.jl +++ b/benchmark/run.jl @@ -1,8 +1,8 @@ using Pkg Pkg.develop(PackageSpec(path=dirname(@__DIR__))) -Pkg.update() -using PkgBenchmark, MixedModels +Pkg.instantiate() +using PkgBenchmark, MixedModels, Statistics # Pkg.update() allows us to benchmark even when dependencies/compat requirements change juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"` config = BenchmarkConfig(; id="origin/HEAD", juliacmd) -export_markdown("benchmark.md", judge(MixedModels, config; verbose=true)) +export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=true, f=median)) From c021b945736e286ed79a9e4e538c5ca6c6804b5e Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Tue, 22 Aug 2023 22:36:11 -0500 Subject: [PATCH 04/22] disable retune --- benchmark/run.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/run.jl b/benchmark/run.jl index 0a7b99e62..c7eda4a4c 100644 --- a/benchmark/run.jl +++ b/benchmark/run.jl @@ -5,4 +5,4 @@ using PkgBenchmark, MixedModels, Statistics # Pkg.update() allows us to benchmark even when dependencies/compat requirements change juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"` config = BenchmarkConfig(; id="origin/HEAD", juliacmd) -export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=true, f=median)) +export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=median)) From 01a1bf6867a2a2e50d364b5579e13aeede13c80e Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Tue, 22 Aug 2023 22:43:01 -0500 Subject: [PATCH 05/22] increase time tolerance --- benchmark/run.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmark/run.jl b/benchmark/run.jl index c7eda4a4c..690d966eb 100644 --- a/benchmark/run.jl +++ b/benchmark/run.jl @@ -5,4 +5,6 @@ using PkgBenchmark, MixedModels, Statistics # Pkg.update() allows us to benchmark even when dependencies/compat requirements change juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"` config = BenchmarkConfig(; id="origin/HEAD", juliacmd) -export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=median)) +# for many of the smaller models, we get a lot of noise at the default 5% tolerance +# TODO: specify a tune.json with per model time tolerances +export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=median, judgekwargs=(;time_tolerance=0.1))) From 155baf2aefebd8e6cb36c940083a2dbdbd296f1f Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Tue, 22 Aug 2023 23:20:59 -0500 Subject: [PATCH 06/22] robustify --- benchmark/Project.toml | 4 ++-- benchmark/benchmarks.jl | 4 ++-- benchmark/run.jl | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/benchmark/Project.toml b/benchmark/Project.toml index cfb2253a8..6c7445036 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -2,10 +2,10 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" MixedModels = "ff71e718-51f3-5ec2-a782-8ffcbfa3c316" PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" -Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -StatsModels = "3eaba693-59b7-5ba5-a881-562e759f1c8d" +StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" [compat] BenchmarkTools = "1" PkgBenchmark = "0.2" +StatsBase = "0.33, 0.34" julia = "1" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 4b53e6bae..6f20e2a0c 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,9 +1,9 @@ -using BenchmarkTools, MixedModels, StatsModels +using BenchmarkTools, MixedModels using MixedModels: dataset const SUITE = BenchmarkGroup() -const global contrasts = Dict{Symbol,Any}( +const contrasts = Dict{Symbol,Any}( :batch => Grouping(), # dyestuff, dyestuff2, pastes :cask => Grouping(), # pastes :d => Grouping(), # insteval diff --git a/benchmark/run.jl b/benchmark/run.jl index 690d966eb..05a8c70a3 100644 --- a/benchmark/run.jl +++ b/benchmark/run.jl @@ -1,10 +1,11 @@ using Pkg Pkg.develop(PackageSpec(path=dirname(@__DIR__))) Pkg.instantiate() -using PkgBenchmark, MixedModels, Statistics +using PkgBenchmark, MixedModels, StatsBase # Pkg.update() allows us to benchmark even when dependencies/compat requirements change juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"` config = BenchmarkConfig(; id="origin/HEAD", juliacmd) # for many of the smaller models, we get a lot of noise at the default 5% tolerance # TODO: specify a tune.json with per model time tolerances -export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=median, judgekwargs=(;time_tolerance=0.1))) +robustify(f) = x -> f(trim(x; prop=0.10)) +export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=robustify(maximum), judgekwargs=(;time_tolerance=0.1))) From 72e41efc88c3c6a7c677203247eb10c001165a2e Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Tue, 22 Aug 2023 23:41:28 -0500 Subject: [PATCH 07/22] wip --- benchmark/run.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmark/run.jl b/benchmark/run.jl index 05a8c70a3..41a7155d8 100644 --- a/benchmark/run.jl +++ b/benchmark/run.jl @@ -2,10 +2,12 @@ using Pkg Pkg.develop(PackageSpec(path=dirname(@__DIR__))) Pkg.instantiate() using PkgBenchmark, MixedModels, StatsBase +using BenchmarkTools: BenchmarkGroup # Pkg.update() allows us to benchmark even when dependencies/compat requirements change juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"` config = BenchmarkConfig(; id="origin/HEAD", juliacmd) # for many of the smaller models, we get a lot of noise at the default 5% tolerance # TODO: specify a tune.json with per model time tolerances +StatsBase.trim(group::BenchmarkGroup; kwargs...) = PkgBenchmark.mapvals(x -> trim(x; kwargs...), group) robustify(f) = x -> f(trim(x; prop=0.10)) export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=robustify(maximum), judgekwargs=(;time_tolerance=0.1))) From 023eba5b6ad9795d289ee20b6f8790994a014f07 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Tue, 22 Aug 2023 23:45:46 -0500 Subject: [PATCH 08/22] d'oh --- benchmark/run.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/run.jl b/benchmark/run.jl index 41a7155d8..d63c3fcde 100644 --- a/benchmark/run.jl +++ b/benchmark/run.jl @@ -2,12 +2,12 @@ using Pkg Pkg.develop(PackageSpec(path=dirname(@__DIR__))) Pkg.instantiate() using PkgBenchmark, MixedModels, StatsBase -using BenchmarkTools: BenchmarkGroup +using BenchmarkTools: BenchmarkGroup, mapvals # Pkg.update() allows us to benchmark even when dependencies/compat requirements change juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"` config = BenchmarkConfig(; id="origin/HEAD", juliacmd) # for many of the smaller models, we get a lot of noise at the default 5% tolerance # TODO: specify a tune.json with per model time tolerances -StatsBase.trim(group::BenchmarkGroup; kwargs...) = PkgBenchmark.mapvals(x -> trim(x; kwargs...), group) +StatsBase.trim(group::BenchmarkGroup; kwargs...) = mapvals(x -> trim(x; kwargs...), group) robustify(f) = x -> f(trim(x; prop=0.10)) export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=robustify(maximum), judgekwargs=(;time_tolerance=0.1))) From 758b43ae8ac3ff7cd521521bdc4d2b52b9d2e596 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 00:15:38 -0500 Subject: [PATCH 09/22] doh --- benchmark/Project.toml | 3 +-- benchmark/run.jl | 7 ++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/benchmark/Project.toml b/benchmark/Project.toml index 6c7445036..4d99f1bda 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -2,10 +2,9 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" MixedModels = "ff71e718-51f3-5ec2-a782-8ffcbfa3c316" PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" -StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [compat] BenchmarkTools = "1" PkgBenchmark = "0.2" -StatsBase = "0.33, 0.34" julia = "1" diff --git a/benchmark/run.jl b/benchmark/run.jl index d63c3fcde..2debb7e24 100644 --- a/benchmark/run.jl +++ b/benchmark/run.jl @@ -1,13 +1,10 @@ using Pkg Pkg.develop(PackageSpec(path=dirname(@__DIR__))) Pkg.instantiate() -using PkgBenchmark, MixedModels, StatsBase -using BenchmarkTools: BenchmarkGroup, mapvals +using PkgBenchmark, MixedModels, Statistics # Pkg.update() allows us to benchmark even when dependencies/compat requirements change juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"` config = BenchmarkConfig(; id="origin/HEAD", juliacmd) # for many of the smaller models, we get a lot of noise at the default 5% tolerance # TODO: specify a tune.json with per model time tolerances -StatsBase.trim(group::BenchmarkGroup; kwargs...) = mapvals(x -> trim(x; kwargs...), group) -robustify(f) = x -> f(trim(x; prop=0.10)) -export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=robustify(maximum), judgekwargs=(;time_tolerance=0.1))) +export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=maximum, judgekwargs=(;time_tolerance=0.1))) From 053260040457755bb09ee4774331639dbcf8ef35 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 00:28:35 -0500 Subject: [PATCH 10/22] boop --- benchmark/benchmarks.jl | 16 ++++++++-------- benchmark/run.jl | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 6f20e2a0c..927f4e862 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -29,7 +29,7 @@ const contrasts = Dict{Symbol,Any}( :spkr => HelmertCoding(), # kb07 ) -const global fms = Dict( +const fms = Dict( :dyestuff => [ @formula(yield ~ 1 + (1 | batch)) ], @@ -81,8 +81,8 @@ const global fms = Dict( ], ) -function fitbobyqa(dsnm::Symbol, i::Integer) - return fit(MixedModel, fms[dsnm][i], dataset(dsnm); contrasts, progress=false) +function fitbobyqa(dsnm::Symbol, i::Integer, ds=dataset(dsnm)) + return fit(MixedModel, fms[dsnm][i], ds; contrasts, progress=false) end # these tests are so fast that they can be very noisy because the denominator is so small, @@ -103,14 +103,14 @@ for (ds, i) in [ (:sleepstudy, 3), (:sleepstudy, 4), ] - SUITE["singlevector"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i) + SUITE["singlevector"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i, $(dataset(ds))) end SUITE["nested"] = BenchmarkGroup(["multiple", "nested", "scalar"]) for (ds, i) in [ (:pastes, 2) ] - SUITE["nested"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i) + SUITE["nested"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i, $(dataset(ds))) end SUITE["crossed"] = BenchmarkGroup(["multiple", "crossed", "scalar"]) @@ -120,10 +120,10 @@ for (ds, i) in [ (:kb07, 1), (:machines, 1), (:ml1m, 1), - (:mrk17_exp1, 1), + # (:mrk17_exp1, 1), (:penicillin, 1), ] - SUITE["crossed"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i) + SUITE["crossed"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i, $(dataset(ds))) end SUITE["crossedvector"] = BenchmarkGroup(["multiple", "crossed", "vector"]) @@ -133,5 +133,5 @@ for (ds, i) in [ (:kb07, 3), (:mrk17_exp1, 2), ] - SUITE["crossedvector"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i) + SUITE["crossedvector"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i, $(dataset(ds))) end diff --git a/benchmark/run.jl b/benchmark/run.jl index 2debb7e24..690d966eb 100644 --- a/benchmark/run.jl +++ b/benchmark/run.jl @@ -7,4 +7,4 @@ juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"` config = BenchmarkConfig(; id="origin/HEAD", juliacmd) # for many of the smaller models, we get a lot of noise at the default 5% tolerance # TODO: specify a tune.json with per model time tolerances -export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=maximum, judgekwargs=(;time_tolerance=0.1))) +export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=median, judgekwargs=(;time_tolerance=0.1))) From 38ca5fbf420996889c1fb98560f378979380b09b Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 00:38:24 -0500 Subject: [PATCH 11/22] tryagain --- benchmark/benchmarks.jl | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 927f4e862..9169ff359 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,9 +1,9 @@ -using BenchmarkTools, MixedModels +using BenchmarkTools, MixedModels, StatsModels using MixedModels: dataset const SUITE = BenchmarkGroup() -const contrasts = Dict{Symbol,Any}( +const global contrasts = Dict{Symbol,Any}( :batch => Grouping(), # dyestuff, dyestuff2, pastes :cask => Grouping(), # pastes :d => Grouping(), # insteval @@ -29,7 +29,7 @@ const contrasts = Dict{Symbol,Any}( :spkr => HelmertCoding(), # kb07 ) -const fms = Dict( +const global fms = Dict( :dyestuff => [ @formula(yield ~ 1 + (1 | batch)) ], @@ -81,8 +81,9 @@ const fms = Dict( ], ) -function fitbobyqa(dsnm::Symbol, i::Integer, ds=dataset(dsnm)) - return fit(MixedModel, fms[dsnm][i], ds; contrasts, progress=false) +function fitbobyqa(dsnm::Symbol, i::Integer) + model = LinearMixedModel(fms[dsnm][i], dataset(dsnm); contrasts) + return @benchmarkable fit!($(model); progress=false) end # these tests are so fast that they can be very noisy because the denominator is so small, @@ -103,14 +104,14 @@ for (ds, i) in [ (:sleepstudy, 3), (:sleepstudy, 4), ] - SUITE["singlevector"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i, $(dataset(ds))) + SUITE["singlevector"][string(ds, ':', i)] = fitbobyqa(ds, i) end SUITE["nested"] = BenchmarkGroup(["multiple", "nested", "scalar"]) for (ds, i) in [ (:pastes, 2) ] - SUITE["nested"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i, $(dataset(ds))) + SUITE["nested"][string(ds, ':', i)] = fitbobyqa(ds, i) end SUITE["crossed"] = BenchmarkGroup(["multiple", "crossed", "scalar"]) @@ -120,10 +121,10 @@ for (ds, i) in [ (:kb07, 1), (:machines, 1), (:ml1m, 1), - # (:mrk17_exp1, 1), + (:mrk17_exp1, 1), (:penicillin, 1), ] - SUITE["crossed"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i, $(dataset(ds))) + SUITE["crossed"][string(ds, ':', i)] = fitbobyqa(ds, i) end SUITE["crossedvector"] = BenchmarkGroup(["multiple", "crossed", "vector"]) @@ -133,5 +134,5 @@ for (ds, i) in [ (:kb07, 3), (:mrk17_exp1, 2), ] - SUITE["crossedvector"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i, $(dataset(ds))) + SUITE["crossedvector"][string(ds, ':', i)] = fitbobyqa(ds, i) end From 8c50d98f424d06ea17f581b73f76ed0d009e3b5c Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 00:39:28 -0500 Subject: [PATCH 12/22] hmmm --- benchmark/benchmarks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 9169ff359..f1cee7e1f 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -83,7 +83,7 @@ const global fms = Dict( function fitbobyqa(dsnm::Symbol, i::Integer) model = LinearMixedModel(fms[dsnm][i], dataset(dsnm); contrasts) - return @benchmarkable fit!($(model); progress=false) + return @benchmarkable fit!(model; progress=false) end # these tests are so fast that they can be very noisy because the denominator is so small, From 5a592ffc49be7113cac145cff00db07233cf6257 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 00:40:20 -0500 Subject: [PATCH 13/22] hmmm --- benchmark/benchmarks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index f1cee7e1f..c5d723960 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -83,7 +83,7 @@ const global fms = Dict( function fitbobyqa(dsnm::Symbol, i::Integer) model = LinearMixedModel(fms[dsnm][i], dataset(dsnm); contrasts) - return @benchmarkable fit!(model; progress=false) + return @benchmarkable fit!($(deepcopy(model)); progress=false) end # these tests are so fast that they can be very noisy because the denominator is so small, From 071b4ec9493e5feb4b012964467d8aab157201b5 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 00:41:30 -0500 Subject: [PATCH 14/22] hmmm --- benchmark/benchmarks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index c5d723960..9eef3d38c 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -83,7 +83,7 @@ const global fms = Dict( function fitbobyqa(dsnm::Symbol, i::Integer) model = LinearMixedModel(fms[dsnm][i], dataset(dsnm); contrasts) - return @benchmarkable fit!($(deepcopy(model)); progress=false) + return @benchmarkable fit!(deepcopy(model); progress=false) end # these tests are so fast that they can be very noisy because the denominator is so small, From 5ae72e8bc03d4f4f97c030d9622eda6981ee1499 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 00:44:38 -0500 Subject: [PATCH 15/22] x --- benchmark/benchmarks.jl | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 9eef3d38c..904c48f9c 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -81,11 +81,9 @@ const global fms = Dict( ], ) -function fitbobyqa(dsnm::Symbol, i::Integer) - model = LinearMixedModel(fms[dsnm][i], dataset(dsnm); contrasts) - return @benchmarkable fit!(deepcopy(model); progress=false) +function get_model(dsnm::Symbol, i::Integer) + return LinearMixedModel(fms[dsnm][i], dataset(dsnm); contrasts) end - # these tests are so fast that they can be very noisy because the denominator is so small, # so we disable them by default for auto-benchmarking # SUITE["simplescalar"] = BenchmarkGroup(["single", "simple", "scalar"]) @@ -104,14 +102,14 @@ for (ds, i) in [ (:sleepstudy, 3), (:sleepstudy, 4), ] - SUITE["singlevector"][string(ds, ':', i)] = fitbobyqa(ds, i) + SUITE["singlevector"][string(ds, ':', i)] = @benchmarkable fit!(get_model(ds, i); progress=false) end SUITE["nested"] = BenchmarkGroup(["multiple", "nested", "scalar"]) for (ds, i) in [ (:pastes, 2) ] - SUITE["nested"][string(ds, ':', i)] = fitbobyqa(ds, i) + SUITE["nested"][string(ds, ':', i)] = @benchmarkable fit!(get_model(ds, i); progress=false) end SUITE["crossed"] = BenchmarkGroup(["multiple", "crossed", "scalar"]) @@ -124,7 +122,7 @@ for (ds, i) in [ (:mrk17_exp1, 1), (:penicillin, 1), ] - SUITE["crossed"][string(ds, ':', i)] = fitbobyqa(ds, i) + SUITE["crossed"][string(ds, ':', i)] = @benchmarkable fit!(get_model(ds, i); progress=false) end SUITE["crossedvector"] = BenchmarkGroup(["multiple", "crossed", "vector"]) @@ -134,5 +132,5 @@ for (ds, i) in [ (:kb07, 3), (:mrk17_exp1, 2), ] - SUITE["crossedvector"][string(ds, ':', i)] = fitbobyqa(ds, i) + SUITE["crossedvector"][string(ds, ':', i)] = @benchmarkable fit!(get_model(ds, i); progress=false) end From f256e299741db909d0febe2bac4180dc90277b14 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 00:45:41 -0500 Subject: [PATCH 16/22] a --- benchmark/benchmarks.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 904c48f9c..6ca05f323 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -102,14 +102,14 @@ for (ds, i) in [ (:sleepstudy, 3), (:sleepstudy, 4), ] - SUITE["singlevector"][string(ds, ':', i)] = @benchmarkable fit!(get_model(ds, i); progress=false) + SUITE["singlevector"][string(ds, ':', i)] = @benchmarkable fit!($(get_model(ds, i)); progress=false) end SUITE["nested"] = BenchmarkGroup(["multiple", "nested", "scalar"]) for (ds, i) in [ (:pastes, 2) ] - SUITE["nested"][string(ds, ':', i)] = @benchmarkable fit!(get_model(ds, i); progress=false) + SUITE["nested"][string(ds, ':', i)] = @benchmarkable ffit!($(get_model(ds, i)); progress=false) end SUITE["crossed"] = BenchmarkGroup(["multiple", "crossed", "scalar"]) @@ -122,7 +122,7 @@ for (ds, i) in [ (:mrk17_exp1, 1), (:penicillin, 1), ] - SUITE["crossed"][string(ds, ':', i)] = @benchmarkable fit!(get_model(ds, i); progress=false) + SUITE["crossed"][string(ds, ':', i)] = @benchmarkable fit!($(get_model(ds, i)); progress=false) end SUITE["crossedvector"] = BenchmarkGroup(["multiple", "crossed", "vector"]) @@ -132,5 +132,5 @@ for (ds, i) in [ (:kb07, 3), (:mrk17_exp1, 2), ] - SUITE["crossedvector"][string(ds, ':', i)] = @benchmarkable fit!(get_model(ds, i); progress=false) + SUITE["crossedvector"][string(ds, ':', i)] = @benchmarkable fit!($(get_model(ds, i)); progress=false) end From c44e6d1c8a822d4d2b7818556f671c59599d28fc Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 00:46:59 -0500 Subject: [PATCH 17/22] a --- benchmark/benchmarks.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 6ca05f323..c9d91d8f4 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -102,14 +102,14 @@ for (ds, i) in [ (:sleepstudy, 3), (:sleepstudy, 4), ] - SUITE["singlevector"][string(ds, ':', i)] = @benchmarkable fit!($(get_model(ds, i)); progress=false) + SUITE["singlevector"][string(ds, ':', i)] = @benchmarkable fit!(get_model($ds, $i); progress=false) end SUITE["nested"] = BenchmarkGroup(["multiple", "nested", "scalar"]) for (ds, i) in [ (:pastes, 2) ] - SUITE["nested"][string(ds, ':', i)] = @benchmarkable ffit!($(get_model(ds, i)); progress=false) + SUITE["nested"][string(ds, ':', i)] = @benchmarkable fit!(get_model($ds, $i); progress=false) end SUITE["crossed"] = BenchmarkGroup(["multiple", "crossed", "scalar"]) @@ -122,7 +122,7 @@ for (ds, i) in [ (:mrk17_exp1, 1), (:penicillin, 1), ] - SUITE["crossed"][string(ds, ':', i)] = @benchmarkable fit!($(get_model(ds, i)); progress=false) + SUITE["crossed"][string(ds, ':', i)] = @benchmarkable fit!(get_model($ds, $i); progress=false) end SUITE["crossedvector"] = BenchmarkGroup(["multiple", "crossed", "vector"]) @@ -132,5 +132,5 @@ for (ds, i) in [ (:kb07, 3), (:mrk17_exp1, 2), ] - SUITE["crossedvector"][string(ds, ':', i)] = @benchmarkable fit!($(get_model(ds, i)); progress=false) + SUITE["crossedvector"][string(ds, ':', i)] = @benchmarkable fit!(get_model($ds, $i); progress=false) end From bf67b163afb6dc685788602ad16d6ae679169251 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 00:51:18 -0500 Subject: [PATCH 18/22] a --- benchmark/run.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/run.jl b/benchmark/run.jl index 690d966eb..44a84d5ef 100644 --- a/benchmark/run.jl +++ b/benchmark/run.jl @@ -7,4 +7,4 @@ juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"` config = BenchmarkConfig(; id="origin/HEAD", juliacmd) # for many of the smaller models, we get a lot of noise at the default 5% tolerance # TODO: specify a tune.json with per model time tolerances -export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=median, judgekwargs=(;time_tolerance=0.1))) +export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=median, judgekwargs=(;time_tolerance=0.1, memory_tolerance=0.05))) From b2fb27439ff3d0579194b5d1010c00434047fb7c Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 01:00:54 -0500 Subject: [PATCH 19/22] x --- benchmark/benchmarks.jl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index c9d91d8f4..626146397 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -81,10 +81,11 @@ const global fms = Dict( ], ) -function get_model(dsnm::Symbol, i::Integer) - return LinearMixedModel(fms[dsnm][i], dataset(dsnm); contrasts) +function fitbobyqa(dsnm::Symbol, i::Integer) + return fit(MixedModel, fms[dsnm][i], dataset(dsnm); contrasts, progress=false) end -# these tests are so fast that they can be very noisy because the denominator is so small, + +# these tests are so fast that they can be very noisy because the denominator is so small, # so we disable them by default for auto-benchmarking # SUITE["simplescalar"] = BenchmarkGroup(["single", "simple", "scalar"]) # for (ds, i) in [ @@ -102,14 +103,14 @@ for (ds, i) in [ (:sleepstudy, 3), (:sleepstudy, 4), ] - SUITE["singlevector"][string(ds, ':', i)] = @benchmarkable fit!(get_model($ds, $i); progress=false) + SUITE["singlevector"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i) end SUITE["nested"] = BenchmarkGroup(["multiple", "nested", "scalar"]) for (ds, i) in [ (:pastes, 2) ] - SUITE["nested"][string(ds, ':', i)] = @benchmarkable fit!(get_model($ds, $i); progress=false) + SUITE["nested"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i) end SUITE["crossed"] = BenchmarkGroup(["multiple", "crossed", "scalar"]) @@ -122,7 +123,7 @@ for (ds, i) in [ (:mrk17_exp1, 1), (:penicillin, 1), ] - SUITE["crossed"][string(ds, ':', i)] = @benchmarkable fit!(get_model($ds, $i); progress=false) + SUITE["crossed"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i) end SUITE["crossedvector"] = BenchmarkGroup(["multiple", "crossed", "vector"]) @@ -132,5 +133,5 @@ for (ds, i) in [ (:kb07, 3), (:mrk17_exp1, 2), ] - SUITE["crossedvector"][string(ds, ':', i)] = @benchmarkable fit!(get_model($ds, $i); progress=false) + SUITE["crossedvector"][string(ds, ':', i)] = @benchmarkable fitbobyqa($ds, $i) end From e90cd312f41754321a3b102e17cdd315ca5aa159 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 01:08:55 -0500 Subject: [PATCH 20/22] x --- benchmark/benchmarks.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 626146397..4c05d5c0d 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,4 +1,4 @@ -using BenchmarkTools, MixedModels, StatsModels +using BenchmarkTools, MixedModels using MixedModels: dataset const SUITE = BenchmarkGroup() @@ -85,7 +85,7 @@ function fitbobyqa(dsnm::Symbol, i::Integer) return fit(MixedModel, fms[dsnm][i], dataset(dsnm); contrasts, progress=false) end -# these tests are so fast that they can be very noisy because the denominator is so small, +# these tests are so fast that they can be very noisy because the denominator is so small, # so we disable them by default for auto-benchmarking # SUITE["simplescalar"] = BenchmarkGroup(["single", "simple", "scalar"]) # for (ds, i) in [ From 1f3107195cbccb700f97a70c17bc89fe08b74ab2 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 23 Aug 2023 01:34:13 -0500 Subject: [PATCH 21/22] try forcing tuning --- benchmark/tune.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 benchmark/tune.json diff --git a/benchmark/tune.json b/benchmark/tune.json new file mode 100644 index 000000000..969ce60ed --- /dev/null +++ b/benchmark/tune.json @@ -0,0 +1 @@ +[{"Julia":"1.9.2","BenchmarkTools":"1.0.0"},[["BenchmarkGroup",{"data":{"singlevector":["BenchmarkGroup",{"data":{"sleepstudy:3":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"sleepstudy:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"sleepstudy:4":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["single","vector"]}],"crossed":["BenchmarkGroup",{"data":{"kb07:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"ml1m:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"insteval:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"insteval:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"machines:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"mrk17_exp1:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"penicillin:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["multiple","crossed","scalar"]}],"crossedvector":["BenchmarkGroup",{"data":{"kb07:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"d3:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"kb07:3":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"mrk17_exp1:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["multiple","crossed","vector"]}],"nested":["BenchmarkGroup",{"data":{"pastes:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["multiple","nested","scalar"]}]},"tags":[]}]]] \ No newline at end of file From 6eb1197bba39ec1d525e5be748f9d1cbb5283893 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Tue, 5 Mar 2024 16:15:57 -0600 Subject: [PATCH 22/22] only run benchmark ci manually --- .github/workflows/benchmark.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a2c31ae77..96d974cea 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -3,16 +3,17 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true on: - pull_request: - branches: - - main - paths-ignore: - - 'LICENSE.md' - - 'README.md' - - 'docs/**' - - 'issues/**' - - 'format/**' - - 'test/**' + workflow_dispatch: +# pull_request: +# branches: +# - main +# paths-ignore: +# - 'LICENSE.md' +# - 'README.md' +# - 'docs/**' +# - 'issues/**' +# - 'format/**' +# - 'test/**' jobs: benchmarks: runs-on: ubuntu-latest