unify filter criteria

benchmark/Project.toml

@@ -1,9 +1,10 @@
 [deps]
+ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 Finch = "9177782c-1635-4eb9-9bfb-d9dfa25e6bce"
 MatrixDepot = "b51810bb-c9f3-55da-ae3c-350fc1fbce05"
 MatrixMarket = "4d4711f2-db25-561a-b6b3-d35e7d4047d3"
 PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
 
 [compat]
-BenchmarkTools = "1.5"
+BenchmarkTools = "1.5"

benchmark/benchmarks.jl

@@ -10,6 +10,29 @@ using BenchmarkTools
 using MatrixDepot
 using SparseArrays
 using Random
+using ArgParse
+
+s = ArgParseSettings("Run Finch.jl benchmarks. By default, all tests are run unless --include or --exclude options are provided. 
+If the environment variable FINCH_BENCHMARK_ARGS is set, it will override the given arguments.")
+
+@add_arg_table! s begin
+    "--include", "-i"
+    nargs = '*'
+    default = []
+    help = "list of benchmark suites to include, e.g., --include high-level structures"
+
+    "--exclude", "-e"
+    nargs = '*'
+    default = []
+    help = "list of benchmark suites to exclude, e.g., --exclude compile graphs"
+end
+
+if "FINCH_BENCHMARK_ARGS" in keys(ENV)
+    ARGS = split(ENV["FINCH_BENCHMARK_ARGS"], " ")
+end
+
+parsed_args = parse_args(ARGS, s)
+
 include(joinpath(@__DIR__, "../docs/examples/bfs.jl"))
 include(joinpath(@__DIR__, "../docs/examples/pagerank.jl"))
 include(joinpath(@__DIR__, "../docs/examples/shortest_paths.jl"))
@@ -374,6 +397,13 @@ SUITE["structure"]["banded"]["SparseList"] = @benchmarkable spmv_serial($A_ref,
 SUITE["structure"]["banded"]["SparseBand"] = @benchmarkable spmv_serial($A, $x)
 SUITE["structure"]["banded"]["SparseInterval"] = @benchmarkable spmv_serial($A2, $x)
 
-if !isnothing(ENV["BENCHMARK_FILTER"])
-    SUITE = eval(:(SUITE[@tagged $(Meta.parse(ENV["BENCHMARK_FILTER"]))]))
+if !isempty(parsed_args["include"])
+    inc = reduce((a, b) -> :($a || $b), parsed_args["include"])
+    println(:(SUITE[@tagged $inc]))
+    SUITE = eval(:(SUITE[@tagged $inc]))
+end
+if !isempty(parsed_args["exclude"])
+    exc = reduce((a, b) -> :($a || $b), parsed_args["exclude"])
+    println(:(SUITE[@tagged !$exc]))
+    SUITE = eval(:(SUITE[@tagged !$exc]))
 end

benchmark/runbenchmarks.jl

@@ -11,7 +11,7 @@ end
 using PkgBenchmark
 benchmarkpkg(
     dirname(@__DIR__),
-    BenchmarkConfig(env = Dict("JULIA_NUM_THREADS" => "8")),
+    BenchmarkConfig(env = Dict("JULIA_NUM_THREADS" => "8", "FINCH_BENCHMARK_ARGS" => get(ENV, "FINCH_BENCHMARK_ARGS", join(ARGS, " ")))),
     resultfile = joinpath(@__DIR__, "result.json"),
 )
 

benchmark/runjudge.jl

@@ -11,7 +11,7 @@ end
 using PkgBenchmark
 
 function mkconfig(; kwargs...)
-    return BenchmarkConfig(env = Dict("JULIA_NUM_THREADS" => "1"); kwargs...)
+    return BenchmarkConfig(env = Dict("JULIA_NUM_THREADS" => "1", "FINCH_BENCHMARK_ARGS" => get(ENV, "FINCH_BENCHMARK_ARGS", join(ARGS, " "))); kwargs...)
 end
 
 script = tempname(joinpath(@__DIR__))

test/runtests.jl

@@ -19,16 +19,20 @@ If the environment variable FINCH_TEST_ARGS is set, it will override the given a
     help = "overwrite reference output for $(Sys.WORD_SIZE)-bit systems"
     "--include", "-i"
     nargs = '*'
-    default = nothing
+    default = []
     help = "list of test suites to include, e.g., --include constructors merges"
 
     "--exclude", "-e"
     nargs = '*'
-    default = nothing
+    default = []
     help = "list of test suites to exclude, e.g., --exclude parallel algebra"
 end
 
-parsed_args = parse_args(get(ENV, "FINCH_TEST_ARGS", ARGS), s)
+if "FINCH_TEST_ARGS" in keys(ENV)
+    ARGS = split(ENV["FINCH_TEST_ARGS"], " ")
+end
+
+parsed_args = parse_args(ARGS, s)
 
 """
     check_output(fname, arg)
@@ -66,8 +70,8 @@ end
 function should_run(name)
     global parsed_args
     inc = parsed_args["include"]
-    exc = something(parsed_args["exclude"], [])
-    return (isnothing(inc) || !isempty(intersect(test_names, inc))) && isempty(intersect(test_names, exc))
+    exc = parsed_args["exclude"]
+    return (isempty(inc) || name in inc) && !(name in exc)
 end
 
 macro repl(io, ex, quiet = false)