From fe28b42977638a51d0b5dd3a81693914c1d8351a Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Mon, 12 Aug 2024 12:58:43 +0200 Subject: [PATCH 01/17] Add profile run to ci --- .github/workflows/profile.yaml | 85 +++++++++++++++++++ Cargo.lock | 35 ++++++++ node-graph/graph-craft/Cargo.toml | 4 + .../graph-craft/benches/compile_demo_art.rs | 51 ++++++++--- 4 files changed, 164 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/profile.yaml diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml new file mode 100644 index 0000000000..78c0715769 --- /dev/null +++ b/.github/workflows/profile.yaml @@ -0,0 +1,85 @@ +name: Rust Profiling with iai-callgrind + +on: + pull_request: + branches: [ master ] + +env: + CARGO_TERM_COLOR: always + +jobs: + profile: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + + - name: Install Valgrind + run: | + sudo apt-get update + sudo apt-get install -y valgrind + + - name: Cache dependencies + uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install iai-callgrind + run: | + cargo install iai-callgrind-runner@0.12.3 + cargo add --dev iai-callgrind@0.12.3 + + - name: Checkout main branch + run: | + git fetch origin main:main + git checkout main + + - name: Run baseline benchmarks + run: | + cargo bench --bench compile_demo_art --features=iai -- --save-baseline=main + + - name: Checkout PR branch + run: | + git checkout ${{ github.event.pull_request.head.sha }} + + - name: Run PR benchmarks + run: | + cargo bench --bench compile_demo_art --features=iai -- --baseline main > benchmark_results.txt + + - name: Parse benchmark results + id: parse_results + run: | + COMMENT="## Performance Benchmark Results\n\n" + while IFS= read -r line; do + if [[ $line == *"Comparison with"* ]]; then + COMMENT+="### $line\n" + elif [[ $line == *"Instructions:"* || $line == *"L1 Hits:"* || $line == *"Estimated Cycles:"* ]]; then + COMMENT+="$line\n" + fi + done < benchmark_results.txt + echo "comment<> $GITHUB_OUTPUT + echo -e "$COMMENT" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Comment PR + uses: actions/github-script@v6 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.name, + body: ${{steps.parse_results.outputs.comment}} + }) diff --git a/Cargo.lock b/Cargo.lock index 580745f358..987b791ff7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2432,6 +2432,7 @@ dependencies = [ "glob", "graph-craft", "graphene-core", + "iai-callgrind", "js-sys", "log", "num-traits", @@ -3042,6 +3043,40 @@ dependencies = [ "tracing", ] +[[package]] +name = "iai-callgrind" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "283f598a969822c70af13aae1272ba09c97014c7344d3b24652e5b1d7b771c36" +dependencies = [ + "bincode", + "iai-callgrind-macros", + "iai-callgrind-runner", +] + +[[package]] +name = "iai-callgrind-macros" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e2ff2e86bdba764b66d94b65f2caa03da60d971a6930fdc2e67f12582c5bb8" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.72", +] + +[[package]] +name = "iai-callgrind-runner" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb92a65def0d3a0ef41029c411dc2ecdd3518708c062f8bd576fd4143be1c56b" +dependencies = [ + "serde", +] + [[package]] name = "iana-time-zone" version = "0.1.60" diff --git a/node-graph/graph-craft/Cargo.toml b/node-graph/graph-craft/Cargo.toml index 4d88564287..8e3b5315d8 100644 --- a/node-graph/graph-craft/Cargo.toml +++ b/node-graph/graph-craft/Cargo.toml @@ -11,6 +11,8 @@ dealloc_nodes = [] wgpu = [] tokio = ["dep:tokio"] wayland = [] +criterion = [] +iai = [] [dependencies] # Local dependencies @@ -56,11 +58,13 @@ glob = "0.3" pprof = { version = "0.13", features = ["flamegraph"] } serde_json = { workspace = true } graph-craft = { workspace = true, features = ["serde"] } +iai-callgrind = "0.12.3" [[bench]] name = "compile_demo_art" harness = false + # [[bench]] # name = "exec_demo_art" # harness = false diff --git a/node-graph/graph-craft/benches/compile_demo_art.rs b/node-graph/graph-craft/benches/compile_demo_art.rs index 28c33683e4..42c85152d9 100644 --- a/node-graph/graph-craft/benches/compile_demo_art.rs +++ b/node-graph/graph-craft/benches/compile_demo_art.rs @@ -1,28 +1,57 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; use graph_craft::document::NodeNetwork; use graph_craft::graphene_compiler::Compiler; use graph_craft::proto::ProtoNetwork; +use std::path::PathBuf; -pub fn compile_to_proto(c: &mut Criterion) { - let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern"); - for path in artworks { - let Ok(path) = path else { continue }; - let content = std::fs::read(&path).expect("failed to read file"); - let network = load_network(std::str::from_utf8(&content).unwrap()); - let name = path.file_stem().unwrap().to_str().unwrap(); +#[cfg(feature = "criterion")] +use criterion::{black_box, criterion_group, criterion_main, Criterion}; - c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput)); - } -} +#[cfg(feature = "iai")] +use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main}; fn load_network(document_string: &str) -> NodeNetwork { let document: serde_json::Value = serde_json::from_str(document_string).expect("Failed to parse document"); serde_json::from_value::(document["network_interface"]["network"].clone()).expect("Failed to parse document") } + fn compile(network: NodeNetwork) -> ProtoNetwork { let compiler = Compiler {}; compiler.compile_single(network).unwrap() } +fn bench_compile(path: PathBuf) { + let content = std::fs::read(&path).expect("failed to read file"); + let network = load_network(std::str::from_utf8(&content).unwrap()); + black_box(compile(black_box(network))); +} + +#[cfg(feature = "criterion")] +fn compile_to_proto(c: &mut Criterion) { + let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern"); + for path in artworks { + let Ok(path) = path else { continue }; + let name = path.file_stem().unwrap().to_str().unwrap(); + c.bench_function(name, |b| b.iter(|| bench_compile(path.clone()))); + } +} + +#[cfg_attr(feature = "iai", library_benchmark)] +fn iai_compile_to_proto() { + let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern"); + for path in artworks { + let Ok(path) = path else { continue }; + bench_compile(path); + } +} + +#[cfg(feature = "criterion")] criterion_group!(benches, compile_to_proto); + +#[cfg(feature = "criterion")] criterion_main!(benches); + +#[cfg(feature = "iai")] +library_benchmark_group!(name = compile_group; benchmarks = iai_compile_to_proto); + +#[cfg(feature = "iai")] +main!(library_benchmark_groups = compile_group); From 909d1002b48c3e64864eabccf249733ce2241eef Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Mon, 12 Aug 2024 13:04:40 +0200 Subject: [PATCH 02/17] Remove cargo add --- .github/workflows/profile.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index 78c0715769..3e5b7383b7 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -38,7 +38,6 @@ jobs: - name: Install iai-callgrind run: | cargo install iai-callgrind-runner@0.12.3 - cargo add --dev iai-callgrind@0.12.3 - name: Checkout main branch run: | From 3d50bcc74accced8769f8f5906d1e6f43649acbe Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Mon, 12 Aug 2024 13:09:21 +0200 Subject: [PATCH 03/17] Rename more main to master --- .github/workflows/profile.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index 3e5b7383b7..20b5652aa1 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -39,14 +39,14 @@ jobs: run: | cargo install iai-callgrind-runner@0.12.3 - - name: Checkout main branch + - name: Checkout master branch run: | - git fetch origin main:main - git checkout main + git fetch origin master:master + git checkout master - name: Run baseline benchmarks run: | - cargo bench --bench compile_demo_art --features=iai -- --save-baseline=main + cargo bench --bench compile_demo_art --features=iai -- --save-baseline=master - name: Checkout PR branch run: | @@ -54,7 +54,7 @@ jobs: - name: Run PR benchmarks run: | - cargo bench --bench compile_demo_art --features=iai -- --baseline main > benchmark_results.txt + cargo bench --bench compile_demo_art --features=iai -- --baseline master > benchmark_results.txt - name: Parse benchmark results id: parse_results From a2f5ae294cdf7192ccdd86bd0620242530371229 Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Mon, 12 Aug 2024 13:18:27 +0200 Subject: [PATCH 04/17] Don't run on master before the pr has been merged --- .github/workflows/profile.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index 20b5652aa1..f1fa803673 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -39,10 +39,10 @@ jobs: run: | cargo install iai-callgrind-runner@0.12.3 - - name: Checkout master branch - run: | - git fetch origin master:master - git checkout master + # - name: Checkout master branch + # run: | + # git fetch origin master:master + # git checkout master - name: Run baseline benchmarks run: | From 3bb71e0e56ac3e8820a478ee188bbe011a2b88a6 Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Mon, 12 Aug 2024 13:28:01 +0200 Subject: [PATCH 05/17] Fix pr comment step --- .github/workflows/profile.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index f1fa803673..e6f1cf7062 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -76,9 +76,14 @@ jobs: with: github-token: ${{secrets.GITHUB_TOKEN}} script: | + const fs = require('fs'); + const benchmarkResults = fs.readFileSync('benchmark_results.txt', 'utf8'); github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.name, - body: ${{steps.parse_results.outputs.comment}} + body: `## Performance Benchmark Results + + ${benchmarkResults}` }) + From f8e87f30ce76a173646e7d6dddc3f5ac453e4f86 Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Mon, 12 Aug 2024 13:42:23 +0200 Subject: [PATCH 06/17] Fix comment v2 --- .github/workflows/profile.yaml | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index e6f1cf7062..d0e4d4b979 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -56,21 +56,6 @@ jobs: run: | cargo bench --bench compile_demo_art --features=iai -- --baseline master > benchmark_results.txt - - name: Parse benchmark results - id: parse_results - run: | - COMMENT="## Performance Benchmark Results\n\n" - while IFS= read -r line; do - if [[ $line == *"Comparison with"* ]]; then - COMMENT+="### $line\n" - elif [[ $line == *"Instructions:"* || $line == *"L1 Hits:"* || $line == *"Estimated Cycles:"* ]]; then - COMMENT+="$line\n" - fi - done < benchmark_results.txt - echo "comment<> $GITHUB_OUTPUT - echo -e "$COMMENT" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - name: Comment PR uses: actions/github-script@v6 with: @@ -78,12 +63,14 @@ jobs: script: | const fs = require('fs'); const benchmarkResults = fs.readFileSync('benchmark_results.txt', 'utf8'); - github.rest.issues.createComment({ - issue_number: context.issue.number, + + await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.name, + issue_number: context.issue.number, body: `## Performance Benchmark Results - ${benchmarkResults}` - }) - + \`\`\` + ${benchmarkResults} + \`\`\`` + }); From 45a50106ca6d77ba9dfc18aee77861eb052a6a56 Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Mon, 12 Aug 2024 13:52:10 +0200 Subject: [PATCH 07/17] Fix v3 --- .github/workflows/profile.yaml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index d0e4d4b979..0a3abe4a8d 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -53,24 +53,27 @@ jobs: git checkout ${{ github.event.pull_request.head.sha }} - name: Run PR benchmarks + id: benchmark run: | - cargo bench --bench compile_demo_art --features=iai -- --baseline master > benchmark_results.txt + cargo bench --bench compile_demo_art_iai -- --baseline main > benchmark_results.txt + echo "BENCHMARK_OUTPUT<> $GITHUB_OUTPUT + cat benchmark_results.txt >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - name: Comment PR uses: actions/github-script@v6 with: github-token: ${{secrets.GITHUB_TOKEN}} script: | - const fs = require('fs'); - const benchmarkResults = fs.readFileSync('benchmark_results.txt', 'utf8'); + const output = `#### Performance Benchmark Results + \`\`\` + ${{ steps.benchmark.outputs.BENCHMARK_OUTPUT }} + \`\`\` + `; - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.name, + github.rest.issues.createComment({ issue_number: context.issue.number, - body: `## Performance Benchmark Results - - \`\`\` - ${benchmarkResults} - \`\`\`` - }); + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) From 112d73bc0d9ba9504eac314f779c5ae651e44bfd Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Mon, 12 Aug 2024 14:02:26 +0200 Subject: [PATCH 08/17] Fix missing features --- .github/workflows/profile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index 0a3abe4a8d..0ded579442 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -55,7 +55,7 @@ jobs: - name: Run PR benchmarks id: benchmark run: | - cargo bench --bench compile_demo_art_iai -- --baseline main > benchmark_results.txt + cargo bench --bench compile_demo_art --features=iai -- --baseline main > benchmark_results.txt echo "BENCHMARK_OUTPUT<> $GITHUB_OUTPUT cat benchmark_results.txt >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT From a16fe26687842a668106cabd803fcf75f97899a7 Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Wed, 14 Aug 2024 11:39:16 +0200 Subject: [PATCH 09/17] Fix string interpolation --- .github/workflows/profile.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index 0ded579442..4521a9385e 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -55,10 +55,9 @@ jobs: - name: Run PR benchmarks id: benchmark run: | - cargo bench --bench compile_demo_art --features=iai -- --baseline main > benchmark_results.txt - echo "BENCHMARK_OUTPUT<> $GITHUB_OUTPUT - cat benchmark_results.txt >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT + BENCH_OUTPUT=$(cargo bench --bench compile_demo_art --features=iai -- --baseline=master) + ESCAPED_OUTPUT=$(echo "$BENCH_OUTPUT" | jq -sR .) + echo "BENCHMARK_OUTPUT=$ESCAPED_OUTPUT" >> $GITHUB_OUTPUT - name: Comment PR uses: actions/github-script@v6 From a2191f05345f4b2f25ae1bb05570eca416681e8b Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Wed, 14 Aug 2024 12:38:16 +0200 Subject: [PATCH 10/17] Only post comment on performance diff --- .github/workflows/profile.yaml | 49 ++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index 4521a9385e..6a03fef9a2 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -55,24 +55,45 @@ jobs: - name: Run PR benchmarks id: benchmark run: | - BENCH_OUTPUT=$(cargo bench --bench compile_demo_art --features=iai -- --baseline=master) - ESCAPED_OUTPUT=$(echo "$BENCH_OUTPUT" | jq -sR .) - echo "BENCHMARK_OUTPUT=$ESCAPED_OUTPUT" >> $GITHUB_OUTPUT + BENCH_OUTPUT=$(cargo bench --bench compile_demo_art --features=iai -- --baseline=master --output-format=json) + echo "BENCHMARK_OUTPUT<> $GITHUB_OUTPUT + echo "$BENCH_OUTPUT" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - name: Comment PR uses: actions/github-script@v6 with: github-token: ${{secrets.GITHUB_TOKEN}} script: | - const output = `#### Performance Benchmark Results - \`\`\` - ${{ steps.benchmark.outputs.BENCHMARK_OUTPUT }} - \`\`\` - `; + const benchmarkOutput = JSON.parse('${{ steps.benchmark.outputs.BENCHMARK_OUTPUT }}'); - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: output - }) + let significantChanges = false; + let commentBody = "#### Performance Benchmark Results\n\n"; + + for (const benchmark of benchmarkOutput) { + if (benchmark.callgrind_summary && benchmark.callgrind_summary.summaries) { + for (const summary of benchmark.callgrind_summary.summaries) { + for (const [eventKind, costsDiff] of Object.entries(summary.events)) { + if (costsDiff.diff_pct !== null && Math.abs(costsDiff.diff_pct) > 5) { + significantChanges = true; + const changeDirection = costsDiff.diff_pct > 0 ? "increase" : "decrease"; + const color = costsDiff.diff_pct > 0 ? "red" : "green"; + commentBody += `\`${benchmark.module_path}\` - ${eventKind}:\n`; + commentBody += `\\color{${color}}${changeDirection} of ${Math.abs(costsDiff.diff_pct).toFixed(2)}%\n`; + commentBody += `Old: ${costsDiff.old}, New: ${costsDiff.new}\n\n`; + } + } + } + } + } + + if (significantChanges) { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: commentBody + }); + } else { + console.log("No significant performance changes detected. Skipping comment."); + } From a43f359f9ca0e34c47e08165aad99848c9f16daa Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Wed, 14 Aug 2024 13:33:15 +0200 Subject: [PATCH 11/17] Fix benchmark runner --- .github/workflows/profile.yaml | 2 +- .../graph-craft/benches/compile_demo_art.rs | 21 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index 6a03fef9a2..bd89f90f3b 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -55,7 +55,7 @@ jobs: - name: Run PR benchmarks id: benchmark run: | - BENCH_OUTPUT=$(cargo bench --bench compile_demo_art --features=iai -- --baseline=master --output-format=json) + BENCH_OUTPUT=$(cargo bench --bench compile_demo_art --features=iai -- --baseline=master --output-format=json | jq -s) echo "BENCHMARK_OUTPUT<> $GITHUB_OUTPUT echo "$BENCH_OUTPUT" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT diff --git a/node-graph/graph-craft/benches/compile_demo_art.rs b/node-graph/graph-craft/benches/compile_demo_art.rs index 42c85152d9..f0336226b9 100644 --- a/node-graph/graph-craft/benches/compile_demo_art.rs +++ b/node-graph/graph-craft/benches/compile_demo_art.rs @@ -19,10 +19,10 @@ fn compile(network: NodeNetwork) -> ProtoNetwork { compiler.compile_single(network).unwrap() } -fn bench_compile(path: PathBuf) { - let content = std::fs::read(&path).expect("failed to read file"); - let network = load_network(std::str::from_utf8(&content).unwrap()); - black_box(compile(black_box(network))); +fn load_from_name(name: &str) -> NodeNetwork { + let content = std::fs::read(&format!("../../demo-artwork/{name}.graphite")).expect("failed to read file"); + let content = std::str::from_utf8(&content).unwrap(); + load_network(content) } #[cfg(feature = "criterion")] @@ -31,17 +31,16 @@ fn compile_to_proto(c: &mut Criterion) { for path in artworks { let Ok(path) = path else { continue }; let name = path.file_stem().unwrap().to_str().unwrap(); - c.bench_function(name, |b| b.iter(|| bench_compile(path.clone()))); + let content = std::fs::read(&path).expect("failed to read file"); + let network = load_network(std::str::from_utf8(&content).unwrap()); + c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput)); } } #[cfg_attr(feature = "iai", library_benchmark)] -fn iai_compile_to_proto() { - let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern"); - for path in artworks { - let Ok(path) = path else { continue }; - bench_compile(path); - } +#[cfg_attr(feature = "iai", benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "red-dress", "valley-of-spires"], setup = load_from_name))] +fn iai_compile_to_proto(input: NodeNetwork) { + black_box(compile(input)); } #[cfg(feature = "criterion")] From 4bc8b4719b26d237e386d1ce2764da52f5bc4596 Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Wed, 14 Aug 2024 14:33:22 +0200 Subject: [PATCH 12/17] Try adding escaping --- .github/workflows/profile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index bd89f90f3b..0275c568ba 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -55,7 +55,7 @@ jobs: - name: Run PR benchmarks id: benchmark run: | - BENCH_OUTPUT=$(cargo bench --bench compile_demo_art --features=iai -- --baseline=master --output-format=json | jq -s) + BENCH_OUTPUT=$(cargo bench --bench compile_demo_art --features=iai -- --baseline=master --output-format=json | jq -sc) echo "BENCHMARK_OUTPUT<> $GITHUB_OUTPUT echo "$BENCH_OUTPUT" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT @@ -65,7 +65,7 @@ jobs: with: github-token: ${{secrets.GITHUB_TOKEN}} script: | - const benchmarkOutput = JSON.parse('${{ steps.benchmark.outputs.BENCHMARK_OUTPUT }}'); + const benchmarkOutput = JSON.parse(`${{ steps.benchmark.outputs.BENCHMARK_OUTPUT }}`); let significantChanges = false; let commentBody = "#### Performance Benchmark Results\n\n"; From 532f1bc8f0e30f3128ec541ca851f2eef4748842 Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Wed, 14 Aug 2024 16:23:00 +0200 Subject: [PATCH 13/17] Remove escaped quotes --- .github/workflows/profile.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index 0275c568ba..6b4cfc5fd4 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -55,7 +55,7 @@ jobs: - name: Run PR benchmarks id: benchmark run: | - BENCH_OUTPUT=$(cargo bench --bench compile_demo_art --features=iai -- --baseline=master --output-format=json | jq -sc) + BENCH_OUTPUT=$(cargo bench --bench compile_demo_art --features=iai -- --baseline=master --output-format=json | jq -sc | sed 's/\\"//g') echo "BENCHMARK_OUTPUT<> $GITHUB_OUTPUT echo "$BENCH_OUTPUT" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT @@ -96,4 +96,5 @@ jobs: }); } else { console.log("No significant performance changes detected. Skipping comment."); + console.log(commentBody); } From f56018c6668ab56bdd835c9cceea891ffcf48889 Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Wed, 14 Aug 2024 17:04:13 +0200 Subject: [PATCH 14/17] Use proper master baseline --- .github/workflows/profile.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profile.yaml index 6b4cfc5fd4..9fa282406b 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profile.yaml @@ -39,10 +39,10 @@ jobs: run: | cargo install iai-callgrind-runner@0.12.3 - # - name: Checkout master branch - # run: | - # git fetch origin master:master - # git checkout master + - name: Checkout master branch + run: | + git fetch origin master:master + git checkout master - name: Run baseline benchmarks run: | From 0e389b4f83af0dbde4c12b8a7c891dea062a2d26 Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Wed, 14 Aug 2024 17:04:13 +0200 Subject: [PATCH 15/17] Use proper master baseline --- .../{profile.yaml => profiling.yaml} | 2 +- .../graph-craft/benches/compile_demo_art.rs | 39 ++++++++++--------- 2 files changed, 21 insertions(+), 20 deletions(-) rename .github/workflows/{profile.yaml => profiling.yaml} (98%) diff --git a/.github/workflows/profile.yaml b/.github/workflows/profiling.yaml similarity index 98% rename from .github/workflows/profile.yaml rename to .github/workflows/profiling.yaml index 9fa282406b..b275118d89 100644 --- a/.github/workflows/profile.yaml +++ b/.github/workflows/profiling.yaml @@ -1,4 +1,4 @@ -name: Rust Profiling with iai-callgrind +name: Profiling on: pull_request: diff --git a/node-graph/graph-craft/benches/compile_demo_art.rs b/node-graph/graph-craft/benches/compile_demo_art.rs index f0336226b9..293cdd9845 100644 --- a/node-graph/graph-craft/benches/compile_demo_art.rs +++ b/node-graph/graph-craft/benches/compile_demo_art.rs @@ -3,54 +3,55 @@ use graph_craft::graphene_compiler::Compiler; use graph_craft::proto::ProtoNetwork; use std::path::PathBuf; -#[cfg(feature = "criterion")] +#[cfg(all(feature = "criterion", not(feature = "iai")))] use criterion::{black_box, criterion_group, criterion_main, Criterion}; -#[cfg(feature = "iai")] +#[cfg(all(not(feature = "criterion"), feature = "iai"))] use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main}; -fn load_network(document_string: &str) -> NodeNetwork { +pub fn load_network(document_string: &str) -> NodeNetwork { let document: serde_json::Value = serde_json::from_str(document_string).expect("Failed to parse document"); serde_json::from_value::(document["network_interface"]["network"].clone()).expect("Failed to parse document") } -fn compile(network: NodeNetwork) -> ProtoNetwork { +pub fn compile(network: NodeNetwork) -> ProtoNetwork { let compiler = Compiler {}; compiler.compile_single(network).unwrap() } -fn load_from_name(name: &str) -> NodeNetwork { - let content = std::fs::read(&format!("../../demo-artwork/{name}.graphite")).expect("failed to read file"); - let content = std::str::from_utf8(&content).unwrap(); - load_network(content) +pub fn bench_compile(path: PathBuf) { + let content = std::fs::read(&path).expect("failed to read file"); + let network = load_network(std::str::from_utf8(&content).unwrap()); + black_box(compile(black_box(network))); } -#[cfg(feature = "criterion")] +#[cfg(all(feature = "criterion", not(feature = "iai")))] fn compile_to_proto(c: &mut Criterion) { let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern"); for path in artworks { let Ok(path) = path else { continue }; let name = path.file_stem().unwrap().to_str().unwrap(); - let content = std::fs::read(&path).expect("failed to read file"); - let network = load_network(std::str::from_utf8(&content).unwrap()); - c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput)); + c.bench_function(name, |b| b.iter(|| bench_compile(path.clone()))); } } #[cfg_attr(feature = "iai", library_benchmark)] -#[cfg_attr(feature = "iai", benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "red-dress", "valley-of-spires"], setup = load_from_name))] -fn iai_compile_to_proto(input: NodeNetwork) { - black_box(compile(input)); +fn iai_compile_to_proto() { + let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern"); + for path in artworks { + let Ok(path) = path else { continue }; + bench_compile(path); + } } -#[cfg(feature = "criterion")] +#[cfg(all(feature = "criterion", not(feature = "iai")))] criterion_group!(benches, compile_to_proto); -#[cfg(feature = "criterion")] +#[cfg(all(feature = "criterion", not(feature = "iai")))] criterion_main!(benches); -#[cfg(feature = "iai")] +#[cfg(all(not(feature = "criterion"), feature = "iai"))] library_benchmark_group!(name = compile_group; benchmarks = iai_compile_to_proto); -#[cfg(feature = "iai")] +#[cfg(all(not(feature = "criterion"), feature = "iai"))] main!(library_benchmark_groups = compile_group); From fa3ed7ab78148ec7982d1ace72176b17b4afe225 Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Thu, 15 Aug 2024 12:18:26 +0200 Subject: [PATCH 16/17] Fix rebase regression --- .../graph-craft/benches/compile_demo_art.rs | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/node-graph/graph-craft/benches/compile_demo_art.rs b/node-graph/graph-craft/benches/compile_demo_art.rs index 293cdd9845..ba6faa5386 100644 --- a/node-graph/graph-craft/benches/compile_demo_art.rs +++ b/node-graph/graph-craft/benches/compile_demo_art.rs @@ -1,53 +1,55 @@ use graph_craft::document::NodeNetwork; use graph_craft::graphene_compiler::Compiler; use graph_craft::proto::ProtoNetwork; -use std::path::PathBuf; -#[cfg(all(feature = "criterion", not(feature = "iai")))] +#[cfg(feature = "criterion")] use criterion::{black_box, criterion_group, criterion_main, Criterion}; #[cfg(all(not(feature = "criterion"), feature = "iai"))] use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main}; -pub fn load_network(document_string: &str) -> NodeNetwork { +fn load_network(document_string: &str) -> NodeNetwork { let document: serde_json::Value = serde_json::from_str(document_string).expect("Failed to parse document"); serde_json::from_value::(document["network_interface"]["network"].clone()).expect("Failed to parse document") } -pub fn compile(network: NodeNetwork) -> ProtoNetwork { +fn compile(network: NodeNetwork) -> ProtoNetwork { let compiler = Compiler {}; compiler.compile_single(network).unwrap() } -pub fn bench_compile(path: PathBuf) { - let content = std::fs::read(&path).expect("failed to read file"); +#[cfg(all(not(feature = "criterion"), feature = "iai"))] +fn load_from_name(name: &str) -> NodeNetwork { + let content = std::fs::read(&format!("../../demo-artwork/{name}.graphite")).expect("failed to read file"); let network = load_network(std::str::from_utf8(&content).unwrap()); + let content = std::str::from_utf8(&content).unwrap(); black_box(compile(black_box(network))); + load_network(content) } -#[cfg(all(feature = "criterion", not(feature = "iai")))] +#[cfg(feature = "criterion")] fn compile_to_proto(c: &mut Criterion) { let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern"); for path in artworks { let Ok(path) = path else { continue }; let name = path.file_stem().unwrap().to_str().unwrap(); - c.bench_function(name, |b| b.iter(|| bench_compile(path.clone()))); + let content = std::fs::read(&path).expect("failed to read file"); + let network = load_network(std::str::from_utf8(&content).unwrap()); + c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput)); } } -#[cfg_attr(feature = "iai", library_benchmark)] -fn iai_compile_to_proto() { - let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern"); - for path in artworks { - let Ok(path) = path else { continue }; - bench_compile(path); - } +#[cfg(all(not(feature = "criterion"), feature = "iai"))] +#[cfg_attr(all(feature = "iai", not(feature = "criterion")), library_benchmark)] +#[cfg_attr(all(feature = "iai", not(feature="criterion")), benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "red-dress", "valley-of-spires"], setup = load_from_name))] +fn iai_compile_to_proto(input: NodeNetwork) { + black_box(compile(input)); } -#[cfg(all(feature = "criterion", not(feature = "iai")))] +#[cfg(feature = "criterion")] criterion_group!(benches, compile_to_proto); -#[cfg(all(feature = "criterion", not(feature = "iai")))] +#[cfg(feature = "criterion")] criterion_main!(benches); #[cfg(all(not(feature = "criterion"), feature = "iai"))] From 3356ab33a6bb9a448482da319886ff68fcc67f88 Mon Sep 17 00:00:00 2001 From: "dennis@kobert.dev" Date: Thu, 15 Aug 2024 13:12:37 +0200 Subject: [PATCH 17/17] Remove unused dependency --- Cargo.lock | 118 ------------------------------ node-graph/graph-craft/Cargo.toml | 15 ++-- 2 files changed, 8 insertions(+), 125 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 987b791ff7..a434c8cc85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1207,15 +1207,6 @@ dependencies = [ "libm", ] -[[package]] -name = "cpp_demangle" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119" -dependencies = [ - "cfg-if", -] - [[package]] name = "cpufeatures" version = "0.2.12" @@ -1415,15 +1406,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" -[[package]] -name = "debugid" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" -dependencies = [ - "uuid", -] - [[package]] name = "deranged" version = "0.3.11" @@ -1775,18 +1757,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "findshlibs" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" -dependencies = [ - "cc", - "lazy_static", - "libc", - "winapi", -] - [[package]] name = "fixedbitset" version = "0.4.2" @@ -2436,7 +2406,6 @@ dependencies = [ "js-sys", "log", "num-traits", - "pprof", "reqwest 0.12.5", "rustc-hash 2.0.0", "serde", @@ -3229,24 +3198,6 @@ dependencies = [ "cfb", ] -[[package]] -name = "inferno" -version = "0.11.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" -dependencies = [ - "ahash", - "indexmap 2.2.6", - "is-terminal", - "itoa 1.0.11", - "log", - "num-format", - "once_cell", - "quick-xml 0.26.0", - "rgb", - "str_stack", -] - [[package]] name = "instant" version = "0.1.13" @@ -4014,16 +3965,6 @@ dependencies = [ "syn 2.0.72", ] -[[package]] -name = "num-format" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" -dependencies = [ - "arrayvec", - "itoa 1.0.11", -] - [[package]] name = "num-integer" version = "0.1.46" @@ -4790,27 +4731,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" -[[package]] -name = "pprof" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb" -dependencies = [ - "backtrace", - "cfg-if", - "findshlibs", - "inferno", - "libc", - "log", - "nix 0.26.4", - "once_cell", - "parking_lot", - "smallvec", - "symbolic-demangle", - "tempfile", - "thiserror", -] - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -4893,15 +4813,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43d84d1d7a6ac92673717f9f6d1518374ef257669c24ebc5ac25d5033828be58" -[[package]] -name = "quick-xml" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" -dependencies = [ - "memchr", -] - [[package]] name = "quick-xml" version = "0.31.0" @@ -6129,12 +6040,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "str_stack" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" - [[package]] name = "strict-num" version = "0.1.1" @@ -6198,29 +6103,6 @@ dependencies = [ "siphasher 1.0.1", ] -[[package]] -name = "symbolic-common" -version = "12.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16629323a4ec5268ad23a575110a724ad4544aae623451de600c747bf87b36cf" -dependencies = [ - "debugid", - "memmap2", - "stable_deref_trait", - "uuid", -] - -[[package]] -name = "symbolic-demangle" -version = "12.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c043a45f08f41187414592b3ceb53fb0687da57209cc77401767fb69d5b596" -dependencies = [ - "cpp_demangle", - "rustc-demangle", - "symbolic-common", -] - [[package]] name = "syn" version = "1.0.109" diff --git a/node-graph/graph-craft/Cargo.toml b/node-graph/graph-craft/Cargo.toml index 8e3b5315d8..12781e6a45 100644 --- a/node-graph/graph-craft/Cargo.toml +++ b/node-graph/graph-craft/Cargo.toml @@ -41,30 +41,31 @@ wgpu-executor = { workspace = true } serde = { workspace = true, optional = true } tokio = { workspace = true, optional = true } -[target.'cfg(target_arch = "wasm32")'.dependencies] # Workspace dependencies +[target.'cfg(target_arch = "wasm32")'.dependencies] web-sys = { workspace = true } js-sys = { workspace = true } wasm-bindgen = { workspace = true } wasm-bindgen-futures = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -# Workspace dependencies winit = { workspace = true } [dev-dependencies] -criterion = { version = "0.5", features = ["html_reports"] } -glob = "0.3" -pprof = { version = "0.13", features = ["flamegraph"] } +# Workspace dependencies serde_json = { workspace = true } graph-craft = { workspace = true, features = ["serde"] } -iai-callgrind = "0.12.3" +# Required dependencies +criterion = { version = "0.5", features = ["html_reports"]} +glob = "0.3" +iai-callgrind = { version = "0.12.3"} + +# Benchmarks [[bench]] name = "compile_demo_art" harness = false - # [[bench]] # name = "exec_demo_art" # harness = false