Add a profiling action to CI which comments on PRs with notable demo art performance variances

.github/workflows/profiling.yaml

@@ -0,0 +1,100 @@
+name: Profiling
+
+on:
+  pull_request:
+    branches: [ master ]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  profile:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        profile: minimal
+        toolchain: stable
+
+    - name: Install Valgrind
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y valgrind
+
+    - name: Cache dependencies
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          target
+        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Install iai-callgrind
+      run: |
+        cargo install iai-callgrind-runner@0.12.3
+
+    - name: Checkout master branch
+      run: |
+        git fetch origin master:master
+        git checkout master
+
+    - name: Run baseline benchmarks
+      run: |
+        cargo bench --bench compile_demo_art --features=iai -- --save-baseline=master
+
+    - name: Checkout PR branch
+      run: |
+        git checkout ${{ github.event.pull_request.head.sha }}
+
+    - name: Run PR benchmarks
+      id: benchmark
+      run: |
+        BENCH_OUTPUT=$(cargo bench --bench compile_demo_art --features=iai -- --baseline=master --output-format=json  | jq -sc | sed 's/\\"//g')
+        echo "BENCHMARK_OUTPUT<<EOF" >> $GITHUB_OUTPUT
+        echo "$BENCH_OUTPUT" >> $GITHUB_OUTPUT
+        echo "EOF" >> $GITHUB_OUTPUT
+
+    - name: Comment PR
+      uses: actions/github-script@v6
+      with:
+        github-token: ${{secrets.GITHUB_TOKEN}}
+        script: |
+          const benchmarkOutput = JSON.parse(`${{ steps.benchmark.outputs.BENCHMARK_OUTPUT }}`);
+
+          let significantChanges = false;
+          let commentBody = "#### Performance Benchmark Results\n\n";
+
+          for (const benchmark of benchmarkOutput) {
+            if (benchmark.callgrind_summary && benchmark.callgrind_summary.summaries) {
+              for (const summary of benchmark.callgrind_summary.summaries) {
+                for (const [eventKind, costsDiff] of Object.entries(summary.events)) {
+                  if (costsDiff.diff_pct !== null && Math.abs(costsDiff.diff_pct) > 5) {
+                    significantChanges = true;
+                    const changeDirection = costsDiff.diff_pct > 0 ? "increase" : "decrease";
+                    const color = costsDiff.diff_pct > 0 ? "red" : "green";
+                    commentBody += `\`${benchmark.module_path}\` - ${eventKind}:\n`;
+                    commentBody += `\\color{${color}}${changeDirection} of ${Math.abs(costsDiff.diff_pct).toFixed(2)}%\n`;
+                    commentBody += `Old: ${costsDiff.old}, New: ${costsDiff.new}\n\n`;
+                  }
+                }
+              }
+            }
+          }
+
+          if (significantChanges) {
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: commentBody
+            });
+          } else {
+            console.log("No significant performance changes detected. Skipping comment.");
+            console.log(commentBody);
+          }

node-graph/graph-craft/Cargo.toml

@@ -11,6 +11,8 @@ dealloc_nodes = []
 wgpu = []
 tokio = ["dep:tokio"]
 wayland = []
+criterion = []
+iai = []
 
 [dependencies]
 # Local dependencies
@@ -39,24 +41,27 @@ wgpu-executor = { workspace = true }
 serde = { workspace = true, optional = true }
 tokio = { workspace = true, optional = true }
 
-[target.'cfg(target_arch = "wasm32")'.dependencies]
 # Workspace dependencies
+[target.'cfg(target_arch = "wasm32")'.dependencies]
 web-sys = { workspace = true }
 js-sys = { workspace = true }
 wasm-bindgen = { workspace = true }
 wasm-bindgen-futures = { workspace = true }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-# Workspace dependencies
 winit = { workspace = true }
 
 [dev-dependencies]
-criterion = { version = "0.5", features = ["html_reports"] }
-glob = "0.3"
-pprof = { version = "0.13", features = ["flamegraph"] }
+# Workspace dependencies
 serde_json = { workspace = true }
 graph-craft = { workspace = true, features = ["serde"] }
 
+# Required dependencies
+criterion = { version = "0.5", features = ["html_reports"]}
+glob = "0.3"
+iai-callgrind = { version = "0.12.3"}
+
+# Benchmarks
 [[bench]]
 name = "compile_demo_art"
 harness = false

node-graph/graph-craft/benches/compile_demo_art.rs

@@ -1,28 +1,59 @@
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use graph_craft::document::NodeNetwork;
 use graph_craft::graphene_compiler::Compiler;
 use graph_craft::proto::ProtoNetwork;
 
-pub fn compile_to_proto(c: &mut Criterion) {
-	let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern");
-	for path in artworks {
-		let Ok(path) = path else { continue };
-		let content = std::fs::read(&path).expect("failed to read file");
-		let network = load_network(std::str::from_utf8(&content).unwrap());
-		let name = path.file_stem().unwrap().to_str().unwrap();
+#[cfg(feature = "criterion")]
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
 
-		c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput));
-	}
-}
+#[cfg(all(not(feature = "criterion"), feature = "iai"))]
+use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main};
 
 fn load_network(document_string: &str) -> NodeNetwork {
 	let document: serde_json::Value = serde_json::from_str(document_string).expect("Failed to parse document");
 	serde_json::from_value::<NodeNetwork>(document["network_interface"]["network"].clone()).expect("Failed to parse document")
 }
+
 fn compile(network: NodeNetwork) -> ProtoNetwork {
 	let compiler = Compiler {};
 	compiler.compile_single(network).unwrap()
 }
 
+#[cfg(all(not(feature = "criterion"), feature = "iai"))]
+fn load_from_name(name: &str) -> NodeNetwork {
+	let content = std::fs::read(&format!("../../demo-artwork/{name}.graphite")).expect("failed to read file");
+	let network = load_network(std::str::from_utf8(&content).unwrap());
+	let content = std::str::from_utf8(&content).unwrap();
+	black_box(compile(black_box(network)));
+	load_network(content)
+}
+
+#[cfg(feature = "criterion")]
+fn compile_to_proto(c: &mut Criterion) {
+	let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern");
+	for path in artworks {
+		let Ok(path) = path else { continue };
+		let name = path.file_stem().unwrap().to_str().unwrap();
+		let content = std::fs::read(&path).expect("failed to read file");
+		let network = load_network(std::str::from_utf8(&content).unwrap());
+		c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput));
+	}
+}
+
+#[cfg(all(not(feature = "criterion"), feature = "iai"))]
+#[cfg_attr(all(feature = "iai", not(feature = "criterion")), library_benchmark)]
+#[cfg_attr(all(feature = "iai", not(feature="criterion")), benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "red-dress", "valley-of-spires"], setup = load_from_name))]
+fn iai_compile_to_proto(input: NodeNetwork) {
+	black_box(compile(input));
+}
+
+#[cfg(feature = "criterion")]
 criterion_group!(benches, compile_to_proto);
+
+#[cfg(feature = "criterion")]
 criterion_main!(benches);
+
+#[cfg(all(not(feature = "criterion"), feature = "iai"))]
+library_benchmark_group!(name = compile_group; benchmarks = iai_compile_to_proto);
+
+#[cfg(all(not(feature = "criterion"), feature = "iai"))]
+main!(library_benchmark_groups = compile_group);