add reflink support

CHANGELOG-foldiff.md

@@ -6,11 +6,12 @@
 - replace `anyhow` with custom error types
 - write custom threading utilities
 
-## pending
+## 1.3.0
 - `foldiff upgrade` - upgrade older manifests to new ones
 - move core `foldiff` functionality to `libfoldiff`
   * significant refactors
   * decouple logic from `indicatif` and `cliutils`
+- use reflinks when copying unchanged files to reduce disk usage on filesystems such as btrfs, xfs, and apfs
 
 ## 1.2.0
 - switch to FLDF v1.1.0

foldiff/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "foldiff"
 authors = ["Hazel Atkinson"]
-version = "1.2.0"
+version = "1.3.0"
 edition = "2021"
 license-file = "../LICENSE.md"
 description = "A general purpose diffing tool that operates on folders of mixed text/binary files."

foldiff/src/main.rs

@@ -9,9 +9,9 @@ mod cliutils;
 
 #[derive(Parser, Debug)]
 #[command(
-	version = "v1.2.0",
+	version = "v1.3.0",
 	about,
-	long_version = "v1.2.0
+	long_version = "v1.3.0
    writing fldf v1.1.0
    reading fldf 1.0.0-r, v1.1.0"
 )]

libfoldiff/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "libfoldiff"
 authors = ["Hazel Atkinson"]
-version = "1.0.0"
+version = "1.3.0"
 edition = "2021"
 license-file = "../LICENSE.md"
 description = "A general purpose diffing library for the FDLF format."
@@ -18,6 +18,7 @@ twox-hash = "1.6.3"
 zstd = { version = "0.13.2", features = ["zstdmt"] }
 rayon = "1.10.0"
 memmap2 = "0.9.4"
+reflink = "0.1.3"
 
 [dev-dependencies]
 tempfile = "3.12.0"

libfoldiff/src/applying.rs

@@ -70,16 +70,30 @@ impl ApplyingDiff {
 						self.manifest.untouched_files
 							.par_iter()
 							.filter_map(|(h, p)| {
-								// std::fs::copy would be faster, but we want to verify the hash
-								let mut src = handle_res_parit!(File::open(self.old_root.join(p)), "Failed to open file to copy from {}", p);
-								let mut dst = handle_res_parit!(create_file(&self.new_root.join(p)), "Failed to create file to copy to {}", p);
-
-								let mut hw = hash::XXHashStreamer::new(&mut dst);
-								handle_res_parit!(std::io::copy(&mut src, &mut hw), "Failed to copy file {}", p);
-
-								let rh = hw.finish();
-								if rh != *h {
-									return Some(anyhow!("Found {p} was different to expected (hash was {rh}, not {})", *h));
+								let h = *h;
+								let old_path = self.old_root.join(p);
+								let new_path = self.new_root.join(p);
+
+								let real_hash =
+									// if we're on *nix, try reflinking
+									if cfg!(unix) && reflink::reflink(&old_path, &new_path).is_ok() {
+										// reflinked, check the hash
+										handle_res_parit!(hash::hash_file(&old_path), "Failed to hash file copied from {}", p)
+									}
+									else {
+										// reflink failed or we're on windows, copy
+										// copying in kernel space would be slightly faster but we have to check the hash
+										let mut src = handle_res_parit!(File::open(&old_path), "Failed to open file to copy from {}", p);
+										let mut dst = handle_res_parit!(create_file(&new_path), "Failed to create file to copy to {}", p);
+
+										let mut hw = hash::XXHashStreamer::new(&mut dst);
+										handle_res_parit!(std::io::copy(&mut src, &mut hw), "Failed to copy file {}", p);
+
+										hw.finish()
+									};
+
+								if real_hash != h {
+									return Some(anyhow!("Found {p} was different to expected (hash was {real_hash}, not {})", h));
 								}
 
 								inc(&bar_untouched);
@@ -143,6 +157,7 @@ impl ApplyingDiff {
 									let len = u64::from_be_bytes(*diff_map[blob..].first_chunk().unwrap()) as usize;
 									let blob = blob + 8; // advance past length
 
+									// TODO: reflink
 									// copy
 									let mut read = Cursor::new(&diff_map[blob..(blob + len)]);
 									let f = handle_res_parit!(create_file(&self.new_root.join(p)), "Failed to create new file {p} to write to");