From 90fe626da7eeeebd9eee4cfa242eb4a046744b3f Mon Sep 17 00:00:00 2001 From: Valentin Lorentz Date: Mon, 16 Dec 2024 13:42:05 +0100 Subject: [PATCH] LLP: mmap labels instead of loading them in memory LLP on SWH's graph now needs more than 3TB, which makes it crash in this step when there is anything else running on our 4TB machine that needs significant memory, like a previous version of the graph. --- src/algo/llp/mod.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/algo/llp/mod.rs b/src/algo/llp/mod.rs index 89c4e57b..6ffd5330 100644 --- a/src/algo/llp/mod.rs +++ b/src/algo/llp/mod.rs @@ -363,16 +363,17 @@ pub fn layered_label_propagation( .context("Could not load labels from best gammar")? .to_vec(); + let mmap_flags = Flags::TRANSPARENT_HUGE_PAGES | Flags::RANDOM_ACCESS; for (i, gamma_index) in gamma_indices.iter().enumerate() { info!("Starting step {}...", i); - let labels = - >::load_mem(labels_path(*gamma_index)).context("Could not load labels")?; + let labels = >::load_mmap(labels_path(*gamma_index), mmap_flags) + .context("Could not load labels")?; combine(&mut result_labels, *labels, &mut temp_perm).context("Could not combine labels")?; // This recombination with the best labels does not appear in the paper, but // it is not harmful and fixes a few corner cases in which experimentally // LLP does not perform well. It was introduced by Marco Rosa in the Java // LAW code. - let best_labels = >::load_mem(labels_path(best_gamma_index)) + let best_labels = >::load_mmap(labels_path(best_gamma_index), mmap_flags) .context("Could not load labels from best gamma")?; let number_of_labels = combine(&mut result_labels, *best_labels, &mut temp_perm)?; info!("Number of labels: {}", number_of_labels);