diff --git a/crates/core/src/commands/config.rs b/crates/core/src/commands/config.rs index fa6c0065..81ffd3d7 100644 --- a/crates/core/src/commands/config.rs +++ b/crates/core/src/commands/config.rs @@ -82,7 +82,32 @@ pub(crate) fn save_config( let dbe = DecryptBackend::new(repo.be.clone(), key); // for hot/cold backend, this only saves the config to the cold repo. _ = dbe.save_file_uncompressed(&new_config)?; + save_config_hot(repo, new_config, key) +} +/// Save a [`ConfigFile`] only to the hot part of a repository +/// +/// # Type Parameters +/// +/// * `P` - The progress bar type. +/// * `S` - The state the repository is in. +/// +/// # Arguments +/// +/// * `repo` - The repository to save the config to +/// * `new_config` - The config to save +/// * `key` - The key to encrypt the config with +/// +/// # Errors +/// +/// * [`CryptBackendErrorKind::SerializingToJsonByteVectorFailed`] - If the file could not be serialized to json. +/// +/// [`CryptBackendErrorKind::SerializingToJsonByteVectorFailed`]: crate::error::CryptBackendErrorKind::SerializingToJsonByteVectorFailed +pub(crate) fn save_config_hot( + repo: &Repository, + mut new_config: ConfigFile, + key: impl CryptoKey, +) -> RusticResult<()> { if let Some(hot_be) = repo.be_hot.clone() { // save config to hot repo let dbe = DecryptBackend::new(hot_be.clone(), key); diff --git a/crates/core/src/commands/init.rs b/crates/core/src/commands/init.rs index 46037348..e9cc90f3 100644 --- a/crates/core/src/commands/init.rs +++ b/crates/core/src/commands/init.rs @@ -60,7 +60,7 @@ pub(crate) fn init( Ok((key, config)) } -/// Initialize a new repository with a given config. +/// Save a [`ConfigFile`] only to the hot part of a repository /// /// # Type Parameters /// @@ -69,14 +69,15 @@ pub(crate) fn init( /// /// # Arguments /// -/// * `repo` - The repository to initialize. -/// * `pass` - The password to encrypt the key with. -/// * `key_opts` - The options to create the key with. -/// * `config` - The config to use. +/// * `repo` - The repository to save the config to +/// * `new_config` - The config to save +/// * `key` - The key to encrypt the config with /// -/// # Returns +/// # Errors +/// +/// * [`CryptBackendErrorKind::SerializingToJsonByteVectorFailed`] - If the file could not be serialized to json. /// -/// The key used to encrypt the config. +/// [`CryptBackendErrorKind::SerializingToJsonByteVectorFailed`]: crate::error::CryptBackendErrorKind::SerializingToJsonByteVectorFailed pub(crate) fn init_with_config( repo: &Repository, pass: &str, diff --git a/crates/core/src/commands/repair.rs b/crates/core/src/commands/repair.rs index 62294334..b07b1b26 100644 --- a/crates/core/src/commands/repair.rs +++ b/crates/core/src/commands/repair.rs @@ -1,2 +1,3 @@ +pub mod hotcold; pub mod index; pub mod snapshots; diff --git a/crates/core/src/commands/repair/hotcold.rs b/crates/core/src/commands/repair/hotcold.rs new file mode 100644 index 00000000..275fadde --- /dev/null +++ b/crates/core/src/commands/repair/hotcold.rs @@ -0,0 +1,180 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use log::{debug, info, warn}; + +use crate::{ + backend::decrypt::DecryptReadBackend, + repofile::{BlobType, IndexFile, PackId}, + repository::Open, + ErrorKind, FileType, Id, Progress, ProgressBars, ReadBackend, Repository, RusticError, + RusticResult, WriteBackend, ALL_FILE_TYPES, +}; + +pub(crate) fn repair_hotcold( + repo: &Repository, + dry_run: bool, +) -> RusticResult<()> { + for file_type in ALL_FILE_TYPES { + if file_type != FileType::Pack { + correct_missing_files(repo, file_type, |_| true, dry_run)?; + } + } + Ok(()) +} + +pub(crate) fn repair_hotcold_packs( + repo: &Repository, + dry_run: bool, +) -> RusticResult<()> { + let tree_packs = get_tree_packs(repo)?; + correct_missing_files( + repo, + FileType::Pack, + |id| tree_packs.contains(&PackId::from(*id)), + dry_run, + ) +} + +pub(crate) fn correct_missing_files( + repo: &Repository, + file_type: FileType, + is_relevant: impl Fn(&Id) -> bool, + dry_run: bool, +) -> RusticResult<()> { + let Some(repo_hot) = &repo.be_hot else { + return Err(RusticError::new( + ErrorKind::Repository, + "Repository is no hot/cold repository.", + )); + }; + + let (missing_hot, missing_hot_size, missing_cold, missing_cold_size) = + get_missing_files(repo, file_type, is_relevant)?; + + // copy missing files from hot to cold repo + if !missing_cold.is_empty() { + if dry_run { + info!( + "would have copied {} hot {file_type:?} files to cold", + missing_cold.len() + ); + debug!("files: {missing_cold:?}"); + } else { + let p = repo + .pb + .progress_bytes(format!("copying missing cold {file_type:?} files...")); + p.set_length(missing_cold_size); + copy(missing_cold, file_type, repo_hot, &repo.be_cold)?; + p.finish(); + } + } + + if !missing_hot.is_empty() { + if dry_run { + info!( + "would have copied {} cold {file_type:?} files to hot", + missing_hot.len() + ); + debug!("files: {missing_hot:?}"); + } else { + // TODO: warm-up + // copy missing files from cold to hot repo + let p = repo + .pb + .progress_bytes(format!("copying missing hot {file_type:?} files...")); + p.set_length(missing_hot_size); + copy(missing_hot, file_type, &repo.be_cold, repo_hot)?; + p.finish(); + } + } + + Ok(()) +} + +fn copy( + files: Vec, + file_type: FileType, + from: &impl ReadBackend, + to: &impl WriteBackend, +) -> RusticResult<()> { + for id in files { + let file = from.read_full(file_type, &id)?; + to.write_bytes(file_type, &id, false, file)?; + } + Ok(()) +} + +pub(crate) fn get_tree_packs( + repo: &Repository, +) -> RusticResult> { + let p = repo.pb.progress_counter("reading index..."); + let mut tree_packs = BTreeSet::new(); + for index in repo.dbe().stream_all::(&p)? { + let index = index?.1; + for (p, _) in index.all_packs() { + let blob_type = p.blob_type(); + if blob_type == BlobType::Tree { + _ = tree_packs.insert(p.id); + } + } + } + Ok(tree_packs) +} + +pub(crate) fn get_missing_files( + repo: &Repository, + file_type: FileType, + is_relevant: impl Fn(&Id) -> bool, +) -> RusticResult<(Vec, u64, Vec, u64)> { + let Some(repo_hot) = &repo.be_hot else { + return Err(RusticError::new( + ErrorKind::Repository, + "Repository is no hot/cold repository.", + )); + }; + + let p = repo + .pb + .progress_spinner(format!("listing hot {file_type:?} files...")); + let hot_files: BTreeMap<_, _> = repo_hot.list_with_size(file_type)?.into_iter().collect(); + p.finish(); + + let p = repo + .pb + .progress_spinner(format!("listing cold {file_type:?} files...")); + let cold_files: BTreeMap<_, _> = repo + .be_cold + .list_with_size(file_type)? + .into_iter() + .collect(); + p.finish(); + + let common: BTreeSet<_> = hot_files + .iter() + .filter_map(|(id, size_hot)| match cold_files.get(id) { + Some(size_cold) if size_cold == size_hot => Some(*id), + Some(size_cold) => { + warn!("sizes mismatch: type {file_type:?}, id: {id}, size hot: {size_hot}, size cold: {size_cold}. Ignoring..."); + None + } + None => None, + }) + .collect(); + + let retain = |files: BTreeMap<_, _>| { + let mut retain_size: u64 = 0; + let only: Vec<_> = files + .into_iter() + .filter(|(id, _)| !common.contains(id) && is_relevant(id)) + .map(|(id, size)| { + retain_size += u64::from(size); + id + }) + .collect(); + (only, retain_size) + }; + + let (cold_only, cold_only_size) = retain(cold_files); + let (hot_only, hot_only_size) = retain(hot_files); + Ok((cold_only, cold_only_size, hot_only, hot_only_size)) +} diff --git a/crates/core/src/repository.rs b/crates/core/src/repository.rs index a0299ddd..250b2899 100644 --- a/crates/core/src/repository.rs +++ b/crates/core/src/repository.rs @@ -33,12 +33,13 @@ use crate::{ self, backup::BackupOptions, check::{check_repository, CheckOptions}, - config::ConfigOptions, + config::{save_config_hot, ConfigOptions}, copy::CopySnapshot, forget::{ForgetGroups, KeepOptions}, key::{add_current_key_to_repo, KeyOptions}, prune::{prune_repository, PruneOptions, PrunePlan}, repair::{ + hotcold::{repair_hotcold, repair_hotcold_packs}, index::{index_checked_from_collector, repair_index, RepairIndexOptions}, snapshots::{repair_snapshots, RepairSnapshotsOptions}, }, @@ -302,6 +303,9 @@ pub struct Repository { /// The Backend to use for hot files pub(crate) be_hot: Option>, + /// The Backend to use for cold files + pub(crate) be_cold: Arc, + /// The options used for this repository opts: RepositoryOptions, @@ -367,6 +371,8 @@ impl

Repository { info!("using warm-up command {warm_up}"); } + let be_cold = be.clone(); + if opts.warm_up { be = WarmUpAccessBackend::new_warm_up(be); } @@ -382,6 +388,7 @@ impl

Repository { name, be, be_hot, + be_cold, opts: opts.clone(), pb, status: (), @@ -498,6 +505,37 @@ impl Repository { /// * If listing the repository config file failed /// * If there is more than one repository config file pub fn open_with_password(self, password: &str) -> RusticResult> { + self.open_with_password_may_use_hot(password, true) + } + + /// Open the repository with a given password using only the cold repository. + /// + /// This gets the decryption key and reads the config file + /// + /// # Arguments + /// + /// * `password` - The password to use + /// + /// # Errors + /// + /// * If no repository config file is found + /// * If the keys of the hot and cold backend don't match + /// * If the password is incorrect + /// * If no suitable key is found + /// * If listing the repository config file failed + /// * If there is more than one repository config file + pub fn open_with_password_only_cold( + self, + password: &str, + ) -> RusticResult> { + self.open_with_password_may_use_hot(password, false) + } + + fn open_with_password_may_use_hot( + self, + password: &str, + may_use_hot: bool, + ) -> RusticResult> { let config_id = self.config_id()?.ok_or_else(|| { RusticError::new( ErrorKind::Configuration, @@ -506,26 +544,34 @@ impl Repository { .attach_context("name", self.name.clone()) })?; - if let Some(be_hot) = &self.be_hot { - let mut keys = self.be.list_with_size(FileType::Key)?; - keys.sort_unstable_by_key(|key| key.0); - let mut hot_keys = be_hot.list_with_size(FileType::Key)?; - hot_keys.sort_unstable_by_key(|key| key.0); - if keys != hot_keys { - return Err(RusticError::new( + if may_use_hot { + if let Some(be_hot) = &self.be_hot { + let mut keys = self.be.list_with_size(FileType::Key)?; + keys.sort_unstable_by_key(|key| key.0); + let mut hot_keys = be_hot.list_with_size(FileType::Key)?; + hot_keys.sort_unstable_by_key(|key| key.0); + if keys != hot_keys { + return Err(RusticError::new( ErrorKind::Key, "Keys of hot and cold repositories don't match for `{name}`. Please check the keys.", ) .attach_context("name", self.name.clone())); + } } } + let be = if may_use_hot { + self.be.clone() + } else { + self.be_cold.clone() + }; - let key = find_key_in_backend(&self.be, &password, None)?; - + let key = find_key_in_backend(&be, &password, None)?; info!("repository {}: password is correct.", self.name); - - let dbe = DecryptBackend::new(self.be.clone(), key); - let config: ConfigFile = dbe.get_file(&config_id)?; + let dbe = DecryptBackend::new(be, key); + let mut config: ConfigFile = dbe.get_file(&config_id)?; + if !may_use_hot && self.be_hot.is_some() { + config.is_hot = Some(true); + } self.open_raw(key, config) } @@ -690,6 +736,7 @@ impl Repository { name: self.name, be: self.be, be_hot: self.be_hot, + be_cold: self.be_cold, opts: self.opts, pb: self.pb, status: open, @@ -764,6 +811,21 @@ impl Repository { pub fn warm_up_wait(&self, packs: impl ExactSizeIterator) -> RusticResult<()> { warm_up_wait(self, packs) } + + /// Repair hotcold files except packs + /// + /// This compares the pack files in the hot and cold repo part and copies missing ones. + /// + /// # Arguments + /// + /// * `dry_run` - If true, only print what would be done + /// + /// # Errors + /// + // TODO: Document errors + pub fn repair_hotcold_except_packs(&self, dry_run: bool) -> RusticResult<()> { + repair_hotcold(self, dry_run) + } } /// A repository which is open, i.e. the password has been checked and the decryption key is available. @@ -883,6 +945,33 @@ impl Repository { pub(crate) fn dbe(&self) -> &DecryptBackend { self.status.dbe() } + + /// Save a [`ConfigFile`] only to the hot part of a repository + /// + /// # Type Parameters + /// + /// * `P` - The progress bar type. + /// * `S` - The state the repository is in. + /// + /// # Arguments + /// + /// * `repo` - The repository to save the config to + /// * `new_config` - The config to save + /// * `key` - The key to encrypt the config with + /// + /// # Errors + /// + /// * [`CryptBackendErrorKind::SerializingToJsonByteVectorFailed`] - If the file could not be serialized to json. + /// + /// [`CryptBackendErrorKind::SerializingToJsonByteVectorFailed`]: crate::error::CryptBackendErrorKind::SerializingToJsonByteVectorFailed + pub fn init_hot(&self) -> RusticResult<()> { + if let Some(hot_be) = self.be_hot.clone() { + hot_be.create()?; + } + let config = self.config().clone(); + let key = *self.dbe().key(); + save_config_hot(self, config, key) + } } impl Repository { @@ -1280,6 +1369,7 @@ impl Repository { name: self.name, be: self.be, be_hot: self.be_hot, + be_cold: self.be_cold, opts: self.opts, pb: self.pb, status, @@ -1342,6 +1432,7 @@ impl Repository { name: self.name, be: self.be, be_hot: self.be_hot, + be_cold: self.be_cold, opts: self.opts, pb: self.pb, status, @@ -1400,6 +1491,21 @@ impl Repository { pub fn repair_index(&self, opts: &RepairIndexOptions, dry_run: bool) -> RusticResult<()> { repair_index(self, *opts, dry_run) } + + /// Repair hotcold packs + /// + /// This compares the pack files in the hot and cold repo part and copies missing ones. + /// + /// # Arguments + /// + /// * `dry_run` - If true, only print what would be done + /// + /// # Errors + /// + // TODO: Document errors + pub fn repair_hotcold_packs(&self, dry_run: bool) -> RusticResult<()> { + repair_hotcold_packs(self, dry_run) + } } /// A repository which is indexed such that all tree blobs are contained in the index. @@ -1728,6 +1834,7 @@ impl Repository { name: self.name, be: self.be, be_hot: self.be_hot, + be_cold: self.be_cold, opts: self.opts, pb: self.pb, status: self.status.into_open(), @@ -1959,6 +2066,7 @@ impl Repository { name: self.name, be: self.be, be_hot: self.be_hot, + be_cold: self.be_cold, opts: self.opts, pb: self.pb, status: self.status.into_indexed_tree(),