From b7260b61118a1ef1b52ba2e92b55c70bc77cc820 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 20:10:19 -0600
Subject: [PATCH 01/83] create a backup of the snapshot for the PV's

---
 functions/backup_restore/backup/backup.py |  33 +++++--
 functions/backup_restore/zfs/snapshot.py  | 112 +++++++++++++++++++---
 2 files changed, 124 insertions(+), 21 deletions(-)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index 49a600c..0a2bbec 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -145,9 +145,17 @@ def backup_all(self):
             dataset_paths = self.kube_pvc_fetcher.get_volume_paths_by_namespace(f"ix-{app_name}")
             if dataset_paths:
                 self.logger.info(f"Backing up {app_name} PVCs...")
-                snapshot_errors = self.snapshot_manager.create_snapshots(self.snapshot_name, dataset_paths, self.retention_number)
-                if snapshot_errors:
-                    failures[app_name].extend(snapshot_errors)
+                snapshot_result = self.snapshot_manager.create_snapshots(self.snapshot_name, dataset_paths, self.retention_number)
+                if snapshot_result["errors"]:
+                    failures[app_name].extend(snapshot_result["errors"])
+
+                if snapshot_result["success"]:
+                    for snapshot in snapshot_result["snapshots"]:
+                        self.logger.info(f"Sending snapshot {snapshot} to backup directory...")
+                        backup_path = app_backup_dir / "snapshots" / f"{snapshot.replace('/', '_')}.zfs"
+                        send_result = self.snapshot_manager.zfs_send(snapshot, backup_path, compress=True)
+                        if not send_result["success"]:
+                            failures[app_name].append(send_result["message"])
 
         self._create_backup_snapshot()
         self._log_failures(failures)
@@ -175,10 +183,14 @@ def _create_backup_snapshot(self):
         Create a snapshot of the backup dataset after all backups are completed.
         """
         self.logger.info(f"\nCreating snapshot for backup: {self.backup_dataset}")
-        if self.snapshot_manager.create_snapshots(self.snapshot_name, [self.backup_dataset], self.retention_number):
-            self.logger.error("Failed to create snapshot for backup dataset.")
-        else:
+        snapshot_result = self.snapshot_manager.create_snapshots(self.snapshot_name, [self.backup_dataset], self.retention_number)
+
+        if snapshot_result.get("success"):
             self.logger.info("Snapshot created successfully for backup dataset.")
+        else:
+            self.logger.error("Failed to create snapshot for backup dataset.")
+            for error in snapshot_result.get("errors", []):
+                self.logger.error(error)
 
     def _cleanup_old_backups(self):
         """
@@ -217,7 +229,8 @@ def _backup_application_datasets(self):
         datasets_to_backup = [ds for ds in all_datasets if ds.startswith(self.kubeconfig.dataset) and ds not in datasets_to_ignore]
         self.logger.debug(f"Snapshotting datasets: {datasets_to_backup}")
 
-        snapshot_errors = self.snapshot_manager.create_snapshots(self.snapshot_name, datasets_to_backup, self.retention_number)
-        if snapshot_errors:
-            self.logger.error(f"Failed to create snapshots for application datasets: {snapshot_errors}")
-
+        snapshot_result = self.snapshot_manager.create_snapshots(self.snapshot_name, datasets_to_backup, self.retention_number)
+        if not snapshot_result.get("success"):
+            self.logger.error("Failed to create snapshots for application datasets.")
+            for error in snapshot_result.get("errors", []):
+                self.logger.error(error)
\ No newline at end of file
diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index 251492d..713e922 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -61,7 +61,7 @@ def _cleanup_snapshots(self, dataset_paths: list, retention_number: int) -> list
         return errors
 
     @type_check
-    def create_snapshots(self, snapshot_name, dataset_paths: list, retention_number: int) -> list:
+    def create_snapshots(self, snapshot_name, dataset_paths: list, retention_number: int) -> dict:
         """
         Create snapshots for specified ZFS datasets and cleanup old snapshots.
 
@@ -71,30 +71,44 @@ def create_snapshots(self, snapshot_name, dataset_paths: list, retention_number:
         - retention_number (int): Number of recent snapshots to retain.
 
         Returns:
-        - list: A list of error messages, if any.
+        - dict: Result containing status, messages, and list of created snapshots.
         """
-        errors = []
+        result = {
+            "success": False,
+            "message": "",
+            "errors": [],
+            "snapshots": []
+        }
+
         for path in dataset_paths:
             if path not in self.cache.datasets:
                 error_msg = f"Dataset {path} does not exist."
                 self.logger.error(error_msg)
-                errors.append(error_msg)
+                result["errors"].append(error_msg)
                 continue
 
             snapshot_full_name = f"{path}@{snapshot_name}"
             command = f"/sbin/zfs snapshot \"{snapshot_full_name}\""
-            result = run_command(command)
-            if result.is_success():
+            snapshot_result = run_command(command)
+            if snapshot_result.is_success():
                 self.cache.add_snapshot(snapshot_full_name)
                 self.logger.debug(f"Created snapshot: {snapshot_full_name}")
+                result["snapshots"].append(snapshot_full_name)
             else:
-                error_msg = f"Failed to create snapshot for {snapshot_full_name}: {result.get_error()}"
+                error_msg = f"Failed to create snapshot for {snapshot_full_name}: {snapshot_result.get_error()}"
                 self.logger.error(error_msg)
-                errors.append(error_msg)
+                result["errors"].append(error_msg)
         
         cleanup_errors = self._cleanup_snapshots(dataset_paths, retention_number)
-        errors.extend(cleanup_errors)
-        return errors
+        result["errors"].extend(cleanup_errors)
+
+        if not result["errors"]:
+            result["success"] = True
+            result["message"] = "All snapshots created and cleaned up successfully."
+        else:
+            result["message"] = "Some errors occurred during snapshot creation or cleanup."
+
+        return result
 
     @type_check
     def delete_snapshots(self, snapshot_name: str) -> list:
@@ -206,4 +220,80 @@ def rollback_all_snapshots(self, snapshot_name: str, dataset_path: str) -> None:
                 else:
                     self.logger.error(f"Failed to rollback {snapshot}: {rollback_result.get_error()}")
         except Exception as e:
-            self.logger.error(f"Failed to rollback snapshots for {dataset_path}: {e}", exc_info=True)
\ No newline at end of file
+            self.logger.error(f"Failed to rollback snapshots for {dataset_path}: {e}", exc_info=True)
+
+    @type_check
+    def zfs_send(self, source: str, destination: Path, compress: bool = False) -> dict:
+        """
+        Send a ZFS snapshot to a destination file, with optional gzip compression.
+
+        Parameters:
+        - source (str): The source ZFS snapshot to send.
+        - destination (Path): The destination file to send the snapshot to.
+        - compress (bool): Whether to use gzip compression. Default is False.
+
+        Returns:
+        - dict: Result containing status and message.
+        """
+        result = {
+            "success": False,
+            "message": ""
+        }
+
+        try:
+            if compress:
+                command = f"/sbin/zfs send \"{source}\" | gzip > \"{destination}\""
+            else:
+                command = f"/sbin/zfs send \"{source}\" > \"{destination}\""
+
+            send_result = run_command(command)
+            if send_result.is_success():
+                self.logger.debug(f"Successfully sent snapshot {source} to {destination}")
+                result["success"] = True
+                result["message"] = f"Successfully sent snapshot {source} to {destination}"
+            else:
+                result["message"] = f"Failed to send snapshot {source}: {send_result.get_error()}"
+                self.logger.error(result["message"])
+        except Exception as e:
+            result["message"] = f"Exception occurred while sending snapshot {source}: {e}"
+            self.logger.error(result["message"], exc_info=True)
+
+        return result
+
+    @type_check
+    def zfs_receive(self, source: Path, destination: str, decompress: bool = False) -> dict:
+        """
+        Receive a ZFS snapshot from a source file and restore it to the destination dataset.
+
+        Parameters:
+        - source (Path): The source file containing the snapshot.
+        - destination (str): The destination ZFS dataset to receive the snapshot to.
+        - decompress (bool): Whether the source file is gzip compressed. Default is False.
+
+        Returns:
+        - dict: Result containing status and message.
+        """
+        result = {
+            "success": False,
+            "message": ""
+        }
+
+        try:
+            if decompress:
+                command = f"gunzip -c \"{source}\" | /sbin/zfs recv \"{destination}\""
+            else:
+                command = f"/sbin/zfs recv \"{destination}\" < \"{source}\""
+
+            receive_result = run_command(command)
+            if receive_result.is_success():
+                self.logger.debug(f"Successfully received snapshot from {source} to {destination}")
+                result["success"] = True
+                result["message"] = f"Successfully received snapshot from {source} to {destination}"
+            else:
+                result["message"] = f"Failed to receive snapshot from {source}: {receive_result.get_error()}"
+                self.logger.error(result["message"])
+        except Exception as e:
+            result["message"] = f"Exception occurred while receiving snapshot from {source}: {e}"
+            self.logger.error(result["message"], exc_info=True)
+
+        return result

From d92fa3974a8c822e06afd38aec28ce055090cdc3 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 20:13:47 -0600
Subject: [PATCH 02/83] mkdir

---
 functions/backup_restore/backup/backup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index 0a2bbec..8fdbb50 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -153,6 +153,7 @@ def backup_all(self):
                     for snapshot in snapshot_result["snapshots"]:
                         self.logger.info(f"Sending snapshot {snapshot} to backup directory...")
                         backup_path = app_backup_dir / "snapshots" / f"{snapshot.replace('/', '_')}.zfs"
+                        backup_path.parent.mkdir(parents=True, exist_ok=True)
                         send_result = self.snapshot_manager.zfs_send(snapshot, backup_path, compress=True)
                         if not send_result["success"]:
                             failures[app_name].append(send_result["message"])

From 1c27a97688c9cc663c28cf5edd6158d28d673b18 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 20:23:47 -0600
Subject: [PATCH 03/83] use a different replace

---
 functions/backup_restore/backup/backup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index 8fdbb50..0a0f843 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -152,7 +152,7 @@ def backup_all(self):
                 if snapshot_result["success"]:
                     for snapshot in snapshot_result["snapshots"]:
                         self.logger.info(f"Sending snapshot {snapshot} to backup directory...")
-                        backup_path = app_backup_dir / "snapshots" / f"{snapshot.replace('/', '_')}.zfs"
+                        backup_path = app_backup_dir / "snapshots" / f"{snapshot.replace('/', '%%')}.zfs"
                         backup_path.parent.mkdir(parents=True, exist_ok=True)
                         send_result = self.snapshot_manager.zfs_send(snapshot, backup_path, compress=True)
                         if not send_result["success"]:

From d1f293aef98fc731d2d57f7a7f6c5cbe9ae0e512 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 20:56:10 -0600
Subject: [PATCH 04/83] attempt restore of snapshots

---
 functions/backup_restore/backup/backup.py     |  2 +-
 .../backup_restore/charts/backup_fetch.py     |  3 +++
 .../backup_restore/restore/restore_all.py     |  2 ++
 .../backup_restore/restore/restore_base.py    | 22 +++++++++++++++++++
 .../backup_restore/restore/restore_single.py  |  1 +
 5 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index 0a0f843..72b2487 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -150,8 +150,8 @@ def backup_all(self):
                     failures[app_name].extend(snapshot_result["errors"])
 
                 if snapshot_result["success"]:
+                    self.logger.info(f"Sending snapshots to backup directory...")
                     for snapshot in snapshot_result["snapshots"]:
-                        self.logger.info(f"Sending snapshot {snapshot} to backup directory...")
                         backup_path = app_backup_dir / "snapshots" / f"{snapshot.replace('/', '%%')}.zfs"
                         backup_path.parent.mkdir(parents=True, exist_ok=True)
                         send_result = self.snapshot_manager.zfs_send(snapshot, backup_path, compress=True)
diff --git a/functions/backup_restore/charts/backup_fetch.py b/functions/backup_restore/charts/backup_fetch.py
index 333774f..de95f73 100644
--- a/functions/backup_restore/charts/backup_fetch.py
+++ b/functions/backup_restore/charts/backup_fetch.py
@@ -66,6 +66,7 @@ def _parse_chart(self, chart_base_dir: Path, app_name: str) -> Dict[str, Union[s
                 'secrets': [],
                 'crds': [],
                 'pv_zfs_volumes': [],
+                'snapshots': []
             }
         }
 
@@ -73,6 +74,7 @@ def _parse_chart(self, chart_base_dir: Path, app_name: str) -> Dict[str, Union[s
         kubernetes_objects_dir = chart_base_dir / 'kubernetes_objects'
         database_dir = chart_base_dir / 'database'
         versions_dir = chart_base_dir / 'chart_versions'
+        snapshots_dir = chart_base_dir / 'snapshots'
 
         # Parse metadata and config
         metadata = self._parse_metadata(chart_info_dir)
@@ -96,6 +98,7 @@ def _parse_chart(self, chart_base_dir: Path, app_name: str) -> Dict[str, Union[s
             chart_info['files']['crds'] = self._get_files(kubernetes_objects_dir / 'crds')
             chart_info['files']['pv_zfs_volumes'] = self._get_files(kubernetes_objects_dir / 'pv_zfs_volumes')
             chart_info['files']['cnpg_pvcs_to_delete'] = self._get_file(kubernetes_objects_dir / 'cnpg_pvcs_to_delete.txt')
+            chart_info['files']['snapshots'] = self._get_files(snapshots_dir)
 
             return chart_info
 
diff --git a/functions/backup_restore/restore/restore_all.py b/functions/backup_restore/restore/restore_all.py
index d7bca1a..1fece7d 100644
--- a/functions/backup_restore/restore/restore_all.py
+++ b/functions/backup_restore/restore/restore_all.py
@@ -42,6 +42,8 @@ def restore(self):
                 self.logger.error(f"Failed to rollback snapshots for {app_name}: {e}\n")
                 self.failures[app_name].append(f"Failed to rollback volume snapshots: {e}")
 
+            self.restore_snapshots(app_name)
+
         self.logger.info("\nStarting Kubernetes Services\n"
                          "----------------------------")
         try:
diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 6a8ab70..098a325 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -108,6 +108,28 @@ def _rollback_volumes(self, app_name: str):
                 else:
                     self.logger.debug(result["message"])
 
+    @type_check
+    def restore_snapshots(self, app_name: str):
+        """
+        Restore snapshots from the backup directory for a specific application.
+
+        Parameters:
+        - app_name (str): The name of the application to restore snapshots for.
+        """
+        snapshot_files = self.chart_info.get_file(app_name, "snapshots")
+        if snapshot_files:
+            self.logger.debug(f"Restoring snapshots for {app_name}...")
+            for snapshot_file in snapshot_files:
+                original_path = snapshot_file.stem.replace('%%', '/')
+                dataset_path, _ = original_path.split('@', 1)
+
+                restore_result = self.snapshot_manager.zfs_receive(snapshot_file, dataset_path, decompress=True)
+                if not restore_result["success"]:
+                    self.failures[app_name].append(restore_result["message"])
+                    self.logger.error(f"Failed to restore snapshot from {snapshot_file} for {app_name}: {restore_result['message']}")
+                else:
+                    self.logger.debug(f"Successfully restored snapshot from {snapshot_file} for {app_name}")
+
     @type_check
     def _restore_application(self, app_name: str) -> bool:
         """Restore a single application."""
diff --git a/functions/backup_restore/restore/restore_single.py b/functions/backup_restore/restore/restore_single.py
index 502fbe8..2791b48 100644
--- a/functions/backup_restore/restore/restore_single.py
+++ b/functions/backup_restore/restore/restore_single.py
@@ -26,6 +26,7 @@ def restore(self, app_names: list):
         for app_name in app_names:
             try:
                 self._rollback_volumes(app_name)
+                self.restore_snapshots(app_name)
             except Exception as e:
                 self.logger.error(f"Failed to rollback snapshots for {app_name}: {e}\n")
                 self.failures[app_name].append(f"Failed to rollback volume snapshots: {e}")

From 0cdef6abc0fbebd806aeeafa0e7f903cb6cc71f4 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 21:15:19 -0600
Subject: [PATCH 05/83] Refactor ZFS snapshot receive command to force
 overwrite destination

---
 functions/backup_restore/zfs/snapshot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index 713e922..9dd6f5d 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -282,7 +282,7 @@ def zfs_receive(self, source: Path, destination: str, decompress: bool = False)
             if decompress:
                 command = f"gunzip -c \"{source}\" | /sbin/zfs recv \"{destination}\""
             else:
-                command = f"/sbin/zfs recv \"{destination}\" < \"{source}\""
+                command = f"/sbin/zfs recv -F \"{destination}\" < \"{source}\""
 
             receive_result = run_command(command)
             if receive_result.is_success():

From e15c38861d093bcd8fe85d56a97498df816d01f2 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 21:25:41 -0600
Subject: [PATCH 06/83] Refactor ZFS snapshot receive command to force
 overwrite destination

---
 functions/backup_restore/zfs/snapshot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index 9dd6f5d..d8b54c5 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -280,7 +280,7 @@ def zfs_receive(self, source: Path, destination: str, decompress: bool = False)
 
         try:
             if decompress:
-                command = f"gunzip -c \"{source}\" | /sbin/zfs recv \"{destination}\""
+                command = f"gunzip -c \"{source}\" | /sbin/zfs recv -F \"{destination}\""
             else:
                 command = f"/sbin/zfs recv -F \"{destination}\" < \"{source}\""
 

From fcc0ff5470322aa777359da018e0b3f5e58fc62f Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 22:01:59 -0600
Subject: [PATCH 07/83] Refactor ZFS snapshot receive command to improve
 restore process

---
 .../backup_restore/restore/restore_base.py    |  2 ++
 functions/backup_restore/zfs/snapshot.py      | 28 +++++++++----------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 098a325..43ba826 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -121,7 +121,9 @@ def restore_snapshots(self, app_name: str):
             self.logger.debug(f"Restoring snapshots for {app_name}...")
             for snapshot_file in snapshot_files:
                 original_path = snapshot_file.stem.replace('%%', '/')
+                self.logger.debug(f"Original path: {original_path}")
                 dataset_path, _ = original_path.split('@', 1)
+                self.logger.debug(f"Dataset path: {dataset_path}")
 
                 restore_result = self.snapshot_manager.zfs_receive(snapshot_file, dataset_path, decompress=True)
                 if not restore_result["success"]:
diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index d8b54c5..ea6618f 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -261,14 +261,14 @@ def zfs_send(self, source: str, destination: Path, compress: bool = False) -> di
         return result
 
     @type_check
-    def zfs_receive(self, source: Path, destination: str, decompress: bool = False) -> dict:
+    def zfs_receive(self, snapshot_file: Path, dataset_path: str, decompress: bool = False) -> dict:
         """
-        Receive a ZFS snapshot from a source file and restore it to the destination dataset.
+        Receive a ZFS snapshot from a file and restore it to the specified dataset path.
 
         Parameters:
-        - source (Path): The source file containing the snapshot.
-        - destination (str): The destination ZFS dataset to receive the snapshot to.
-        - decompress (bool): Whether the source file is gzip compressed. Default is False.
+        - snapshot_file (Path): The path to the snapshot file.
+        - dataset_path (str): The ZFS dataset path to restore to.
+        - decompress (bool): Whether the snapshot file is gzip compressed. Default is False.
 
         Returns:
         - dict: Result containing status and message.
@@ -279,21 +279,21 @@ def zfs_receive(self, source: Path, destination: str, decompress: bool = False)
         }
 
         try:
+            receive_command = f'zfs recv -F "{dataset_path}"'
             if decompress:
-                command = f"gunzip -c \"{source}\" | /sbin/zfs recv -F \"{destination}\""
+                command = f'gunzip < "{snapshot_file}" | {receive_command}'
             else:
-                command = f"/sbin/zfs recv -F \"{destination}\" < \"{source}\""
-
+                command = f'cat "{snapshot_file}" | {receive_command}'
+            
+            self.logger.debug(f"Executing command: {command}")
             receive_result = run_command(command)
             if receive_result.is_success():
-                self.logger.debug(f"Successfully received snapshot from {source} to {destination}")
                 result["success"] = True
-                result["message"] = f"Successfully received snapshot from {source} to {destination}"
+                result["message"] = f"Successfully restored snapshot from {snapshot_file} to {dataset_path}"
             else:
-                result["message"] = f"Failed to receive snapshot from {source}: {receive_result.get_error()}"
-                self.logger.error(result["message"])
+                result["message"] = receive_result.get_error()
         except Exception as e:
-            result["message"] = f"Exception occurred while receiving snapshot from {source}: {e}"
+            result["message"] = f"Exception occurred while restoring snapshot from {snapshot_file}: {e}"
             self.logger.error(result["message"], exc_info=True)
 
-        return result
+        return result
\ No newline at end of file

From 195f1dddcc7674c977870f16a46872cd39eafcd1 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 22:20:23 -0600
Subject: [PATCH 08/83] Refactor ZFS snapshot receive command to improve
 restore process

---
 functions/backup_restore/restore/restore_base.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 43ba826..3f5d2b7 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -120,18 +120,24 @@ def restore_snapshots(self, app_name: str):
         if snapshot_files:
             self.logger.debug(f"Restoring snapshots for {app_name}...")
             for snapshot_file in snapshot_files:
-                original_path = snapshot_file.stem.replace('%%', '/')
+                # Replace '%%' with '/' in the full file path
+                snapshot_file_path = str(snapshot_file).replace('%%', '/')
+                self.logger.debug(f"Modified snapshot file path: {snapshot_file_path}")
+                
+                # Extract the dataset path from the modified file path
+                original_path = Path(snapshot_file_path).stem
                 self.logger.debug(f"Original path: {original_path}")
                 dataset_path, _ = original_path.split('@', 1)
                 self.logger.debug(f"Dataset path: {dataset_path}")
 
-                restore_result = self.snapshot_manager.zfs_receive(snapshot_file, dataset_path, decompress=True)
+                restore_result = self.snapshot_manager.zfs_receive(Path(snapshot_file_path), dataset_path, decompress=True)
                 if not restore_result["success"]:
                     self.failures[app_name].append(restore_result["message"])
                     self.logger.error(f"Failed to restore snapshot from {snapshot_file} for {app_name}: {restore_result['message']}")
                 else:
                     self.logger.debug(f"Successfully restored snapshot from {snapshot_file} for {app_name}")
 
+
     @type_check
     def _restore_application(self, app_name: str) -> bool:
         """Restore a single application."""

From 5ee0b65a30a6a60137440762e7ba94e569a200c8 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 22:31:49 -0600
Subject: [PATCH 09/83] Refactor ZFS snapshot receive command to improve
 restore process

---
 functions/backup_restore/restore/restore_base.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 3f5d2b7..853d9e4 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -120,23 +120,23 @@ def restore_snapshots(self, app_name: str):
         if snapshot_files:
             self.logger.debug(f"Restoring snapshots for {app_name}...")
             for snapshot_file in snapshot_files:
-                # Replace '%%' with '/' in the full file path
-                snapshot_file_path = str(snapshot_file).replace('%%', '/')
+                # Perform the replacement of '%%' with '/' on the filename
+                snapshot_file_name = snapshot_file.name.replace('%%', '/')
+                snapshot_file_path = snapshot_file.with_name(snapshot_file_name)
                 self.logger.debug(f"Modified snapshot file path: {snapshot_file_path}")
-                
+
                 # Extract the dataset path from the modified file path
-                original_path = Path(snapshot_file_path).stem
+                original_path = snapshot_file_path.stem
                 self.logger.debug(f"Original path: {original_path}")
                 dataset_path, _ = original_path.split('@', 1)
                 self.logger.debug(f"Dataset path: {dataset_path}")
 
-                restore_result = self.snapshot_manager.zfs_receive(Path(snapshot_file_path), dataset_path, decompress=True)
+                restore_result = self.snapshot_manager.zfs_receive(snapshot_file_path, dataset_path, decompress=True)
                 if not restore_result["success"]:
                     self.failures[app_name].append(restore_result["message"])
-                    self.logger.error(f"Failed to restore snapshot from {snapshot_file} for {app_name}: {restore_result['message']}")
+                    self.logger.error(f"Failed to restore snapshot from {snapshot_file_path} for {app_name}: {restore_result['message']}")
                 else:
-                    self.logger.debug(f"Successfully restored snapshot from {snapshot_file} for {app_name}")
-
+                    self.logger.debug(f"Successfully restored snapshot from {snapshot_file_path} for {app_name}")
 
     @type_check
     def _restore_application(self, app_name: str) -> bool:

From 4e7680ad926896c2c19c7beb133ae650c9203085 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 22:34:48 -0600
Subject: [PATCH 10/83] Refactor ZFS snapshot receive command to improve
 restore process

---
 functions/backup_restore/restore/restore_base.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 853d9e4..03193b4 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -120,9 +120,9 @@ def restore_snapshots(self, app_name: str):
         if snapshot_files:
             self.logger.debug(f"Restoring snapshots for {app_name}...")
             for snapshot_file in snapshot_files:
-                # Perform the replacement of '%%' with '/' on the filename
-                snapshot_file_name = snapshot_file.name.replace('%%', '/')
-                snapshot_file_path = snapshot_file.with_name(snapshot_file_name)
+                # Perform the replacement of '%%' with '/' on the full file path
+                snapshot_file_path_str = str(snapshot_file).replace('%%', '/')
+                snapshot_file_path = Path(snapshot_file_path_str)
                 self.logger.debug(f"Modified snapshot file path: {snapshot_file_path}")
 
                 # Extract the dataset path from the modified file path

From 93eb26515f1365ec5e4650ebdb86d7b3626fe67c Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 22:53:32 -0600
Subject: [PATCH 11/83] Refactor ZFS snapshot receive command to improve
 restore process

---
 functions/backup_restore/restore/restore_base.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 03193b4..da1664f 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -120,15 +120,13 @@ def restore_snapshots(self, app_name: str):
         if snapshot_files:
             self.logger.debug(f"Restoring snapshots for {app_name}...")
             for snapshot_file in snapshot_files:
-                # Perform the replacement of '%%' with '/' on the full file path
-                snapshot_file_path_str = str(snapshot_file).replace('%%', '/')
-                snapshot_file_path = Path(snapshot_file_path_str)
-                self.logger.debug(f"Modified snapshot file path: {snapshot_file_path}")
-
-                # Extract the dataset path from the modified file path
-                original_path = snapshot_file_path.stem
-                self.logger.debug(f"Original path: {original_path}")
-                dataset_path, _ = original_path.split('@', 1)
+                # Keep the snapshot file path intact
+                snapshot_file_path = snapshot_file
+                self.logger.debug(f"Snapshot file path: {snapshot_file_path}")
+
+                # Extract the dataset path from the snapshot file name after replacing '%%' with '/'
+                snapshot_file_name = snapshot_file.stem.replace('%%', '/')
+                dataset_path, _ = snapshot_file_name.split('@', 1)
                 self.logger.debug(f"Dataset path: {dataset_path}")
 
                 restore_result = self.snapshot_manager.zfs_receive(snapshot_file_path, dataset_path, decompress=True)

From c97a5a67e0fd88ab60a6ad306cee5311653beb9c Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 23:03:49 -0600
Subject: [PATCH 12/83] remove any existing snapshots on restoration

---
 functions/backup_restore/zfs/snapshot.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index ea6618f..76c70b8 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -279,7 +279,14 @@ def zfs_receive(self, snapshot_file: Path, dataset_path: str, decompress: bool =
         }
 
         try:
-            receive_command = f'zfs recv -F "{dataset_path}"'
+            destroy_snapshots_command = f'/sbin/zfs list -H -t snapshot -o name -r "{dataset_path}" | xargs -n1 /sbin/zfs destroy'
+            destroy_result = run_command(destroy_snapshots_command)
+            if not destroy_result.is_success():
+                result["message"] = f"Failed to destroy existing snapshots in {dataset_path}: {destroy_result.get_error()}"
+                self.logger.error(result["message"])
+                return result
+
+            receive_command = f'/sbin/zfs recv -F "{dataset_path}"'
             if decompress:
                 command = f'gunzip < "{snapshot_file}" | {receive_command}'
             else:
@@ -296,4 +303,4 @@ def zfs_receive(self, snapshot_file: Path, dataset_path: str, decompress: bool =
             result["message"] = f"Exception occurred while restoring snapshot from {snapshot_file}: {e}"
             self.logger.error(result["message"], exc_info=True)
 
-        return result
\ No newline at end of file
+        return result

From 409e237a28b6c80a6dfcbb9bd433cdc69637f712 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 23:22:22 -0600
Subject: [PATCH 13/83] Refactor ZFS snapshot destroy command to handle null
 characters in snapshot names

---
 functions/backup_restore/zfs/snapshot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index 76c70b8..943d05b 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -279,7 +279,7 @@ def zfs_receive(self, snapshot_file: Path, dataset_path: str, decompress: bool =
         }
 
         try:
-            destroy_snapshots_command = f'/sbin/zfs list -H -t snapshot -o name -r "{dataset_path}" | xargs -n1 /sbin/zfs destroy'
+            destroy_snapshots_command = f'/sbin/zfs list -H -t snapshot -o name -r "{dataset_path}" | xargs -0 -n1 /sbin/zfs destroy'
             destroy_result = run_command(destroy_snapshots_command)
             if not destroy_result.is_success():
                 result["message"] = f"Failed to destroy existing snapshots in {dataset_path}: {destroy_result.get_error()}"

From 71afb87939b0416b8084c61412704f984039de39 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Sun, 26 May 2024 23:31:54 -0600
Subject: [PATCH 14/83] Refactor ZFS snapshot destroy command to handle null
 characters in snapshot names

---
 functions/backup_restore/zfs/snapshot.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index 943d05b..b2e9de9 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -279,12 +279,19 @@ def zfs_receive(self, snapshot_file: Path, dataset_path: str, decompress: bool =
         }
 
         try:
-            destroy_snapshots_command = f'/sbin/zfs list -H -t snapshot -o name -r "{dataset_path}" | xargs -0 -n1 /sbin/zfs destroy'
-            destroy_result = run_command(destroy_snapshots_command)
-            if not destroy_result.is_success():
-                result["message"] = f"Failed to destroy existing snapshots in {dataset_path}: {destroy_result.get_error()}"
-                self.logger.error(result["message"])
-                return result
+            all_snapshots = self.list_snapshots()
+            dataset_snapshots = [snap for snap in all_snapshots if snap.startswith(f"{dataset_path}@")]
+
+            for snapshot in dataset_snapshots:
+                destroy_command = f'/sbin/zfs destroy "{snapshot}"'
+                destroy_result = run_command(destroy_command)
+                if destroy_result.is_success():
+                    self.cache.remove_snapshot(snapshot)
+                    self.logger.debug(f"Deleted snapshot: {snapshot}")
+                else:
+                    result["message"] = f"Failed to destroy existing snapshot {snapshot}: {destroy_result.get_error()}"
+                    self.logger.error(result["message"])
+                    return result
 
             receive_command = f'/sbin/zfs recv -F "{dataset_path}"'
             if decompress:

From 6585c22543ace1903380fcb8204be555f4deb7d5 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 00:47:20 -0600
Subject: [PATCH 15/83] create parent dataset if needed

---
 functions/backup_restore/restore/restore_base.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index da1664f..4d3c3c7 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -129,6 +129,17 @@ def restore_snapshots(self, app_name: str):
                 dataset_path, _ = snapshot_file_name.split('@', 1)
                 self.logger.debug(f"Dataset path: {dataset_path}")
 
+                # Ensure the parent dataset exists
+                parent_dataset_path = '/'.join(dataset_path.split('/')[:-1])
+                if not self.zfs_manager.dataset_exists(parent_dataset_path):
+                    self.logger.debug(f"Parent dataset {parent_dataset_path} does not exist. Creating it...")
+                    create_result = self.zfs_manager.create_dataset(parent_dataset_path)
+                    if not create_result:
+                        self.failures[app_name].append(f"Failed to create parent dataset {parent_dataset_path}")
+                        self.logger.error(f"Failed to create parent dataset {parent_dataset_path}")
+                        continue
+
+                # Restore the snapshot
                 restore_result = self.snapshot_manager.zfs_receive(snapshot_file_path, dataset_path, decompress=True)
                 if not restore_result["success"]:
                     self.failures[app_name].append(restore_result["message"])

From d4192e298ef2fe687bcdb2ae124d672ece58c720 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 14:33:13 -0600
Subject: [PATCH 16/83] restore ix_volumes as well

---
 functions/backup_restore/backup/backup.py     |  8 ++
 functions/backup_restore/charts/api_fetch.py  | 21 ++++-
 .../backup_restore/charts/backup_fetch.py     | 20 +++++
 .../backup_restore/restore/restore_base.py    | 85 +++++++++++++++----
 functions/backup_restore/zfs/snapshot.py      | 58 +++++++------
 5 files changed, 151 insertions(+), 41 deletions(-)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index 72b2487..ce0d9a2 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -158,6 +158,14 @@ def backup_all(self):
                         if not send_result["success"]:
                             failures[app_name].append(send_result["message"])
 
+            if chart_info.ix_volumes_dataset:
+                self.logger.info(f"Sending snapshots to backup directory...")
+                backup_path = app_backup_dir / "snapshots" / f"{snapshot.replace('/', '%%')}.zfs"
+                backup_path.parent.mkdir(parents=True, exist_ok=True)
+                send_result = self.snapshot_manager.zfs_send(snapshot, backup_path, compress=True)
+                if not send_result["success"]:
+                    failures[app_name].append(send_result["message"])
+
         self._create_backup_snapshot()
         self._log_failures(failures)
         self._cleanup_old_backups()
diff --git a/functions/backup_restore/charts/api_fetch.py b/functions/backup_restore/charts/api_fetch.py
index 04d8adb..fbb0e38 100644
--- a/functions/backup_restore/charts/api_fetch.py
+++ b/functions/backup_restore/charts/api_fetch.py
@@ -1,6 +1,7 @@
 import threading
+from pathlib import Path
 from abc import ABC, abstractmethod
-from typing import Dict, List
+from typing import Dict, List, Union
 from utils.logger import get_logger
 from utils.type_check import type_check
 from utils.singletons import MiddlewareClientManager
@@ -254,6 +255,24 @@ def has_pvc(self) -> bool:
         self.logger.debug(f"Has PVCs for app {self.app_name}: {has_pvc}")
         return has_pvc
 
+    @property
+    def ix_volumes_dataset(self) -> Union[str, None]:
+        """
+        Get the ixVolumes dataset path for a given application.
+
+        Returns:
+        - str: The ixVolumes dataset path if it exists, else None.
+        """
+        ix_volumes = self._chart_data.get("ixVolumes", [])
+        if ix_volumes:
+            host_path = ix_volumes[0].get("hostPath")
+            if host_path:
+                if host_path.startswith("/mnt/"):
+                    host_path = host_path[5:]
+                dataset_path = str(Path(host_path).parent)
+                return dataset_path
+        return None
+
 class APIChartCollection(ChartObserver):
     @type_check
     def __init__(self, refresh_on_update: bool = False):
diff --git a/functions/backup_restore/charts/backup_fetch.py b/functions/backup_restore/charts/backup_fetch.py
index de95f73..4747737 100644
--- a/functions/backup_restore/charts/backup_fetch.py
+++ b/functions/backup_restore/charts/backup_fetch.py
@@ -362,6 +362,26 @@ def get_dataset(self, app_name: str) -> str:
         """
         return self.charts_info.get(app_name, {}).get('metadata', {}).get('dataset', '')
 
+    @type_check
+    def get_ix_volumes_dataset(self, app_name: str) -> Union[str, None]:
+        """
+        Get the ixVolumes dataset path for a given application.
+
+        Returns:
+        - str: The ixVolumes dataset path if it exists, else None.
+        """
+        ix_volumes = self.charts_info.get(app_name, {}).get("ixVolumes", [])
+        if ix_volumes:
+            host_path = ix_volumes[0].get("hostPath")
+            if host_path:
+                # Remove the "/mnt/" prefix
+                if host_path.startswith("/mnt/"):
+                    host_path = host_path[5:]
+                # Remove the last directory to get the dataset path
+                dataset_path = str(Path(host_path).parent)
+                return dataset_path
+        return None
+
     def handle_critical_failure(self, app_name: str) -> None:
         """
         Remove the application from all_releases and other relevant lists.
diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 4d3c3c7..22aca0e 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -1,4 +1,5 @@
 import os
+import yaml
 from pathlib import Path
 from collections import defaultdict
 from app.app_manager import AppManager
@@ -95,26 +96,80 @@ def _restore_crds(self, app_name):
 
     @type_check
     def _rollback_volumes(self, app_name: str):
-        """Rollback persistent volumes."""
+        """
+        Rollback persistent volumes or restore from backups if necessary.
+
+        Parameters:
+        - app_name (str): The name of the application to restore volumes for.
+        """
         pv_files = self.chart_info.get_file(app_name, "pv_zfs_volumes")
         pv_only_files = [file for file in pv_files if file.name.endswith('-pv.yaml')]
+
+        snapshots_to_rollback = []
+        snapshots_to_restore = []
+
+        self.logger.info(f"Preparing to rollback or restore ZFS snapshots for {app_name}...")
+
+        # Process PV files
         if pv_only_files:
-            self.logger.info(f"Rolling back ZFS snapshots for {app_name}...")
             for pv_file in pv_only_files:
-                result = self.snapshot_manager.rollback_persistent_volume(self.snapshot_name, pv_file)
-                if not result["success"]:
-                    self.failures[app_name].append(result["message"])
-                    self.logger.error(f"Failed to rollback {pv_file} for {app_name}: {result['message']}")
+                try:
+                    with pv_file.open('r') as file:
+                        pv_data = yaml.safe_load(file)
+                    pool_name = pv_data['spec']['csi']['volumeAttributes']['openebs.io/poolname']
+                    volume_handle = pv_data['spec']['csi']['volumeHandle']
+                    dataset_path = f"{pool_name}/{volume_handle}"
+                    snapshot = f"{dataset_path}@{self.snapshot_name}"
+
+                    if self.snapshot_manager.snapshot_exists(snapshot):
+                        snapshots_to_rollback.append(snapshot)
+                    else:
+                        snapshots_to_restore.append(snapshot)
+                except Exception as e:
+                    message = f"Failed to process PV file {pv_file}: {e}"
+                    self.logger.error(message, exc_info=True)
+                    self.failures[app_name].append(message)
+                    continue
+
+        # Process ix_volumes
+        ix_volumes_dataset = self.chart_info.get_ix_volumes_dataset()
+        if ix_volumes_dataset:
+            snapshot = f"{ix_volumes_dataset}@{self.snapshot_name}"
+            if self.snapshot_manager.snapshot_exists(snapshot):
+                snapshots_to_rollback.append(snapshot)
+            else:
+                snapshots_to_restore.append(snapshot)
+
+        # Rollback snapshots
+        if snapshots_to_rollback:
+            self.logger.info(f"Rolling back snapshots for {app_name}...")
+            rollback_result = self.snapshot_manager.rollback_snapshots(snapshots_to_rollback)
+            for message in rollback_result["messages"]:
+                if not rollback_result["success"]:
+                    self.failures[app_name].append(message)
+                    self.logger.error(message)
+                else:
+                    self.logger.debug(message)
+
+        # Restore snapshots
+        if snapshots_to_restore:
+            self.logger.info(f"Restoring snapshots for {app_name}...")
+            restore_result = self._restore_snapshots(app_name, snapshots_to_restore)
+            for message in restore_result["messages"]:
+                if not restore_result["success"]:
+                    self.failures[app_name].append(message)
+                    self.logger.error(message)
                 else:
-                    self.logger.debug(result["message"])
+                    self.logger.debug(message)
 
     @type_check
-    def restore_snapshots(self, app_name: str):
+    def _restore_snapshots(self, app_name: str, snapshots_to_restore: list):
         """
         Restore snapshots from the backup directory for a specific application.
 
         Parameters:
         - app_name (str): The name of the application to restore snapshots for.
+        - snapshots_to_restore (list): List of snapshots to restore.
         """
         snapshot_files = self.chart_info.get_file(app_name, "snapshots")
         if snapshot_files:
@@ -139,13 +194,13 @@ def restore_snapshots(self, app_name: str):
                         self.logger.error(f"Failed to create parent dataset {parent_dataset_path}")
                         continue
 
-                # Restore the snapshot
-                restore_result = self.snapshot_manager.zfs_receive(snapshot_file_path, dataset_path, decompress=True)
-                if not restore_result["success"]:
-                    self.failures[app_name].append(restore_result["message"])
-                    self.logger.error(f"Failed to restore snapshot from {snapshot_file_path} for {app_name}: {restore_result['message']}")
-                else:
-                    self.logger.debug(f"Successfully restored snapshot from {snapshot_file_path} for {app_name}")
+                if snapshot_file_name in snapshots_to_restore:
+                    restore_result = self.snapshot_manager.zfs_receive(snapshot_file_path, dataset_path, decompress=True)
+                    if not restore_result["success"]:
+                        self.failures[app_name].append(restore_result["message"])
+                        self.logger.error(f"Failed to restore snapshot from {snapshot_file_path} for {app_name}: {restore_result['message']}")
+                    else:
+                        self.logger.debug(f"Successfully restored snapshot from {snapshot_file_path} for {app_name}")
 
     @type_check
     def _restore_application(self, app_name: str) -> bool:
diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index b2e9de9..110809c 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -136,51 +136,46 @@ def delete_snapshots(self, snapshot_name: str) -> list:
         return errors
 
     @type_check
-    def rollback_persistent_volume(self, snapshot_name: str, pv_file: Path) -> dict:
+    def rollback_snapshots(self, snapshots: list) -> dict:
         """
-        Restore a PV from a backup YAML file and rollback to a specified snapshot.
+        Rollback multiple snapshots.
 
         Parameters:
-        - snapshot_name (str): Name of the snapshot to rollback to.
-        - pv_file (Path): Path to the PV file.
+        - snapshots (list): List of snapshots to rollback.
 
         Returns:
-        - dict: Result containing status and message.
+        - dict: Result containing status, messages, and list of rolled back snapshots.
         """
         result = {
-            "success": False,
-            "message": ""
+            "success": True,
+            "messages": [],
+            "rolled_back_snapshots": []
         }
 
-        try:
-            with pv_file.open('r') as file:
-                pv_data = yaml.safe_load(file)
-
-            pool_name = pv_data['spec']['csi']['volumeAttributes']['openebs.io/poolname']
-            volume_handle = pv_data['spec']['csi']['volumeHandle']
-            dataset_path = f"{pool_name}/{volume_handle}"
-
+        for snapshot in snapshots:
+            dataset_path, snapshot_name = snapshot.split('@', 1)
             if dataset_path not in self.cache.datasets:
                 message = f"Dataset {dataset_path} does not exist. Cannot restore snapshot."
                 self.logger.warning(message)
-                result["message"] = message
-                return result
+                result["messages"].append(message)
+                result["success"] = False
+                continue
 
-            rollback_command = f"/sbin/zfs rollback -r -f \"{dataset_path}@{snapshot_name}\""
+            rollback_command = f"/sbin/zfs rollback -r -f \"{snapshot}\""
             rollback_result = run_command(rollback_command)
             if rollback_result.is_success():
                 message = f"Successfully rolled back {dataset_path} to snapshot {snapshot_name}."
                 self.logger.debug(message)
-                result["success"] = True
-                result["message"] = message
+                result["rolled_back_snapshots"].append({
+                    "dataset": dataset_path,
+                    "snapshot": snapshot_name
+                })
+                result["messages"].append(message)
             else:
                 message = f"Failed to rollback {dataset_path} to snapshot {snapshot_name}: {rollback_result.get_error()}"
                 self.logger.error(message)
-                result["message"] = message
-        except Exception as e:
-            message = f"Failed to process PV file {pv_file}: {e}"
-            self.logger.error(message, exc_info=True)
-            result["message"] = message
+                result["messages"].append(message)
+                result["success"] = False
 
         return result
 
@@ -196,6 +191,19 @@ def list_snapshots(self) -> list:
         self.logger.debug(f"Listing all snapshots: {snapshots}")
         return snapshots
 
+    @type_check
+    def snapshot_exists(self, snapshot_name: str) -> bool:
+        """
+        Check if a snapshot exists in the cache.
+
+        Parameters:
+        - snapshot_name (str): The name of the snapshot to check.
+
+        Returns:
+        - bool: True if the snapshot exists, False otherwise.
+        """
+        return snapshot_name in self.cache.snapshots
+
     @type_check
     def rollback_all_snapshots(self, snapshot_name: str, dataset_path: str) -> None:
         """

From a1d147e28f2d81dfd90529e1bcf0bcdbd2d1c303 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 14:37:52 -0600
Subject: [PATCH 17/83] use config for backup

---
 functions/backup_restore/charts/api_fetch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/charts/api_fetch.py b/functions/backup_restore/charts/api_fetch.py
index fbb0e38..670924f 100644
--- a/functions/backup_restore/charts/api_fetch.py
+++ b/functions/backup_restore/charts/api_fetch.py
@@ -263,7 +263,7 @@ def ix_volumes_dataset(self) -> Union[str, None]:
         Returns:
         - str: The ixVolumes dataset path if it exists, else None.
         """
-        ix_volumes = self._chart_data.get("ixVolumes", [])
+        ix_volumes = self.chart_config.get("ixVolumes", [])
         if ix_volumes:
             host_path = ix_volumes[0].get("hostPath")
             if host_path:

From 19669ea6e29784e39c2d5017757d1278dbe6d7a3 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 14:41:12 -0600
Subject: [PATCH 18/83] define the snapshot name for ix_volume

---
 functions/backup_restore/backup/backup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index ce0d9a2..a41c1e0 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -160,6 +160,7 @@ def backup_all(self):
 
             if chart_info.ix_volumes_dataset:
                 self.logger.info(f"Sending snapshots to backup directory...")
+                snapshot = chart_info.ix_volumes_dataset + "@" + self.snapshot_name
                 backup_path = app_backup_dir / "snapshots" / f"{snapshot.replace('/', '%%')}.zfs"
                 backup_path.parent.mkdir(parents=True, exist_ok=True)
                 send_result = self.snapshot_manager.zfs_send(snapshot, backup_path, compress=True)

From 4fb54f7d7103fd335be1d756b3c2b44b84fb9af5 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 14:47:33 -0600
Subject: [PATCH 19/83] fix method calling

---
 functions/backup_restore/restore/restore_all.py    | 2 --
 functions/backup_restore/restore/restore_single.py | 1 -
 2 files changed, 3 deletions(-)

diff --git a/functions/backup_restore/restore/restore_all.py b/functions/backup_restore/restore/restore_all.py
index 1fece7d..d7bca1a 100644
--- a/functions/backup_restore/restore/restore_all.py
+++ b/functions/backup_restore/restore/restore_all.py
@@ -42,8 +42,6 @@ def restore(self):
                 self.logger.error(f"Failed to rollback snapshots for {app_name}: {e}\n")
                 self.failures[app_name].append(f"Failed to rollback volume snapshots: {e}")
 
-            self.restore_snapshots(app_name)
-
         self.logger.info("\nStarting Kubernetes Services\n"
                          "----------------------------")
         try:
diff --git a/functions/backup_restore/restore/restore_single.py b/functions/backup_restore/restore/restore_single.py
index 2791b48..502fbe8 100644
--- a/functions/backup_restore/restore/restore_single.py
+++ b/functions/backup_restore/restore/restore_single.py
@@ -26,7 +26,6 @@ def restore(self, app_names: list):
         for app_name in app_names:
             try:
                 self._rollback_volumes(app_name)
-                self.restore_snapshots(app_name)
             except Exception as e:
                 self.logger.error(f"Failed to rollback snapshots for {app_name}: {e}\n")
                 self.failures[app_name].append(f"Failed to rollback volume snapshots: {e}")

From e34a20d573c5f26f088da93eda87cee62b7ddcb1 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 14:49:12 -0600
Subject: [PATCH 20/83] pass parameter

---
 functions/backup_restore/restore/restore_base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 22aca0e..eaa1051 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -132,7 +132,7 @@ def _rollback_volumes(self, app_name: str):
                     continue
 
         # Process ix_volumes
-        ix_volumes_dataset = self.chart_info.get_ix_volumes_dataset()
+        ix_volumes_dataset = self.chart_info.get_ix_volumes_dataset(app_name)
         if ix_volumes_dataset:
             snapshot = f"{ix_volumes_dataset}@{self.snapshot_name}"
             if self.snapshot_manager.snapshot_exists(snapshot):

From 35b1b243e3d105d3458d6aadc0fd4f0657fa0059 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 14:56:08 -0600
Subject: [PATCH 21/83] Refactor ZFS snapshot restore process for improved
 efficiency

---
 functions/backup_restore/restore/restore_base.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index eaa1051..6f298a0 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -108,8 +108,6 @@ def _rollback_volumes(self, app_name: str):
         snapshots_to_rollback = []
         snapshots_to_restore = []
 
-        self.logger.info(f"Preparing to rollback or restore ZFS snapshots for {app_name}...")
-
         # Process PV files
         if pv_only_files:
             for pv_file in pv_only_files:

From f67c75c2546eb070bc4a738f731edfbc0e9fba4f Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 15:56:31 -0600
Subject: [PATCH 22/83] correct getting ix volumes dataset from backup parser

---
 functions/backup_restore/charts/backup_fetch.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/functions/backup_restore/charts/backup_fetch.py b/functions/backup_restore/charts/backup_fetch.py
index 4747737..e773439 100644
--- a/functions/backup_restore/charts/backup_fetch.py
+++ b/functions/backup_restore/charts/backup_fetch.py
@@ -57,6 +57,9 @@ def _parse_chart(self, chart_base_dir: Path, app_name: str) -> Dict[str, Union[s
                 'dataset': '',
                 'is_cnpg': False,
             },
+            'config': {
+                'ixVolumes': []
+            },
             'files': {
                 'database': None,
                 'namespace': None,
@@ -88,6 +91,8 @@ def _parse_chart(self, chart_base_dir: Path, app_name: str) -> Dict[str, Union[s
             chart_info['metadata']['dataset'] = metadata.get('dataset', '')
             chart_info['metadata']['is_cnpg'] = self._is_cnpg(config)
 
+            chart_info['config']['ixVolumes'] = config.get('ixVolumes', [])
+
             # Add files
             chart_info['files']['database'] = self._get_database_file(database_dir, app_name)
             chart_info['files']['namespace'] = self._get_file(kubernetes_objects_dir / 'namespace' / 'namespace.yaml')
@@ -370,7 +375,7 @@ def get_ix_volumes_dataset(self, app_name: str) -> Union[str, None]:
         Returns:
         - str: The ixVolumes dataset path if it exists, else None.
         """
-        ix_volumes = self.charts_info.get(app_name, {}).get("ixVolumes", [])
+        ix_volumes = self.charts_info.get(app_name, {}).get("config", {}).get("ixVolumes", [])
         if ix_volumes:
             host_path = ix_volumes[0].get("hostPath")
             if host_path:

From e2c6721ced4a7aaab00b24daf828f039a7ceb079 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 16:08:50 -0600
Subject: [PATCH 23/83] Refactor backup_fetch.py to handle config values as
 strings

---
 functions/backup_restore/charts/backup_fetch.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/functions/backup_restore/charts/backup_fetch.py b/functions/backup_restore/charts/backup_fetch.py
index e773439..a65e572 100644
--- a/functions/backup_restore/charts/backup_fetch.py
+++ b/functions/backup_restore/charts/backup_fetch.py
@@ -418,6 +418,10 @@ def _serialize_charts_info(self) -> dict:
                     k: str(v) if isinstance(v, Path) else (
                         [str(i) for i in v] if v is not None else 'None'
                     ) for k, v in info['files'].items()
+                },
+                'config': {
+                    k: v if not isinstance(v, Path) else str(v)
+                    for k, v in info.get('config', {}).items()
                 }
             } for app, info in self.charts_info.items()
-        }
+        }
\ No newline at end of file

From da3884ff48b726ab30e6c48c23e81c7cf40dce9e Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 17:00:47 -0600
Subject: [PATCH 24/83] add more debugging logs

---
 .../backup_restore/restore/restore_base.py    | 56 ++++++++++++++++---
 1 file changed, 47 insertions(+), 9 deletions(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 6f298a0..52f3182 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -102,27 +102,36 @@ def _rollback_volumes(self, app_name: str):
         Parameters:
         - app_name (str): The name of the application to restore volumes for.
         """
+        self.logger.debug(f"Starting rollback process for {app_name}...")
+        
         pv_files = self.chart_info.get_file(app_name, "pv_zfs_volumes")
+        self.logger.debug(f"Found PV files for {app_name}: {pv_files}")
         pv_only_files = [file for file in pv_files if file.name.endswith('-pv.yaml')]
 
         snapshots_to_rollback = []
         snapshots_to_restore = []
 
+        self.logger.debug(f"Preparing to process PV files for {app_name}...")
+
         # Process PV files
         if pv_only_files:
             for pv_file in pv_only_files:
                 try:
                     with pv_file.open('r') as file:
                         pv_data = yaml.safe_load(file)
+                    self.logger.debug(f"Loaded PV data from {pv_file}: {pv_data}")
                     pool_name = pv_data['spec']['csi']['volumeAttributes']['openebs.io/poolname']
                     volume_handle = pv_data['spec']['csi']['volumeHandle']
                     dataset_path = f"{pool_name}/{volume_handle}"
                     snapshot = f"{dataset_path}@{self.snapshot_name}"
+                    self.logger.debug(f"Constructed snapshot path: {snapshot}")
 
                     if self.snapshot_manager.snapshot_exists(snapshot):
                         snapshots_to_rollback.append(snapshot)
+                        self.logger.debug(f"Snapshot exists and will be rolled back: {snapshot}")
                     else:
                         snapshots_to_restore.append(snapshot)
+                        self.logger.debug(f"Snapshot does not exist and will be restored: {snapshot}")
                 except Exception as e:
                     message = f"Failed to process PV file {pv_file}: {e}"
                     self.logger.error(message, exc_info=True)
@@ -131,19 +140,23 @@ def _rollback_volumes(self, app_name: str):
 
         # Process ix_volumes
         ix_volumes_dataset = self.chart_info.get_ix_volumes_dataset(app_name)
+        self.logger.debug(f"Found ix_volumes dataset for {app_name}: {ix_volumes_dataset}")
         if ix_volumes_dataset:
             snapshot = f"{ix_volumes_dataset}@{self.snapshot_name}"
+            self.logger.debug(f"Constructed ix_volumes snapshot path: {snapshot}")
             if self.snapshot_manager.snapshot_exists(snapshot):
                 snapshots_to_rollback.append(snapshot)
+                self.logger.debug(f"Snapshot exists and will be rolled back: {snapshot}")
             else:
                 snapshots_to_restore.append(snapshot)
+                self.logger.debug(f"Snapshot does not exist and will be restored: {snapshot}")
 
         # Rollback snapshots
         if snapshots_to_rollback:
             self.logger.info(f"Rolling back snapshots for {app_name}...")
             rollback_result = self.snapshot_manager.rollback_snapshots(snapshots_to_rollback)
-            for message in rollback_result["messages"]:
-                if not rollback_result["success"]:
+            for message in rollback_result.get("messages", []):
+                if not rollback_result.get("success", False):
                     self.failures[app_name].append(message)
                     self.logger.error(message)
                 else:
@@ -153,25 +166,35 @@ def _rollback_volumes(self, app_name: str):
         if snapshots_to_restore:
             self.logger.info(f"Restoring snapshots for {app_name}...")
             restore_result = self._restore_snapshots(app_name, snapshots_to_restore)
-            for message in restore_result["messages"]:
-                if not restore_result["success"]:
+            for message in restore_result.get("messages", []):
+                if not restore_result.get("success", False):
                     self.failures[app_name].append(message)
                     self.logger.error(message)
                 else:
                     self.logger.debug(message)
 
     @type_check
-    def _restore_snapshots(self, app_name: str, snapshots_to_restore: list):
+    def _restore_snapshots(self, app_name: str, snapshots_to_restore: list) -> dict:
         """
         Restore snapshots from the backup directory for a specific application.
 
         Parameters:
         - app_name (str): The name of the application to restore snapshots for.
         - snapshots_to_restore (list): List of snapshots to restore.
+
+        Returns:
+        - dict: Result containing status and messages.
         """
+        result = {
+            "success": True,
+            "messages": []
+        }
+
+        self.logger.debug(f"Starting snapshot restore process for {app_name} with snapshots: {snapshots_to_restore}")
+
         snapshot_files = self.chart_info.get_file(app_name, "snapshots")
         if snapshot_files:
-            self.logger.debug(f"Restoring snapshots for {app_name}...")
+            self.logger.debug(f"Found snapshot files for {app_name}: {snapshot_files}")
             for snapshot_file in snapshot_files:
                 # Keep the snapshot file path intact
                 snapshot_file_path = snapshot_file
@@ -180,7 +203,7 @@ def _restore_snapshots(self, app_name: str, snapshots_to_restore: list):
                 # Extract the dataset path from the snapshot file name after replacing '%%' with '/'
                 snapshot_file_name = snapshot_file.stem.replace('%%', '/')
                 dataset_path, _ = snapshot_file_name.split('@', 1)
-                self.logger.debug(f"Dataset path: {dataset_path}")
+                self.logger.debug(f"Dataset path extracted: {dataset_path}")
 
                 # Ensure the parent dataset exists
                 parent_dataset_path = '/'.join(dataset_path.split('/')[:-1])
@@ -188,17 +211,32 @@ def _restore_snapshots(self, app_name: str, snapshots_to_restore: list):
                     self.logger.debug(f"Parent dataset {parent_dataset_path} does not exist. Creating it...")
                     create_result = self.zfs_manager.create_dataset(parent_dataset_path)
                     if not create_result:
-                        self.failures[app_name].append(f"Failed to create parent dataset {parent_dataset_path}")
-                        self.logger.error(f"Failed to create parent dataset {parent_dataset_path}")
+                        message = f"Failed to create parent dataset {parent_dataset_path}"
+                        result["success"] = False
+                        result["messages"].append(message)
+                        self.failures[app_name].append(message)
+                        self.logger.error(message)
                         continue
 
                 if snapshot_file_name in snapshots_to_restore:
+                    self.logger.debug(f"Restoring snapshot {snapshot_file_name} from {snapshot_file_path} to {dataset_path}")
                     restore_result = self.snapshot_manager.zfs_receive(snapshot_file_path, dataset_path, decompress=True)
                     if not restore_result["success"]:
+                        result["success"] = False
+                        result["messages"].append(restore_result["message"])
                         self.failures[app_name].append(restore_result["message"])
                         self.logger.error(f"Failed to restore snapshot from {snapshot_file_path} for {app_name}: {restore_result['message']}")
                     else:
+                        result["messages"].append(f"Successfully restored snapshot from {snapshot_file_path} for {app_name}")
                         self.logger.debug(f"Successfully restored snapshot from {snapshot_file_path} for {app_name}")
+        else:
+            message = f"No snapshot files found for {app_name}"
+            result["success"] = False
+            result["messages"].append(message)
+            self.failures[app_name].append(message)
+            self.logger.error(message)
+
+        return result
 
     @type_check
     def _restore_application(self, app_name: str) -> bool:

From aa30dbe657605e96bed66dcb1cf73ecff813a80e Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 17:32:25 -0600
Subject: [PATCH 25/83] improve logs slightly,

---
 functions/backup_restore/database/restore.py | 6 +++---
 functions/backup_restore/zfs/snapshot.py     | 1 -
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/functions/backup_restore/database/restore.py b/functions/backup_restore/database/restore.py
index 069021e..6d41d80 100644
--- a/functions/backup_restore/database/restore.py
+++ b/functions/backup_restore/database/restore.py
@@ -139,7 +139,7 @@ def _get_restore_command(self) -> Tuple[str, str]:
                 "--if-exists",
                 "--no-owner",
                 "--no-privileges",
-                "--single-transaction"
+                "--disable-triggers"
             ]
             open_mode = 'rb'
 
@@ -201,7 +201,7 @@ def _execute_restore_command(self, retries=3, wait=5) -> Dict[str, str]:
                 return result
 
             # Check for deadlock and retry if detected
-            if 'deadlock detected' in stderr:
+            if b'deadlock detected' in stderr:
                 message = f"Deadlock detected. Retrying {attempt + 1}/{retries}..."
                 self.logger.warning(message)
                 result["message"] = f"{result['message']} {message}"
@@ -211,4 +211,4 @@ def _execute_restore_command(self, retries=3, wait=5) -> Dict[str, str]:
 
         result["message"] = f"{result['message']} Restore failed after retrying."
         self.logger.error(result["message"])
-        return result
+        return result
\ No newline at end of file
diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index 110809c..2b7e660 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -188,7 +188,6 @@ def list_snapshots(self) -> list:
         - list: A list of all snapshot names.
         """
         snapshots = list(self.cache.snapshots)
-        self.logger.debug(f"Listing all snapshots: {snapshots}")
         return snapshots
 
     @type_check

From cf6bc1f13f117b3abafb7477b3faa8f001747b1a Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 18:35:30 -0600
Subject: [PATCH 26/83] try just dropping the table each time

---
 functions/backup_restore/database/restore.py | 69 +++++++++++++++++++-
 1 file changed, 68 insertions(+), 1 deletion(-)

diff --git a/functions/backup_restore/database/restore.py b/functions/backup_restore/database/restore.py
index 6d41d80..77d7a4e 100644
--- a/functions/backup_restore/database/restore.py
+++ b/functions/backup_restore/database/restore.py
@@ -71,17 +71,24 @@ def restore(self, timeout=300, interval=5) -> Dict[str, str]:
         if not self.primary_pod:
             message = "Primary pod not found."
             self.logger.error(message)
-            result["message"] = message
 
             if was_stopped:
                 self.logger.debug(f"Stopping app {self.app_name} after restore failure.")
                 self.stop_app(self.app_name)
 
+            result["message"] = message
             return result
 
         self.command, self.open_mode = self._get_restore_command()
 
         try:
+            # Drop and recreate the database before restoring
+            drop_create_result = self._drop_and_create_database()
+            if not drop_create_result["success"]:
+                result["message"] = drop_create_result["message"]
+                self.logger.error(result["message"])
+                return result
+
             result = self._execute_restore_command()
 
             if not result["success"]:
@@ -146,6 +153,66 @@ def _get_restore_command(self) -> Tuple[str, str]:
         self.logger.debug(f"Restore command for app {self.app_name}: {command}")
         return command, open_mode
 
+    def _drop_and_create_database(self) -> Dict[str, str]:
+        """
+        Drop and recreate the database.
+
+        Returns:
+            dict: Result containing status and message.
+        """
+        result = {
+            "success": False,
+            "message": ""
+        }
+
+        drop_command = [
+            "k3s", "kubectl", "exec",
+            "--namespace", self.namespace,
+            "--stdin",
+            "--container", "postgres",
+            self.primary_pod,
+            "--",
+            "psql",
+            "--command",
+            f"DROP DATABASE IF EXISTS {self.database_name};"
+        ]
+
+        create_command = [
+            "k3s", "kubectl", "exec",
+            "--namespace", self.namespace,
+            "--stdin",
+            "--container", "postgres",
+            self.primary_pod,
+            "--",
+            "psql",
+            "--command",
+            f"CREATE DATABASE {self.database_name} OWNER {self.database_user};"
+        ]
+
+        try:
+            drop_process = subprocess.run(drop_command, capture_output=True, text=True)
+            if drop_process.returncode != 0:
+                result["message"] = f"Failed to drop database: {drop_process.stderr}"
+                self.logger.error(result["message"])
+                return result
+
+            create_process = subprocess.run(create_command, capture_output=True, text=True)
+            if create_process.returncode != 0:
+                result["message"] = f"Failed to create database: {create_process.stderr}"
+                self.logger.error(result["message"])
+                return result
+
+            result["success"] = True
+            result["message"] = "Database dropped and recreated successfully."
+            self.logger.debug(result["message"])
+
+        except Exception as e:
+            message = f"Failed to drop and create database: {e}"
+            self.logger.error(message, exc_info=True)
+            result["message"] = message
+
+        return result
+
     def _execute_restore_command(self, retries=3, wait=5) -> Dict[str, str]:
         """
         Execute the restore command on the primary pod with retry logic in case of deadlock.

From da3d82547af67ff388c43a999bc495e06b8afe31 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 18:47:18 -0600
Subject: [PATCH 27/83] drop all objects instead?

---
 functions/backup_restore/database/base.py    |  8 +--
 functions/backup_restore/database/restore.py | 66 +++++++++++---------
 2 files changed, 39 insertions(+), 35 deletions(-)

diff --git a/functions/backup_restore/database/base.py b/functions/backup_restore/database/base.py
index b93e1e5..50694f0 100644
--- a/functions/backup_restore/database/base.py
+++ b/functions/backup_restore/database/base.py
@@ -33,10 +33,10 @@ def __init__(self, app_name: str):
         self.dump_command = None
         self.error = None
 
-        # Fetch database name and user if needed
-        if self.chart_info.chart_name != "immich":
-            self.database_name = self.fetch_database_name()
-            self.database_user = self.fetch_database_user() or self.database_name
+        # # Fetch database name and user if needed
+        # if self.chart_info.chart_name != "immich":
+        self.database_name = self.fetch_database_name()
+        self.database_user = self.fetch_database_user() or self.database_name
 
     def fetch_primary_pod(self, timeout=600, interval=5) -> str:
         """
diff --git a/functions/backup_restore/database/restore.py b/functions/backup_restore/database/restore.py
index 77d7a4e..2a753e4 100644
--- a/functions/backup_restore/database/restore.py
+++ b/functions/backup_restore/database/restore.py
@@ -82,10 +82,10 @@ def restore(self, timeout=300, interval=5) -> Dict[str, str]:
         self.command, self.open_mode = self._get_restore_command()
 
         try:
-            # Drop and recreate the database before restoring
-            drop_create_result = self._drop_and_create_database()
-            if not drop_create_result["success"]:
-                result["message"] = drop_create_result["message"]
+            # Drop all objects in the database before restoring
+            drop_all_objects_result = self._drop_all_objects()
+            if not drop_all_objects_result["success"]:
+                result["message"] = drop_all_objects_result["message"]
                 self.logger.error(result["message"])
                 return result
 
@@ -153,9 +153,9 @@ def _get_restore_command(self) -> Tuple[str, str]:
         self.logger.debug(f"Restore command for app {self.app_name}: {command}")
         return command, open_mode
 
-    def _drop_and_create_database(self) -> Dict[str, str]:
+    def _drop_all_objects(self) -> Dict[str, str]:
         """
-        Drop and recreate the database.
+        Drop all objects in the database.
 
         Returns:
             dict: Result containing status and message.
@@ -165,7 +165,7 @@ def _drop_and_create_database(self) -> Dict[str, str]:
             "message": ""
         }
 
-        drop_command = [
+        drop_all_objects_command = [
             "k3s", "kubectl", "exec",
             "--namespace", self.namespace,
             "--stdin",
@@ -173,41 +173,45 @@ def _drop_and_create_database(self) -> Dict[str, str]:
             self.primary_pod,
             "--",
             "psql",
+            "--dbname", self.database_name,
             "--command",
-            f"DROP DATABASE IF EXISTS {self.database_name};"
-        ]
-
-        create_command = [
-            "k3s", "kubectl", "exec",
-            "--namespace", self.namespace,
-            "--stdin",
-            "--container", "postgres",
-            self.primary_pod,
-            "--",
-            "psql",
-            "--command",
-            f"CREATE DATABASE {self.database_name} OWNER {self.database_user};"
+            """
+            DO $$ DECLARE
+                r RECORD;
+            BEGIN
+                -- drop all tables
+                FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP
+                    EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE';
+                END LOOP;
+                -- drop all sequences
+                FOR r IN (SELECT sequencename FROM pg_sequences WHERE schemaname = current_schema()) LOOP
+                    EXECUTE 'DROP SEQUENCE IF EXISTS ' || quote_ident(r.sequencename) || ' CASCADE';
+                END LOOP;
+                -- drop all views
+                FOR r IN (SELECT viewname FROM pg_views WHERE schemaname = current_schema()) LOOP
+                    EXECUTE 'DROP VIEW IF EXISTS ' || quote_ident(r.viewname) || ' CASCADE';
+                END LOOP;
+                -- drop all functions
+                FOR r IN (SELECT proname FROM pg_proc p JOIN pg_namespace n ON p.pronamespace = n.oid WHERE n.nspname = current_schema()) LOOP
+                    EXECUTE 'DROP FUNCTION IF EXISTS ' || quote_ident(r.proname) || ' CASCADE';
+                END LOOP;
+            END $$;
+            """
         ]
 
         try:
-            drop_process = subprocess.run(drop_command, capture_output=True, text=True)
+            drop_process = subprocess.run(drop_all_objects_command, capture_output=True, text=True)
             if drop_process.returncode != 0:
-                result["message"] = f"Failed to drop database: {drop_process.stderr}"
-                self.logger.error(result["message"])
-                return result
-
-            create_process = subprocess.run(create_command, capture_output=True, text=True)
-            if create_process.returncode != 0:
-                result["message"] = f"Failed to create database: {create_process.stderr}"
+                result["message"] = f"Failed to drop all objects in database: {drop_process.stderr}"
                 self.logger.error(result["message"])
                 return result
 
             result["success"] = True
-            result["message"] = "Database dropped and recreated successfully."
+            result["message"] = "All objects in database dropped successfully."
             self.logger.debug(result["message"])
 
         except Exception as e:
-            message = f"Failed to drop and create database: {e}"
+            message = f"Failed to drop all objects in database: {e}"
             self.logger.error(message, exc_info=True)
             result["message"] = message
 
@@ -278,4 +282,4 @@ def _execute_restore_command(self, retries=3, wait=5) -> Dict[str, str]:
 
         result["message"] = f"{result['message']} Restore failed after retrying."
         self.logger.error(result["message"])
-        return result
\ No newline at end of file
+        return result

From 9c73d691d328eb2dcb2e42acd5f26a4a8f0a3283 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 19:02:34 -0600
Subject: [PATCH 28/83] dont restore cnpg databases for testing

---
 .../backup_restore/restore/restore_all.py     | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/functions/backup_restore/restore/restore_all.py b/functions/backup_restore/restore/restore_all.py
index d7bca1a..d45e082 100644
--- a/functions/backup_restore/restore/restore_all.py
+++ b/functions/backup_restore/restore/restore_all.py
@@ -106,21 +106,21 @@ def restore(self):
                 self.logger.error(f"Job for {app_name} failed: {e}")
                 self._handle_critical_failure(app_name, str(f"Job failed: {e}"))
 
-        if self.chart_info.cnpg_apps:
-            self.logger.info("\nRestoring CNPG Databases\n"
-                            "------------------------")
-        for app_name in self.chart_info.cnpg_apps:
-            try:
-                self.logger.info(f"Restoring database for {app_name}...")
-                db_manager = RestoreCNPGDatabase(app_name, self.chart_info.get_file(app_name, "database"))
-                result = db_manager.restore()
-                if not result["success"]:
-                    self.failures[app_name].append(result["message"])
-                else:
-                    self.logger.info(result["message"])
-            except Exception as e:
-                self.logger.error(f"Failed to restore database for {app_name}: {e}")
-                self.failures[app_name].append(f"Database restore failed: {e}")
+        # if self.chart_info.cnpg_apps:
+        #     self.logger.info("\nRestoring CNPG Databases\n"
+        #                     "------------------------")
+        # for app_name in self.chart_info.cnpg_apps:
+        #     try:
+        #         self.logger.info(f"Restoring database for {app_name}...")
+        #         db_manager = RestoreCNPGDatabase(app_name, self.chart_info.get_file(app_name, "database"))
+        #         result = db_manager.restore()
+        #         if not result["success"]:
+        #             self.failures[app_name].append(result["message"])
+        #         else:
+        #             self.logger.info(result["message"])
+        #     except Exception as e:
+        #         self.logger.error(f"Failed to restore database for {app_name}: {e}")
+        #         self.failures[app_name].append(f"Database restore failed: {e}")
 
         self._log_failures()
 

From 330065885475bcd0cdf37b6a3c1f49fb9bdb77a0 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 19:16:24 -0600
Subject: [PATCH 29/83] try killing active connections

---
 functions/backup_restore/database/restore.py  | 94 ++++++++++++-------
 .../backup_restore/restore/restore_all.py     | 30 +++---
 2 files changed, 74 insertions(+), 50 deletions(-)

diff --git a/functions/backup_restore/database/restore.py b/functions/backup_restore/database/restore.py
index 2a753e4..f9a65ce 100644
--- a/functions/backup_restore/database/restore.py
+++ b/functions/backup_restore/database/restore.py
@@ -79,16 +79,15 @@ def restore(self, timeout=300, interval=5) -> Dict[str, str]:
             result["message"] = message
             return result
 
-        self.command, self.open_mode = self._get_restore_command()
-
         try:
-            # Drop all objects in the database before restoring
-            drop_all_objects_result = self._drop_all_objects()
-            if not drop_all_objects_result["success"]:
-                result["message"] = drop_all_objects_result["message"]
+            # Terminate active connections and drop the database
+            drop_database_result = self._drop_and_recreate_database()
+            if not drop_database_result["success"]:
+                result["message"] = drop_database_result["message"]
                 self.logger.error(result["message"])
                 return result
 
+            # Restore the database from the backup file
             result = self._execute_restore_command()
 
             if not result["success"]:
@@ -153,9 +152,9 @@ def _get_restore_command(self) -> Tuple[str, str]:
         self.logger.debug(f"Restore command for app {self.app_name}: {command}")
         return command, open_mode
 
-    def _drop_all_objects(self) -> Dict[str, str]:
+    def _drop_and_recreate_database(self) -> Dict[str, str]:
         """
-        Drop all objects in the database.
+        Terminate active connections, drop the database, and recreate it.
 
         Returns:
             dict: Result containing status and message.
@@ -165,7 +164,7 @@ def _drop_all_objects(self) -> Dict[str, str]:
             "message": ""
         }
 
-        drop_all_objects_command = [
+        terminate_connections_command = [
             "k3s", "kubectl", "exec",
             "--namespace", self.namespace,
             "--stdin",
@@ -173,45 +172,70 @@ def _drop_all_objects(self) -> Dict[str, str]:
             self.primary_pod,
             "--",
             "psql",
-            "--dbname", self.database_name,
+            "--dbname", "postgres",
             "--command",
+            f"""
+            SELECT pg_terminate_backend(pg_stat_activity.pid)
+            FROM pg_stat_activity
+            WHERE pg_stat_activity.datname = '{self.database_name}'
+              AND pid <> pg_backend_pid();
             """
-            DO $$ DECLARE
-                r RECORD;
-            BEGIN
-                -- drop all tables
-                FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = current_schema()) LOOP
-                    EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE';
-                END LOOP;
-                -- drop all sequences
-                FOR r IN (SELECT sequencename FROM pg_sequences WHERE schemaname = current_schema()) LOOP
-                    EXECUTE 'DROP SEQUENCE IF EXISTS ' || quote_ident(r.sequencename) || ' CASCADE';
-                END LOOP;
-                -- drop all views
-                FOR r IN (SELECT viewname FROM pg_views WHERE schemaname = current_schema()) LOOP
-                    EXECUTE 'DROP VIEW IF EXISTS ' || quote_ident(r.viewname) || ' CASCADE';
-                END LOOP;
-                -- drop all functions
-                FOR r IN (SELECT proname FROM pg_proc p JOIN pg_namespace n ON p.pronamespace = n.oid WHERE n.nspname = current_schema()) LOOP
-                    EXECUTE 'DROP FUNCTION IF EXISTS ' || quote_ident(r.proname) || ' CASCADE';
-                END LOOP;
-            END $$;
-            """
+        ]
+
+        drop_database_command = [
+            "k3s", "kubectl", "exec",
+            "--namespace", self.namespace,
+            "--stdin",
+            "--container", "postgres",
+            self.primary_pod,
+            "--",
+            "psql",
+            "--dbname", "postgres",
+            "--command",
+            f"DROP DATABASE IF EXISTS {self.database_name};"
+        ]
+
+        create_database_command = [
+            "k3s", "kubectl", "exec",
+            "--namespace", self.namespace,
+            "--stdin",
+            "--container", "postgres",
+            self.primary_pod,
+            "--",
+            "psql",
+            "--dbname", "postgres",
+            "--command",
+            f"CREATE DATABASE {self.database_name} OWNER {self.database_user};"
         ]
 
         try:
-            drop_process = subprocess.run(drop_all_objects_command, capture_output=True, text=True)
+            # Terminate active connections
+            terminate_process = subprocess.run(terminate_connections_command, capture_output=True, text=True)
+            if terminate_process.returncode != 0:
+                result["message"] = f"Failed to terminate active connections: {terminate_process.stderr}"
+                self.logger.error(result["message"])
+                return result
+
+            # Drop the database
+            drop_process = subprocess.run(drop_database_command, capture_output=True, text=True)
             if drop_process.returncode != 0:
-                result["message"] = f"Failed to drop all objects in database: {drop_process.stderr}"
+                result["message"] = f"Failed to drop the database: {drop_process.stderr}"
+                self.logger.error(result["message"])
+                return result
+
+            # Recreate the database
+            create_process = subprocess.run(create_database_command, capture_output=True, text=True)
+            if create_process.returncode != 0:
+                result["message"] = f"Failed to recreate the database: {create_process.stderr}"
                 self.logger.error(result["message"])
                 return result
 
             result["success"] = True
-            result["message"] = "All objects in database dropped successfully."
+            result["message"] = "Database dropped and recreated successfully."
             self.logger.debug(result["message"])
 
         except Exception as e:
-            message = f"Failed to drop all objects in database: {e}"
+            message = f"Failed to drop and recreate database: {e}"
             self.logger.error(message, exc_info=True)
             result["message"] = message
 
diff --git a/functions/backup_restore/restore/restore_all.py b/functions/backup_restore/restore/restore_all.py
index d45e082..d7bca1a 100644
--- a/functions/backup_restore/restore/restore_all.py
+++ b/functions/backup_restore/restore/restore_all.py
@@ -106,21 +106,21 @@ def restore(self):
                 self.logger.error(f"Job for {app_name} failed: {e}")
                 self._handle_critical_failure(app_name, str(f"Job failed: {e}"))
 
-        # if self.chart_info.cnpg_apps:
-        #     self.logger.info("\nRestoring CNPG Databases\n"
-        #                     "------------------------")
-        # for app_name in self.chart_info.cnpg_apps:
-        #     try:
-        #         self.logger.info(f"Restoring database for {app_name}...")
-        #         db_manager = RestoreCNPGDatabase(app_name, self.chart_info.get_file(app_name, "database"))
-        #         result = db_manager.restore()
-        #         if not result["success"]:
-        #             self.failures[app_name].append(result["message"])
-        #         else:
-        #             self.logger.info(result["message"])
-        #     except Exception as e:
-        #         self.logger.error(f"Failed to restore database for {app_name}: {e}")
-        #         self.failures[app_name].append(f"Database restore failed: {e}")
+        if self.chart_info.cnpg_apps:
+            self.logger.info("\nRestoring CNPG Databases\n"
+                            "------------------------")
+        for app_name in self.chart_info.cnpg_apps:
+            try:
+                self.logger.info(f"Restoring database for {app_name}...")
+                db_manager = RestoreCNPGDatabase(app_name, self.chart_info.get_file(app_name, "database"))
+                result = db_manager.restore()
+                if not result["success"]:
+                    self.failures[app_name].append(result["message"])
+                else:
+                    self.logger.info(result["message"])
+            except Exception as e:
+                self.logger.error(f"Failed to restore database for {app_name}: {e}")
+                self.failures[app_name].append(f"Database restore failed: {e}")
 
         self._log_failures()
 

From f8e00f2548b852bdc797b7f3af1c721a1bcf8b72 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 19:24:12 -0600
Subject: [PATCH 30/83] Refactor restore.py to improve restore command handling

---
 functions/backup_restore/database/restore.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/functions/backup_restore/database/restore.py b/functions/backup_restore/database/restore.py
index f9a65ce..52d267a 100644
--- a/functions/backup_restore/database/restore.py
+++ b/functions/backup_restore/database/restore.py
@@ -258,6 +258,8 @@ def _execute_restore_command(self, retries=3, wait=5) -> Dict[str, str]:
         }
         self.logger.debug(f"Executing restore command on pod: {self.primary_pod} with dump file: {self.backup_file}")
 
+        self.command, self.open_mode = self._get_restore_command()
+
         for attempt in range(retries):
             try:
                 if self.backup_file.suffix == '.gz':

From 1ee93292d70cd0373bf94a1e4adbab73b14e6ec8 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 20:20:15 -0600
Subject: [PATCH 31/83] revert but keep binary read

---
 functions/backup_restore/database/restore.py | 107 +------------------
 1 file changed, 5 insertions(+), 102 deletions(-)

diff --git a/functions/backup_restore/database/restore.py b/functions/backup_restore/database/restore.py
index 52d267a..b25b4cd 100644
--- a/functions/backup_restore/database/restore.py
+++ b/functions/backup_restore/database/restore.py
@@ -71,23 +71,17 @@ def restore(self, timeout=300, interval=5) -> Dict[str, str]:
         if not self.primary_pod:
             message = "Primary pod not found."
             self.logger.error(message)
+            result["message"] = message
 
             if was_stopped:
                 self.logger.debug(f"Stopping app {self.app_name} after restore failure.")
                 self.stop_app(self.app_name)
 
-            result["message"] = message
             return result
 
-        try:
-            # Terminate active connections and drop the database
-            drop_database_result = self._drop_and_recreate_database()
-            if not drop_database_result["success"]:
-                result["message"] = drop_database_result["message"]
-                self.logger.error(result["message"])
-                return result
+        self.command, self.open_mode = self._get_restore_command()
 
-            # Restore the database from the backup file
+        try:
             result = self._execute_restore_command()
 
             if not result["success"]:
@@ -145,102 +139,13 @@ def _get_restore_command(self) -> Tuple[str, str]:
                 "--if-exists",
                 "--no-owner",
                 "--no-privileges",
-                "--disable-triggers"
+                "--single-transaction"
             ]
             open_mode = 'rb'
 
         self.logger.debug(f"Restore command for app {self.app_name}: {command}")
         return command, open_mode
 
-    def _drop_and_recreate_database(self) -> Dict[str, str]:
-        """
-        Terminate active connections, drop the database, and recreate it.
-
-        Returns:
-            dict: Result containing status and message.
-        """
-        result = {
-            "success": False,
-            "message": ""
-        }
-
-        terminate_connections_command = [
-            "k3s", "kubectl", "exec",
-            "--namespace", self.namespace,
-            "--stdin",
-            "--container", "postgres",
-            self.primary_pod,
-            "--",
-            "psql",
-            "--dbname", "postgres",
-            "--command",
-            f"""
-            SELECT pg_terminate_backend(pg_stat_activity.pid)
-            FROM pg_stat_activity
-            WHERE pg_stat_activity.datname = '{self.database_name}'
-              AND pid <> pg_backend_pid();
-            """
-        ]
-
-        drop_database_command = [
-            "k3s", "kubectl", "exec",
-            "--namespace", self.namespace,
-            "--stdin",
-            "--container", "postgres",
-            self.primary_pod,
-            "--",
-            "psql",
-            "--dbname", "postgres",
-            "--command",
-            f"DROP DATABASE IF EXISTS {self.database_name};"
-        ]
-
-        create_database_command = [
-            "k3s", "kubectl", "exec",
-            "--namespace", self.namespace,
-            "--stdin",
-            "--container", "postgres",
-            self.primary_pod,
-            "--",
-            "psql",
-            "--dbname", "postgres",
-            "--command",
-            f"CREATE DATABASE {self.database_name} OWNER {self.database_user};"
-        ]
-
-        try:
-            # Terminate active connections
-            terminate_process = subprocess.run(terminate_connections_command, capture_output=True, text=True)
-            if terminate_process.returncode != 0:
-                result["message"] = f"Failed to terminate active connections: {terminate_process.stderr}"
-                self.logger.error(result["message"])
-                return result
-
-            # Drop the database
-            drop_process = subprocess.run(drop_database_command, capture_output=True, text=True)
-            if drop_process.returncode != 0:
-                result["message"] = f"Failed to drop the database: {drop_process.stderr}"
-                self.logger.error(result["message"])
-                return result
-
-            # Recreate the database
-            create_process = subprocess.run(create_database_command, capture_output=True, text=True)
-            if create_process.returncode != 0:
-                result["message"] = f"Failed to recreate the database: {create_process.stderr}"
-                self.logger.error(result["message"])
-                return result
-
-            result["success"] = True
-            result["message"] = "Database dropped and recreated successfully."
-            self.logger.debug(result["message"])
-
-        except Exception as e:
-            message = f"Failed to drop and recreate database: {e}"
-            self.logger.error(message, exc_info=True)
-            result["message"] = message
-
-        return result
-
     def _execute_restore_command(self, retries=3, wait=5) -> Dict[str, str]:
         """
         Execute the restore command on the primary pod with retry logic in case of deadlock.
@@ -258,8 +163,6 @@ def _execute_restore_command(self, retries=3, wait=5) -> Dict[str, str]:
         }
         self.logger.debug(f"Executing restore command on pod: {self.primary_pod} with dump file: {self.backup_file}")
 
-        self.command, self.open_mode = self._get_restore_command()
-
         for attempt in range(retries):
             try:
                 if self.backup_file.suffix == '.gz':
@@ -308,4 +211,4 @@ def _execute_restore_command(self, retries=3, wait=5) -> Dict[str, str]:
 
         result["message"] = f"{result['message']} Restore failed after retrying."
         self.logger.error(result["message"])
-        return result
+        return result
\ No newline at end of file

From 0e942a667978e8f1dee26f577eed832152ad5946 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Mon, 27 May 2024 23:55:09 -0600
Subject: [PATCH 32/83] check to see if snapshots exist in backup before
 declaring we will use backup

---
 functions/backup_restore/restore/restore_base.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 52f3182..8c64eb8 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -129,9 +129,13 @@ def _rollback_volumes(self, app_name: str):
                     if self.snapshot_manager.snapshot_exists(snapshot):
                         snapshots_to_rollback.append(snapshot)
                         self.logger.debug(f"Snapshot exists and will be rolled back: {snapshot}")
-                    else:
+                    elif any(snap.stem.replace('%%', '/') == snapshot for snap in self.chart_info.get_file(app_name, "snapshots") or []):
                         snapshots_to_restore.append(snapshot)
-                        self.logger.debug(f"Snapshot does not exist and will be restored: {snapshot}")
+                        self.logger.debug(f"Snapshot to restore found in backups: {snapshot}")
+                    else:
+                        message = f"Snapshot {snapshot} cannot be rolled back or restored from backup."
+                        self.failures[app_name].append(message)
+                        self.logger.error(message)
                 except Exception as e:
                     message = f"Failed to process PV file {pv_file}: {e}"
                     self.logger.error(message, exc_info=True)
@@ -147,9 +151,13 @@ def _rollback_volumes(self, app_name: str):
             if self.snapshot_manager.snapshot_exists(snapshot):
                 snapshots_to_rollback.append(snapshot)
                 self.logger.debug(f"Snapshot exists and will be rolled back: {snapshot}")
-            else:
+            elif any(snap.stem.replace('%%', '/') == snapshot for snap in self.chart_info.get_file(app_name, "snapshots") or []):
                 snapshots_to_restore.append(snapshot)
-                self.logger.debug(f"Snapshot does not exist and will be restored: {snapshot}")
+                self.logger.debug(f"Snapshot to restore found in backups: {snapshot}")
+            else:
+                message = f"Snapshot {snapshot} for ix_volumes cannot be rolled back or restored from backup."
+                self.failures[app_name].append(message)
+                self.logger.error(message)
 
         # Rollback snapshots
         if snapshots_to_rollback:

From ab32cf40f59e5ef10d4a4dd92b1c089bff956fd0 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 19:41:34 -0600
Subject: [PATCH 33/83] refactor snapshot backup

---
 functions/backup_restore/backup/backup.py |  66 +++---
 functions/backup_restore/zfs/cache.py     |  53 +++--
 functions/backup_restore/zfs/snapshot.py  | 251 ++++++++++------------
 3 files changed, 188 insertions(+), 182 deletions(-)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index a41c1e0..c64c47e 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -145,19 +145,30 @@ def backup_all(self):
             dataset_paths = self.kube_pvc_fetcher.get_volume_paths_by_namespace(f"ix-{app_name}")
             if dataset_paths:
                 self.logger.info(f"Backing up {app_name} PVCs...")
-                snapshot_result = self.snapshot_manager.create_snapshots(self.snapshot_name, dataset_paths, self.retention_number)
-                if snapshot_result["errors"]:
-                    failures[app_name].extend(snapshot_result["errors"])
-
-                if snapshot_result["success"]:
-                    self.logger.info(f"Sending snapshots to backup directory...")
-                    for snapshot in snapshot_result["snapshots"]:
-                        backup_path = app_backup_dir / "snapshots" / f"{snapshot.replace('/', '%%')}.zfs"
-                        backup_path.parent.mkdir(parents=True, exist_ok=True)
-                        send_result = self.snapshot_manager.zfs_send(snapshot, backup_path, compress=True)
-                        if not send_result["success"]:
-                            failures[app_name].append(send_result["message"])
-
+                for dataset_path in dataset_paths:
+                    self.logger.info(f"Backing up PVCs for dataset {dataset_path}...")
+
+                    # Check to see if dataset exists
+                    if not self.lifecycle_manager.dataset_exists(dataset_path):
+                        self.logger.error(f"Dataset {dataset_path} does not exist.")
+                        failures[app_name].append(f"Dataset {dataset_path} does not exist.")
+                        continue
+
+                    # Create the snapshot for the current dataset
+                    snapshot_result = self.snapshot_manager.create_snapshot(self.snapshot_name, dataset_path)
+                    if not snapshot_result["success"]:
+                        failures[app_name].append(snapshot_result["message"])
+                        continue
+                    
+                    # Send the snapshot to the backup directory
+                    snapshot_name = f"{dataset_path}@{self.snapshot_name}"
+                    backup_path = app_backup_dir / "snapshots" / f"{snapshot_name.replace('/', '%%')}.zfs"
+                    backup_path.parent.mkdir(parents=True, exist_ok=True)
+                    send_result = self.snapshot_manager.zfs_send(snapshot_name, backup_path, compress=True)
+                    if not send_result["success"]:
+                        failures[app_name].append(send_result["message"])
+
+            # Handle ix_volumes_dataset separately
             if chart_info.ix_volumes_dataset:
                 self.logger.info(f"Sending snapshots to backup directory...")
                 snapshot = chart_info.ix_volumes_dataset + "@" + self.snapshot_name
@@ -193,7 +204,7 @@ def _create_backup_snapshot(self):
         Create a snapshot of the backup dataset after all backups are completed.
         """
         self.logger.info(f"\nCreating snapshot for backup: {self.backup_dataset}")
-        snapshot_result = self.snapshot_manager.create_snapshots(self.snapshot_name, [self.backup_dataset], self.retention_number)
+        snapshot_result = self.snapshot_manager.create_snapshot(self.snapshot_name, self.backup_dataset)
 
         if snapshot_result.get("success"):
             self.logger.info("Snapshot created successfully for backup dataset.")
@@ -210,21 +221,28 @@ def _cleanup_old_backups(self):
             (ds for ds in self.lifecycle_manager.list_datasets() if ds.startswith(f"{self.backup_dataset_parent}/HeavyScript--")),
             key=lambda ds: datetime.strptime(ds.replace(f"{self.backup_dataset_parent}/HeavyScript--", ""), '%Y-%m-%d_%H:%M:%S')
         )
-        
+
         if len(backup_datasets) > self.retention_number:
             for old_backup_dataset in backup_datasets[:-self.retention_number]:
                 snapshot_name = old_backup_dataset.split("/")[-1]
                 self.logger.info(f"Deleting oldest backup due to retention limit: {snapshot_name}")
+
                 try:
                     self.lifecycle_manager.delete_dataset(old_backup_dataset)
                     self.logger.debug(f"Removed old backup: {old_backup_dataset}")
                 except Exception as e:
                     self.logger.error(f"Failed to delete old backup dataset {old_backup_dataset}: {e}", exc_info=True)
-                
+
                 self.logger.debug(f"Deleting snapshots for: {snapshot_name}")
-                snapshot_errors = self.snapshot_manager.delete_snapshots(snapshot_name)
-                if snapshot_errors:
-                    self.logger.error(f"Failed to delete snapshots for {snapshot_name}: {snapshot_errors}")
+                all_snapshots = self.snapshot_manager.list_snapshots()
+                matching_snapshots = [snap for snap in all_snapshots if snap.endswith(f"@{snapshot_name}")]
+
+                for snapshot in matching_snapshots:
+                    delete_result = self.snapshot_manager.delete_snapshot(snapshot)
+                    if not delete_result["success"]:
+                        self.logger.error(f"Failed to delete snapshot {snapshot}: {delete_result['message']}")
+
+                self.logger.info(f"Cleanup completed for backup: {snapshot_name}")
 
     def _backup_application_datasets(self):
         """
@@ -239,8 +257,8 @@ def _backup_application_datasets(self):
         datasets_to_backup = [ds for ds in all_datasets if ds.startswith(self.kubeconfig.dataset) and ds not in datasets_to_ignore]
         self.logger.debug(f"Snapshotting datasets: {datasets_to_backup}")
 
-        snapshot_result = self.snapshot_manager.create_snapshots(self.snapshot_name, datasets_to_backup, self.retention_number)
-        if not snapshot_result.get("success"):
-            self.logger.error("Failed to create snapshots for application datasets.")
-            for error in snapshot_result.get("errors", []):
-                self.logger.error(error)
\ No newline at end of file
+        for dataset in datasets_to_backup:
+            # Create snapshot for each dataset
+            snapshot_result = self.snapshot_manager.create_snapshot(self.snapshot_name, dataset)
+            if not snapshot_result.get("success"):
+                self.logger.error(f"Failed to create snapshot for dataset {dataset}: {snapshot_result['message']}")
\ No newline at end of file
diff --git a/functions/backup_restore/zfs/cache.py b/functions/backup_restore/zfs/cache.py
index bf158e2..0c5151e 100644
--- a/functions/backup_restore/zfs/cache.py
+++ b/functions/backup_restore/zfs/cache.py
@@ -12,7 +12,7 @@ class ZFSCache:
     _instance = None
     _lock = threading.Lock()
     _datasets = set()
-    _snapshots = set()
+    _snapshots = {}
 
     def __new__(cls):
         if not cls._instance:
@@ -47,22 +47,35 @@ def _load_datasets(self) -> set:
             self.logger.error("Failed to load datasets.")
             return set()
 
-    def _load_snapshots(self) -> set:
+    def _load_snapshots(self) -> dict:
         """
-        Load all ZFS snapshots.
+        Load all ZFS snapshots and their refer sizes.
 
         Returns:
-            set: A set of all ZFS snapshots.
+            dict: A dictionary of all ZFS snapshots with their details.
         """
-        command = "/sbin/zfs list -H -t snapshot -o name"
+        command = "/sbin/zfs list -H -t snapshot -o name,refer"
         result = run_command(command, suppress_output=True)
         if result.is_success():
-            snapshots = set(result.get_output().split('\n'))
+            snapshots = {}
+            for line in result.get_output().split('\n'):
+                if line:
+                    parts = line.split()
+                    snapshot_name = parts[0]
+                    refer_size = self._convert_size_to_bytes(parts[1])
+                    snapshots[snapshot_name] = {"refer": refer_size}
             self.logger.debug(f"Loaded {len(snapshots)} snapshots.")
             return snapshots
         else:
             self.logger.error("Failed to load snapshots.")
-            return set()
+            return {}
+
+    def _convert_size_to_bytes(self, size_str):
+        size_units = {"K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}
+        if size_str[-1] in size_units:
+            return int(float(size_str[:-1]) * size_units[size_str[-1]])
+        else:
+            return int(size_str)
 
     def hard_refresh(self):
         """
@@ -71,7 +84,7 @@ def hard_refresh(self):
         ZFSCache._datasets = self._load_datasets()
         ZFSCache._snapshots = self._load_snapshots()
 
-    def get_snapshots_for_dataset(self, dataset: str) -> set:
+    def get_snapshots_for_dataset(self, dataset: str) -> dict:
         """
         Get all snapshots associated with a specific dataset.
 
@@ -79,10 +92,10 @@ def get_snapshots_for_dataset(self, dataset: str) -> set:
             dataset (str): The name of the dataset.
 
         Returns:
-            set: A set of snapshots associated with the dataset.
+            dict: A dictionary of snapshots associated with the dataset.
         """
         with self._lock:
-            return {snap for snap in ZFSCache._snapshots if snap.startswith(dataset + '@')}
+            return {snap: details for snap, details in ZFSCache._snapshots.items() if snap.startswith(dataset + '@')}
 
     @property
     def datasets(self) -> set:
@@ -107,23 +120,23 @@ def datasets(self, value: set):
             ZFSCache._datasets = value
 
     @property
-    def snapshots(self) -> set:
+    def snapshots(self) -> dict:
         """
         Get the current set of snapshots.
 
         Returns:
-            set: The current set of snapshots.
+            dict: The current set of snapshots.
         """
         with self._lock:
             return ZFSCache._snapshots
 
     @snapshots.setter
-    def snapshots(self, value: set):
+    def snapshots(self, value: dict):
         """
         Set the current set of snapshots.
 
         Parameters:
-            value (set): The new set of snapshots.
+            value (dict): The new set of snapshots.
         """
         with self._lock:
             ZFSCache._snapshots = value
@@ -153,16 +166,17 @@ def remove_dataset(self, dataset: str):
             self.logger.debug(f"Removed dataset: {dataset}")
 
     @type_check
-    def add_snapshot(self, snapshot: str):
+    def add_snapshot(self, snapshot: str, refer_size: int):
         """
         Add a snapshot to the cache.
 
         Parameters:
             snapshot (str): The snapshot to add.
+            refer_size (int): The refer size of the snapshot.
         """
         with self._lock:
-            ZFSCache._snapshots.add(snapshot)
-            self.logger.debug(f"Added snapshot: {snapshot}")
+            ZFSCache._snapshots[snapshot] = {"refer": refer_size}
+            self.logger.debug(f"Added snapshot: {snapshot} with refer size: {refer_size}")
 
     @type_check
     def remove_snapshot(self, snapshot: str):
@@ -173,5 +187,6 @@ def remove_snapshot(self, snapshot: str):
             snapshot (str): The snapshot to remove.
         """
         with self._lock:
-            ZFSCache._snapshots.discard(snapshot)
-            self.logger.debug(f"Removed snapshot: {snapshot}")
+            if snapshot in ZFSCache._snapshots:
+                del ZFSCache._snapshots[snapshot]
+                self.logger.debug(f"Removed snapshot: {snapshot}")
diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index 2b7e660..d7a8c4e 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -1,7 +1,5 @@
-import re
-import yaml
+import subprocess
 from pathlib import Path
-from datetime import datetime
 
 from zfs.cache import ZFSCache
 from utils.shell import run_command
@@ -22,160 +20,138 @@ def __init__(self):
         self.cache = ZFSCache()
 
     @type_check
-    def _cleanup_snapshots(self, dataset_paths: list, retention_number: int) -> list:
+    def create_snapshot(self, snapshot_name: str, dataset: str) -> dict:
         """
-        Cleanup older snapshots, retaining only a specified number of the most recent ones.
+        Create a single ZFS snapshot for the specified dataset.
 
         Parameters:
-        - dataset_paths (list): List of paths to datasets.
-        - retention_number (int): Number of recent snapshots to retain.
+        - snapshot_name (str): Name of the snapshot.
+        - dataset (str): Dataset to create the snapshot for.
 
         Returns:
-        - list: A list of error messages, if any.
+        - dict: Result containing status and message.
         """
-        errors = []
-        for path in dataset_paths:
-            if path not in self.cache.datasets:
-                error_msg = f"Dataset {path} does not exist."
-                self.logger.error(error_msg)
-                errors.append(error_msg)
-                continue
-
-            matching_snapshots = [snap for snap in self.cache.snapshots if snap.startswith(f"{path}@HeavyScript--")]
-            matching_snapshots.sort(key=lambda x: datetime.strptime(re.search(r'HeavyScript--\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}', x).group(), "HeavyScript--%Y-%m-%d_%H:%M:%S"))
-
-            self.logger.debug(f"Found {len(matching_snapshots)} snapshots for dataset path {path}.")
-
-            if len(matching_snapshots) > retention_number:
-                snapshots_to_delete = matching_snapshots[:-retention_number]
-                for snapshot in snapshots_to_delete:
-                    delete_command = f"/sbin/zfs destroy \"{snapshot}\""
-                    delete_result = run_command(delete_command)
-                    if delete_result.is_success():
-                        self.cache.remove_snapshot(snapshot)
-                        self.logger.debug(f"Deleted snapshot: {snapshot}")
-                    else:
-                        error_msg = f"Failed to delete snapshot {snapshot}: {delete_result.get_error()}"
-                        self.logger.error(error_msg)
-                        errors.append(error_msg)
-        return errors
+        result = {
+            "success": False,
+            "message": ""
+        }
+
+        if dataset not in self.cache.datasets:
+            result["message"] = f"Dataset {dataset} does not exist."
+            self.logger.error(result["message"])
+            return result
+
+        snapshot_full_name = f"{dataset}@{snapshot_name}"
+        command = f"/sbin/zfs snapshot \"{snapshot_full_name}\""
+        snapshot_result = run_command(command)
+        if snapshot_result.is_success():
+            refer_size = self.get_snapshot_refer_size(snapshot_full_name)
+            self.cache.add_snapshot(snapshot_full_name, refer_size)
+            self.logger.debug(f"Created snapshot: {snapshot_full_name} with refer size: {refer_size}")
+            result["success"] = True
+            result["message"] = f"Snapshot {snapshot_full_name} created successfully."
+        else:
+            result["message"] = f"Failed to create snapshot for {snapshot_full_name}: {snapshot_result.get_error()}"
+            self.logger.error(result["message"])
+
+        return result
 
     @type_check
-    def create_snapshots(self, snapshot_name, dataset_paths: list, retention_number: int) -> dict:
+    def get_snapshot_refer_size(self, snapshot: str) -> int:
         """
-        Create snapshots for specified ZFS datasets and cleanup old snapshots.
+        Get the refer size of a ZFS snapshot.
 
         Parameters:
-        - snapshot_name (str): Name of the snapshot.
-        - dataset_paths (list): List of paths to create snapshots for.
-        - retention_number (int): Number of recent snapshots to retain.
+        - snapshot (str): The name of the snapshot.
 
         Returns:
-        - dict: Result containing status, messages, and list of created snapshots.
+        - int: The refer size of the snapshot in bytes.
         """
-        result = {
-            "success": False,
-            "message": "",
-            "errors": [],
-            "snapshots": []
-        }
-
-        for path in dataset_paths:
-            if path not in self.cache.datasets:
-                error_msg = f"Dataset {path} does not exist."
-                self.logger.error(error_msg)
-                result["errors"].append(error_msg)
-                continue
-
-            snapshot_full_name = f"{path}@{snapshot_name}"
-            command = f"/sbin/zfs snapshot \"{snapshot_full_name}\""
-            snapshot_result = run_command(command)
-            if snapshot_result.is_success():
-                self.cache.add_snapshot(snapshot_full_name)
-                self.logger.debug(f"Created snapshot: {snapshot_full_name}")
-                result["snapshots"].append(snapshot_full_name)
-            else:
-                error_msg = f"Failed to create snapshot for {snapshot_full_name}: {snapshot_result.get_error()}"
-                self.logger.error(error_msg)
-                result["errors"].append(error_msg)
-        
-        cleanup_errors = self._cleanup_snapshots(dataset_paths, retention_number)
-        result["errors"].extend(cleanup_errors)
+        try:
+            result = subprocess.run(["zfs", "list", "-H", "-o", "refer", snapshot], capture_output=True, text=True, check=True)
+            size_str = result.stdout.strip()
+            size = self._convert_size_to_bytes(size_str)
+            return size
+        except subprocess.CalledProcessError as e:
+            self.logger.error(f"Failed to get refer size for snapshot {snapshot}: {e}")
+            return 0
 
-        if not result["errors"]:
-            result["success"] = True
-            result["message"] = "All snapshots created and cleaned up successfully."
+    @type_check
+    def _convert_size_to_bytes(self, size_str):
+        size_units = {"K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}
+        if size_str[-1] in size_units:
+            return int(float(size_str[:-1]) * size_units[size_str[-1]])
         else:
-            result["message"] = "Some errors occurred during snapshot creation or cleanup."
-
-        return result
+            return int(size_str)
 
     @type_check
-    def delete_snapshots(self, snapshot_name: str) -> list:
+    def delete_snapshot(self, snapshot: str) -> dict:
         """
-        Delete all snapshots matching a specific name.
+        Delete a single ZFS snapshot.
 
         Parameters:
-        - snapshot_name (str): The name of the snapshot to delete.
+        - snapshot (str): The name of the snapshot to delete.
 
         Returns:
-        - list: A list of error messages, if any.
+        - dict: Result containing status and message.
         """
-        errors = []
-        matching_snapshots = [snap for snap in self.cache.snapshots if snap.endswith(f"@{snapshot_name}")]
-        for snapshot in matching_snapshots:
-            delete_command = f"/sbin/zfs destroy \"{snapshot}\""
-            delete_result = run_command(delete_command)
-            if delete_result.is_success():
-                self.cache.remove_snapshot(snapshot)
-                self.logger.debug(f"Deleted snapshot: {snapshot}")
-            else:
-                error_msg = f"Failed to delete snapshot {snapshot}: {delete_result.get_error()}"
-                self.logger.error(error_msg)
-                errors.append(error_msg)
-        return errors
+        result = {
+            "success": False,
+            "message": ""
+        }
+
+        delete_command = f"/sbin/zfs destroy \"{snapshot}\""
+        delete_result = run_command(delete_command)
+        if delete_result.is_success():
+            self.cache.remove_snapshot(snapshot)
+            self.logger.debug(f"Deleted snapshot: {snapshot}")
+            result["success"] = True
+            result["message"] = f"Snapshot {snapshot} deleted successfully."
+        else:
+            result["message"] = f"Failed to delete snapshot {snapshot}: {delete_result.get_error()}"
+            self.logger.error(result["message"])
+
+        return result
 
     @type_check
-    def rollback_snapshots(self, snapshots: list) -> dict:
+    def rollback_snapshot(self, snapshot: str, recursive: bool = False, force: bool = False) -> dict:
         """
-        Rollback multiple snapshots.
+        Rollback a single ZFS snapshot.
 
         Parameters:
-        - snapshots (list): List of snapshots to rollback.
+        - snapshot (str): The name of the snapshot to rollback.
+        - recursive (bool): Whether to rollback recursively. Default is False.
+        - force (bool): Whether to force the rollback. Default is False.
 
         Returns:
-        - dict: Result containing status, messages, and list of rolled back snapshots.
+        - dict: Result containing status and message.
         """
         result = {
-            "success": True,
-            "messages": [],
-            "rolled_back_snapshots": []
+            "success": False,
+            "message": ""
         }
 
-        for snapshot in snapshots:
-            dataset_path, snapshot_name = snapshot.split('@', 1)
-            if dataset_path not in self.cache.datasets:
-                message = f"Dataset {dataset_path} does not exist. Cannot restore snapshot."
-                self.logger.warning(message)
-                result["messages"].append(message)
-                result["success"] = False
-                continue
-
-            rollback_command = f"/sbin/zfs rollback -r -f \"{snapshot}\""
-            rollback_result = run_command(rollback_command)
-            if rollback_result.is_success():
-                message = f"Successfully rolled back {dataset_path} to snapshot {snapshot_name}."
-                self.logger.debug(message)
-                result["rolled_back_snapshots"].append({
-                    "dataset": dataset_path,
-                    "snapshot": snapshot_name
-                })
-                result["messages"].append(message)
-            else:
-                message = f"Failed to rollback {dataset_path} to snapshot {snapshot_name}: {rollback_result.get_error()}"
-                self.logger.error(message)
-                result["messages"].append(message)
-                result["success"] = False
+        dataset_path, snapshot_name = snapshot.split('@', 1)
+        if dataset_path not in self.cache.datasets:
+            result["message"] = f"Dataset {dataset_path} does not exist. Cannot restore snapshot."
+            self.logger.warning(result["message"])
+            return result
+
+        rollback_command = f"/sbin/zfs rollback"
+        if recursive:
+            rollback_command += " -r"
+        if force:
+            rollback_command += " -f"
+        rollback_command += f" \"{snapshot}\""
+        
+        rollback_result = run_command(rollback_command)
+        if rollback_result.is_success():
+            result["success"] = True
+            result["message"] = f"Successfully rolled back {dataset_path} to snapshot {snapshot_name}."
+            self.logger.debug(result["message"])
+        else:
+            result["message"] = f"Failed to rollback {dataset_path} to snapshot {snapshot_name}: {rollback_result.get_error()}"
+            self.logger.error(result["message"])
 
         return result
 
@@ -187,7 +163,7 @@ def list_snapshots(self) -> list:
         Returns:
         - list: A list of all snapshot names.
         """
-        snapshots = list(self.cache.snapshots)
+        snapshots = list(self.cache.snapshots.keys())
         return snapshots
 
     @type_check
@@ -204,13 +180,15 @@ def snapshot_exists(self, snapshot_name: str) -> bool:
         return snapshot_name in self.cache.snapshots
 
     @type_check
-    def rollback_all_snapshots(self, snapshot_name: str, dataset_path: str) -> None:
+    def rollback_all_snapshots(self, snapshot_name: str, dataset_path: str, recursive: bool = False, force: bool = False) -> None:
         """
         Rollback all snapshots under a given path recursively that match the snapshot name.
 
         Parameters:
         - snapshot_name (str): The name of the snapshot to rollback to.
         - dataset_path (str): The path of the dataset to rollback snapshots for.
+        - recursive (bool): Whether to rollback recursively. Default is False.
+        - force (bool): Whether to force the rollback. Default is False.
         """
         if dataset_path not in self.cache.datasets:
             self.logger.error(f"Dataset {dataset_path} does not exist. Cannot rollback snapshots.")
@@ -220,12 +198,7 @@ def rollback_all_snapshots(self, snapshot_name: str, dataset_path: str) -> None:
             all_snapshots = [snap for snap in self.cache.snapshots if snap.startswith(dataset_path)]
             matching_snapshots = [snap for snap in all_snapshots if snap.endswith(f"@{snapshot_name}")]
             for snapshot in matching_snapshots:
-                rollback_command = f"/sbin/zfs rollback -r -f \"{snapshot}\""
-                rollback_result = run_command(rollback_command)
-                if rollback_result.is_success():
-                    self.logger.debug(f"Successfully rolled back {snapshot}.")
-                else:
-                    self.logger.error(f"Failed to rollback {snapshot}: {rollback_result.get_error()}")
+                self.rollback_snapshot(snapshot, recursive, force)
         except Exception as e:
             self.logger.error(f"Failed to rollback snapshots for {dataset_path}: {e}", exc_info=True)
 
@@ -289,16 +262,16 @@ def zfs_receive(self, snapshot_file: Path, dataset_path: str, decompress: bool =
             all_snapshots = self.list_snapshots()
             dataset_snapshots = [snap for snap in all_snapshots if snap.startswith(f"{dataset_path}@")]
 
+            delete_errors = []
             for snapshot in dataset_snapshots:
-                destroy_command = f'/sbin/zfs destroy "{snapshot}"'
-                destroy_result = run_command(destroy_command)
-                if destroy_result.is_success():
-                    self.cache.remove_snapshot(snapshot)
-                    self.logger.debug(f"Deleted snapshot: {snapshot}")
-                else:
-                    result["message"] = f"Failed to destroy existing snapshot {snapshot}: {destroy_result.get_error()}"
-                    self.logger.error(result["message"])
-                    return result
+                delete_result = self.delete_snapshot(snapshot)
+                if not delete_result["success"]:
+                    delete_errors.append(delete_result["message"])
+
+            if delete_errors:
+                result["message"] = f"Failed to destroy existing snapshots: {delete_errors}"
+                self.logger.error(result["message"])
+                return result
 
             receive_command = f'/sbin/zfs recv -F "{dataset_path}"'
             if decompress:

From 429bc38e7e685a65b5da132de490ed24d3c69350 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 19:48:11 -0600
Subject: [PATCH 34/83] Refactor ZFSSnapshotManager to handle exceptions when
 getting refer size

---
 functions/backup_restore/zfs/snapshot.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index d7a8c4e..a9b9320 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -68,12 +68,16 @@ def get_snapshot_refer_size(self, snapshot: str) -> int:
         - int: The refer size of the snapshot in bytes.
         """
         try:
-            result = subprocess.run(["zfs", "list", "-H", "-o", "refer", snapshot], capture_output=True, text=True, check=True)
-            size_str = result.stdout.strip()
-            size = self._convert_size_to_bytes(size_str)
-            return size
-        except subprocess.CalledProcessError as e:
-            self.logger.error(f"Failed to get refer size for snapshot {snapshot}: {e}")
+            result = run_command(f"zfs list -H -o refer \"{snapshot}\"")
+            if result.is_success():
+                size_str = result.get_output()
+                size = self._convert_size_to_bytes(size_str)
+                return size
+            else:
+                self.logger.error(f"Failed to get refer size for snapshot {snapshot}: {result.get_error()}")
+                return 0
+        except Exception as e:
+            self.logger.error(f"Exception occurred while getting refer size for snapshot {snapshot}: {e}")
             return 0
 
     @type_check

From 7c868e9754a6cf88a9fb30e6519a80e16b8e28d6 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 19:53:11 -0600
Subject: [PATCH 35/83] update refer logic

---
 functions/backup_restore/zfs/cache.py    | 14 +++++++++-----
 functions/backup_restore/zfs/snapshot.py | 12 ++++++++----
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/functions/backup_restore/zfs/cache.py b/functions/backup_restore/zfs/cache.py
index 0c5151e..7a14f28 100644
--- a/functions/backup_restore/zfs/cache.py
+++ b/functions/backup_restore/zfs/cache.py
@@ -72,10 +72,14 @@ def _load_snapshots(self) -> dict:
 
     def _convert_size_to_bytes(self, size_str):
         size_units = {"K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}
-        if size_str[-1] in size_units:
-            return int(float(size_str[:-1]) * size_units[size_str[-1]])
-        else:
-            return int(size_str)
+        try:
+            if size_str[-1] in size_units:
+                return int(float(size_str[:-1]) * size_units[size_str[-1]])
+            else:
+                return int(size_str)
+        except ValueError:
+            self.logger.error(f"Invalid size string: {size_str}")
+            return 0
 
     def hard_refresh(self):
         """
@@ -189,4 +193,4 @@ def remove_snapshot(self, snapshot: str):
         with self._lock:
             if snapshot in ZFSCache._snapshots:
                 del ZFSCache._snapshots[snapshot]
-                self.logger.debug(f"Removed snapshot: {snapshot}")
+                self.logger.debug(f"Removed snapshot: {snapshot}")
\ No newline at end of file
diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index a9b9320..f8f354b 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -83,10 +83,14 @@ def get_snapshot_refer_size(self, snapshot: str) -> int:
     @type_check
     def _convert_size_to_bytes(self, size_str):
         size_units = {"K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}
-        if size_str[-1] in size_units:
-            return int(float(size_str[:-1]) * size_units[size_str[-1]])
-        else:
-            return int(size_str)
+        try:
+            if size_str[-1] in size_units:
+                return int(float(size_str[:-1]) * size_units[size_str[-1]])
+            else:
+                return int(size_str)
+        except ValueError:
+            self.logger.error(f"Invalid size string: {size_str}")
+            return 0
 
     @type_check
     def delete_snapshot(self, snapshot: str) -> dict:

From 45eb7b08cfd851446b761a815e0eca4d07566b36 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 20:02:01 -0600
Subject: [PATCH 36/83] try using a tab character to split instead

---
 functions/backup_restore/zfs/snapshot.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index f8f354b..87321ad 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -1,4 +1,3 @@
-import subprocess
 from pathlib import Path
 
 from zfs.cache import ZFSCache

From 98d891cbdeceb33a66bece1b0c65b56ac6cdd780 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 20:05:42 -0600
Subject: [PATCH 37/83] Refactor ZFSCache to improve snapshot refer size
 handling

---
 functions/backup_restore/zfs/cache.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/zfs/cache.py b/functions/backup_restore/zfs/cache.py
index 7a14f28..d75e383 100644
--- a/functions/backup_restore/zfs/cache.py
+++ b/functions/backup_restore/zfs/cache.py
@@ -60,7 +60,7 @@ def _load_snapshots(self) -> dict:
             snapshots = {}
             for line in result.get_output().split('\n'):
                 if line:
-                    parts = line.split()
+                    parts = line.rsplit('\t', 1)
                     snapshot_name = parts[0]
                     refer_size = self._convert_size_to_bytes(parts[1])
                     snapshots[snapshot_name] = {"refer": refer_size}

From de2278c3d5450214aed6034f5f79f867323d30fb Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 20:30:18 -0600
Subject: [PATCH 38/83] Refactor backup and restore code to improve cleanup
 process

---
 functions/backup_restore/backup/backup.py   | 32 ----------
 functions/backup_restore/backup_manager.py  | 67 ++++-----------------
 functions/backup_restore/base_manager.py    | 44 ++++++++++++++
 functions/backup_restore/restore_manager.py |  3 +-
 4 files changed, 58 insertions(+), 88 deletions(-)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index c64c47e..9b2262a 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -180,7 +180,6 @@ def backup_all(self):
 
         self._create_backup_snapshot()
         self._log_failures(failures)
-        self._cleanup_old_backups()
 
     def _log_failures(self, failures):
         """
@@ -213,37 +212,6 @@ def _create_backup_snapshot(self):
             for error in snapshot_result.get("errors", []):
                 self.logger.error(error)
 
-    def _cleanup_old_backups(self):
-        """
-        Cleanup old backups and their associated snapshots if the number of backups exceeds the retention limit.
-        """
-        backup_datasets = sorted(
-            (ds for ds in self.lifecycle_manager.list_datasets() if ds.startswith(f"{self.backup_dataset_parent}/HeavyScript--")),
-            key=lambda ds: datetime.strptime(ds.replace(f"{self.backup_dataset_parent}/HeavyScript--", ""), '%Y-%m-%d_%H:%M:%S')
-        )
-
-        if len(backup_datasets) > self.retention_number:
-            for old_backup_dataset in backup_datasets[:-self.retention_number]:
-                snapshot_name = old_backup_dataset.split("/")[-1]
-                self.logger.info(f"Deleting oldest backup due to retention limit: {snapshot_name}")
-
-                try:
-                    self.lifecycle_manager.delete_dataset(old_backup_dataset)
-                    self.logger.debug(f"Removed old backup: {old_backup_dataset}")
-                except Exception as e:
-                    self.logger.error(f"Failed to delete old backup dataset {old_backup_dataset}: {e}", exc_info=True)
-
-                self.logger.debug(f"Deleting snapshots for: {snapshot_name}")
-                all_snapshots = self.snapshot_manager.list_snapshots()
-                matching_snapshots = [snap for snap in all_snapshots if snap.endswith(f"@{snapshot_name}")]
-
-                for snapshot in matching_snapshots:
-                    delete_result = self.snapshot_manager.delete_snapshot(snapshot)
-                    if not delete_result["success"]:
-                        self.logger.error(f"Failed to delete snapshot {snapshot}: {delete_result['message']}")
-
-                self.logger.info(f"Cleanup completed for backup: {snapshot_name}")
-
     def _backup_application_datasets(self):
         """
         Backup all datasets within the specified application dataset, except for those specified in to_ignore_datasets_on_backup.
diff --git a/functions/backup_restore/backup_manager.py b/functions/backup_restore/backup_manager.py
index 1ddf066..52785ae 100644
--- a/functions/backup_restore/backup_manager.py
+++ b/functions/backup_restore/backup_manager.py
@@ -1,17 +1,14 @@
-import re
 import shutil
 from pathlib import Path
 from base_manager import BaseManager
 from backup.backup import Backup
 from backup.export_ import ChartInfoExporter
-from zfs.snapshot import ZFSSnapshotManager
 from utils.logger import get_logger
 
 class BackupManager(BaseManager):
     def __init__(self, backup_abs_path: Path):
         super().__init__(backup_abs_path)
         self.logger = get_logger()
-        self.snapshot_manager = ZFSSnapshotManager()
         self.logger.info(f"BackupManager initialized for {self.backup_abs_path}")
 
     def backup_all(self, retention=None):
@@ -37,27 +34,11 @@ def export_chart_info(self, retention=None):
     def delete_backup_by_name(self, backup_name: str):
         """Delete a specific backup by name."""
         self.logger.info(f"Attempting to delete backup: {backup_name}")
-        full_backups, export_dirs = self.list_backups()
-
-        for backup in full_backups:
-            if backup.endswith(backup_name):
-                self.logger.info(f"Deleting full backup: {backup}")
-                self.lifecycle_manager.delete_dataset(backup)
-                self.snapshot_manager.delete_snapshots(backup_name)
-                self.logger.info(f"Deleted full backup: {backup} and associated snapshots")
-                self.cleanup_dangling_snapshots()
-                return True
-
-        for export in export_dirs:
-            if export.name == backup_name:
-                self.logger.info(f"Deleting export: {export}")
-                shutil.rmtree(export)
-                self.logger.info(f"Deleted export: {export}")
-                self.cleanup_dangling_snapshots()
-                return True
-
-        self.logger.info(f"Backup {backup_name} not found")
-        return False
+        result = self.delete_backup(backup_name)
+        if result:
+            self.logger.info(f"Deleted backup: {backup_name}")
+        else:
+            self.logger.info(f"Backup {backup_name} not found")
 
     def delete_backup_by_index(self, backup_index: int):
         """Delete a specific backup by index."""
@@ -67,30 +48,20 @@ def delete_backup_by_index(self, backup_index: int):
 
         if 0 <= backup_index < len(all_backups):
             backup = all_backups[backup_index]
-            if backup in full_backups:
-                backup_name = Path(backup).name
-                self.logger.info(f"Deleting full backup: {backup_name}")
-                self.lifecycle_manager.delete_dataset(backup)
-                self.snapshot_manager.delete_snapshots(backup_name)
-                self.logger.info(f"Deleted full backup: {backup_name} and associated snapshots")
-            elif backup in export_dirs:
-                self.logger.info(f"Deleting export: {backup.name}")
-                shutil.rmtree(backup)
-                self.logger.info(f"Deleted export: {backup.name}")
-            self.cleanup_dangling_snapshots()
-            return True
-
-        self.logger.info(f"Invalid backup index: {backup_index}")
-        return False
+            backup_name = Path(backup).name
+            self.logger.info(f"Deleting backup: {backup_name}")
+            self.delete_backup(backup_name)
+            self.logger.info(f"Deleted backup: {backup_name}")
+        else:
+            self.logger.info(f"Invalid backup index: {backup_index}")
 
     def interactive_delete_backup(self):
         """Offer an interactive selection to delete backups."""
         self.logger.info("Starting interactive backup deletion")
         selected_backup = self.interactive_select_backup()
         if selected_backup:
-            all_backups = self.list_backups()[0] + self.list_backups()[1]
-            backup_index = all_backups.index(selected_backup)
-            self.delete_backup_by_index(backup_index)
+            backup_name = Path(selected_backup).name
+            self.delete_backup_by_name(backup_name)
 
     def display_backups(self):
         """Display all backups without deleting them."""
@@ -132,18 +103,6 @@ def cleanup_dangling_snapshots(self):
                     self.logger.info(f"Deleted snapshot: {snapshot_name}")
                     deleted_snapshots.add(snapshot_name)
 
-    def delete_old_backups(self, retention):
-        """Delete backups that exceed the retention limit."""
-        self.logger.debug(f"Deleting old backups exceeding retention: {retention}")
-        full_backups, _ = self.list_backups()
-        if len(full_backups) > retention:
-            for backup in full_backups[retention:]:
-                backup_name = Path(backup).name
-                self.logger.info(f"Deleting old backup: {backup_name}")
-                self.lifecycle_manager.delete_dataset(backup)
-                self.snapshot_manager.delete_snapshots(backup_name)
-                self.logger.info(f"Deleted old backup: {backup_name} and associated snapshots")
-
     def delete_old_exports(self, retention):
         """Delete exports that exceed the retention limit."""
         self.logger.debug(f"Deleting old exports exceeding retention: {retention}")
diff --git a/functions/backup_restore/base_manager.py b/functions/backup_restore/base_manager.py
index c83610c..04ffc8f 100644
--- a/functions/backup_restore/base_manager.py
+++ b/functions/backup_restore/base_manager.py
@@ -1,3 +1,5 @@
+import re
+import shutil
 from datetime import datetime
 from pathlib import Path
 from zfs.lifecycle import ZFSLifecycleManager
@@ -41,6 +43,48 @@ def list_backups(self):
         self.logger.debug(f"Found {len(full_backups)} full backups and {len(export_dirs)} export directories")
         return full_backups, export_dirs
 
+    def _list_snapshots_for_backup(self, backup_name: str):
+        """List all snapshots matching a specific backup name."""
+        self.logger.debug(f"Listing snapshots for backup: {backup_name}")
+        all_snapshots = self.snapshot_manager.list_snapshots()
+        pattern = re.compile(r'HeavyScript--\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}')
+        matching_snapshots = [snap for snap in all_snapshots if pattern.search(snap) and snap.endswith(f"@{backup_name}")]
+        self.logger.debug(f"Found {len(matching_snapshots)} snapshots for backup: {backup_name}")
+        return matching_snapshots
+
+    def delete_backup(self, backup_name: str):
+        """Delete a specific backup and its associated snapshots by name."""
+        full_backups, export_dirs = self.list_backups()
+
+        for backup in full_backups:
+            if backup.endswith(backup_name):
+                self.logger.info(f"Deleting full backup: {backup}")
+                self.lifecycle_manager.delete_dataset(backup)
+                snapshots = self._list_snapshots_for_backup(backup_name)
+                for snapshot in snapshots:
+                    self.snapshot_manager.delete_snapshot(snapshot)
+                self.logger.info(f"Deleted full backup: {backup} and associated snapshots")
+                return True
+
+        for export in export_dirs:
+            if export.name == backup_name:
+                self.logger.info(f"Deleting export: {export}")
+                shutil.rmtree(export)
+                self.logger.info(f"Deleted export: {export}")
+                return True
+
+        self.logger.info(f"Backup {backup_name} not found")
+        return False
+
+    def delete_old_backups(self, retention: int):
+        """Delete backups that exceed the retention limit."""
+        self.logger.debug(f"Deleting old backups exceeding retention: {retention}")
+        full_backups, _ = self.list_backups()
+        if len(full_backups) > retention:
+            for backup in full_backups[retention:]:
+                backup_name = Path(backup).name
+                self.delete_backup(backup_name)
+
     def interactive_select_backup(self, backup_type="all"):
         """
         Offer an interactive selection of backups.
diff --git a/functions/backup_restore/restore_manager.py b/functions/backup_restore/restore_manager.py
index b09c2e8..fb98e4f 100644
--- a/functions/backup_restore/restore_manager.py
+++ b/functions/backup_restore/restore_manager.py
@@ -39,8 +39,7 @@ def remove_newer_backups(self, backup_name: str):
             for backup in newer_backups:
                 newer_backup_name = Path(backup).name
                 self.logger.info(f"Deleting newer backup due to restore: {newer_backup_name}")
-                self.lifecycle_manager.delete_dataset(backup)
-                self.snapshot_manager.delete_snapshots(newer_backup_name)
+                self.delete_backup(newer_backup_name)
                 self.logger.info(f"Deleted backup: {newer_backup_name} and associated snapshots.")
 
         return True

From 041d45afe88986a2aa9133d1d3d33ad8325918ad Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 20:31:52 -0600
Subject: [PATCH 39/83] Refactor backup_manager.py to improve cleanup process
 and retention handling

---
 functions/backup_restore/backup_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/backup_manager.py b/functions/backup_restore/backup_manager.py
index 52785ae..d15dc11 100644
--- a/functions/backup_restore/backup_manager.py
+++ b/functions/backup_restore/backup_manager.py
@@ -1,3 +1,4 @@
+import re
 import shutil
 from pathlib import Path
 from base_manager import BaseManager
@@ -27,7 +28,6 @@ def export_chart_info(self, retention=None):
         exporter = ChartInfoExporter(self.backup_abs_path)
         exporter.export()
         self.logger.info("Chart information export completed successfully")
-        self.cleanup_dangling_snapshots()
         if retention is not None:
             self.delete_old_exports(retention)
 

From 737a5eb91af756caa118ec30a2edc648844a142b Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 20:33:51 -0600
Subject: [PATCH 40/83] Refactor delete_snapshots method in BackupManager to
 handle dangling snapshots

---
 functions/backup_restore/backup_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/backup_manager.py b/functions/backup_restore/backup_manager.py
index d15dc11..b87da78 100644
--- a/functions/backup_restore/backup_manager.py
+++ b/functions/backup_restore/backup_manager.py
@@ -99,7 +99,7 @@ def cleanup_dangling_snapshots(self):
                 snapshot_name = match.group()
                 if snapshot_name not in full_backup_names and snapshot_name not in deleted_snapshots:
                     self.logger.info(f"Deleting dangling snapshot: {snapshot_name}")
-                    self.snapshot_manager.delete_snapshots(snapshot_name)
+                    self.snapshot_manager.delete_snapshot(snapshot_name)
                     self.logger.info(f"Deleted snapshot: {snapshot_name}")
                     deleted_snapshots.add(snapshot_name)
 

From 99203a6eb1bdd92add336cc9eb5dcab7e983f100 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 20:54:57 -0600
Subject: [PATCH 41/83] use a property for all datasets instead of a function

---
 functions/backup_restore/backup/backup.py  |  3 +--
 functions/backup_restore/backup_manager.py |  4 +---
 functions/backup_restore/base_manager.py   |  2 +-
 functions/backup_restore/zfs/lifecycle.py  | 15 ++++++++++++---
 4 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index 9b2262a..19d66cb 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -220,9 +220,8 @@ def _backup_application_datasets(self):
         - applications_dataset (str): The root dataset under which Kubernetes operates.
         """
         datasets_to_ignore = KubeUtils().to_ignore_datasets_on_backup(self.kubeconfig.dataset)
-        all_datasets = self.lifecycle_manager.list_datasets()
 
-        datasets_to_backup = [ds for ds in all_datasets if ds.startswith(self.kubeconfig.dataset) and ds not in datasets_to_ignore]
+        datasets_to_backup = [ds for ds in self.lifecycle_manager.datasets if ds.startswith(self.kubeconfig.dataset) and ds not in datasets_to_ignore]
         self.logger.debug(f"Snapshotting datasets: {datasets_to_backup}")
 
         for dataset in datasets_to_backup:
diff --git a/functions/backup_restore/backup_manager.py b/functions/backup_restore/backup_manager.py
index b87da78..61c0123 100644
--- a/functions/backup_restore/backup_manager.py
+++ b/functions/backup_restore/backup_manager.py
@@ -91,17 +91,15 @@ def cleanup_dangling_snapshots(self):
 
         all_snapshots = self.snapshot_manager.list_snapshots()
         pattern = re.compile(r'HeavyScript--\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}')
-        deleted_snapshots = set()
 
         for snapshot in all_snapshots:
             match = pattern.search(snapshot)
             if match:
                 snapshot_name = match.group()
-                if snapshot_name not in full_backup_names and snapshot_name not in deleted_snapshots:
+                if snapshot_name not in full_backup_names:
                     self.logger.info(f"Deleting dangling snapshot: {snapshot_name}")
                     self.snapshot_manager.delete_snapshot(snapshot_name)
                     self.logger.info(f"Deleted snapshot: {snapshot_name}")
-                    deleted_snapshots.add(snapshot_name)
 
     def delete_old_exports(self, retention):
         """Delete exports that exceed the retention limit."""
diff --git a/functions/backup_restore/base_manager.py b/functions/backup_restore/base_manager.py
index 04ffc8f..ce1b6ea 100644
--- a/functions/backup_restore/base_manager.py
+++ b/functions/backup_restore/base_manager.py
@@ -29,7 +29,7 @@ def list_backups(self):
         """List all backups in the parent dataset, separated into full backups and exports."""
         self.logger.debug("Listing all backups")
         full_backups = sorted(
-            (ds for ds in self.lifecycle_manager.list_datasets() if ds.startswith(f"{self.backup_dataset_parent}/HeavyScript--")),
+            (ds for ds in self.lifecycle_manager.datasets if ds.startswith(f"{self.backup_dataset_parent}/HeavyScript--")),
             key=lambda ds: datetime.strptime(ds.split('/')[-1].replace("HeavyScript--", ""), '%Y-%m-%d_%H:%M:%S'),
             reverse=True
         )
diff --git a/functions/backup_restore/zfs/lifecycle.py b/functions/backup_restore/zfs/lifecycle.py
index fd0b75c..af028b5 100644
--- a/functions/backup_restore/zfs/lifecycle.py
+++ b/functions/backup_restore/zfs/lifecycle.py
@@ -60,6 +60,15 @@ def create_dataset(self, dataset: str, options: dict = None) -> bool:
 
     @type_check
     def delete_dataset(self, dataset: str) -> bool:
+        """
+        Delete a ZFS dataset, including all its snapshots.
+
+        Parameters:
+        - dataset (str): The name of the dataset to delete.
+
+        Returns:
+        - bool: True if the dataset was successfully deleted, False otherwise.
+        """
         if not self.dataset_exists(dataset):
             self.logger.warning(f"Dataset \"{dataset}\" does not exist. Cannot delete.")
             return False
@@ -87,10 +96,10 @@ def delete_dataset(self, dataset: str) -> bool:
             self.logger.error(f"Failed to delete dataset \"{dataset}\": {result.get_error()}")
             return False
 
-    @type_check
-    def list_datasets(self) -> list:
+    @property
+    def datasets(self) -> list:
         """
-        List all cached ZFS datasets.
+        Property to get the current list of cached ZFS datasets.
 
         Returns:
         - list: A list of all dataset names.

From b0d1add390bc91cbc216d3a7ccb83f4edcf9684f Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 21:41:16 -0600
Subject: [PATCH 42/83] Refactor snapshot deletion logic in BackupManager to
 handle dangling snapshots

---
 functions/backup_restore/backup_manager.py | 12 ++---
 functions/backup_restore/base_manager.py   |  3 +-
 functions/backup_restore/zfs/lifecycle.py  | 20 ++++-----
 functions/backup_restore/zfs/snapshot.py   | 51 ++++++----------------
 4 files changed, 30 insertions(+), 56 deletions(-)

diff --git a/functions/backup_restore/backup_manager.py b/functions/backup_restore/backup_manager.py
index 61c0123..9fa3478 100644
--- a/functions/backup_restore/backup_manager.py
+++ b/functions/backup_restore/backup_manager.py
@@ -89,17 +89,19 @@ def cleanup_dangling_snapshots(self):
         full_backups, _ = self.list_backups()
         full_backup_names = {Path(backup).name for backup in full_backups}
 
-        all_snapshots = self.snapshot_manager.list_snapshots()
         pattern = re.compile(r'HeavyScript--\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}')
 
-        for snapshot in all_snapshots:
+        for snapshot in self.snapshot_manager.snapshots:
             match = pattern.search(snapshot)
             if match:
                 snapshot_name = match.group()
                 if snapshot_name not in full_backup_names:
-                    self.logger.info(f"Deleting dangling snapshot: {snapshot_name}")
-                    self.snapshot_manager.delete_snapshot(snapshot_name)
-                    self.logger.info(f"Deleted snapshot: {snapshot_name}")
+                    self.logger.info(f"Deleting dangling snapshot: {snapshot}")
+                    delete_result = self.snapshot_manager.delete_snapshot(snapshot)
+                    if delete_result["success"]:
+                        self.logger.info(f"Deleted snapshot: {snapshot}")
+                    else:
+                        self.logger.error(f"Failed to delete snapshot {snapshot}: {delete_result['message']}")
 
     def delete_old_exports(self, retention):
         """Delete exports that exceed the retention limit."""
diff --git a/functions/backup_restore/base_manager.py b/functions/backup_restore/base_manager.py
index ce1b6ea..84fb050 100644
--- a/functions/backup_restore/base_manager.py
+++ b/functions/backup_restore/base_manager.py
@@ -46,9 +46,8 @@ def list_backups(self):
     def _list_snapshots_for_backup(self, backup_name: str):
         """List all snapshots matching a specific backup name."""
         self.logger.debug(f"Listing snapshots for backup: {backup_name}")
-        all_snapshots = self.snapshot_manager.list_snapshots()
         pattern = re.compile(r'HeavyScript--\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}')
-        matching_snapshots = [snap for snap in all_snapshots if pattern.search(snap) and snap.endswith(f"@{backup_name}")]
+        matching_snapshots = [snap for snap in self.snapshot_manager.snapshots if pattern.search(snap) and snap.endswith(f"@{backup_name}")]
         self.logger.debug(f"Found {len(matching_snapshots)} snapshots for backup: {backup_name}")
         return matching_snapshots
 
diff --git a/functions/backup_restore/zfs/lifecycle.py b/functions/backup_restore/zfs/lifecycle.py
index af028b5..76a12db 100644
--- a/functions/backup_restore/zfs/lifecycle.py
+++ b/functions/backup_restore/zfs/lifecycle.py
@@ -1,17 +1,13 @@
 from zfs.cache import ZFSCache
 from utils.shell import run_command
 from utils.type_check import type_check
-from utils.logger import get_logger
+from .cache import ZFSCache
 
-class ZFSLifecycleManager:
+class ZFSLifecycleManager(ZFSCache):
     """
     Class responsible for lifecycle operations of ZFS datasets, such as checking existence, creating, and deleting datasets.
     """
 
-    def __init__(self):
-        self.logger = get_logger()
-        self.cache = ZFSCache()
-
     @type_check
     def dataset_exists(self, dataset: str) -> bool:
         """
@@ -23,7 +19,7 @@ def dataset_exists(self, dataset: str) -> bool:
         Returns:
         - bool: True if the dataset exists, False otherwise.
         """
-        exists = dataset in self.cache.datasets
+        exists = dataset in self.datasets
         self.logger.debug(f"Dataset \"{dataset}\" exists: {exists}")
         return exists
 
@@ -51,7 +47,7 @@ def create_dataset(self, dataset: str, options: dict = None) -> bool:
         result = run_command(command, suppress_output=True)
 
         if result.is_success():
-            self.cache.add_dataset(dataset)
+            self.add_dataset(dataset)
             self.logger.debug(f"Dataset \"{dataset}\" created successfully.")
             return True
         else:
@@ -74,12 +70,12 @@ def delete_dataset(self, dataset: str) -> bool:
             return False
 
         # Delete all associated snapshots first
-        snapshots_to_delete = self.cache.get_snapshots_for_dataset(dataset)
+        snapshots_to_delete = self.get_snapshots_for_dataset(dataset)
         for snapshot in snapshots_to_delete:
             command = f"/sbin/zfs destroy \"{snapshot}\""
             result = run_command(command, suppress_output=True)
             if result.is_success():
-                self.cache.remove_snapshot(snapshot)
+                self.remove_snapshot(snapshot)
                 self.logger.debug(f"Snapshot \"{snapshot}\" deleted successfully.")
             else:
                 self.logger.error(f"Failed to delete snapshot \"{snapshot}\": {result.get_error()}")
@@ -89,7 +85,7 @@ def delete_dataset(self, dataset: str) -> bool:
         command = f"/sbin/zfs destroy -r \"{dataset}\""
         result = run_command(command, suppress_output=True)
         if result.is_success():
-            self.cache.remove_dataset(dataset)
+            self.remove_dataset(dataset)
             self.logger.debug(f"Dataset \"{dataset}\" deleted successfully.")
             return True
         else:
@@ -104,6 +100,6 @@ def datasets(self) -> list:
         Returns:
         - list: A list of all dataset names.
         """
-        datasets = list(self.cache.datasets)
+        datasets = list(self._datasets)
         self.logger.debug(f"Listing all datasets: {datasets}")
         return datasets
diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index 87321ad..4721e97 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -1,23 +1,13 @@
 from pathlib import Path
-
-from zfs.cache import ZFSCache
 from utils.shell import run_command
 from utils.type_check import type_check
-from utils.logger import get_logger
+from .cache import ZFSCache
 
-class ZFSSnapshotManager:
+class ZFSSnapshotManager(ZFSCache):
     """
     Class responsible for managing ZFS snapshots, including creation, deletion, and rollback operations.
     """
 
-    @type_check
-    def __init__(self):
-        """
-        Initialize the ZFSSnapshotManager class.
-        """
-        self.logger = get_logger()
-        self.cache = ZFSCache()
-
     @type_check
     def create_snapshot(self, snapshot_name: str, dataset: str) -> dict:
         """
@@ -35,7 +25,7 @@ def create_snapshot(self, snapshot_name: str, dataset: str) -> dict:
             "message": ""
         }
 
-        if dataset not in self.cache.datasets:
+        if dataset not in self.datasets:
             result["message"] = f"Dataset {dataset} does not exist."
             self.logger.error(result["message"])
             return result
@@ -45,7 +35,7 @@ def create_snapshot(self, snapshot_name: str, dataset: str) -> dict:
         snapshot_result = run_command(command)
         if snapshot_result.is_success():
             refer_size = self.get_snapshot_refer_size(snapshot_full_name)
-            self.cache.add_snapshot(snapshot_full_name, refer_size)
+            self.add_snapshot(snapshot_full_name, refer_size)
             self.logger.debug(f"Created snapshot: {snapshot_full_name} with refer size: {refer_size}")
             result["success"] = True
             result["message"] = f"Snapshot {snapshot_full_name} created successfully."
@@ -67,7 +57,7 @@ def get_snapshot_refer_size(self, snapshot: str) -> int:
         - int: The refer size of the snapshot in bytes.
         """
         try:
-            result = run_command(f"zfs list -H -o refer \"{snapshot}\"")
+            result = run_command(f"/sbin/zfs list -H -o refer \"{snapshot}\"")
             if result.is_success():
                 size_str = result.get_output()
                 size = self._convert_size_to_bytes(size_str)
@@ -79,18 +69,6 @@ def get_snapshot_refer_size(self, snapshot: str) -> int:
             self.logger.error(f"Exception occurred while getting refer size for snapshot {snapshot}: {e}")
             return 0
 
-    @type_check
-    def _convert_size_to_bytes(self, size_str):
-        size_units = {"K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}
-        try:
-            if size_str[-1] in size_units:
-                return int(float(size_str[:-1]) * size_units[size_str[-1]])
-            else:
-                return int(size_str)
-        except ValueError:
-            self.logger.error(f"Invalid size string: {size_str}")
-            return 0
-
     @type_check
     def delete_snapshot(self, snapshot: str) -> dict:
         """
@@ -110,7 +88,7 @@ def delete_snapshot(self, snapshot: str) -> dict:
         delete_command = f"/sbin/zfs destroy \"{snapshot}\""
         delete_result = run_command(delete_command)
         if delete_result.is_success():
-            self.cache.remove_snapshot(snapshot)
+            self.remove_snapshot(snapshot)
             self.logger.debug(f"Deleted snapshot: {snapshot}")
             result["success"] = True
             result["message"] = f"Snapshot {snapshot} deleted successfully."
@@ -139,7 +117,7 @@ def rollback_snapshot(self, snapshot: str, recursive: bool = False, force: bool
         }
 
         dataset_path, snapshot_name = snapshot.split('@', 1)
-        if dataset_path not in self.cache.datasets:
+        if dataset_path not in self.datasets:
             result["message"] = f"Dataset {dataset_path} does not exist. Cannot restore snapshot."
             self.logger.warning(result["message"])
             return result
@@ -162,15 +140,15 @@ def rollback_snapshot(self, snapshot: str, recursive: bool = False, force: bool
 
         return result
 
-    @type_check
-    def list_snapshots(self) -> list:
+    @property
+    def snapshots(self) -> list:
         """
         List all cached ZFS snapshots.
 
         Returns:
         - list: A list of all snapshot names.
         """
-        snapshots = list(self.cache.snapshots.keys())
+        snapshots = list(self.snapshots.keys())
         return snapshots
 
     @type_check
@@ -184,7 +162,7 @@ def snapshot_exists(self, snapshot_name: str) -> bool:
         Returns:
         - bool: True if the snapshot exists, False otherwise.
         """
-        return snapshot_name in self.cache.snapshots
+        return snapshot_name in self.snapshots
 
     @type_check
     def rollback_all_snapshots(self, snapshot_name: str, dataset_path: str, recursive: bool = False, force: bool = False) -> None:
@@ -197,12 +175,12 @@ def rollback_all_snapshots(self, snapshot_name: str, dataset_path: str, recursiv
         - recursive (bool): Whether to rollback recursively. Default is False.
         - force (bool): Whether to force the rollback. Default is False.
         """
-        if dataset_path not in self.cache.datasets:
+        if dataset_path not in self.datasets:
             self.logger.error(f"Dataset {dataset_path} does not exist. Cannot rollback snapshots.")
             return
 
         try:
-            all_snapshots = [snap for snap in self.cache.snapshots if snap.startswith(dataset_path)]
+            all_snapshots = [snap for snap in self.snapshots if snap.startswith(dataset_path)]
             matching_snapshots = [snap for snap in all_snapshots if snap.endswith(f"@{snapshot_name}")]
             for snapshot in matching_snapshots:
                 self.rollback_snapshot(snapshot, recursive, force)
@@ -266,8 +244,7 @@ def zfs_receive(self, snapshot_file: Path, dataset_path: str, decompress: bool =
         }
 
         try:
-            all_snapshots = self.list_snapshots()
-            dataset_snapshots = [snap for snap in all_snapshots if snap.startswith(f"{dataset_path}@")]
+            dataset_snapshots = [snap for snap in self.snapshots if snap.startswith(f"{dataset_path}@")]
 
             delete_errors = []
             for snapshot in dataset_snapshots:

From 7457acd0b856c654cc869405704221aca5138e8f Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 21:45:43 -0600
Subject: [PATCH 43/83] reference correct parent list

---
 functions/backup_restore/zfs/snapshot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/zfs/snapshot.py b/functions/backup_restore/zfs/snapshot.py
index 4721e97..cb41c92 100644
--- a/functions/backup_restore/zfs/snapshot.py
+++ b/functions/backup_restore/zfs/snapshot.py
@@ -148,7 +148,7 @@ def snapshots(self) -> list:
         Returns:
         - list: A list of all snapshot names.
         """
-        snapshots = list(self.snapshots.keys())
+        snapshots = list(self._snapshots.keys())
         return snapshots
 
     @type_check

From 09a640e99959b1b7556ce4812be9be381aef44d9 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 21:48:15 -0600
Subject: [PATCH 44/83] delete dangling snapshots AFTER we delete the full
 backups

---
 functions/backup_restore/backup_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/backup_manager.py b/functions/backup_restore/backup_manager.py
index 9fa3478..2d1b8fa 100644
--- a/functions/backup_restore/backup_manager.py
+++ b/functions/backup_restore/backup_manager.py
@@ -18,9 +18,9 @@ def backup_all(self, retention=None):
         backup = Backup(self.backup_abs_path)
         backup.backup_all()
         self.logger.info("Backup completed successfully")
-        self.cleanup_dangling_snapshots()
         if retention is not None:
             self.delete_old_backups(retention)
+        self.cleanup_dangling_snapshots()
 
     def export_chart_info(self, retention=None):
         """Export chart information with optional retention."""

From 2691de0fa542e86fb85bdf28f04067cbcbc2859d Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 23:31:23 -0600
Subject: [PATCH 45/83] Refactor backup and restore code to handle dangling
 snapshots

---
 functions/backup_restore/backup/backup.py       |  1 +
 functions/backup_restore/restore/restore_all.py |  2 +-
 .../backup_restore/restore/restore_base.py      | 17 +++++++++--------
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index 19d66cb..4bc8c4b 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -142,6 +142,7 @@ def backup_all(self):
                     self.logger.error(f"Failed to backup database for {app_name}: {result['message']}")
                     failures[app_name].append(result["message"])
 
+            # TODO: Print off better messages for each of the two types
             dataset_paths = self.kube_pvc_fetcher.get_volume_paths_by_namespace(f"ix-{app_name}")
             if dataset_paths:
                 self.logger.info(f"Backing up {app_name} PVCs...")
diff --git a/functions/backup_restore/restore/restore_all.py b/functions/backup_restore/restore/restore_all.py
index d7bca1a..d2b69ba 100644
--- a/functions/backup_restore/restore/restore_all.py
+++ b/functions/backup_restore/restore/restore_all.py
@@ -157,7 +157,7 @@ def _initial_kubernetes_setup(self):
 
         try:
             self.logger.info(f"Rolling back snapshots under {self.kube_config_reader.dataset}")
-            self.snapshot_manager.rollback_all_snapshots(self.snapshot_name, self.kube_config_reader.dataset)
+            self.snapshot_manager.rollback_all_snapshots(self.snapshot_name, self.kube_config_reader.dataset, recursive=True, force=True)
         except Exception as e:
             self.logger.error(f"Failed to rollback snapshots: {e}")
             raise Exception("Initial Kubernetes setup failed.")
diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 8c64eb8..915bc2b 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -36,7 +36,7 @@ def __init__(self, backup_dir: Path):
 
             print("Rolling back snapshot for backup dataset, ensuring integrity...")
             self.snapshot_manager = ZFSSnapshotManager()
-            self.snapshot_manager.rollback_all_snapshots(self.snapshot_name, self.backup_dataset)
+            self.snapshot_manager.rollback_all_snapshots(self.snapshot_name, self.backup_dataset, recursive=True, force=True)
 
             self.middleware = MiddlewareClientManager.fetch()
             self.kubernetes_config_file = self.backup_dir / "kubernetes_config" / "kubernetes_config.json"
@@ -162,13 +162,14 @@ def _rollback_volumes(self, app_name: str):
         # Rollback snapshots
         if snapshots_to_rollback:
             self.logger.info(f"Rolling back snapshots for {app_name}...")
-            rollback_result = self.snapshot_manager.rollback_snapshots(snapshots_to_rollback)
-            for message in rollback_result.get("messages", []):
-                if not rollback_result.get("success", False):
-                    self.failures[app_name].append(message)
-                    self.logger.error(message)
-                else:
-                    self.logger.debug(message)
+            for snapshot in snapshots_to_rollback:
+                rollback_result = self.snapshot_manager.rollback_snapshot(snapshot, recursive=True, force=True)
+                for message in rollback_result.get("messages", []):
+                    if not rollback_result.get("success", False):
+                        self.failures[app_name].append(message)
+                        self.logger.error(message)
+                    else:
+                        self.logger.debug(message)
 
         # Restore snapshots
         if snapshots_to_restore:

From ab0fe91511b8df84b2bfd1d65ab69eb8ffce66a2 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Tue, 28 May 2024 23:40:22 -0600
Subject: [PATCH 46/83] Refactor export cleanup process and retention handling

---
 functions/backup_restore/backup/export_.py | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/functions/backup_restore/backup/export_.py b/functions/backup_restore/backup/export_.py
index b558bbd..75d5e47 100644
--- a/functions/backup_restore/backup/export_.py
+++ b/functions/backup_restore/backup/export_.py
@@ -66,25 +66,6 @@ def export(self):
             self._convert_json_to_yaml(chart_info_dir / 'values.json')
         
         self.logger.info("Chart information export completed.")
-        self._cleanup_old_exports()
-
-    def _cleanup_old_exports(self):
-        """
-        Cleanup old exports if the number of exports exceeds the retention limit.
-        """
-        export_dirs = sorted(
-            (d for d in self.export_dir.iterdir() if d.is_dir() and d.name.startswith("Export--")),
-            key=lambda d: datetime.strptime(d.name.replace("Export--", ""), '%Y-%m-%d_%H:%M:%S')
-        )
-
-        if len(export_dirs) > self.retention_number:
-            for old_export_dir in export_dirs[:-self.retention_number]:
-                self.logger.info(f"Deleting oldest export due to retention limit: {old_export_dir.name}")
-                try:
-                    shutil.rmtree(old_export_dir)
-                    self.logger.debug(f"Removed old export: {old_export_dir}")
-                except Exception as e:
-                    self.logger.error(f"Failed to delete old export directory {old_export_dir}: {e}", exc_info=True)
 
     def _convert_json_to_yaml(self, json_file: Path):
         """

From d6d8ef520242310f0f31cb2f9648cda165f6d48b Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Wed, 29 May 2024 07:55:00 -0600
Subject: [PATCH 47/83] refactor restore message and volume handling

---
 .../backup_restore/kube/resources_restore.py  | 102 ++++-----
 .../backup_restore/restore/restore_all.py     |  10 +-
 .../backup_restore/restore/restore_base.py    | 210 +++++++-----------
 3 files changed, 132 insertions(+), 190 deletions(-)

diff --git a/functions/backup_restore/kube/resources_restore.py b/functions/backup_restore/kube/resources_restore.py
index fa688a8..39c0ee7 100644
--- a/functions/backup_restore/kube/resources_restore.py
+++ b/functions/backup_restore/kube/resources_restore.py
@@ -93,69 +93,69 @@ def restore_namespace(self, namespace_file: Path) -> bool:
             return False
 
     @type_check
-    def restore_secrets(self, secret_files: List[Path]) -> list:
+    def restore_secret(self, secret_file: Path) -> dict:
         """
-        Restore secrets for the application from its backup directory.
+        Restore a single secret for the application from its backup directory.
 
         Parameters:
-        - secret_files (List[Path]): List of secret file paths to restore.
+        - secret_file (Path): Path of the secret file to restore.
 
         Returns:
-        - list: List of files that failed to restore. If everything succeeds, returns an empty list.
+        - dict: Result containing status and message.
         """
-        self.logger.debug("Restoring secrets from provided file list...")
-        failures = []
+        result = {
+            "success": False,
+            "message": ""
+        }
 
-        if not secret_files:
-            self.logger.warning("No secret files provided.")
-            return []
-
-        for secret_file in secret_files:
-            self.logger.debug(f"Restoring secret from file: {secret_file}")
-            try:
-                with open(secret_file, 'r') as f:
-                    secret_body = yaml.safe_load(f)
-                    secret_body['metadata'].pop('resourceVersion', None)
-                    secret_body['metadata'].pop('uid', None)
-                    secret_body['metadata']['annotations'] = secret_body['metadata'].get('annotations', {})
-                    secret_body['metadata']['annotations']['kubectl.kubernetes.io/last-applied-configuration'] = yaml.dump(secret_body)
-                with open(secret_file, 'w') as f:
-                    yaml.dump(secret_body, f)
-                restoreResult = run_command(f"k3s kubectl apply -f \"{secret_file}\" --validate=false")
-                if restoreResult.is_success():
-                    self.logger.debug(f"Restored {secret_file.name}")
-                else:
-                    self.logger.error(f"Failed to restore {secret_file.name}: {restoreResult.get_error()}")
-                    failures.append(secret_file.name)
-            except Exception as e:
-                self.logger.error(f"Error processing secret file {secret_file}: {e}")
-                failures.append(secret_file.name)
-        return failures
+        self.logger.debug(f"Restoring secret from file: {secret_file}")
+        try:
+            with open(secret_file, 'r') as f:
+                secret_body = yaml.safe_load(f)
+                secret_body['metadata'].pop('resourceVersion', None)
+                secret_body['metadata'].pop('uid', None)
+                secret_body['metadata']['annotations'] = secret_body['metadata'].get('annotations', {})
+                secret_body['metadata']['annotations']['kubectl.kubernetes.io/last-applied-configuration'] = yaml.dump(secret_body)
+            with open(secret_file, 'w') as f:
+                yaml.dump(secret_body, f)
+            restore_result = run_command(f"k3s kubectl apply -f \"{secret_file}\" --validate=false")
+            if restore_result.is_success():
+                self.logger.debug(f"Restored {secret_file.name}")
+                result["success"] = True
+                result["message"] = f"Restored {secret_file.name} successfully."
+            else:
+                self.logger.error(f"Failed to restore {secret_file.name}: {restore_result.get_error()}")
+                result["message"] = f"Failed to restore {secret_file.name}: {restore_result.get_error()}"
+        except Exception as e:
+            self.logger.error(f"Error processing secret file {secret_file}: {e}")
+            result["message"] = f"Error processing secret file {secret_file}: {e}"
+        
+        return result
 
     @type_check
-    def restore_crd(self, crd_files: List[Path]) -> list:
+    def restore_crd(self, crd_file: Path) -> dict:
         """
-        Restore CRDs for the application from its backup directory.
+        Restore a single CRD for the application from its backup directory.
 
         Parameters:
-        - crd_files (List[Path]): List of CRD file paths to restore.
+        - crd_file (Path): Path of the CRD file to restore.
 
         Returns:
-        - list: List of files that failed to restore. If everything succeeds, returns an empty list.
+        - dict: Result containing status and message.
         """
-        self.logger.debug("Restoring CRDs from provided file list...")
-        failures = []
-
-        if not crd_files:
-            self.logger.warning("No CRD files provided.")
-            return []
-
-        for file in crd_files:
-            self.logger.debug(f"Restoring CRD from file: {file}")
-            restoreResult = run_command(f"k3s kubectl apply -f \"{file}\" --validate=false")
-            if restoreResult.is_success():
-                self.logger.debug(f"Restored {file.name}")
-            else:
-                self.logger.error(f"Failed to restore {file.name}: {restoreResult.get_error()}")
-                failures.append(file.name)
-        return failures
\ No newline at end of file
+        result = {
+            "success": False,
+            "message": ""
+        }
+
+        self.logger.debug(f"Restoring CRD from file: {crd_file}")
+        restore_result = run_command(f"k3s kubectl apply -f \"{crd_file}\" --validate=false")
+        if restore_result.is_success():
+            self.logger.debug(f"Restored {crd_file.name}")
+            result["success"] = True
+            result["message"] = f"Restored {crd_file.name} successfully."
+        else:
+            self.logger.error(f"Failed to restore {crd_file.name}: {restore_result.get_error()}")
+            result["message"] = f"Failed to restore {crd_file.name}: {restore_result.get_error()}"
+
+        return result
\ No newline at end of file
diff --git a/functions/backup_restore/restore/restore_all.py b/functions/backup_restore/restore/restore_all.py
index d2b69ba..54b2527 100644
--- a/functions/backup_restore/restore/restore_all.py
+++ b/functions/backup_restore/restore/restore_all.py
@@ -40,7 +40,7 @@ def restore(self):
                 self._rollback_volumes(app_name)
             except Exception as e:
                 self.logger.error(f"Failed to rollback snapshots for {app_name}: {e}\n")
-                self.failures[app_name].append(f"Failed to rollback volume snapshots: {e}")
+                self.failures.setdefault(app_name, []).append(f"Failed to rollback volume snapshots: {e}")
 
         self.logger.info("\nStarting Kubernetes Services\n"
                          "----------------------------")
@@ -57,7 +57,7 @@ def restore(self):
             CatalogRestoreManager(self.catalog_dir).restore()
         except Exception as e:
             self.logger.warning(f"Failed to restore catalog: {e}")
-            self.failures["Catalog"].append(f"Restoration failed: {e}")
+            self.failures.setdefault(app_name, []).append(f"Restoration failed: {e}")
 
         if self.chart_info.apps_with_crds:
             self.logger.info("\nRestoring Custom Resource Definitions\n"
@@ -90,7 +90,7 @@ def restore(self):
                     continue
             except Exception as e:
                 self.logger.error(f"Failed to restore {app_name}: {e}\n")
-                self.failures[app_name].append(f"Restoration failed: {e}")
+                self.failures.setdefault(app_name, []).append(f"Restoration failed: {e}")
                 continue
 
             self.logger.info("")
@@ -115,12 +115,12 @@ def restore(self):
                 db_manager = RestoreCNPGDatabase(app_name, self.chart_info.get_file(app_name, "database"))
                 result = db_manager.restore()
                 if not result["success"]:
-                    self.failures[app_name].append(result["message"])
+                    self.failures.setdefault(app_name, []).append(result["message"])
                 else:
                     self.logger.info(result["message"])
             except Exception as e:
                 self.logger.error(f"Failed to restore database for {app_name}: {e}")
-                self.failures[app_name].append(f"Database restore failed: {e}")
+                self.failures.setdefault(app_name, []).append(f"Database restore failed: {e}")
 
         self._log_failures()
 
diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 915bc2b..f87ce4f 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -83,7 +83,7 @@ def _log_failures(self):
     def _handle_critical_failure(self, app_name: str, error: str):
         """Handle a critical failure that prevents further restoration."""
         self.logger.error(f"Critical error for {app_name}: {error}")
-        self.failures[app_name].append(error)
+        self.failures.setdefault(app_name, []).append(error)
         if app_name not in self.critical_failures:
             self.critical_failures.append(app_name)
             self.chart_info.handle_critical_failure(app_name)
@@ -104,43 +104,72 @@ def _rollback_volumes(self, app_name: str):
         """
         self.logger.debug(f"Starting rollback process for {app_name}...")
         
+        def rollback_snapshot(snapshot: str, name: str, volume_type: str):
+            self.logger.info(f"Rolling back {volume_type} {name}...")
+            rollback_result = self.snapshot_manager.rollback_snapshot(snapshot, recursive=True, force=True)
+            if not rollback_result.get("success", False):
+                self.failures.setdefault(app_name, []).append(rollback_result.get("message", "Unknown error"))
+                self.logger.error(rollback_result.get("message", "Unknown error"))
+
+        def restore_snapshot(snapshot: str, name: str, volume_type: str):
+            self.logger.info(f"Restoring {volume_type} {name} from backup...")
+            snapshot_files = self.chart_info.get_file(app_name, "snapshots")
+            if snapshot_files:
+                for snapshot_file in snapshot_files:
+                    snapshot_file_path = snapshot_file
+                    snapshot_file_name = snapshot_file.stem.replace('%%', '/')
+                    dataset_path, _ = snapshot_file_name.split('@', 1)
+                    parent_dataset_path = '/'.join(dataset_path.split('/')[:-1])
+
+                    if not self.zfs_manager.dataset_exists(parent_dataset_path):
+                        self.logger.debug(f"Parent dataset {parent_dataset_path} does not exist. Creating it...")
+                        create_result = self.zfs_manager.create_dataset(parent_dataset_path)
+                        if not create_result["success"]:
+                            message = f"Failed to create parent dataset {parent_dataset_path}"
+                            self.failures.setdefault(app_name, []).append(message)
+                            self.logger.error(message)
+                            continue
+
+                    if snapshot_file_name == snapshot:
+                        restore_result = self.snapshot_manager.zfs_receive(snapshot_file_path, dataset_path, decompress=True)
+                        if not restore_result["success"]:
+                            self.failures.setdefault(app_name, []).append(restore_result["message"])
+                            self.logger.error(f"Failed to restore snapshot from {snapshot_file_path} for {app_name}: {restore_result['message']}")
+            else:
+                message = f"No snapshot files found for {app_name}"
+                self.failures.setdefault(app_name, []).append(message)
+                self.logger.error(message)
+
+        # Process PV files
         pv_files = self.chart_info.get_file(app_name, "pv_zfs_volumes")
         self.logger.debug(f"Found PV files for {app_name}: {pv_files}")
         pv_only_files = [file for file in pv_files if file.name.endswith('-pv.yaml')]
-
-        snapshots_to_rollback = []
-        snapshots_to_restore = []
-
-        self.logger.debug(f"Preparing to process PV files for {app_name}...")
-
-        # Process PV files
-        if pv_only_files:
-            for pv_file in pv_only_files:
-                try:
-                    with pv_file.open('r') as file:
-                        pv_data = yaml.safe_load(file)
-                    self.logger.debug(f"Loaded PV data from {pv_file}: {pv_data}")
-                    pool_name = pv_data['spec']['csi']['volumeAttributes']['openebs.io/poolname']
-                    volume_handle = pv_data['spec']['csi']['volumeHandle']
-                    dataset_path = f"{pool_name}/{volume_handle}"
-                    snapshot = f"{dataset_path}@{self.snapshot_name}"
-                    self.logger.debug(f"Constructed snapshot path: {snapshot}")
-
-                    if self.snapshot_manager.snapshot_exists(snapshot):
-                        snapshots_to_rollback.append(snapshot)
-                        self.logger.debug(f"Snapshot exists and will be rolled back: {snapshot}")
-                    elif any(snap.stem.replace('%%', '/') == snapshot for snap in self.chart_info.get_file(app_name, "snapshots") or []):
-                        snapshots_to_restore.append(snapshot)
-                        self.logger.debug(f"Snapshot to restore found in backups: {snapshot}")
-                    else:
-                        message = f"Snapshot {snapshot} cannot be rolled back or restored from backup."
-                        self.failures[app_name].append(message)
-                        self.logger.error(message)
-                except Exception as e:
-                    message = f"Failed to process PV file {pv_file}: {e}"
-                    self.logger.error(message, exc_info=True)
-                    self.failures[app_name].append(message)
-                    continue
+        
+        for pv_file in pv_only_files:
+            try:
+                with pv_file.open('r') as file:
+                    pv_data = yaml.safe_load(file)
+                self.logger.debug(f"Loaded PV data from {pv_file}: {pv_data}")
+                pool_name = pv_data['spec']['csi']['volumeAttributes']['openebs.io/poolname']
+                volume_handle = pv_data['spec']['csi']['volumeHandle']
+                dataset_path = f"{pool_name}/{volume_handle}"
+                snapshot = f"{dataset_path}@{self.snapshot_name}"
+                pv_name = pv_file.stem
+
+                self.logger.debug(f"Constructed snapshot path: {snapshot}")
+
+                if self.snapshot_manager.snapshot_exists(snapshot):
+                    rollback_snapshot(snapshot, pv_name, "PVC")
+                elif any(snap.stem.replace('%%', '/') == snapshot for snap in self.chart_info.get_file(app_name, "snapshots") or []):
+                    restore_snapshot(snapshot, pv_name, "PVC")
+                else:
+                    message = f"Snapshot {snapshot} for PVC {pv_name} cannot be rolled back or restored from backup."
+                    self.failures.setdefault(app_name, []).append(message)
+                    self.logger.error(message)
+            except Exception as e:
+                message = f"Failed to process PV file {pv_file}: {e}"
+                self.logger.error(message, exc_info=True)
+                self.failures.setdefault(app_name, []).append(message)
 
         # Process ix_volumes
         ix_volumes_dataset = self.chart_info.get_ix_volumes_dataset(app_name)
@@ -148,105 +177,16 @@ def _rollback_volumes(self, app_name: str):
         if ix_volumes_dataset:
             snapshot = f"{ix_volumes_dataset}@{self.snapshot_name}"
             self.logger.debug(f"Constructed ix_volumes snapshot path: {snapshot}")
+
             if self.snapshot_manager.snapshot_exists(snapshot):
-                snapshots_to_rollback.append(snapshot)
-                self.logger.debug(f"Snapshot exists and will be rolled back: {snapshot}")
+                rollback_snapshot(snapshot, "ix_volumes", "ix_volumes")
             elif any(snap.stem.replace('%%', '/') == snapshot for snap in self.chart_info.get_file(app_name, "snapshots") or []):
-                snapshots_to_restore.append(snapshot)
-                self.logger.debug(f"Snapshot to restore found in backups: {snapshot}")
+                restore_snapshot(snapshot, "ix_volumes", "ix_volumes")
             else:
                 message = f"Snapshot {snapshot} for ix_volumes cannot be rolled back or restored from backup."
-                self.failures[app_name].append(message)
+                self.failures.setdefault(app_name, []).append(message)
                 self.logger.error(message)
 
-        # Rollback snapshots
-        if snapshots_to_rollback:
-            self.logger.info(f"Rolling back snapshots for {app_name}...")
-            for snapshot in snapshots_to_rollback:
-                rollback_result = self.snapshot_manager.rollback_snapshot(snapshot, recursive=True, force=True)
-                for message in rollback_result.get("messages", []):
-                    if not rollback_result.get("success", False):
-                        self.failures[app_name].append(message)
-                        self.logger.error(message)
-                    else:
-                        self.logger.debug(message)
-
-        # Restore snapshots
-        if snapshots_to_restore:
-            self.logger.info(f"Restoring snapshots for {app_name}...")
-            restore_result = self._restore_snapshots(app_name, snapshots_to_restore)
-            for message in restore_result.get("messages", []):
-                if not restore_result.get("success", False):
-                    self.failures[app_name].append(message)
-                    self.logger.error(message)
-                else:
-                    self.logger.debug(message)
-
-    @type_check
-    def _restore_snapshots(self, app_name: str, snapshots_to_restore: list) -> dict:
-        """
-        Restore snapshots from the backup directory for a specific application.
-
-        Parameters:
-        - app_name (str): The name of the application to restore snapshots for.
-        - snapshots_to_restore (list): List of snapshots to restore.
-
-        Returns:
-        - dict: Result containing status and messages.
-        """
-        result = {
-            "success": True,
-            "messages": []
-        }
-
-        self.logger.debug(f"Starting snapshot restore process for {app_name} with snapshots: {snapshots_to_restore}")
-
-        snapshot_files = self.chart_info.get_file(app_name, "snapshots")
-        if snapshot_files:
-            self.logger.debug(f"Found snapshot files for {app_name}: {snapshot_files}")
-            for snapshot_file in snapshot_files:
-                # Keep the snapshot file path intact
-                snapshot_file_path = snapshot_file
-                self.logger.debug(f"Snapshot file path: {snapshot_file_path}")
-
-                # Extract the dataset path from the snapshot file name after replacing '%%' with '/'
-                snapshot_file_name = snapshot_file.stem.replace('%%', '/')
-                dataset_path, _ = snapshot_file_name.split('@', 1)
-                self.logger.debug(f"Dataset path extracted: {dataset_path}")
-
-                # Ensure the parent dataset exists
-                parent_dataset_path = '/'.join(dataset_path.split('/')[:-1])
-                if not self.zfs_manager.dataset_exists(parent_dataset_path):
-                    self.logger.debug(f"Parent dataset {parent_dataset_path} does not exist. Creating it...")
-                    create_result = self.zfs_manager.create_dataset(parent_dataset_path)
-                    if not create_result:
-                        message = f"Failed to create parent dataset {parent_dataset_path}"
-                        result["success"] = False
-                        result["messages"].append(message)
-                        self.failures[app_name].append(message)
-                        self.logger.error(message)
-                        continue
-
-                if snapshot_file_name in snapshots_to_restore:
-                    self.logger.debug(f"Restoring snapshot {snapshot_file_name} from {snapshot_file_path} to {dataset_path}")
-                    restore_result = self.snapshot_manager.zfs_receive(snapshot_file_path, dataset_path, decompress=True)
-                    if not restore_result["success"]:
-                        result["success"] = False
-                        result["messages"].append(restore_result["message"])
-                        self.failures[app_name].append(restore_result["message"])
-                        self.logger.error(f"Failed to restore snapshot from {snapshot_file_path} for {app_name}: {restore_result['message']}")
-                    else:
-                        result["messages"].append(f"Successfully restored snapshot from {snapshot_file_path} for {app_name}")
-                        self.logger.debug(f"Successfully restored snapshot from {snapshot_file_path} for {app_name}")
-        else:
-            message = f"No snapshot files found for {app_name}"
-            result["success"] = False
-            result["messages"].append(message)
-            self.failures[app_name].append(message)
-            self.logger.error(message)
-
-        return result
-
     @type_check
     def _restore_application(self, app_name: str) -> bool:
         """Restore a single application."""
@@ -294,12 +234,14 @@ def _restore_application(self, app_name: str) -> bool:
                     self.logger.error(f"Critical failure in restoring {app_name}, skipping further processing.\n")
                     return False
 
-            app_secrets = self.chart_info.get_file(app_name, "secrets")
-            if app_secrets:
-                self.logger.info(f"Restoring secrets for {app_name}...")
-                secret_failures = self.restore_resources.restore_secrets(app_secrets)
-                if secret_failures:
-                    self.failures[app_name].extend(secret_failures)
+                secret_files = self.chart_info.get_file(app_name, "secrets")
+                if secret_files:
+                    self.logger.info(f"Restoring secrets for {app_name}...")
+                    for secret_file in secret_files:
+                        secret_result = self.restore_resources.restore_secret(secret_file)
+                        if not secret_result.get("success", False):
+                            self.failures.setdefault(app_name, []).append(secret_result.get("message"))
+                            self.logger.error(secret_result.get("message"))
 
             if app_name not in self.create_list:
                 try:

From 51836ce709b4b04c33ec2528e7c90801db2cce74 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Wed, 29 May 2024 08:14:53 -0600
Subject: [PATCH 48/83] Refactor backup and restore code to create ZFS dataset
 for backups

---
 functions/backup_restore/backup/backup.py | 34 +++++++++++++++--------
 functions/backup_restore/base_manager.py  |  9 +++++-
 2 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index 4bc8c4b..61644c9 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -49,7 +49,7 @@ def __init__(self, backup_dir: Path, retention_number: int = 15):
 
         self.backup_dataset_parent = self.backup_dir.relative_to("/mnt")
         self.backup_dataset = str(self.backup_dataset_parent)
-        self._create_backup_dataset(self.backup_dataset)
+        self._create_backup_dataset()
 
         self.chart_collection = APIChartCollection()
         self.all_chart_names = self.chart_collection.all_chart_names
@@ -57,13 +57,20 @@ def __init__(self, backup_dir: Path, retention_number: int = 15):
 
         self.kube_pvc_fetcher = KubePVCFetcher()
 
-    def _create_backup_dataset(self, dataset):
+    def _create_backup_dataset(self):
         """
-        Create a ZFS dataset for backups if it doesn't already exist.
+        Create a ZFS dataset for backups.
         """
-        if not self.lifecycle_manager.dataset_exists(dataset):
-            if not self.lifecycle_manager.create_dataset(dataset):
-                raise RuntimeError(f"Failed to create backup dataset: {dataset}")
+        if not self.lifecycle_manager.dataset_exists(self.backup_dataset):
+            if not self.lifecycle_manager.create_dataset(
+                self.backup_dataset,
+                options={
+                    "atime": "off",
+                    "compression": "zstd-19",
+                    "recordsize": "1M"
+                }
+            ):
+                raise RuntimeError(f"Failed to create backup dataset: {self.backup_dataset}")
 
     def backup_all(self):
         """
@@ -145,14 +152,15 @@ def backup_all(self):
             # TODO: Print off better messages for each of the two types
             dataset_paths = self.kube_pvc_fetcher.get_volume_paths_by_namespace(f"ix-{app_name}")
             if dataset_paths:
-                self.logger.info(f"Backing up {app_name} PVCs...")
                 for dataset_path in dataset_paths:
-                    self.logger.info(f"Backing up PVCs for dataset {dataset_path}...")
+                    pvc_name = dataset_path.split('/')[-1]
+                    self.logger.info(f"Snapshotting PVC: {pvc_name}...")
 
                     # Check to see if dataset exists
                     if not self.lifecycle_manager.dataset_exists(dataset_path):
-                        self.logger.error(f"Dataset {dataset_path} does not exist.")
-                        failures[app_name].append(f"Dataset {dataset_path} does not exist.")
+                        error_msg = f"Dataset {dataset_path} does not exist."
+                        self.logger.error(error_msg)
+                        failures[app_name].append(error_msg)
                         continue
 
                     # Create the snapshot for the current dataset
@@ -160,8 +168,9 @@ def backup_all(self):
                     if not snapshot_result["success"]:
                         failures[app_name].append(snapshot_result["message"])
                         continue
-                    
+
                     # Send the snapshot to the backup directory
+                    self.logger.info(f"Sending snapshot stream to backup file...")
                     snapshot_name = f"{dataset_path}@{self.snapshot_name}"
                     backup_path = app_backup_dir / "snapshots" / f"{snapshot_name.replace('/', '%%')}.zfs"
                     backup_path.parent.mkdir(parents=True, exist_ok=True)
@@ -171,10 +180,11 @@ def backup_all(self):
 
             # Handle ix_volumes_dataset separately
             if chart_info.ix_volumes_dataset:
-                self.logger.info(f"Sending snapshots to backup directory...")
+                self.logger.info(f"Snapshotting ix_volumes...")
                 snapshot = chart_info.ix_volumes_dataset + "@" + self.snapshot_name
                 backup_path = app_backup_dir / "snapshots" / f"{snapshot.replace('/', '%%')}.zfs"
                 backup_path.parent.mkdir(parents=True, exist_ok=True)
+                self.logger.info(f"Sending snapshot stream to backup file...")
                 send_result = self.snapshot_manager.zfs_send(snapshot, backup_path, compress=True)
                 if not send_result["success"]:
                     failures[app_name].append(send_result["message"])
diff --git a/functions/backup_restore/base_manager.py b/functions/backup_restore/base_manager.py
index 84fb050..095a655 100644
--- a/functions/backup_restore/base_manager.py
+++ b/functions/backup_restore/base_manager.py
@@ -16,7 +16,14 @@ def __init__(self, backup_abs_path: Path):
         self.logger.debug(f"Initializing BaseManager for path: {self.backup_abs_path}")
 
         if not self.lifecycle_manager.dataset_exists(self.backup_dataset_parent):
-            self.lifecycle_manager.create_dataset(self.backup_dataset_parent)
+            self.lifecycle_manager.create_dataset(
+                self.backup_dataset_parent,
+                options={
+                    "atime": "off",
+                    "compression": "zstd-19",
+                    "recordsize": "1M"
+                }
+            )
             self.logger.debug(f"Created dataset: {self.backup_dataset_parent}")
 
     def _derive_dataset_parent(self):

From 6a4bf69e218eea9ed2f80cd93a01e6d0dd18363b Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Wed, 29 May 2024 10:53:15 -0600
Subject: [PATCH 49/83] Refactor restore_base.py to improve dataset creation
 and logging

---
 functions/backup_restore/restore/restore_base.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index f87ce4f..710d3f3 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -123,8 +123,7 @@ def restore_snapshot(snapshot: str, name: str, volume_type: str):
 
                     if not self.zfs_manager.dataset_exists(parent_dataset_path):
                         self.logger.debug(f"Parent dataset {parent_dataset_path} does not exist. Creating it...")
-                        create_result = self.zfs_manager.create_dataset(parent_dataset_path)
-                        if not create_result["success"]:
+                        if not self.zfs_manager.create_dataset(parent_dataset_path):
                             message = f"Failed to create parent dataset {parent_dataset_path}"
                             self.failures.setdefault(app_name, []).append(message)
                             self.logger.error(message)
@@ -173,8 +172,8 @@ def restore_snapshot(snapshot: str, name: str, volume_type: str):
 
         # Process ix_volumes
         ix_volumes_dataset = self.chart_info.get_ix_volumes_dataset(app_name)
-        self.logger.debug(f"Found ix_volumes dataset for {app_name}: {ix_volumes_dataset}")
         if ix_volumes_dataset:
+            self.logger.debug(f"Found ix_volumes dataset for {app_name}: {ix_volumes_dataset}")
             snapshot = f"{ix_volumes_dataset}@{self.snapshot_name}"
             self.logger.debug(f"Constructed ix_volumes snapshot path: {snapshot}")
 

From cc3881da6634a3ef7b739d55fa8bb03eeab220f1 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Wed, 29 May 2024 11:03:18 -0600
Subject: [PATCH 50/83] Refactor restore_base.py to restore CRDs for specified
 applications

---
 .../backup_restore/restore/restore_base.py      | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 710d3f3..7bf2b60 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -89,10 +89,21 @@ def _handle_critical_failure(self, app_name: str, error: str):
             self.chart_info.handle_critical_failure(app_name)
 
     def _restore_crds(self, app_name):
+        """
+        Restore CRDs for the specified application.
+
+        Parameters:
+        - app_name (str): The name of the application to restore CRDs for.
+        """
         self.logger.info(f"Restoring CRDs for {app_name}...")
-        crd_failures = self.restore_resources.restore_crd(self.chart_info.get_file(app_name, "crds"))
-        if crd_failures:
-            self.failures[app_name].extend(crd_failures)
+        crd_files = self.chart_info.get_file(app_name, "crds")
+
+        for crd_file in crd_files:
+            self.logger.info(f"Restoring CRD from file: {crd_file}")
+            restore_result = self.restore_resources.restore_crd(crd_file)
+            if not restore_result["success"]:
+                self.failures.setdefault(app_name, []).append(restore_result["message"])
+                self.logger.error(f"Failed to restore CRD from {crd_file}: {restore_result['message']}")
 
     @type_check
     def _rollback_volumes(self, app_name: str):

From c07a7cf0ee2230c81254b5fe15f88efb76f14a38 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Wed, 29 May 2024 11:06:58 -0600
Subject: [PATCH 51/83] Refactor restore_base.py logging to debug level for CRD
 restoration

---
 functions/backup_restore/restore/restore_base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 7bf2b60..8bac539 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -99,7 +99,7 @@ def _restore_crds(self, app_name):
         crd_files = self.chart_info.get_file(app_name, "crds")
 
         for crd_file in crd_files:
-            self.logger.info(f"Restoring CRD from file: {crd_file}")
+            self.logger.debug(f"Restoring CRD from file: {crd_file}")
             restore_result = self.restore_resources.restore_crd(crd_file)
             if not restore_result["success"]:
                 self.failures.setdefault(app_name, []).append(restore_result["message"])

From 920a67b8e8abe5d666bb8bc57cf37328c3cd85a7 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Wed, 29 May 2024 11:09:10 -0600
Subject: [PATCH 52/83] Refactor restore_base.py logging to debug level for CRD
 restoration

---
 functions/backup_restore/restore/restore_base.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 8bac539..fe56831 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -116,14 +116,14 @@ def _rollback_volumes(self, app_name: str):
         self.logger.debug(f"Starting rollback process for {app_name}...")
         
         def rollback_snapshot(snapshot: str, name: str, volume_type: str):
-            self.logger.info(f"Rolling back {volume_type} {name}...")
+            self.logger.info(f"{app_name}: rolling back {volume_type} {name}...")
             rollback_result = self.snapshot_manager.rollback_snapshot(snapshot, recursive=True, force=True)
             if not rollback_result.get("success", False):
                 self.failures.setdefault(app_name, []).append(rollback_result.get("message", "Unknown error"))
                 self.logger.error(rollback_result.get("message", "Unknown error"))
 
         def restore_snapshot(snapshot: str, name: str, volume_type: str):
-            self.logger.info(f"Restoring {volume_type} {name} from backup...")
+            self.logger.info(f"{app_name}: restoring {volume_type} {name} from backup...")
             snapshot_files = self.chart_info.get_file(app_name, "snapshots")
             if snapshot_files:
                 for snapshot_file in snapshot_files:

From 5551e0d138ac57af410bf6ee85ba942c2e14577b Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Wed, 29 May 2024 11:42:53 -0600
Subject: [PATCH 53/83] Refactor restore_base.py to set mountpoint to legacy
 for dataset paths

---
 functions/backup_restore/restore/restore_base.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index fe56831..d078191 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -10,6 +10,7 @@
 from kube.resources_restore import KubeRestoreResources
 from zfs.snapshot import ZFSSnapshotManager
 from zfs.lifecycle import ZFSLifecycleManager
+from utils.shell import run_command
 from utils.logger import setup_global_logger, set_logger
 from utils.singletons import MiddlewareClientManager
 from utils.type_check import type_check
@@ -114,7 +115,17 @@ def _rollback_volumes(self, app_name: str):
         - app_name (str): The name of the application to restore volumes for.
         """
         self.logger.debug(f"Starting rollback process for {app_name}...")
-        
+
+        def set_mountpoint_legacy(dataset_path):
+            command = f"/sbin/zfs set mountpoint=legacy \"{dataset_path}\""
+            result = run_command(command, suppress_output=True)
+            if result.is_success():
+                self.logger.debug(f"Set mountpoint to legacy for {dataset_path}")
+            else:
+                message = f"Failed to set mountpoint to legacy for {dataset_path}: {result.get_error()}"
+                self.failures.setdefault(app_name, []).append(message)
+                self.logger.error(message)
+
         def rollback_snapshot(snapshot: str, name: str, volume_type: str):
             self.logger.info(f"{app_name}: rolling back {volume_type} {name}...")
             rollback_result = self.snapshot_manager.rollback_snapshot(snapshot, recursive=True, force=True)
@@ -176,6 +187,9 @@ def restore_snapshot(snapshot: str, name: str, volume_type: str):
                     message = f"Snapshot {snapshot} for PVC {pv_name} cannot be rolled back or restored from backup."
                     self.failures.setdefault(app_name, []).append(message)
                     self.logger.error(message)
+                    continue
+
+                set_mountpoint_legacy(dataset_path)
             except Exception as e:
                 message = f"Failed to process PV file {pv_file}: {e}"
                 self.logger.error(message, exc_info=True)

From 6e3682c38a9a75910a9f938e7c735d426fb453d2 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Wed, 29 May 2024 11:58:10 -0600
Subject: [PATCH 54/83] Refactor restore_base.py to set mountpoint to legacy
 for dataset paths

---
 functions/backup_restore/database/base.py       | 6 +++---
 functions/backup_restore/restore/restore_all.py | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/functions/backup_restore/database/base.py b/functions/backup_restore/database/base.py
index 50694f0..bd42def 100644
--- a/functions/backup_restore/database/base.py
+++ b/functions/backup_restore/database/base.py
@@ -34,9 +34,9 @@ def __init__(self, app_name: str):
         self.error = None
 
         # # Fetch database name and user if needed
-        # if self.chart_info.chart_name != "immich":
-        self.database_name = self.fetch_database_name()
-        self.database_user = self.fetch_database_user() or self.database_name
+        if self.chart_info.chart_name != "immich":
+            self.database_name = self.fetch_database_name()
+            self.database_user = self.fetch_database_user() or self.database_name
 
     def fetch_primary_pod(self, timeout=600, interval=5) -> str:
         """
diff --git a/functions/backup_restore/restore/restore_all.py b/functions/backup_restore/restore/restore_all.py
index 54b2527..103ca41 100644
--- a/functions/backup_restore/restore/restore_all.py
+++ b/functions/backup_restore/restore/restore_all.py
@@ -11,6 +11,10 @@ def __init__(self, backup_dir: Path):
         super().__init__(backup_dir)
 
     def restore(self):
+        if not self.chart_info.all_releases:
+            self.logger.error("No releases found in backup directory.")
+            return
+
         """Perform the entire restore process."""
         self.logger.info("Building Restore Plan\n"
                         "----------------------")
@@ -20,10 +24,6 @@ def restore(self):
             self.logger.error(str(e))
             return
 
-        if not self.chart_info.all_releases:
-            self.logger.error("No releases found in backup directory.")
-            return
-
         self.logger.info("Performing Initial Kubernetes Operations\n"
                          "----------------------------------------")
         try:

From 3d3a0bddc59fbb487fe66693f1e36ca057f446fc Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Wed, 29 May 2024 12:30:46 -0600
Subject: [PATCH 55/83] Refactor restore_base.py to use logger for rolling back
 snapshot

---
 functions/backup_restore/restore/restore_base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index d078191..5d3098c 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -35,7 +35,7 @@ def __init__(self, backup_dir: Path):
             self.backup_chart_dir = self.backup_dir / "charts"
             self.catalog_dir = self.backup_dir / "catalog"
 
-            print("Rolling back snapshot for backup dataset, ensuring integrity...")
+            self.logger.info("Rolling back snapshot for backup dataset, ensuring integrity...")
             self.snapshot_manager = ZFSSnapshotManager()
             self.snapshot_manager.rollback_all_snapshots(self.snapshot_name, self.backup_dataset, recursive=True, force=True)
 

From 965b6e35521c38895c026b55341cd94d98a99c96 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Wed, 29 May 2024 12:47:43 -0600
Subject: [PATCH 56/83] remove extra indent for secrets

---
 functions/backup_restore/restore/restore_base.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/functions/backup_restore/restore/restore_base.py b/functions/backup_restore/restore/restore_base.py
index 5d3098c..3139aa6 100644
--- a/functions/backup_restore/restore/restore_base.py
+++ b/functions/backup_restore/restore/restore_base.py
@@ -258,14 +258,14 @@ def _restore_application(self, app_name: str) -> bool:
                     self.logger.error(f"Critical failure in restoring {app_name}, skipping further processing.\n")
                     return False
 
-                secret_files = self.chart_info.get_file(app_name, "secrets")
-                if secret_files:
-                    self.logger.info(f"Restoring secrets for {app_name}...")
-                    for secret_file in secret_files:
-                        secret_result = self.restore_resources.restore_secret(secret_file)
-                        if not secret_result.get("success", False):
-                            self.failures.setdefault(app_name, []).append(secret_result.get("message"))
-                            self.logger.error(secret_result.get("message"))
+            secret_files = self.chart_info.get_file(app_name, "secrets")
+            if secret_files:
+                self.logger.info(f"Restoring secrets for {app_name}...")
+                for secret_file in secret_files:
+                    secret_result = self.restore_resources.restore_secret(secret_file)
+                    if not secret_result.get("success", False):
+                        self.failures.setdefault(app_name, []).append(secret_result.get("message"))
+                        self.logger.error(secret_result.get("message"))
 
             if app_name not in self.create_list:
                 try:

From 67747e3dabb7abdb349d671c9c4c95b409ac83b7 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 08:39:22 -0600
Subject: [PATCH 57/83] update config

---
 utils/update_config.py | 40 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 5444174..3e6ff92 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -20,23 +20,55 @@ def update_config(config_file_path):
     # Prepare the new content
     new_content = []
     in_databases_section = False
+    backup_section_exists = False
+    in_backup_section = False
 
     for line in lines:
+        # Detect if we are in the [databases] section
         if line.strip().lower() == '[databases]':
             in_databases_section = True
             continue
         if line.startswith('[') and in_databases_section:
             in_databases_section = False
-        if not in_databases_section:
-            new_content.append(line)
+        if in_databases_section:
+            continue
+
+        # Detect if we are in the [BACKUP] section
+        if line.strip().lower() == '[backup]':
+            backup_section_exists = True
+            in_backup_section = True
+        elif line.startswith('[') and in_backup_section:
+            in_backup_section = False
+
+        # Add lines to the new content, and check for existing keys in the [BACKUP] section
+        if in_backup_section:
+            if 'export_enabled' not in config['BACKUP']:
+                new_content.append('export_enabled=true\n')
+            if 'full_backup_enabled' not in config['BACKUP']:
+                new_content.append('full_backup_enabled=true\n')
+            if 'backup_snapshot_streams' not in config['BACKUP']:
+                new_content.append('backup_snapshot_streams=false\n')
+            if 'max_stream_size' not in config['BACKUP']:
+                new_content.append('# Maximum size of a backup stream, the default is 10GB, be careful when setting this higher\n')
+                new_content.append('# Especially considering PV\'s for plex, sonarr, radarr, etc. can be quite large\n')
+                new_content.append('# Example: max_stream_size=10GB, max_stream_size=20KB, max_stream_size=1TB\n')
+                new_content.append('# max_stream_size=10GB\n')
+        
+        new_content.append(line)
 
-    # Ensure the [BACKUP] section is added if it does not exist
-    if 'BACKUP' not in config:
+    # If the [BACKUP] section does not exist, add it
+    if not backup_section_exists:
         new_content.append('\n[BACKUP]\n')
         new_content.append('export_enabled=true\n')
         new_content.append('full_backup_enabled=true\n')
+        new_content.append('backup_snapshot_streams=false\n')
+        new_content.append('\n## String options ##\n')
         new_content.append('# Uncomment the following line to specify a custom dataset location for backups\n')
         new_content.append('# custom_dataset_location=\n')
+        new_content.append('\n# Maximum size of a backup stream, the default is 10GB, be careful when setting this higher\n')
+        new_content.append('# Especially considering PV\'s for plex, sonarr, radarr, etc. can be quite large\n')
+        new_content.append('# Example: max_stream_size=10GB, max_stream_size=20KB, max_stream_size=1TB\n')
+        new_content.append('# max_stream_size=10GB\n')
 
     # Write the new content back to the config file
     with config_file_path.open('w') as file:

From f51b7fa2f1d8b205e5c513f43aea21f87418e825 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 08:45:08 -0600
Subject: [PATCH 58/83] attempt new method

---
 utils/update_config.py | 48 ++++++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 23 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 3e6ff92..86a1d7a 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -22,6 +22,18 @@ def update_config(config_file_path):
     in_databases_section = False
     backup_section_exists = False
     in_backup_section = False
+    backup_options = {
+        'export_enabled': 'export_enabled=true\n',
+        'full_backup_enabled': 'full_backup_enabled=true\n',
+        'backup_snapshot_streams': 'backup_snapshot_streams=false\n',
+        'max_stream_size': (
+            '# Maximum size of a backup stream, the default is 10GB, be careful when setting this higher\n'
+            '# Especially considering PV\'s for plex, sonarr, radarr, etc. can be quite large\n'
+            '# Example: max_stream_size=10GB, max_stream_size=20KB, max_stream_size=1TB\n'
+            '# max_stream_size=10GB\n'
+        )
+    }
+    backup_keys = set(backup_options.keys())
 
     for line in lines:
         # Detect if we are in the [databases] section
@@ -40,35 +52,25 @@ def update_config(config_file_path):
         elif line.startswith('[') and in_backup_section:
             in_backup_section = False
 
-        # Add lines to the new content, and check for existing keys in the [BACKUP] section
+        # Add lines to the new content
         if in_backup_section:
-            if 'export_enabled' not in config['BACKUP']:
-                new_content.append('export_enabled=true\n')
-            if 'full_backup_enabled' not in config['BACKUP']:
-                new_content.append('full_backup_enabled=true\n')
-            if 'backup_snapshot_streams' not in config['BACKUP']:
-                new_content.append('backup_snapshot_streams=false\n')
-            if 'max_stream_size' not in config['BACKUP']:
-                new_content.append('# Maximum size of a backup stream, the default is 10GB, be careful when setting this higher\n')
-                new_content.append('# Especially considering PV\'s for plex, sonarr, radarr, etc. can be quite large\n')
-                new_content.append('# Example: max_stream_size=10GB, max_stream_size=20KB, max_stream_size=1TB\n')
-                new_content.append('# max_stream_size=10GB\n')
-        
+            key = line.split('=')[0].strip()
+            if key in backup_keys:
+                backup_keys.discard(key)
+
         new_content.append(line)
 
+    # If the [BACKUP] section exists but is missing some keys, add the missing keys
+    if backup_section_exists and backup_keys:
+        new_content.append('\n')
+        for key in backup_keys:
+            new_content.append(backup_options[key])
+
     # If the [BACKUP] section does not exist, add it
     if not backup_section_exists:
         new_content.append('\n[BACKUP]\n')
-        new_content.append('export_enabled=true\n')
-        new_content.append('full_backup_enabled=true\n')
-        new_content.append('backup_snapshot_streams=false\n')
-        new_content.append('\n## String options ##\n')
-        new_content.append('# Uncomment the following line to specify a custom dataset location for backups\n')
-        new_content.append('# custom_dataset_location=\n')
-        new_content.append('\n# Maximum size of a backup stream, the default is 10GB, be careful when setting this higher\n')
-        new_content.append('# Especially considering PV\'s for plex, sonarr, radarr, etc. can be quite large\n')
-        new_content.append('# Example: max_stream_size=10GB, max_stream_size=20KB, max_stream_size=1TB\n')
-        new_content.append('# max_stream_size=10GB\n')
+        for key in backup_options:
+            new_content.append(backup_options[key])
 
     # Write the new content back to the config file
     with config_file_path.open('w') as file:

From bd9448e560cb1b4bac64d10a6cdde192f16142c6 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 08:50:12 -0600
Subject: [PATCH 59/83] use config parser with allow_no_value

---
 utils/update_config.py | 89 ++++++++++++------------------------------
 1 file changed, 25 insertions(+), 64 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 86a1d7a..c38b973 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -5,76 +5,37 @@
 def update_config(config_file_path):
     config_file_path = Path(config_file_path)
     
-    # Read the original content preserving comments
-    with config_file_path.open('r') as file:
-        lines = file.readlines()
-
-    # Create a new config parser object
+    # Create a new config parser object with comments allowed
     config = configparser.ConfigParser(allow_no_value=True)
+    config.optionxform = str  # Preserve the letter case of keys
     config.read(config_file_path)
-    
+
     # Remove the [databases] section if it exists
     if 'databases' in config:
         config.remove_section('databases')
-
-    # Prepare the new content
-    new_content = []
-    in_databases_section = False
-    backup_section_exists = False
-    in_backup_section = False
-    backup_options = {
-        'export_enabled': 'export_enabled=true\n',
-        'full_backup_enabled': 'full_backup_enabled=true\n',
-        'backup_snapshot_streams': 'backup_snapshot_streams=false\n',
-        'max_stream_size': (
-            '# Maximum size of a backup stream, the default is 10GB, be careful when setting this higher\n'
-            '# Especially considering PV\'s for plex, sonarr, radarr, etc. can be quite large\n'
-            '# Example: max_stream_size=10GB, max_stream_size=20KB, max_stream_size=1TB\n'
-            '# max_stream_size=10GB\n'
-        )
-    }
-    backup_keys = set(backup_options.keys())
-
-    for line in lines:
-        # Detect if we are in the [databases] section
-        if line.strip().lower() == '[databases]':
-            in_databases_section = True
-            continue
-        if line.startswith('[') and in_databases_section:
-            in_databases_section = False
-        if in_databases_section:
-            continue
-
-        # Detect if we are in the [BACKUP] section
-        if line.strip().lower() == '[backup]':
-            backup_section_exists = True
-            in_backup_section = True
-        elif line.startswith('[') and in_backup_section:
-            in_backup_section = False
-
-        # Add lines to the new content
-        if in_backup_section:
-            key = line.split('=')[0].strip()
-            if key in backup_keys:
-                backup_keys.discard(key)
-
-        new_content.append(line)
-
-    # If the [BACKUP] section exists but is missing some keys, add the missing keys
-    if backup_section_exists and backup_keys:
-        new_content.append('\n')
-        for key in backup_keys:
-            new_content.append(backup_options[key])
-
-    # If the [BACKUP] section does not exist, add it
-    if not backup_section_exists:
-        new_content.append('\n[BACKUP]\n')
-        for key in backup_options:
-            new_content.append(backup_options[key])
-
-    # Write the new content back to the config file
+    
+    # Ensure the [BACKUP] section is added and contains the required options
+    if 'BACKUP' not in config:
+        config.add_section('BACKUP')
+
+    backup_section = config['BACKUP']
+
+    # Add required options if they do not exist
+    if 'export_enabled' not in backup_section:
+        backup_section['export_enabled'] = 'true'
+    if 'full_backup_enabled' not in backup_section:
+        backup_section['full_backup_enabled'] = 'true'
+    if 'backup_snapshot_streams' not in backup_section:
+        backup_section['backup_snapshot_streams'] = 'false'
+    if 'max_stream_size' not in backup_section:
+        backup_section['; Maximum size of a backup stream, the default is 10GB, be careful when setting this higher\n'
+                        '# Especially considering PV\'s for plex, sonarr, radarr, etc. can be quite large\n'
+                        '# Example: max_stream_size=10GB, max_stream_size=20KB, max_stream_size=1TB'] = None
+        backup_section['max_stream_size'] = '10GB'
+
+    # Write the updated config back to the file
     with config_file_path.open('w') as file:
-        file.writelines(new_content)
+        config.write(file, space_around_delimiters=False)
 
 if __name__ == "__main__":
     if len(sys.argv) != 2:

From f2a7c72e6e1c9ab07d7a41f65d2ae098042eb02d Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 09:00:46 -0600
Subject: [PATCH 60/83] Update config with missing sections and options from
 default config

---
 utils/update_config.py | 58 +++++++++++++++++++++++++++---------------
 1 file changed, 38 insertions(+), 20 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index c38b973..23b05b9 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -4,8 +4,13 @@
 
 def update_config(config_file_path):
     config_file_path = Path(config_file_path)
+    default_config_path = Path(__file__).parent / '.default.config.ini'
     
-    # Create a new config parser object with comments allowed
+    # Read the original content preserving comments
+    with config_file_path.open('r') as file:
+        lines = file.readlines()
+
+    # Load the existing config
     config = configparser.ConfigParser(allow_no_value=True)
     config.optionxform = str  # Preserve the letter case of keys
     config.read(config_file_path)
@@ -13,29 +18,42 @@ def update_config(config_file_path):
     # Remove the [databases] section if it exists
     if 'databases' in config:
         config.remove_section('databases')
-    
-    # Ensure the [BACKUP] section is added and contains the required options
-    if 'BACKUP' not in config:
-        config.add_section('BACKUP')
 
-    backup_section = config['BACKUP']
+    # Load the default config from .default.config
+    default_config_parser = configparser.ConfigParser(allow_no_value=True)
+    default_config_parser.optionxform = str  # Preserve the letter case of keys
+    default_config_parser.read(default_config_path)
 
-    # Add required options if they do not exist
-    if 'export_enabled' not in backup_section:
-        backup_section['export_enabled'] = 'true'
-    if 'full_backup_enabled' not in backup_section:
-        backup_section['full_backup_enabled'] = 'true'
-    if 'backup_snapshot_streams' not in backup_section:
-        backup_section['backup_snapshot_streams'] = 'false'
-    if 'max_stream_size' not in backup_section:
-        backup_section['; Maximum size of a backup stream, the default is 10GB, be careful when setting this higher\n'
-                        '# Especially considering PV\'s for plex, sonarr, radarr, etc. can be quite large\n'
-                        '# Example: max_stream_size=10GB, max_stream_size=20KB, max_stream_size=1TB'] = None
-        backup_section['max_stream_size'] = '10GB'
+    # Update the existing config with missing sections and options from the default config
+    for section in default_config_parser.sections():
+        if not config.has_section(section):
+            config.add_section(section)
+        for key, value in default_config_parser.items(section):
+            if not config.has_option(section, key):
+                config.set(section, key, value)
 
-    # Write the updated config back to the file
+    # Write the updated config back to the file preserving the original comments
     with config_file_path.open('w') as file:
-        config.write(file, space_around_delimiters=False)
+        in_databases_section = False
+        for line in lines:
+            if line.strip().lower() == '[databases]':
+                in_databases_section = True
+                continue
+            if line.startswith('[') and in_databases_section:
+                in_databases_section = False
+            if not in_databases_section:
+                file.write(line)
+        
+        file.write('\n')
+        for section in default_config_parser.sections():
+            if not config.has_section(section):
+                file.write(f'[{section}]\n')
+            for key, value in default_config_parser.items(section):
+                if not config.has_option(section, key):
+                    if value is None:
+                        file.write(f'{key}\n')
+                    else:
+                        file.write(f'{key}={value}\n')
 
 if __name__ == "__main__":
     if len(sys.argv) != 2:

From 98402df3aed4b4d1e9f18a05d12a15131a0a6168 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 09:03:11 -0600
Subject: [PATCH 61/83] add new options to backup section

---
 .default.config.ini | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.default.config.ini b/.default.config.ini
index 7b2585f..261ff0e 100644
--- a/.default.config.ini
+++ b/.default.config.ini
@@ -54,7 +54,13 @@ ignore=
 ## true/false options ##
 export_enabled=true
 full_backup_enabled=true
+backup_snapshot_streams=false
 
 ## String options ##
 # Uncomment the following line to specify a custom dataset location for backups
 # custom_dataset_location=
+
+# Maximum size of a backup stream, the default is 10GB, be careful when setting this higher
+# Especially considering PV's for plex, sonarr, radarr, etc. can be quite large
+# Example: max_stream_size=10GB, max_stream_size=20KB, max_stream_size=1TB
+# max_stream_size=10GB
\ No newline at end of file

From c3cf732f1ed1518f899126b1acdd08bb42178f98 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 09:05:46 -0600
Subject: [PATCH 62/83] Refactor update_config.py to exclude [databases]
 section when writing back to file

---
 utils/update_config.py | 36 ++++++++++++++++--------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 23b05b9..05083d3 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -19,35 +19,31 @@ def update_config(config_file_path):
     if 'databases' in config:
         config.remove_section('databases')
 
-    # Load the default config from .default.config
+    # Load the default config from .default.config.ini
     default_config_parser = configparser.ConfigParser(allow_no_value=True)
     default_config_parser.optionxform = str  # Preserve the letter case of keys
     default_config_parser.read(default_config_path)
 
-    # Update the existing config with missing sections and options from the default config
-    for section in default_config_parser.sections():
-        if not config.has_section(section):
-            config.add_section(section)
-        for key, value in default_config_parser.items(section):
-            if not config.has_option(section, key):
-                config.set(section, key, value)
+    # Collect lines to write back, excluding [databases] section
+    new_lines = []
+    in_databases_section = False
+    for line in lines:
+        if line.strip().lower() == '[databases]':
+            in_databases_section = True
+            continue
+        if line.startswith('[') and in_databases_section:
+            in_databases_section = False
+        if not in_databases_section:
+            new_lines.append(line)
 
-    # Write the updated config back to the file preserving the original comments
+    # Write the collected lines back to the file
     with config_file_path.open('w') as file:
-        in_databases_section = False
-        for line in lines:
-            if line.strip().lower() == '[databases]':
-                in_databases_section = True
-                continue
-            if line.startswith('[') and in_databases_section:
-                in_databases_section = False
-            if not in_databases_section:
-                file.write(line)
+        file.writelines(new_lines)
         
-        file.write('\n')
+        # Ensure new sections and options are added if missing
         for section in default_config_parser.sections():
             if not config.has_section(section):
-                file.write(f'[{section}]\n')
+                file.write(f'\n[{section}]\n')
             for key, value in default_config_parser.items(section):
                 if not config.has_option(section, key):
                     if value is None:

From 969b0a2a0376f2ad0a800c354ec0d75babc1b986 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 09:07:49 -0600
Subject: [PATCH 63/83] Refactor update_config.py to update missing sections
 and options from default config

---
 utils/update_config.py | 45 ++++++++++++++++++++++++++----------------
 1 file changed, 28 insertions(+), 17 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 05083d3..5e615c1 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -24,33 +24,44 @@ def update_config(config_file_path):
     default_config_parser.optionxform = str  # Preserve the letter case of keys
     default_config_parser.read(default_config_path)
 
-    # Collect lines to write back, excluding [databases] section
-    new_lines = []
-    in_databases_section = False
-    for line in lines:
-        if line.strip().lower() == '[databases]':
-            in_databases_section = True
-            continue
-        if line.startswith('[') and in_databases_section:
-            in_databases_section = False
-        if not in_databases_section:
-            new_lines.append(line)
-
-    # Write the collected lines back to the file
+    # Update the existing config with missing sections and options from the default config
+    for section in default_config_parser.sections():
+        if not config.has_section(section):
+            config.add_section(section)
+        for key, value in default_config_parser.items(section):
+            if not config.has_option(section, key):
+                config.set(section, key, value)
+
+    # Write the updated config back to the file preserving the original comments
     with config_file_path.open('w') as file:
-        file.writelines(new_lines)
+        in_databases_section = False
+        for line in lines:
+            if line.strip().lower() == '[databases]':
+                in_databases_section = True
+                continue
+            if line.startswith('[') and in_databases_section:
+                in_databases_section = False
+            if not in_databases_section:
+                file.write(line)
         
         # Ensure new sections and options are added if missing
         for section in default_config_parser.sections():
-            if not config.has_section(section):
+            if not any(f'[{section}]' in line for line in lines):
                 file.write(f'\n[{section}]\n')
-            for key, value in default_config_parser.items(section):
-                if not config.has_option(section, key):
+                for key, value in default_config_parser.items(section):
                     if value is None:
                         file.write(f'{key}\n')
                     else:
                         file.write(f'{key}={value}\n')
 
+            else:
+                for key, value in default_config_parser.items(section):
+                    if not any(key in line for line in lines):
+                        if value is None:
+                            file.write(f'{key}\n')
+                        else:
+                            file.write(f'{key}={value}\n')
+
 if __name__ == "__main__":
     if len(sys.argv) != 2:
         print("Usage: python update_config.py <config_file_path>")

From 7b25590074237f2d7315bb9d3c4cff8cedd903d1 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 09:14:11 -0600
Subject: [PATCH 64/83] make config updater a bit more general

---
 utils/update_config.py | 52 +++++++++++++++++++++---------------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 5e615c1..6ee2fe5 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -11,51 +11,51 @@ def update_config(config_file_path):
         lines = file.readlines()
 
     # Load the existing config
-    config = configparser.ConfigParser(allow_no_value=True)
-    config.optionxform = str  # Preserve the letter case of keys
-    config.read(config_file_path)
-
-    # Remove the [databases] section if it exists
-    if 'databases' in config:
-        config.remove_section('databases')
+    current_config = configparser.ConfigParser(allow_no_value=True)
+    current_config.optionxform = str  # Preserve the letter case of keys
+    current_config.read(config_file_path)
 
     # Load the default config from .default.config.ini
-    default_config_parser = configparser.ConfigParser(allow_no_value=True)
-    default_config_parser.optionxform = str  # Preserve the letter case of keys
-    default_config_parser.read(default_config_path)
+    default_config = configparser.ConfigParser(allow_no_value=True)
+    default_config.optionxform = str  # Preserve the letter case of keys
+    default_config.read(default_config_path)
+
+    # Remove sections from the current config that are not in the default config
+    sections_to_remove = [section for section in current_config.sections() if section not in default_config.sections()]
+    for section in sections_to_remove:
+        current_config.remove_section(section)
 
     # Update the existing config with missing sections and options from the default config
-    for section in default_config_parser.sections():
-        if not config.has_section(section):
-            config.add_section(section)
-        for key, value in default_config_parser.items(section):
-            if not config.has_option(section, key):
-                config.set(section, key, value)
+    for section in default_config.sections():
+        if not current_config.has_section(section):
+            current_config.add_section(section)
+        for key, value in default_config.items(section):
+            if not current_config.has_option(section, key):
+                current_config.set(section, key, value)
 
     # Write the updated config back to the file preserving the original comments
     with config_file_path.open('w') as file:
-        in_databases_section = False
+        in_removed_section = False
         for line in lines:
-            if line.strip().lower() == '[databases]':
-                in_databases_section = True
+            if any(line.strip().lower() == f'[{s.lower()}]' for s in sections_to_remove):
+                in_removed_section = True
                 continue
-            if line.startswith('[') and in_databases_section:
-                in_databases_section = False
-            if not in_databases_section:
+            if line.startswith('[') and in_removed_section:
+                in_removed_section = False
+            if not in_removed_section:
                 file.write(line)
         
         # Ensure new sections and options are added if missing
-        for section in default_config_parser.sections():
+        for section in default_config.sections():
             if not any(f'[{section}]' in line for line in lines):
                 file.write(f'\n[{section}]\n')
-                for key, value in default_config_parser.items(section):
+                for key, value in default_config.items(section):
                     if value is None:
                         file.write(f'{key}\n')
                     else:
                         file.write(f'{key}={value}\n')
-
             else:
-                for key, value in default_config_parser.items(section):
+                for key, value in default_config.items(section):
                     if not any(key in line for line in lines):
                         if value is None:
                             file.write(f'{key}\n')

From dde6ad9d353634eddf985490c445cca74aaa207d Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 09:19:43 -0600
Subject: [PATCH 65/83] Refactor update_config.py to remove sections not in
 default config

---
 utils/update_config.py | 75 ++++++++++++++++++------------------------
 1 file changed, 32 insertions(+), 43 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 6ee2fe5..4f8b738 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -6,61 +6,50 @@ def update_config(config_file_path):
     config_file_path = Path(config_file_path)
     default_config_path = Path(__file__).parent / '.default.config.ini'
     
-    # Read the original content preserving comments
-    with config_file_path.open('r') as file:
-        lines = file.readlines()
-
-    # Load the existing config
-    current_config = configparser.ConfigParser(allow_no_value=True)
-    current_config.optionxform = str  # Preserve the letter case of keys
-    current_config.read(config_file_path)
-
     # Load the default config from .default.config.ini
     default_config = configparser.ConfigParser(allow_no_value=True)
     default_config.optionxform = str  # Preserve the letter case of keys
     default_config.read(default_config_path)
-
-    # Remove sections from the current config that are not in the default config
+    
+    # Load the existing config
+    current_config = configparser.ConfigParser(allow_no_value=True)
+    current_config.optionxform = str  # Preserve the letter case of keys
+    current_config.read(config_file_path)
+    
+    # Collect sections to be removed (present in current but not in default)
     sections_to_remove = [section for section in current_config.sections() if section not in default_config.sections()]
-    for section in sections_to_remove:
-        current_config.remove_section(section)
-
-    # Update the existing config with missing sections and options from the default config
+    
+    # Prepare the new content by removing sections that are not in the default config
+    new_content = []
+    in_section_to_remove = False
+    
+    with config_file_path.open('r') as file:
+        for line in file:
+            if any(line.strip().lower() == f'[{section.lower()}]' for section in sections_to_remove):
+                in_section_to_remove = True
+                continue
+            if line.startswith('[') and in_section_to_remove:
+                in_section_to_remove = False
+            if not in_section_to_remove:
+                new_content.append(line)
+    
+    # Write the modified content back to the config file
+    with config_file_path.open('w') as file:
+        file.writelines(new_content)
+    
+    # Reload the config to update it with missing sections and options from the default config
+    current_config.read_string(''.join(new_content))
+    
     for section in default_config.sections():
         if not current_config.has_section(section):
             current_config.add_section(section)
         for key, value in default_config.items(section):
             if not current_config.has_option(section, key):
                 current_config.set(section, key, value)
-
-    # Write the updated config back to the file preserving the original comments
+    
+    # Write the final updated config back to the file, ensuring new sections and options are added
     with config_file_path.open('w') as file:
-        in_removed_section = False
-        for line in lines:
-            if any(line.strip().lower() == f'[{s.lower()}]' for s in sections_to_remove):
-                in_removed_section = True
-                continue
-            if line.startswith('[') and in_removed_section:
-                in_removed_section = False
-            if not in_removed_section:
-                file.write(line)
-        
-        # Ensure new sections and options are added if missing
-        for section in default_config.sections():
-            if not any(f'[{section}]' in line for line in lines):
-                file.write(f'\n[{section}]\n')
-                for key, value in default_config.items(section):
-                    if value is None:
-                        file.write(f'{key}\n')
-                    else:
-                        file.write(f'{key}={value}\n')
-            else:
-                for key, value in default_config.items(section):
-                    if not any(key in line for line in lines):
-                        if value is None:
-                            file.write(f'{key}\n')
-                        else:
-                            file.write(f'{key}={value}\n')
+        current_config.write(file, space_around_delimiters=False)
 
 if __name__ == "__main__":
     if len(sys.argv) != 2:

From 5d27615ece6529c7bea9079c403b783b9bbd28be Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 09:45:35 -0600
Subject: [PATCH 66/83] Refactor update_config.py to remove unused sections and
 options

---
 utils/update_config.py | 50 ++++++++++++++++++++++--------------------
 1 file changed, 26 insertions(+), 24 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 4f8b738..db4641e 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -5,48 +5,50 @@
 def update_config(config_file_path):
     config_file_path = Path(config_file_path)
     default_config_path = Path(__file__).parent / '.default.config.ini'
-    
+
     # Load the default config from .default.config.ini
-    default_config = configparser.ConfigParser(allow_no_value=True)
+    default_config = configparser.ConfigParser(allow_no_value=True, delimiters=("=",))
     default_config.optionxform = str  # Preserve the letter case of keys
     default_config.read(default_config_path)
-    
+
     # Load the existing config
-    current_config = configparser.ConfigParser(allow_no_value=True)
+    current_config = configparser.ConfigParser(allow_no_value=True, delimiters=("=",))
     current_config.optionxform = str  # Preserve the letter case of keys
     current_config.read(config_file_path)
-    
-    # Collect sections to be removed (present in current but not in default)
-    sections_to_remove = [section for section in current_config.sections() if section not in default_config.sections()]
-    
+
     # Prepare the new content by removing sections that are not in the default config
-    new_content = []
-    in_section_to_remove = False
-    
+    sections_to_remove = [section for section in current_config.sections() if section not in default_config.sections()]
+
+    # Read the original content preserving comments
     with config_file_path.open('r') as file:
-        for line in file:
-            if any(line.strip().lower() == f'[{section.lower()}]' for section in sections_to_remove):
-                in_section_to_remove = True
-                continue
-            if line.startswith('[') and in_section_to_remove:
-                in_section_to_remove = False
-            if not in_section_to_remove:
-                new_content.append(line)
-    
+        lines = file.readlines()
+
+    new_content = []
+    in_removed_section = False
+
+    for line in lines:
+        if any(line.strip().lower() == f'[{section.lower()}]' for section in sections_to_remove):
+            in_removed_section = True
+            continue
+        if line.startswith('[') and in_removed_section:
+            in_removed_section = False
+        if not in_removed_section:
+            new_content.append(line)
+
     # Write the modified content back to the config file
     with config_file_path.open('w') as file:
         file.writelines(new_content)
-    
+
     # Reload the config to update it with missing sections and options from the default config
     current_config.read_string(''.join(new_content))
-    
+
     for section in default_config.sections():
         if not current_config.has_section(section):
             current_config.add_section(section)
         for key, value in default_config.items(section):
             if not current_config.has_option(section, key):
                 current_config.set(section, key, value)
-    
+
     # Write the final updated config back to the file, ensuring new sections and options are added
     with config_file_path.open('w') as file:
         current_config.write(file, space_around_delimiters=False)
@@ -55,6 +57,6 @@ def update_config(config_file_path):
     if len(sys.argv) != 2:
         print("Usage: python update_config.py <config_file_path>")
         sys.exit(1)
-    
+
     config_file_path = sys.argv[1]
     update_config(config_file_path)

From facf25e7d36dd8324b37743c70d6521b977574e5 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 09:47:48 -0600
Subject: [PATCH 67/83] Refactor update_config.py to remove unused sections and
 options

---
 utils/update_config.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index db4641e..80426b3 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -24,15 +24,18 @@ def update_config(config_file_path):
         lines = file.readlines()
 
     new_content = []
-    in_removed_section = False
+    in_section = None
 
     for line in lines:
-        if any(line.strip().lower() == f'[{section.lower()}]' for section in sections_to_remove):
-            in_removed_section = True
-            continue
-        if line.startswith('[') and in_removed_section:
-            in_removed_section = False
-        if not in_removed_section:
+        section_header = line.strip().lower()
+        if section_header.startswith('[') and section_header.endswith(']'):
+            section_name = section_header[1:-1]
+            if section_name in sections_to_remove:
+                in_section = section_name
+                continue
+            in_section = None
+
+        if in_section is None:
             new_content.append(line)
 
     # Write the modified content back to the config file

From 59ecc7ee376cbb72f43717ce8dc78d730b90608e Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 09:53:06 -0600
Subject: [PATCH 68/83] switch to using configobj

---
 utils/update_config.py | 78 +++++++++++++++---------------------------
 1 file changed, 27 insertions(+), 51 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 80426b3..f8a950d 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -1,65 +1,41 @@
 import sys
 from pathlib import Path
-import configparser
+from configobj import ConfigObj
 
 def update_config(config_file_path):
     config_file_path = Path(config_file_path)
     default_config_path = Path(__file__).parent / '.default.config.ini'
 
-    # Load the default config from .default.config.ini
-    default_config = configparser.ConfigParser(allow_no_value=True, delimiters=("=",))
-    default_config.optionxform = str  # Preserve the letter case of keys
-    default_config.read(default_config_path)
-
-    # Load the existing config
-    current_config = configparser.ConfigParser(allow_no_value=True, delimiters=("=",))
-    current_config.optionxform = str  # Preserve the letter case of keys
-    current_config.read(config_file_path)
-
-    # Prepare the new content by removing sections that are not in the default config
-    sections_to_remove = [section for section in current_config.sections() if section not in default_config.sections()]
-
-    # Read the original content preserving comments
-    with config_file_path.open('r') as file:
-        lines = file.readlines()
-
-    new_content = []
-    in_section = None
-
-    for line in lines:
-        section_header = line.strip().lower()
-        if section_header.startswith('[') and section_header.endswith(']'):
-            section_name = section_header[1:-1]
-            if section_name in sections_to_remove:
-                in_section = section_name
-                continue
-            in_section = None
-
-        if in_section is None:
-            new_content.append(line)
-
-    # Write the modified content back to the config file
-    with config_file_path.open('w') as file:
-        file.writelines(new_content)
-
-    # Reload the config to update it with missing sections and options from the default config
-    current_config.read_string(''.join(new_content))
-
-    for section in default_config.sections():
-        if not current_config.has_section(section):
-            current_config.add_section(section)
-        for key, value in default_config.items(section):
-            if not current_config.has_option(section, key):
-                current_config.set(section, key, value)
-
-    # Write the final updated config back to the file, ensuring new sections and options are added
-    with config_file_path.open('w') as file:
-        current_config.write(file, space_around_delimiters=False)
+    # Load the existing config and default config
+    current_config = ConfigObj(config_file_path, encoding='utf-8', list_values=False)
+    default_config = ConfigObj(default_config_path, encoding='utf-8', list_values=False)
+
+    # Remove sections from current config that are not in the default config
+    for section in list(current_config.keys()):
+        if section not in default_config:
+            del current_config[section]
+
+    # Update sections and keys from the default config
+    for section, default_options in default_config.items():
+        if section not in current_config:
+            current_config[section] = default_options
+        else:
+            # Remove keys not present in the default config
+            for key in list(current_config[section].keys()):
+                if key not in default_options:
+                    del current_config[section][key]
+            # Add keys from the default config
+            for key, value in default_options.items():
+                if key not in current_config[section]:
+                    current_config[section][key] = value
+
+    # Write the updated config back to the file
+    current_config.write()
 
 if __name__ == "__main__":
     if len(sys.argv) != 2:
         print("Usage: python update_config.py <config_file_path>")
         sys.exit(1)
-
+    
     config_file_path = sys.argv[1]
     update_config(config_file_path)

From a145891d718e87bfdf753c70eb5eb77dcfd1094e Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 09:57:54 -0600
Subject: [PATCH 69/83] convert path obj to strings

---
 heavy_script.sh        |  2 +-
 utils/update_config.py | 16 +++++-----------
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/heavy_script.sh b/heavy_script.sh
index 72899db..7189c24 100644
--- a/heavy_script.sh
+++ b/heavy_script.sh
@@ -56,7 +56,7 @@ fi
 # generate the config.ini file if it does not exist
 generate_config_ini
 
-python3 utils/update_config.py "$script_path/config.ini"
+python3 utils/update_config.py
 
 # Separate bundled short options
 args=()
diff --git a/utils/update_config.py b/utils/update_config.py
index f8a950d..b702365 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -1,10 +1,9 @@
-import sys
 from pathlib import Path
 from configobj import ConfigObj
 
-def update_config(config_file_path):
-    config_file_path = Path(config_file_path)
-    default_config_path = Path(__file__).parent / '.default.config.ini'
+def update_config():
+    config_file_path = str(Path(__file__).parent / 'config.ini')
+    default_config_path = str(Path(__file__).parent / '.default.config.ini')
 
     # Load the existing config and default config
     current_config = ConfigObj(config_file_path, encoding='utf-8', list_values=False)
@@ -32,10 +31,5 @@ def update_config(config_file_path):
     # Write the updated config back to the file
     current_config.write()
 
-if __name__ == "__main__":
-    if len(sys.argv) != 2:
-        print("Usage: python update_config.py <config_file_path>")
-        sys.exit(1)
-    
-    config_file_path = sys.argv[1]
-    update_config(config_file_path)
+if __name__ == "__main__":    
+    update_config()

From bc011bdd117ad5a94aefc37d710ddf21f48cb5c1 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:01:46 -0600
Subject: [PATCH 70/83] debugging print statements

---
 utils/update_config.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index b702365..9162640 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -5,31 +5,42 @@ def update_config():
     config_file_path = str(Path(__file__).parent / 'config.ini')
     default_config_path = str(Path(__file__).parent / '.default.config.ini')
 
+    print(f"Loading current config from: {config_file_path}")
+    print(f"Loading default config from: {default_config_path}")
+
     # Load the existing config and default config
     current_config = ConfigObj(config_file_path, encoding='utf-8', list_values=False)
     default_config = ConfigObj(default_config_path, encoding='utf-8', list_values=False)
 
+    print("Current config sections:", list(current_config.keys()))
+    print("Default config sections:", list(default_config.keys()))
+
     # Remove sections from current config that are not in the default config
     for section in list(current_config.keys()):
         if section not in default_config:
+            print(f"Removing section: {section}")
             del current_config[section]
 
     # Update sections and keys from the default config
     for section, default_options in default_config.items():
         if section not in current_config:
+            print(f"Adding missing section: {section}")
             current_config[section] = default_options
         else:
             # Remove keys not present in the default config
             for key in list(current_config[section].keys()):
                 if key not in default_options:
+                    print(f"Removing key: {key} from section: {section}")
                     del current_config[section][key]
             # Add keys from the default config
             for key, value in default_options.items():
                 if key not in current_config[section]:
+                    print(f"Adding missing key: {key} to section: {section}")
                     current_config[section][key] = value
 
     # Write the updated config back to the file
     current_config.write()
+    print(f"Updated config written to: {config_file_path}")
 
-if __name__ == "__main__":    
+if __name__ == "__main__":
     update_config()

From e11b191fd4a185c467d07ce40b99c7b5199c06d8 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:02:28 -0600
Subject: [PATCH 71/83] back up

---
 utils/update_config.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 9162640..970a8d1 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -2,8 +2,8 @@
 from configobj import ConfigObj
 
 def update_config():
-    config_file_path = str(Path(__file__).parent / 'config.ini')
-    default_config_path = str(Path(__file__).parent / '.default.config.ini')
+    config_file_path = str(Path(__file__).parent.parent / 'config.ini')
+    default_config_path = str(Path(__file__).parent.parent / '.default.config.ini')
 
     print(f"Loading current config from: {config_file_path}")
     print(f"Loading default config from: {default_config_path}")

From da5aafa079963a0da481a73c3e22f9c16b775850 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:06:09 -0600
Subject: [PATCH 72/83] Refactor update_config.py to write missing sections and
 options from default config

---
 utils/update_config.py | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 970a8d1..c6c0a5c 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -39,7 +39,30 @@ def update_config():
                     current_config[section][key] = value
 
     # Write the updated config back to the file
-    current_config.write()
+    with open(config_file_path, 'w', encoding='utf-8') as file:
+        # Ensure new sections and options are added correctly
+        for section in default_config.sections():
+            if section not in current_config:
+                file.write(f'\n[{section}]\n')
+                for key, value in default_config[section].items():
+                    if value is None:
+                        file.write(f'{key}\n')
+                    else:
+                        file.write(f'{key}={value}\n')
+            else:
+                file.write(f'\n[{section}]\n')
+                for line in current_config[section].items():
+                    file.write(f'{line[0]}={line[1]}\n')
+                # Add any missing keys from the default config
+                for key, value in default_config[section].items():
+                    if key not in current_config[section]:
+                        print(f"Adding missing key: {key} to section: {section}")
+                        file.write(f'{key}={value}\n')
+                    # Write any comments associated with keys
+                    elif current_config[section][key] == default_config[section][key]:
+                        for comment in default_config.comments[key]:
+                            file.write(f'{comment}\n')
+
     print(f"Updated config written to: {config_file_path}")
 
 if __name__ == "__main__":

From 5193fa3f58faf4f9b02a449ea0547fa1d5206fbb Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:08:05 -0600
Subject: [PATCH 73/83] Refactor update_config.py to handle missing sections
 and options from default config

---
 utils/update_config.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index c6c0a5c..b526930 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -41,7 +41,7 @@ def update_config():
     # Write the updated config back to the file
     with open(config_file_path, 'w', encoding='utf-8') as file:
         # Ensure new sections and options are added correctly
-        for section in default_config.sections():
+        for section in default_config.keys():
             if section not in current_config:
                 file.write(f'\n[{section}]\n')
                 for key, value in default_config[section].items():
@@ -50,9 +50,10 @@ def update_config():
                     else:
                         file.write(f'{key}={value}\n')
             else:
-                file.write(f'\n[{section}]\n')
-                for line in current_config[section].items():
-                    file.write(f'{line[0]}={line[1]}\n')
+                if not any(line.strip() == f'[{section}]' for line in new_content):
+                    file.write(f'\n[{section}]\n')
+                for key, value in current_config[section].items():
+                    file.write(f'{key}={value}\n')
                 # Add any missing keys from the default config
                 for key, value in default_config[section].items():
                     if key not in current_config[section]:

From 301585c96c8bdffc2ee45c9f260d6f6ab3496e76 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:09:53 -0600
Subject: [PATCH 74/83] Refactor update_config.py to write missing sections and
 options from default config

---
 utils/update_config.py | 33 +++++++++++----------------------
 1 file changed, 11 insertions(+), 22 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index b526930..f463446 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -40,29 +40,18 @@ def update_config():
 
     # Write the updated config back to the file
     with open(config_file_path, 'w', encoding='utf-8') as file:
-        # Ensure new sections and options are added correctly
-        for section in default_config.keys():
-            if section not in current_config:
-                file.write(f'\n[{section}]\n')
-                for key, value in default_config[section].items():
-                    if value is None:
-                        file.write(f'{key}\n')
-                    else:
-                        file.write(f'{key}={value}\n')
-            else:
-                if not any(line.strip() == f'[{section}]' for line in new_content):
-                    file.write(f'\n[{section}]\n')
-                for key, value in current_config[section].items():
+        for section in current_config.keys():
+            file.write(f'\n[{section}]\n')
+            for key, value in current_config[section].items():
+                file.write(f'{key}={value}\n')
+
+            # Add any missing keys and comments from the default config
+            for key, value in default_config[section].items():
+                if key not in current_config[section]:
                     file.write(f'{key}={value}\n')
-                # Add any missing keys from the default config
-                for key, value in default_config[section].items():
-                    if key not in current_config[section]:
-                        print(f"Adding missing key: {key} to section: {section}")
-                        file.write(f'{key}={value}\n')
-                    # Write any comments associated with keys
-                    elif current_config[section][key] == default_config[section][key]:
-                        for comment in default_config.comments[key]:
-                            file.write(f'{comment}\n')
+                if key in default_config.comments and default_config.comments[key]:
+                    for comment in default_config.comments[key]:
+                        file.write(f'{comment}\n')
 
     print(f"Updated config written to: {config_file_path}")
 

From b8dfc0eb95ed3346e63068328adcd5927b9aa79e Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:12:41 -0600
Subject: [PATCH 75/83] Refactor update_config.py to preserve comments and
 ensure new lines before new sections

---
 utils/update_config.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index f463446..717545c 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -9,8 +9,8 @@ def update_config():
     print(f"Loading default config from: {default_config_path}")
 
     # Load the existing config and default config
-    current_config = ConfigObj(config_file_path, encoding='utf-8', list_values=False)
-    default_config = ConfigObj(default_config_path, encoding='utf-8', list_values=False)
+    current_config = ConfigObj(config_file_path, encoding='utf-8', list_values=False, preserve_comments=True)
+    default_config = ConfigObj(default_config_path, encoding='utf-8', list_values=False, preserve_comments=True)
 
     print("Current config sections:", list(current_config.keys()))
     print("Default config sections:", list(default_config.keys()))
@@ -39,19 +39,17 @@ def update_config():
                     current_config[section][key] = value
 
     # Write the updated config back to the file
-    with open(config_file_path, 'w', encoding='utf-8') as file:
-        for section in current_config.keys():
-            file.write(f'\n[{section}]\n')
-            for key, value in current_config[section].items():
-                file.write(f'{key}={value}\n')
+    current_config.write()
 
-            # Add any missing keys and comments from the default config
-            for key, value in default_config[section].items():
-                if key not in current_config[section]:
-                    file.write(f'{key}={value}\n')
-                if key in default_config.comments and default_config.comments[key]:
-                    for comment in default_config.comments[key]:
-                        file.write(f'{comment}\n')
+    # Ensure new lines before new sections
+    with open(config_file_path, 'r', encoding='utf-8') as file:
+        lines = file.readlines()
+
+    with open(config_file_path, 'w', encoding='utf-8') as file:
+        for i, line in enumerate(lines):
+            if line.startswith('[') and i != 0 and lines[i-1].strip() != '':
+                file.write('\n')
+            file.write(line)
 
     print(f"Updated config written to: {config_file_path}")
 

From 4fda94654a8621d27e0800417e09ea569e98082c Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:19:38 -0600
Subject: [PATCH 76/83] Refactor update_config.py to handle missing sections
 and options from default config

---
 utils/update_config.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 717545c..9762eda 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -9,8 +9,8 @@ def update_config():
     print(f"Loading default config from: {default_config_path}")
 
     # Load the existing config and default config
-    current_config = ConfigObj(config_file_path, encoding='utf-8', list_values=False, preserve_comments=True)
-    default_config = ConfigObj(default_config_path, encoding='utf-8', list_values=False, preserve_comments=True)
+    current_config = ConfigObj(config_file_path, encoding='utf-8', list_values=False)
+    default_config = ConfigObj(default_config_path, encoding='utf-8', list_values=False)
 
     print("Current config sections:", list(current_config.keys()))
     print("Default config sections:", list(default_config.keys()))
@@ -26,6 +26,7 @@ def update_config():
         if section not in current_config:
             print(f"Adding missing section: {section}")
             current_config[section] = default_options
+            current_config.comments[section] = default_config.comments.get(section, [])
         else:
             # Remove keys not present in the default config
             for key in list(current_config[section].keys()):
@@ -37,6 +38,9 @@ def update_config():
                 if key not in current_config[section]:
                     print(f"Adding missing key: {key} to section: {section}")
                     current_config[section][key] = value
+                    current_config[section].comments[key] = default_config[section].comments.get(key, [])
+                if key in default_options.inline_comments:
+                    current_config[section].inline_comments[key] = default_options.inline_comments[key]
 
     # Write the updated config back to the file
     current_config.write()

From 3ad08a153656e18dd0836284853a69fa37e8b48a Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:22:53 -0600
Subject: [PATCH 77/83] change default config slightly

---
 .default.config.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.default.config.ini b/.default.config.ini
index 261ff0e..12c4714 100644
--- a/.default.config.ini
+++ b/.default.config.ini
@@ -60,7 +60,7 @@ backup_snapshot_streams=false
 # Uncomment the following line to specify a custom dataset location for backups
 # custom_dataset_location=
 
-# Maximum size of a backup stream, the default is 10GB, be careful when setting this higher
+# Maximum size of a backup stream, be careful when setting this higher
 # Especially considering PV's for plex, sonarr, radarr, etc. can be quite large
 # Example: max_stream_size=10GB, max_stream_size=20KB, max_stream_size=1TB
-# max_stream_size=10GB
\ No newline at end of file
+max_stream_size=1GB
\ No newline at end of file

From 7b4723b7a33fc4b586bb5da68213ae986254efcd Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:24:15 -0600
Subject: [PATCH 78/83] update config.

---
 utils/update_config.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/utils/update_config.py b/utils/update_config.py
index 9762eda..ccc4ef0 100644
--- a/utils/update_config.py
+++ b/utils/update_config.py
@@ -5,38 +5,28 @@ def update_config():
     config_file_path = str(Path(__file__).parent.parent / 'config.ini')
     default_config_path = str(Path(__file__).parent.parent / '.default.config.ini')
 
-    print(f"Loading current config from: {config_file_path}")
-    print(f"Loading default config from: {default_config_path}")
-
     # Load the existing config and default config
     current_config = ConfigObj(config_file_path, encoding='utf-8', list_values=False)
     default_config = ConfigObj(default_config_path, encoding='utf-8', list_values=False)
 
-    print("Current config sections:", list(current_config.keys()))
-    print("Default config sections:", list(default_config.keys()))
-
     # Remove sections from current config that are not in the default config
     for section in list(current_config.keys()):
         if section not in default_config:
-            print(f"Removing section: {section}")
             del current_config[section]
 
     # Update sections and keys from the default config
     for section, default_options in default_config.items():
         if section not in current_config:
-            print(f"Adding missing section: {section}")
             current_config[section] = default_options
             current_config.comments[section] = default_config.comments.get(section, [])
         else:
             # Remove keys not present in the default config
             for key in list(current_config[section].keys()):
                 if key not in default_options:
-                    print(f"Removing key: {key} from section: {section}")
                     del current_config[section][key]
             # Add keys from the default config
             for key, value in default_options.items():
                 if key not in current_config[section]:
-                    print(f"Adding missing key: {key} to section: {section}")
                     current_config[section][key] = value
                     current_config[section].comments[key] = default_config[section].comments.get(key, [])
                 if key in default_options.inline_comments:
@@ -55,7 +45,5 @@ def update_config():
                 file.write('\n')
             file.write(line)
 
-    print(f"Updated config written to: {config_file_path}")
-
 if __name__ == "__main__":
     update_config()

From 50e2d2379fda4985708a42f31dcf50993a38dbf9 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:36:25 -0600
Subject: [PATCH 79/83] Refactor backup.py to support configurable snapshot
 streaming size

---
 functions/backup_restore/backup/backup.py | 43 ++++++++++++++++++-----
 1 file changed, 35 insertions(+), 8 deletions(-)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index 61644c9..81a20b1 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -1,4 +1,5 @@
 from datetime import datetime, timezone
+from configobj import ConfigObj
 from pathlib import Path
 from collections import defaultdict
 
@@ -169,14 +170,40 @@ def backup_all(self):
                         failures[app_name].append(snapshot_result["message"])
                         continue
 
-                    # Send the snapshot to the backup directory
-                    self.logger.info(f"Sending snapshot stream to backup file...")
-                    snapshot_name = f"{dataset_path}@{self.snapshot_name}"
-                    backup_path = app_backup_dir / "snapshots" / f"{snapshot_name.replace('/', '%%')}.zfs"
-                    backup_path.parent.mkdir(parents=True, exist_ok=True)
-                    send_result = self.snapshot_manager.zfs_send(snapshot_name, backup_path, compress=True)
-                    if not send_result["success"]:
-                        failures[app_name].append(send_result["message"])
+                    config_file_path = str(Path(__file__).parent.parent.parent.parent / 'config.ini')
+                    config = ConfigObj(config_file_path, encoding='utf-8', list_values=False)
+
+                    backup_snapshot_streams = config['BACKUP'].as_bool('backup_snapshot_streams')
+                    max_stream_size_str = config['BACKUP'].get('max_stream_size', '10G')
+
+                    def size_str_to_bytes(size_str):
+                        size_units = {"K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}
+                        try:
+                            if size_str[-1] in size_units:
+                                return int(float(size_str[:-1]) * size_units[size_str[-1]])
+                            else:
+                                return int(size_str)
+                        except ValueError:
+                            self.logger.error(f"Invalid size string: {size_str}")
+                            return 0
+
+                    max_stream_size_bytes = size_str_to_bytes(max_stream_size_str)
+
+                    if backup_snapshot_streams:
+                        snapshot_refer_size = self.snapshot_manager.get_snapshot_refer_size(snapshot_name)
+                        if snapshot_refer_size <= max_stream_size_bytes:
+                            # Send the snapshot to the backup directory
+                            self.logger.info(f"Sending snapshot stream to backup file...")
+                            snapshot_name = f"{dataset_path}@{self.snapshot_name}"
+                            backup_path = app_backup_dir / "snapshots" / f"{snapshot_name.replace('/', '%%')}.zfs"
+                            backup_path.parent.mkdir(parents=True, exist_ok=True)
+                            send_result = self.snapshot_manager.zfs_send(snapshot_name, backup_path, compress=True)
+                            if not send_result["success"]:
+                                failures[app_name].append(send_result["message"])
+                        else:
+                            self.logger.warning(f"Snapshot refer size {snapshot_refer_size} exceeds the maximum configured size {max_stream_size_bytes}")
+                    else:
+                        self.logger.debug("Backup snapshot streams are disabled in the configuration.")
 
             # Handle ix_volumes_dataset separately
             if chart_info.ix_volumes_dataset:

From b72d7f2e55c7e358c7457a6643df57c3ccc00d8a Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:38:50 -0600
Subject: [PATCH 80/83] Refactor max_stream_size in default.config.ini to use
 shorthand notation

---
 .default.config.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.default.config.ini b/.default.config.ini
index 12c4714..9ceea6a 100644
--- a/.default.config.ini
+++ b/.default.config.ini
@@ -62,5 +62,5 @@ backup_snapshot_streams=false
 
 # Maximum size of a backup stream, be careful when setting this higher
 # Especially considering PV's for plex, sonarr, radarr, etc. can be quite large
-# Example: max_stream_size=10GB, max_stream_size=20KB, max_stream_size=1TB
-max_stream_size=1GB
\ No newline at end of file
+# Example: max_stream_size=10G, max_stream_size=20K, max_stream_size=1T
+max_stream_size=1G
\ No newline at end of file

From a3be86ddfc3ffc69609b6e1c47f24ed7f149a59b Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:43:17 -0600
Subject: [PATCH 81/83] add some debugging

---
 functions/backup_restore/backup/backup.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index 81a20b1..b26fe1b 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -170,6 +170,7 @@ def backup_all(self):
                         failures[app_name].append(snapshot_result["message"])
                         continue
 
+                    # Read configuration settings
                     config_file_path = str(Path(__file__).parent.parent.parent.parent / 'config.ini')
                     config = ConfigObj(config_file_path, encoding='utf-8', list_values=False)
 
@@ -189,8 +190,14 @@ def size_str_to_bytes(size_str):
 
                     max_stream_size_bytes = size_str_to_bytes(max_stream_size_str)
 
+                    self.logger.debug(f"backup_snapshot_streams: {backup_snapshot_streams}")
+                    self.logger.debug(f"max_stream_size_str: {max_stream_size_str}")
+                    self.logger.debug(f"max_stream_size_bytes: {max_stream_size_bytes}")
+
                     if backup_snapshot_streams:
                         snapshot_refer_size = self.snapshot_manager.get_snapshot_refer_size(snapshot_name)
+                        self.logger.debug(f"snapshot_refer_size: {snapshot_refer_size}")
+
                         if snapshot_refer_size <= max_stream_size_bytes:
                             # Send the snapshot to the backup directory
                             self.logger.info(f"Sending snapshot stream to backup file...")

From 2f787a4f1a50bf1245b2dc98c25be50ec3f245c8 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 10:57:04 -0600
Subject: [PATCH 82/83] Refactor backup.py to support configurable snapshot
 streaming size

---
 functions/backup_restore/backup/backup.py | 76 ++++++++++++-----------
 1 file changed, 41 insertions(+), 35 deletions(-)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index b26fe1b..cd5bfbb 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -58,6 +58,14 @@ def __init__(self, backup_dir: Path, retention_number: int = 15):
 
         self.kube_pvc_fetcher = KubePVCFetcher()
 
+        # Read configuration settings
+        config_file_path = str(Path(__file__).parent.parent.parent.parent / 'config.ini')
+        config = ConfigObj(config_file_path, encoding='utf-8', list_values=False)
+
+        self.backup_snapshot_streams = config['BACKUP'].as_bool('backup_snapshot_streams')
+        self.max_stream_size_str = config['BACKUP'].get('max_stream_size', '10G')
+        self.max_stream_size_bytes = self._size_str_to_bytes(self.max_stream_size_str)
+
     def _create_backup_dataset(self):
         """
         Create a ZFS dataset for backups.
@@ -150,7 +158,6 @@ def backup_all(self):
                     self.logger.error(f"Failed to backup database for {app_name}: {result['message']}")
                     failures[app_name].append(result["message"])
 
-            # TODO: Print off better messages for each of the two types
             dataset_paths = self.kube_pvc_fetcher.get_volume_paths_by_namespace(f"ix-{app_name}")
             if dataset_paths:
                 for dataset_path in dataset_paths:
@@ -170,37 +177,17 @@ def backup_all(self):
                         failures[app_name].append(snapshot_result["message"])
                         continue
 
-                    # Read configuration settings
-                    config_file_path = str(Path(__file__).parent.parent.parent.parent / 'config.ini')
-                    config = ConfigObj(config_file_path, encoding='utf-8', list_values=False)
-
-                    backup_snapshot_streams = config['BACKUP'].as_bool('backup_snapshot_streams')
-                    max_stream_size_str = config['BACKUP'].get('max_stream_size', '10G')
-
-                    def size_str_to_bytes(size_str):
-                        size_units = {"K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}
-                        try:
-                            if size_str[-1] in size_units:
-                                return int(float(size_str[:-1]) * size_units[size_str[-1]])
-                            else:
-                                return int(size_str)
-                        except ValueError:
-                            self.logger.error(f"Invalid size string: {size_str}")
-                            return 0
+                    self.logger.debug(f"backup_snapshot_streams: {self.backup_snapshot_streams}")
+                    self.logger.debug(f"max_stream_size_str: {self.max_stream_size_str}")
+                    self.logger.debug(f"max_stream_size_bytes: {self.max_stream_size_bytes}")
 
-                    max_stream_size_bytes = size_str_to_bytes(max_stream_size_str)
-
-                    self.logger.debug(f"backup_snapshot_streams: {backup_snapshot_streams}")
-                    self.logger.debug(f"max_stream_size_str: {max_stream_size_str}")
-                    self.logger.debug(f"max_stream_size_bytes: {max_stream_size_bytes}")
-
-                    if backup_snapshot_streams:
+                    if self.backup_snapshot_streams:
                         snapshot_refer_size = self.snapshot_manager.get_snapshot_refer_size(snapshot_name)
                         self.logger.debug(f"snapshot_refer_size: {snapshot_refer_size}")
 
-                        if snapshot_refer_size <= max_stream_size_bytes:
+                        if snapshot_refer_size <= self.max_stream_size_bytes:
                             # Send the snapshot to the backup directory
-                            self.logger.info(f"Sending snapshot stream to backup file...")
+                            self.logger.info(f"Sending PV snapshot stream to backup file...")
                             snapshot_name = f"{dataset_path}@{self.snapshot_name}"
                             backup_path = app_backup_dir / "snapshots" / f"{snapshot_name.replace('/', '%%')}.zfs"
                             backup_path.parent.mkdir(parents=True, exist_ok=True)
@@ -208,24 +195,43 @@ def size_str_to_bytes(size_str):
                             if not send_result["success"]:
                                 failures[app_name].append(send_result["message"])
                         else:
-                            self.logger.warning(f"Snapshot refer size {snapshot_refer_size} exceeds the maximum configured size {max_stream_size_bytes}")
+                            self.logger.warning(f"Snapshot refer size {snapshot_refer_size} exceeds the maximum configured size {self.max_stream_size_bytes}")
                     else:
                         self.logger.debug("Backup snapshot streams are disabled in the configuration.")
 
             # Handle ix_volumes_dataset separately
             if chart_info.ix_volumes_dataset:
-                self.logger.info(f"Snapshotting ix_volumes...")
                 snapshot = chart_info.ix_volumes_dataset + "@" + self.snapshot_name
-                backup_path = app_backup_dir / "snapshots" / f"{snapshot.replace('/', '%%')}.zfs"
-                backup_path.parent.mkdir(parents=True, exist_ok=True)
-                self.logger.info(f"Sending snapshot stream to backup file...")
-                send_result = self.snapshot_manager.zfs_send(snapshot, backup_path, compress=True)
-                if not send_result["success"]:
-                    failures[app_name].append(send_result["message"])
+                if self.backup_snapshot_streams:
+                    snapshot_refer_size = self.snapshot_manager.get_snapshot_refer_size(snapshot)
+                    self.logger.debug(f"ix_volumes_dataset snapshot_refer_size: {snapshot_refer_size}")
+
+                    if snapshot_refer_size <= self.max_stream_size_bytes:
+                        self.logger.info(f"Sending ix_volumes snapshot stream to backup file...")
+                        backup_path = app_backup_dir / "snapshots" / f"{snapshot.replace('/', '%%')}.zfs"
+                        backup_path.parent.mkdir(parents=True, exist_ok=True)
+                        send_result = self.snapshot_manager.zfs_send(snapshot, backup_path, compress=True)
+                        if not send_result["success"]:
+                            failures[app_name].append(send_result["message"])
+                    else:
+                        self.logger.warning(f"ix_volumes_dataset snapshot refer size {snapshot_refer_size} exceeds the maximum configured size {self.max_stream_size_bytes}")
+                else:
+                    self.logger.debug("Backup snapshot streams are disabled in the configuration.")
 
         self._create_backup_snapshot()
         self._log_failures(failures)
 
+    def _size_str_to_bytes(self, size_str):
+        size_units = {"K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}
+        try:
+            if size_str[-1] in size_units:
+                return int(float(size_str[:-1]) * size_units[size_str[-1]])
+            else:
+                return int(size_str)
+        except ValueError:
+            self.logger.error(f"Invalid size string: {size_str}")
+            return 0
+
     def _log_failures(self, failures):
         """
         Log a summary of all backup failures.

From 20739a22154f737326716b492362e0941622c9cb Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Thu, 30 May 2024 11:01:59 -0600
Subject: [PATCH 83/83] Refactor backup.py to use snapshot name directly in
 backup process

---
 functions/backup_restore/backup/backup.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/functions/backup_restore/backup/backup.py b/functions/backup_restore/backup/backup.py
index cd5bfbb..21a38d3 100644
--- a/functions/backup_restore/backup/backup.py
+++ b/functions/backup_restore/backup/backup.py
@@ -182,16 +182,17 @@ def backup_all(self):
                     self.logger.debug(f"max_stream_size_bytes: {self.max_stream_size_bytes}")
 
                     if self.backup_snapshot_streams:
-                        snapshot_refer_size = self.snapshot_manager.get_snapshot_refer_size(snapshot_name)
+                        snapshot = f"{dataset_path}@{self.snapshot_name}"
+                        snapshot_refer_size = self.snapshot_manager.get_snapshot_refer_size(snapshot)
                         self.logger.debug(f"snapshot_refer_size: {snapshot_refer_size}")
 
                         if snapshot_refer_size <= self.max_stream_size_bytes:
                             # Send the snapshot to the backup directory
                             self.logger.info(f"Sending PV snapshot stream to backup file...")
-                            snapshot_name = f"{dataset_path}@{self.snapshot_name}"
-                            backup_path = app_backup_dir / "snapshots" / f"{snapshot_name.replace('/', '%%')}.zfs"
+                            snapshot = f"{dataset_path}@{self.snapshot_name}"
+                            backup_path = app_backup_dir / "snapshots" / f"{snapshot.replace('/', '%%')}.zfs"
                             backup_path.parent.mkdir(parents=True, exist_ok=True)
-                            send_result = self.snapshot_manager.zfs_send(snapshot_name, backup_path, compress=True)
+                            send_result = self.snapshot_manager.zfs_send(snapshot, backup_path, compress=True)
                             if not send_result["success"]:
                                 failures[app_name].append(send_result["message"])
                         else: