refactor hashing/dedupe

cyberdrop_dl/clients/hash_client.py

@@ -47,37 +47,49 @@
         self.manager = manager
         self.hashes = defaultdict(lambda: None)
         self.prev_hashes = None
+        self.xxhash="xxh128"
+        self.md5="md5"
+        self.sha256="sha256"
 
     async def startup(self) -> None:
-        self.prev_hashes = set(await self.manager.db_manager.hash_table.get_all_unique_hashes())
+        self.prev_hashes = set(await self.manager.db_manager.hash_table.get_all_unique_hashes(self.xxhash))
 
     async def hash_directory(self, path: Path) -> None:
         path = Path(path)
         async with self.manager.live_manager.get_hash_live(stop=True):
             if not path.is_dir():
                 raise NotADirectoryError
             for file in path.rglob("*"):
-                await self.hash_item(file, None, None)
+                await self.hash_item_helper(file, None, None)
 
     @staticmethod
     def _get_key_from_file(file: Path | str):
         return str(Path(file).absolute())
+    async def hash_item_helper(self, file: Path | str, original_filename: str, referer: URL):
+        hash=await self.hash_item(file, original_filename,referer,hash_type=self.xxhash)
+        if self.manager.config_manager.settings_data["Dupe_Cleanup_Options"]["add_md5_hash"]:
+            await self.hash_item(file, original_filename,referer,hash_type=self.md5)
+        if self.manager.config_manager.settings_data["Dupe_Cleanup_Options"]["add_sha256_hash"]:
+            await self.hash_item(file, original_filename, referer, hash_type=self.sha256)
+        return hash
+
 
-    async def hash_item(self, file: Path | str, original_filename: str, referer: URL) -> str:
+    async def hash_item(self, file: Path | str, original_filename: str, referer: URL,hash_type=None) -> str:
         """Generates hash of a file."""
         key = self._get_key_from_file(file)
         file = Path(file)
         if not file.is_file():
             return None
-        if self.hashes[key]:
-            return self.hashes[key]
+        if self.hashes[(key,hash_type)]:
+            return self.hashes[(key,hash_type)]
         self.manager.progress_manager.hash_progress.update_currently_hashing(file)
-        hash = await self.manager.db_manager.hash_table.get_file_hash_exists(file)
+        hash = await self.manager.db_manager.hash_table.get_file_hash_exists(file,hash_type)
         try:
             if not hash:
-                hash = await self.manager.hash_manager.hash_file(file)
+                hash = await self.manager.hash_manager.hash_file(file,hash_type)
                 await self.manager.db_manager.hash_table.insert_or_update_hash_db(
                     hash,
+                    hash_type,
                     file,
                     original_filename,
                     referer,
@@ -87,25 +99,31 @@
                 self.manager.progress_manager.hash_progress.add_prev_hash()
                 await self.manager.db_manager.hash_table.insert_or_update_hash_db(
                     hash,
+                    hash_type,
                     file,
                     original_filename,
                     referer,
                 )
         except Exception as e:
             log(f"Error hashing {file} : {e}", 40, exc_info=True)
-        self.hashes[key] = hash
+        self.hashes[(key,hash_type)] = hash
         return hash
 
     async def hash_item_during_download(self, media_item: MediaItem) -> None:
         try:
-            if self.manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["hash_while_downloading"]:
-                await self.hash_item(media_item.complete_file, media_item.original_filename, media_item.referer)
+
+            if not self.manager.config_manager.settings_data["Dupe_Cleanup_Options"]["delete_after_download"]:
+                return
+            if self.manager.config_manager.settings_data["Dupe_Cleanup_Options"]["disable_individual_download_hashing"]:
+                return
+
+            await self.hash_item_helper(media_item.complete_file, media_item.original_filename, media_item.referer)
         except Exception as e:
             log(f"After hash processing failed: {media_item.complete_file} with error {e}", 40, exc_info=True)
 
     async def cleanup_dupes(self) -> None:
         with self.manager.live_manager.get_hash_live(stop=True):
-            if not self.manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["delete_after_download"]:
+            if not self.manager.config_manager.settings_data["Dupe_Cleanup_Options"]["delete_after_download"]:
                 return
             file_hashes_dict = await self.get_file_hashes_dict()
         with self.manager.live_manager.get_remove_file_via_hash_live(stop=True):
@@ -117,13 +135,11 @@
             for size, data in size_dict.items():
                 selected_file = Path(data["selected"])
                 other_files = data["others"]
-
                 # Get all matches from the database
                 all_matches = [
                     Path(x[0], x[1])
-                    for x in await self.manager.db_manager.hash_table.get_files_with_hash_matches(hash, size)
+                    for x in await self.manager.db_manager.hash_table.get_files_with_hash_matches(hash, size,self.xxhash)
                 ]
-
                 # Filter out files with the same path as any file in other_files
                 other_matches = [match for match in all_matches if str(match) not in other_files]
                 # Filter files based  on if the file exists
@@ -166,7 +182,7 @@
         hashes_dict = defaultdict(lambda: defaultdict(list))
         # first compare downloads to each other
         for media_item in list(self.manager.path_manager.completed_downloads):
-            hash = await self.hash_item(media_item.complete_file, media_item.original_filename, media_item.referer)
+            hash = await self.hash_item_helper(media_item.complete_file, media_item.original_filename, media_item.referer)
             item = media_item.complete_file.absolute()
             try:
                 size = item.stat().st_size
@@ -205,7 +221,7 @@
         return hashes_dict
 
     def send2trash(self, path: Path) -> None:
-        if self.manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["delete_off_disk"]:
+        if self.manager.config_manager.settings_data["Dupe_Cleanup_Options"]["delete_off_disk"]:
             Path(path).unlink(missing_ok=True)
         else:
             send2trash(path)
@@ -218,10 +234,10 @@
 
     def keep_new_download(self, hash: str, selected_file: Path | str) -> bool:
         return bool(
-            self.manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["keep_new_download"]
+            self.manager.config_manager.settings_data["Dupe_Cleanup_Options"]["keep_new_download"]
             or hash not in self.prev_hashes
-            or str(selected_file) in self.manager.path_manager.prev_downloads_paths,
+            or Path(selected_file) in self.manager.path_manager.prev_downloads_paths,
         )
 
     def keep_prev_file(self) -> bool:
-        return self.manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["keep_prev_download"]
+        return self.manager.config_manager.settings_data["Dupe_Cleanup_Options"]["keep_prev_download"]

cyberdrop_dl/managers/hash_manager.py

@@ -8,29 +8,46 @@
 from cyberdrop_dl.clients.hash_client import HashClient
 
 try:
-    from xxhash import xxh128 as hasher
+    from xxhash import xxh128 as xxhasher
 except ImportError:
-    from hashlib import md5 as hasher
+    xxhasher = None
+from hashlib import md5 as md5hasher
+from hashlib import sha256 as sha256hasher
 
 if TYPE_CHECKING:
     from cyberdrop_dl.managers.manager import Manager
 
 
 class HashManager:
     def __init__(self, manager: Manager) -> None:
-        self.hasher = hasher
+        self.xx_hasher = xxhasher
+        self.md5_hasher = md5hasher
+        self.sha_256_hasher = sha256hasher
         self.hash_client = HashClient(manager)  # Initialize hash client in constructor
-
     async def startup(self) -> None:
         await self.hash_client.startup()
 
-    async def hash_file(self, filename: str) -> str:
+
+
+    async def hash_file(self, filename: str,hash_type:str) -> str:
         file_path = Path.cwd() / filename
         async with aiofiles.open(file_path, "rb") as fp:
             CHUNK_SIZE = 1024 * 1024  # 1MB
             filedata = await fp.read(CHUNK_SIZE)
-            current_hasher = hasher()  # Use the initialized hasher
+            current_hasher = self._get_hasher(hash_type) # Use the initialized hasher
             while filedata:
                 current_hasher.update(filedata)
                 filedata = await fp.read(CHUNK_SIZE)
             return current_hasher.hexdigest()
+
+    def _get_hasher(self,hash_type):
+        if hash_type =="xx128" and not self.xx_hasher:
+            raise ImportError("xxhash module is not installed")
+        elif hash_type == "xxh128":
+            return self.xx_hasher()
+        elif hash_type == "md5":
+            return self.md5_hasher()
+        elif hash_type == "sha256":
+            return self.sha_256_hasher()
+        else:
+            raise ValueError("Invalid hash type")

cyberdrop_dl/ui/prompts/settings_global_prompts.py

@@ -219,60 +219,47 @@ def edit_dupe_settings_prompt(manager: Manager) -> None:
 
     delete_after = inquirer.select(
         message="Toggle duplicate files deletion using hashes:",
-        default=manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["delete_after_download"],
-        choices=[Choice(True, "True"), Choice(False, "False")],
-        vi_mode=manager.vi_mode,
-    ).execute()
-    hash_while_downloading = inquirer.select(
-        message="Hash Files during downloading:",
-        long_instruction="""
-        Generate file hashes after each download, instead of batched
-        together during deduplication process
-        """,
-        default=manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["hash_while_downloading"],
+        default=manager.config_manager.settings_data["Dupe_Cleanup_Options"]["delete_after_download"],
         choices=[Choice(True, "True"), Choice(False, "False")],
         vi_mode=manager.vi_mode,
     ).execute()
 
     dedupe_already_downloaded = inquirer.select(
         message="How to handle files already on system: ",
-        default=manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["dedupe_already_downloaded"],
+        default=manager.config_manager.settings_data["Dupe_Cleanup_Options"]["dedupe_already_downloaded"],
         choices=[Choice(True, "Mark Existing Files as 'new'"), Choice(False, "Skip Existing Files")],
         vi_mode=manager.vi_mode,
     ).execute()
 
     keep_current = inquirer.select(
         message="What to do with new file when deduping: ",
         long_instruction="Keep a curent file. Current files are files that were either downloaded or a file was skipped for already existing when dedupe_already_downloaded is true",
-        default=manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["keep_new_download"],
+        default=manager.config_manager.settings_data["Dupe_Cleanup_Options"]["keep_new_download"],
         choices=[Choice(True, "Keep a newfile"), Choice(False, "Delete all new files")],
         vi_mode=manager.vi_mode,
     ).execute()
 
     keep_prev = inquirer.select(
         message="What to do with previous file(s) when deduping:",
-        default=manager.config_manager.global_settings_data["Dupe_Cleanup_Options"],
+        default=manager.config_manager.settings_data["Dupe_Cleanup_Options"],
         long_instruction="Any file that is not in the list of current files with a matching hash",
         choices=[Choice(True, "Keep a previous file "), Choice(False, "Delete all previous files")],
         vi_mode=manager.vi_mode,
     ).execute()
 
     delete_off_disk = inquirer.select(
         message="How to handle removal of files: ",
-        default=manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["delete_off_disk"],
+        default=manager.config_manager.settings_data["Dupe_Cleanup_Options"]["delete_off_disk"],
         choices=[Choice(True, "Permanently Delete File"), Choice(False, "Send to Trash")],
         vi_mode=manager.vi_mode,
     ).execute()
 
-    manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["delete_after_download"] = delete_after
-    manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["hash_while_downloading"] = (
-        hash_while_downloading
-    )
-    manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["keep_prev_download"] = keep_prev
-    manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["keep_new_download"] = keep_current
+    manager.config_manager.settings_data["Dupe_Cleanup_Options"]["delete_after_download"] = delete_after
+    manager.config_manager.settings_data["Dupe_Cleanup_Options"]["keep_prev_download"] = keep_prev
+    manager.config_manager.settings_data["Dupe_Cleanup_Options"]["keep_new_download"] = keep_current
 
-    manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["dedupe_already_downloaded"] = (
+    manager.config_manager.settings_data["Dupe_Cleanup_Options"]["dedupe_already_downloaded"] = (
         dedupe_already_downloaded
     )
 
-    manager.config_manager.global_settings_data["Dupe_Cleanup_Options"]["delete_off_disk"] = delete_off_disk
+    manager.config_manager.settings_data["Dupe_Cleanup_Options"]["delete_off_disk"] = delete_off_disk

cyberdrop_dl/utils/args/config_definitions.py

@@ -130,6 +130,17 @@
         "auto_import": False,
         "sites": None,
     },
+    "Dupe_Cleanup_Options": {
+        "delete_after_download": False,
+        "keep_prev_download": False,
+        "keep_new_download": True,
+        "dedupe_already_downloaded": False,
+        "delete_off_disk": False,
+        "add_md5_hash": False,
+        "disable_individual_download_hashing":False,
+    "add_sha256_hash": False,
+    },
+
 }
 
 global_settings: dict = {
@@ -152,14 +163,6 @@
         "max_simultaneous_downloads_per_domain": 3,
         "download_speed_limit": 0,
     },
-    "Dupe_Cleanup_Options": {
-        "delete_after_download": False,
-        "hash_while_downloading": False,
-        "keep_prev_download": False,
-        "keep_new_download": True,
-        "dedupe_already_downloaded": False,
-        "delete_off_disk": False,
-    },
     "UI_Options": {
         "vi_mode": False,
         "refresh_rate": 10,

cyberdrop_dl/utils/database/table_definitions.py

@@ -29,13 +29,37 @@
 
 create_temp_referer = """CREATE TABLE IF NOT EXISTS temp_referer (referer TEXT);"""
 
-create_hash = """CREATE TABLE IF NOT EXISTS hash (
+create_files= """
+CREATE TABLE IF NOT EXISTS files (
   folder TEXT,
   download_filename TEXT,
   original_filename TEXT,
   file_size INT,
+  referer TEXT,
+  PRIMARY KEY (folder, download_filename)
+);
+
+"""
+
+create_hash= """
+CREATE TABLE IF NOT EXISTS hash (
+  folder TEXT,
+  download_filename TEXT,
+  hash_type TEXT,
+  hash TEXT,
+  PRIMARY KEY (folder, download_filename, hash_type),
+  FOREIGN KEY (folder, download_filename) REFERENCES files(folder, download_filename)
+);
+
+"""
+
+create_temp_hash= """
+CREATE TABLE IF NOT EXISTS temp_hash (
+  folder TEXT,
+  download_filename TEXT,
+  hash_type TEXT,
   hash TEXT,
-  UNIQUE (folder, original_filename)
-  PRIMARY KEY (folder, original_filename,hash)
+  PRIMARY KEY (folder, download_filename, hash_type),
+  FOREIGN KEY (folder, download_filename) REFERENCES files(folder, download_filename)
 );
 """