From 96c7f9a666e5bbc283cc85a18eeaddb97d96952b Mon Sep 17 00:00:00 2001 From: mullertremolo <22122313+mullertremolo@users.noreply.github.com> Date: Fri, 19 Apr 2024 08:13:07 -0700 Subject: [PATCH] Add created_at and completed_at columns to DB (#884) This adds `created_at` and `completed_at` columns to the DB. This is useful for several possible future uses. # Uses There are several uses for this info, but the one that comes up pretty often is when bad files are downloaded during an outage before cyberdrop-dl knows about the new Etag (such as when Bunkr creates a new maintenance video). If you know when the issue was occurring, this could help you find which files might be bad and need to be redownloaded. Knowing when the item was added to the DB can be used to prioritize downloads that are more recently added and therefore more likely to succeed first. Items that have been incomplete for a long time will most likely never finish, so trying them first just burns through your rate limit. # Details | Column | Description |:--------|:------------- | `created_at` | When the item was added to the DB | `completed_at` | When the item was marked completed The columns are obviously unset for items that predate these additions. --- cyberdrop_dl/utils/database/table_definitions.py | 2 ++ .../utils/database/tables/history_table.py | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cyberdrop_dl/utils/database/table_definitions.py b/cyberdrop_dl/utils/database/table_definitions.py index 7cccdfe1f..3b021e840 100644 --- a/cyberdrop_dl/utils/database/table_definitions.py +++ b/cyberdrop_dl/utils/database/table_definitions.py @@ -6,6 +6,8 @@ download_filename TEXT, original_filename TEXT, completed INTEGER NOT NULL, + created_at TIMESTAMP, + completed_at TIMESTAMP, PRIMARY KEY (domain, url_path, original_filename) );""" diff --git a/cyberdrop_dl/utils/database/tables/history_table.py b/cyberdrop_dl/utils/database/tables/history_table.py index a363266eb..11886ca6f 100644 --- a/cyberdrop_dl/utils/database/tables/history_table.py +++ b/cyberdrop_dl/utils/database/tables/history_table.py @@ -105,7 +105,7 @@ async def insert_incompleted(self, domain: str, media_item: MediaItem) -> None: (domain, media_item.album_id, url_path, str(media_item.referer))) except IntegrityError: await self.db_conn.execute("""DELETE FROM media WHERE domain = 'no_crawler' and url_path = ?""", (url_path,)) - await self.db_conn.execute("""INSERT OR IGNORE INTO media (domain, url_path, referer, album_id, download_path, download_filename, original_filename, completed) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + await self.db_conn.execute("""INSERT OR IGNORE INTO media (domain, url_path, referer, album_id, download_path, download_filename, original_filename, completed, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)""", (domain, url_path, str(media_item.referer), media_item.album_id, str(media_item.download_folder), download_filename, media_item.original_filename, 0)) await self.db_conn.execute("""UPDATE media SET download_filename = ? WHERE domain = ? and url_path = ?""", (download_filename, domain, url_path)) @@ -115,7 +115,7 @@ async def mark_complete(self, domain: str, media_item: MediaItem) -> None: """Mark a download as completed in the database""" domain = await get_db_domain(domain) url_path = await get_db_path(media_item.url, str(media_item.referer)) - await self.db_conn.execute("""UPDATE media SET completed = 1 WHERE domain = ? and url_path = ?""", + await self.db_conn.execute("""UPDATE media SET completed = 1, completed_at = CURRENT_TIMESTAMP WHERE domain = ? and url_path = ?""", (domain, url_path)) await self.db_conn.commit() @@ -154,7 +154,7 @@ async def fix_bunkr_v4_entries(self) -> None: for entry in bunkr_entries: entry = list(entry) entry[0] = "bunkrr" - await self.db_conn.execute("""INSERT or REPLACE INTO media VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", entry) + await self.db_conn.execute("""INSERT or REPLACE INTO media VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", entry) await self.db_conn.commit() await self.db_conn.execute("""DELETE FROM media WHERE domain = 'bunkr'""") @@ -177,7 +177,7 @@ async def fix_primary_keys(self) -> None: await self.db_conn.execute("""ALTER TABLE media_copy RENAME TO media""") await self.db_conn.commit() - + async def add_columns(self) -> None: cursor = await self.db_conn.cursor() result = await cursor.execute("""pragma table_info(media)""") @@ -187,3 +187,11 @@ async def add_columns(self) -> None: if "album_id" not in current_cols: await self.db_conn.execute("""ALTER TABLE media ADD COLUMN album_id TEXT""") await self.db_conn.commit() + + if "created_at" not in current_cols: + await self.db_conn.execute("""ALTER TABLE media ADD COLUMN created_at TIMESTAMP""") + await self.db_conn.commit() + + if "completed_at" not in current_cols: + await self.db_conn.execute("""ALTER TABLE media ADD COLUMN completed_at TIMESTAMP""") + await self.db_conn.commit()