Add created_at and completed_at columns to DB (#884)

This adds `created_at` and `completed_at` columns to the DB. This is useful for several possible future uses.

# Uses

There are several uses for this info, but the one that comes up pretty often is when bad files are downloaded during an outage before cyberdrop-dl knows about the new Etag (such as when Bunkr creates a new maintenance video). If you know when the issue was occurring, this could help you find which files might be bad and need to be redownloaded.

Knowing when the item was added to the DB can be used to prioritize downloads that are more recently added and therefore more likely to succeed first. Items that have been incomplete for a long time will most likely never finish, so trying them first just burns through your rate limit.

# Details

| Column | Description
|:--------|:-------------
| `created_at` | When the item was added to the DB
| `completed_at` | When the item was marked completed

The columns are obviously unset for items that predate these additions.

cyberdrop_dl/utils/database/table_definitions.py

@@ -6,6 +6,8 @@
                                                download_filename TEXT,
                                                original_filename TEXT,
                                                completed INTEGER NOT NULL,
+                                               created_at TIMESTAMP,
+                                               completed_at TIMESTAMP,
                                                PRIMARY KEY (domain, url_path, original_filename)
                                                );"""
 

cyberdrop_dl/utils/database/tables/history_table.py

@@ -105,7 +105,7 @@ async def insert_incompleted(self, domain: str, media_item: MediaItem) -> None:
                                        (domain, media_item.album_id, url_path, str(media_item.referer)))
         except IntegrityError:
             await self.db_conn.execute("""DELETE FROM media WHERE domain = 'no_crawler' and url_path = ?""", (url_path,))
-        await self.db_conn.execute("""INSERT OR IGNORE INTO media (domain, url_path, referer, album_id, download_path, download_filename, original_filename, completed) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+        await self.db_conn.execute("""INSERT OR IGNORE INTO media (domain, url_path, referer, album_id, download_path, download_filename, original_filename, completed, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)""",
                                    (domain, url_path, str(media_item.referer), media_item.album_id, str(media_item.download_folder), download_filename, media_item.original_filename, 0))
         await self.db_conn.execute("""UPDATE media SET download_filename = ? WHERE domain = ? and url_path = ?""",
                                    (download_filename, domain, url_path))
@@ -115,7 +115,7 @@ async def mark_complete(self, domain: str, media_item: MediaItem) -> None:
         """Mark a download as completed in the database"""
         domain = await get_db_domain(domain)
         url_path = await get_db_path(media_item.url, str(media_item.referer))
-        await self.db_conn.execute("""UPDATE media SET completed = 1 WHERE domain = ? and url_path = ?""",
+        await self.db_conn.execute("""UPDATE media SET completed = 1, completed_at = CURRENT_TIMESTAMP WHERE domain = ? and url_path = ?""",
                                    (domain, url_path))
         await self.db_conn.commit()
 
@@ -154,7 +154,7 @@ async def fix_bunkr_v4_entries(self) -> None:
         for entry in bunkr_entries:
             entry = list(entry)
             entry[0] = "bunkrr"
-            await self.db_conn.execute("""INSERT or REPLACE INTO media VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", entry)
+            await self.db_conn.execute("""INSERT or REPLACE INTO media VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", entry)
         await self.db_conn.commit()
 
         await self.db_conn.execute("""DELETE FROM media WHERE domain = 'bunkr'""")
@@ -177,7 +177,7 @@ async def fix_primary_keys(self) -> None:
 
             await self.db_conn.execute("""ALTER TABLE media_copy RENAME TO media""")
             await self.db_conn.commit()
-            
+
     async def add_columns(self) -> None:
         cursor = await self.db_conn.cursor()
         result = await cursor.execute("""pragma table_info(media)""")
@@ -187,3 +187,11 @@ async def add_columns(self) -> None:
         if "album_id" not in current_cols:
             await self.db_conn.execute("""ALTER TABLE media ADD COLUMN album_id TEXT""")
             await self.db_conn.commit()
+
+        if "created_at" not in current_cols:
+            await self.db_conn.execute("""ALTER TABLE media ADD COLUMN created_at TIMESTAMP""")
+            await self.db_conn.commit()
+
+        if "completed_at" not in current_cols:
+            await self.db_conn.execute("""ALTER TABLE media ADD COLUMN completed_at TIMESTAMP""")
+            await self.db_conn.commit()