Updated edit history to allow for faster retention pruning

Closes #230
mochidev · Oct 14, 2024 · d48ae95 · d48ae95
1 parent 2c259ec
commit d48ae95
Show file tree

Hide file tree

Showing 5 changed files with 73 additions and 14 deletions.
diff --git a/Sources/CodableDatastore/Persistence/Disk Persistence/Datastore/DatastoreRoot.swift b/Sources/CodableDatastore/Persistence/Disk Persistence/Datastore/DatastoreRoot.swift
@@ -10,12 +10,38 @@ import Foundation
 
 typealias DatastoreRootIdentifier = DatedIdentifier<DiskPersistence<ReadOnly>.Datastore.RootObject>
 
+struct DatastoreRootReference: Codable, Hashable {
+    var datastoreID: DatastoreIdentifier?
+    var datastoreRootID: DatastoreRootIdentifier
+
+    init(datastoreID: DatastoreIdentifier, datastoreRootID: DatastoreRootIdentifier) {
+        self.datastoreID = datastoreID
+        self.datastoreRootID = datastoreRootID
+    }
+
+    init(from decoder: any Decoder) throws {
+        /// Attempt to decode a full object, otherwise fall back to a single value as it was prior to version 0.4 (2024-10-11)
+        do {
+            let container: KeyedDecodingContainer<CodingKeys> = try decoder.container(keyedBy: CodingKeys.self)
+            self.datastoreID = try container.decodeIfPresent(DatastoreIdentifier.self, forKey: .datastoreID)
+            self.datastoreRootID = try container.decode(DatastoreRootIdentifier.self, forKey: .datastoreRootID)
+        } catch {
+            self.datastoreID = nil
+            self.datastoreRootID = try decoder.singleValueContainer().decode(DatastoreRootIdentifier.self)
+        }
+    }
+}
+
 extension DiskPersistence.Datastore {
     actor RootObject: Identifiable {
         let datastore: DiskPersistence<AccessMode>.Datastore
 
         let id: DatastoreRootIdentifier
 
+        nonisolated var referenceID: DatastoreRootReference {
+            DatastoreRootReference(datastoreID: datastore.id, datastoreRootID: id)
+        }
+
         var _rootObject: DatastoreRootManifest?
 
         var isPersisted: Bool

diff --git a/Sources/CodableDatastore/Persistence/Disk Persistence/DiskPersistence.swift b/Sources/CodableDatastore/Persistence/Disk Persistence/DiskPersistence.swift
@@ -505,8 +505,8 @@ extension DiskPersistence {
     func persist(
         actionName: String?,
         roots: [DatastoreKey : Datastore.RootObject],
-        addedDatastoreRoots: Set<DatastoreRootIdentifier>,
-        removedDatastoreRoots: Set<DatastoreRootIdentifier>
+        addedDatastoreRoots: Set<DatastoreRootReference>,
+        removedDatastoreRoots: Set<DatastoreRootReference>
     ) async throws {
         let containsEdits = try await readingCurrentSnapshot { snapshot in
             try await snapshot.readingManifest { manifest, iteration in

diff --git a/Sources/CodableDatastore/Persistence/Disk Persistence/Snapshot/Snapshot.swift b/Sources/CodableDatastore/Persistence/Disk Persistence/Snapshot/Snapshot.swift
@@ -162,7 +162,36 @@ extension Snapshot {
 
         let fileManager = FileManager()
 
-        /// Start by deleting and pruning roots as needed.
+        /// Start by deleting and pruning roots as needed. We attempt to do this twice, as older versions of the persistence (prior to 0.4) didn't record the datastore ID along with the root id, which would therefor require extra work.
+        /// First, delete the root entries we know to be removed.
+        for datastoreRoot in datastoreRootsToPruneAndDelete {
+            guard let datastoreID = datastoreRoot.datastoreID else { continue }
+            let datastore = datastores[datastoreID] ?? DiskPersistence<AccessMode>.Datastore(id: datastoreID, snapshot: self)
+            do {
+                try await datastore.pruneRootObject(with: datastoreRoot.datastoreRootID, mode: mode, shouldDelete: true)
+            } catch URLError.fileDoesNotExist, CocoaError.fileReadNoSuchFile, CocoaError.fileNoSuchFile, POSIXError.ENOENT {
+                /// This datastore root is already gone.
+            } catch {
+                print("Could not delete datastore root \(datastoreRoot): \(error)")
+                throw error
+            }
+            datastoreRootsToPruneAndDelete.remove(datastoreRoot)
+        }
+        /// Prune the root entries that were just added, as they themselves refer to other deleted assets.
+        for datastoreRoot in datastoreRootsToPrune {
+            guard let datastoreID = datastoreRoot.datastoreID else { continue }
+            let datastore = datastores[datastoreID] ?? DiskPersistence<AccessMode>.Datastore(id: datastoreID, snapshot: self)
+            do {
+                try await datastore.pruneRootObject(with: datastoreRoot.datastoreRootID, mode: mode, shouldDelete: false)
+            } catch URLError.fileDoesNotExist, CocoaError.fileReadNoSuchFile, CocoaError.fileNoSuchFile, POSIXError.ENOENT {
+                /// This datastore root is already gone.
+            } catch {
+                print("Could not prune datastore root \(datastoreRoot): \(error)")
+                throw error
+            }
+            datastoreRootsToPrune.remove(datastoreRoot)
+        }
+        /// If any regerences remain, funnel into this code path for very old persistences.
         if !datastoreRootsToPruneAndDelete.isEmpty || !datastoreRootsToPrune.isEmpty {
             for (_, datastoreInfo) in iteration.dataStores {
                 /// Skip any roots for datastores being deleted, since we'll just unlink the whole directory in that case.
@@ -172,23 +201,27 @@ extension Snapshot {
 
                 /// Delete the root entries we know to be removed.
                 for datastoreRoot in datastoreRootsToPruneAndDelete {
-                    // TODO: Clean this up by also storing the datastore ID in with the root ID…
                     do {
-                        try await datastore.pruneRootObject(with: datastoreRoot, mode: mode, shouldDelete: true)
+                        try await datastore.pruneRootObject(with: datastoreRoot.datastoreRootID, mode: mode, shouldDelete: true)
                         datastoreRootsToPruneAndDelete.remove(datastoreRoot)
-                    } catch {
+                    } catch URLError.fileDoesNotExist, CocoaError.fileReadNoSuchFile, CocoaError.fileNoSuchFile, POSIXError.ENOENT {
                         /// This datastore did not contain the specified root, skip it for now.
+                    } catch {
+                        print("Could not delete datastore root \(datastoreRoot): \(error).")
+                        throw error
                     }
                 }
 
                 /// Prune the root entries that were just added, as they themselves refer to other deleted assets.
                 for datastoreRoot in datastoreRootsToPrune {
-                    // TODO: Clean this up by also storing the datastore ID in with the root ID…
                     do {
-                        try await datastore.pruneRootObject(with: datastoreRoot, mode: mode, shouldDelete: false)
+                        try await datastore.pruneRootObject(with: datastoreRoot.datastoreRootID, mode: mode, shouldDelete: false)
                         datastoreRootsToPrune.remove(datastoreRoot)
-                    } catch {
+                    } catch URLError.fileDoesNotExist, CocoaError.fileReadNoSuchFile, CocoaError.fileNoSuchFile, POSIXError.ENOENT {
                         /// This datastore did not contain the specified root, skip it for now.
+                    } catch {
+                        print("Could not prune datastore root \(datastoreRoot): \(error).")
+                        throw error
                     }
                 }
             }

diff --git a/Sources/CodableDatastore/Persistence/Disk Persistence/Snapshot/SnapshotIteration.swift b/Sources/CodableDatastore/Persistence/Disk Persistence/Snapshot/SnapshotIteration.swift
@@ -48,10 +48,10 @@ struct SnapshotIteration: Codable, Equatable, Identifiable {
     var removedDatastores: Set<DatastoreIdentifier> = []
 
     /// The datastore roots that have been added in this iteration of the snapshot.
-    var addedDatastoreRoots: Set<DatastoreRootIdentifier> = []
+    var addedDatastoreRoots: Set<DatastoreRootReference> = []
 
     /// The datastore roots that have been replaced in this iteration of the snapshot.
-    var removedDatastoreRoots: Set<DatastoreRootIdentifier> = []
+    var removedDatastoreRoots: Set<DatastoreRootReference> = []
 }
 
 extension SnapshotIteration {
@@ -94,7 +94,7 @@ extension SnapshotIteration {
     func datastoreRootsToPrune(
         for mode: SnapshotPruneMode,
         options: SnapshotPruneOptions
-    ) -> Set<DatastoreRootIdentifier> {
+    ) -> Set<DatastoreRootReference> {
         switch (mode, options) {
         case (.pruneRemoved, .pruneAndDelete):  removedDatastoreRoots
         case (.pruneAdded, .pruneAndDelete):    addedDatastoreRoots

diff --git a/Sources/CodableDatastore/Persistence/Disk Persistence/Transaction/Transaction.swift b/Sources/CodableDatastore/Persistence/Disk Persistence/Transaction/Transaction.swift
@@ -172,8 +172,8 @@ extension DiskPersistence {
                 try await root.persistIfNeeded()
             }
 
-            let addedDatastoreRoots = Set(createdRootObjects.map(\.id))
-            let removedDatastoreRoots = Set(deletedRootObjects.map(\.id))
+            let addedDatastoreRoots = Set(createdRootObjects.map(\.referenceID))
+            let removedDatastoreRoots = Set(deletedRootObjects.map(\.referenceID))
 
             try await persistence.persist(
                 actionName: actionName,