Merge pull request #9 from bookdude13/dev

Better handling of unexpected map files, and simpler db handling

Assets/Scripts/CustomFileManager.cs

@@ -7,6 +7,7 @@
 using Newtonsoft.Json;
 using System.Threading.Tasks;
 using System.Linq;
+using Newtonsoft.Json.Linq;
 
 public class CustomFileManager : MonoBehaviour
 {
@@ -15,7 +16,6 @@ public class CustomFileManager : MonoBehaviour
 
     private bool isMovingFiles = false;
     private string synthCustomContentDir = "/sdcard/SynthRidersUC/";
-    private Dictionary<string, MapZMetadata> localMaps = new Dictionary<string, MapZMetadata>();
 
     private readonly string MAP_EXTENSION = ".synth";
     private readonly HashSet<string> STAGE_EXTENSIONS = new HashSet<string>() { ".stagequest", ".spinstagequest" };
@@ -29,27 +29,19 @@ private void Awake()
     }
 
     private async void Start() {
-        await ReloadLocalMaps();
+        await RefreshLocalDatabase(synthCustomContentDir);
         displayManager.EnableFetchingLatest();
     }
 
-    /// Loads all local maps from disk and updates the dictionary
-    private async Task ReloadLocalMaps() {
-        localMaps = await GetLocalMaps(synthCustomContentDir);
-        displayManager.DebugLog($"{localMaps.Count} local maps found");
-    }
-
-    /// Parses the map at the given path and adds it to the localMaps dict
+    /// Parses the map at the given path and adds it to the collection
     public async void AddLocalMap(string mapPath) {
         MapZMetadata metadata = await ParseLocalMap(mapPath);
         if (metadata == null) {
             displayManager.ErrorLog("Failed to parse map at " + mapPath);
             return;
         }
 
-        if (!localMaps.TryAdd(metadata.hash, metadata)) {
-            displayManager.DebugLog("Map already exists in dict. Hash: " + metadata.hash);
-        }
+        db.AddMap(metadata, displayManager);
     }
 
     /// Useful for debugging. Clear out all custom songs
@@ -65,7 +57,7 @@ public async void DeleteCustomSongs() {
             displayManager.ErrorLog($"Failed to delete files: {e.Message}");
         }
 
-        await ReloadLocalMaps();
+        await RefreshLocalDatabase(synthCustomContentDir);
     }
 
     public void StartMoveDownloadedFiles()
@@ -288,15 +280,14 @@ public string MoveCustomPlaylist(string filePath) {
 
     /// Returns a new list with all maps in the source list that aren't contained in the user's custom song directory already
     public List<MapItem> FilterOutExistingMaps(List<MapItem> maps) {
-        displayManager.DebugLog($"{localMaps.Count} local maps found");
-        return maps.Where(mapItem => !localMaps.ContainsKey(mapItem.hash)).ToList();
+        displayManager.DebugLog($"{db.GetNumberOfMaps()} local maps found");
+        return maps.Where(mapItem => db.GetFromHash(mapItem.hash) == null).ToList();
     }
 
     /// Refreshes local database metadata. Parses all missing custom map files.
     /// This saves the updated database.
-    /// Returned dictionary is hash: object
-    private async Task<Dictionary<string, MapZMetadata>> GetLocalMaps(string synthCustomContentDir) {
-        var maps = new Dictionary<string, MapZMetadata>();
+    private async Task RefreshLocalDatabase(string synthCustomContentDir) {
+        var localHashes = new HashSet<string>();
 
         // Make sure local db state is up to date
         await db.Load();
@@ -305,11 +296,11 @@ private async Task<Dictionary<string, MapZMetadata>> GetLocalMaps(string synthCu
             var mapsDir = Path.Join(synthCustomContentDir, "CustomSongs");
             if (!Directory.Exists(mapsDir)) {
                 displayManager.ErrorLog("Custom maps directory doesn't exist!");
-                return new Dictionary<string, MapZMetadata>();
+                return;
             }
 
             var files = Directory.GetFiles(mapsDir, $"*{MAP_EXTENSION}");
-            displayManager.DebugLog($"Updating database with map files...");
+            displayManager.DebugLog($"Updating database with map files ({files.Length} found)...");
             // This will implicitly remove any entries that are only present in the db
             int count = 0;
             int totalFiles = files.Length;
@@ -318,7 +309,7 @@ private async Task<Dictionary<string, MapZMetadata>> GetLocalMaps(string synthCu
                 if (dbMetadata != null) {
                     // DB has this version already - good to go
                     // displayManager.DebugLog(Path.GetFileName(filePath) + " already in db");
-                    maps.Add(dbMetadata.hash, dbMetadata);
+                    localHashes.Add(dbMetadata.hash);
                 }
                 else {
                     // DB doesn't have this version; parse and add
@@ -327,8 +318,9 @@ private async Task<Dictionary<string, MapZMetadata>> GetLocalMaps(string synthCu
                         displayManager.ErrorLog("Failed to parse map at " + filePath);
                         continue;
                     }
-                    maps.Add(metadata.hash, metadata);
-                    db.AddMap(metadata);
+
+                    localHashes.Add(metadata.hash);
+                    db.AddMap(metadata, displayManager);
                 }
 
                 count++;
@@ -343,16 +335,19 @@ private async Task<Dictionary<string, MapZMetadata>> GetLocalMaps(string synthCu
         }
         catch (System.Exception e) {
             displayManager.ErrorLog($"Failed to get local maps: {e.Message}");
-            return new Dictionary<string, MapZMetadata>();
+            return;
         }
 
-        // Successfully loaded maps. Save to db for next run
+        // Successfully loaded maps
+        // Remove all db entries that are no longer on the local file system
+        displayManager.DebugLog("Removing database entries that aren't on the local file system...");
+        db.RemoveMissingHashes(localHashes);
+
+        // Save to db for next run
         if (!await db.Save()) {
             displayManager.ErrorLog("Failed to save db");
             // ignore for now; we still loaded everything fine
         }
-
-        return maps;
     }
 
     /// Parses local map file. Returns null if can't parse or no metadata
@@ -374,6 +369,10 @@ private async Task<MapZMetadata> ParseLocalMap(string filePath) {
                         }
                     }
                 }
+
+                // No return, so missing metadata file.
+                // Report this to be fixed on the Z site
+                displayManager.ErrorLog($"Missing synthriderz.meta.json in map {Path.GetFileName(filePath)}. If not a draft map, report to bookdude13");
             }
         }
         catch (System.Exception e) {

Assets/Scripts/LocalDatabase.cs

@@ -21,6 +21,10 @@ public class LocalDatabase {
     /// Faster lookup of maps by path
     private Dictionary<string, MapZMetadata> localMapPathLookup = new Dictionary<string, MapZMetadata>();
 
+    [JsonIgnore]
+    /// Faster lookup of maps by hash
+    private Dictionary<string, MapZMetadata> localMapHashLookup = new Dictionary<string, MapZMetadata>();
+
 
     public LocalDatabase(DisplayManager displayManager) {
         this.displayManager = displayManager;
@@ -36,19 +40,60 @@ public MapZMetadata GetFromPath(string filePath) {
         return null;
     }
 
+    /// Gets locally stored metadata based on map hash.
+    /// Returns null if not found
+    public MapZMetadata GetFromHash(string hash) {
+        if (localMapHashLookup.ContainsKey(hash)) {
+            return localMapHashLookup[hash];
+        }
+
+        return null;
+    }
+
+    public int GetNumberOfMaps() {
+        return localMapMetadata.Count;
+    }
+
     /// Adds map metadata to database.
-    /// If the file path is already present, replace
-    public void AddMap(MapZMetadata mapMeta) {
+    /// If the file path is already present or hash is already present replace
+    public void AddMap(MapZMetadata mapMeta, ILogHandler logger) {
         // Remove existing to replace with new
         if (localMapPathLookup.ContainsKey(mapMeta.FilePath)) {
+            logger.DebugLog($"Removing map with existing path {mapMeta.FilePath}");
             localMapMetadata.Remove(localMapPathLookup[mapMeta.FilePath]);
             localMapPathLookup.Remove(mapMeta.FilePath);
         }
 
+        if (localMapHashLookup.ContainsKey(mapMeta.hash)) {
+            logger.DebugLog($"Removing map with matching hash {mapMeta.hash}");
+            localMapMetadata.Remove(localMapHashLookup[mapMeta.hash]);
+            localMapHashLookup.Remove(mapMeta.hash);
+        }
+
+        logger.DebugLog($"Adding map {Path.GetFileNameWithoutExtension(mapMeta.FilePath)}");
         localMapPathLookup.Add(mapMeta.FilePath, mapMeta);
+        localMapHashLookup.Add(mapMeta.hash, mapMeta);
         localMapMetadata.Add(mapMeta);
     }
 
+    /// Remove maps that aren't in the list of hashes
+    public void RemoveMissingHashes(HashSet<string> savedHashes) {
+        var toRemove = new List<MapZMetadata>();
+        foreach (var mapMeta in localMapMetadata) {
+            if (!savedHashes.Contains(mapMeta.hash)) {
+                // Not saved; remove from db
+                toRemove.Add(mapMeta);
+            }
+        }
+
+        foreach (var mapMeta in toRemove) {
+            displayManager.DebugLog($"db map not found in filesystem; removing {Path.GetFileName(mapMeta.FilePath)}");
+            localMapMetadata.Remove(mapMeta);
+            localMapPathLookup.Remove(mapMeta.FilePath);
+            localMapHashLookup.Remove(mapMeta.hash);
+        }
+    }
+
     /// Loads db state from file.
     /// Note: Not done implicitly upon creation!
     public async Task Load() {
@@ -66,8 +111,10 @@ public async Task Load() {
 
         this.localMapMetadata = localDb.localMapMetadata;
         this.localMapPathLookup.Clear();
+        this.localMapHashLookup.Clear();
         foreach (var mapMeta in localMapMetadata) {
             localMapPathLookup.Add(mapMeta.FilePath, mapMeta);
+            localMapHashLookup.Add(mapMeta.hash, mapMeta);
         }
         displayManager.DebugLog("DB loaded");
     }

OVRAssetBundles/OVRMasterBundle/OVRMasterBundle.manifest

@@ -1,5 +1,5 @@
 ManifestFileVersion: 0
-CRC: 1452989552
+CRC: 594521003
 AssetBundleManifest:
   AssetBundleInfos:
     Info_0:

OVRAssetBundles/OVRMasterBundle/scene_mainscene.manifest

@@ -3,7 +3,7 @@ CRC: 2882445343
 Hashes:
   AssetFileHash:
     serializedVersion: 2
-    Hash: ae616c74e989d9d6157064f5bf20802c
+    Hash: 485492e4e19363b34ce3364a7444fe41
   TypeTreeHash:
     serializedVersion: 2
     Hash: e2035c4c4f0f66f38066d4508842f4d8

OVRAssetBundles/Temp/SceneLoadData.txt

@@ -1,2 +1,2 @@
-1673828421
+1673873857
 MainScene

ProjectSettings/ProjectSettings.asset

@@ -135,7 +135,7 @@ PlayerSettings:
     16:10: 1
     16:9: 1
     Others: 1
-  bundleVersion: 1.0
+  bundleVersion: 1.0.1
   preloadedAssets:
   - {fileID: 0}
   - {fileID: 0}
@@ -215,7 +215,7 @@ PlayerSettings:
     iPhone: 0
     tvOS: 0
   overrideDefaultApplicationIdentifier: 1
-  AndroidBundleVersionCode: 2
+  AndroidBundleVersionCode: 3
   AndroidMinSdkVersion: 29
   AndroidTargetSdkVersion: 0
   AndroidPreferredInstallLocation: 0