Track more asset metadata

Refresh.GameServer/CommandLineManager.cs

@@ -18,7 +18,7 @@ internal CommandLineManager(RefreshGameServer server)
     private class Options
     {
         [Option('i', "import_assets", Required = false, 
-            HelpText = "Re-import all assets from the datastore into the database. This is a destructive action, only use when upgrading to <=v1.5.0")]
+            HelpText = "Re-import all assets from the datastore into the database.")]
         public bool ImportAssets { get; set; }
 
         [Option('I', "import_images", Required = false, HelpText = "Convert all images in the database to .PNGs. Otherwise, images will be converted as they are used")]

Refresh.GameServer/Database/GameDatabaseContext.Assets.cs

@@ -18,15 +18,9 @@ public void AddAssetToDatabase(GameAsset asset) =>
             this._realm.Add(asset);
         });
 
-    public void AddAssetsToDatabase(IEnumerable<GameAsset> assets) =>
+    public void AddOrUpdateAssetsInDatabase(IEnumerable<GameAsset> assets) =>
         this._realm.Write(() =>
         {
-            this._realm.Add(assets);
-        });
-
-    public void DeleteAllAssetMetadata() =>
-        this._realm.Write(() =>
-        {
-            this._realm.RemoveAll<GameAsset>();
+            this._realm.Add(assets, true);
         });
 }

Refresh.GameServer/Database/GameDatabaseProvider.cs

@@ -32,7 +32,7 @@ protected GameDatabaseProvider(IDateTimeProvider time)
         this._time = time;
     }
 
-    protected override ulong SchemaVersion => 97;
+    protected override ulong SchemaVersion => 99;
 
     protected override string Filename => "refreshGameServer.realm";
 

Refresh.GameServer/Endpoints/ApiV3/DataTypes/Response/ApiGameAssetResponse.cs

@@ -9,6 +9,7 @@ public class ApiGameAssetResponse : IApiResponse, IDataConvertableFrom<ApiGameAs
     public required ApiGameUserResponse? OriginalUploader { get; set; }
     public required DateTimeOffset UploadDate { get; set; }
     public required GameAssetType AssetType { get; set; }
+    public required IEnumerable<string> Dependencies { get; set; }
 
     public static ApiGameAssetResponse? FromOld(GameAsset? old)
     {
@@ -20,6 +21,7 @@ public class ApiGameAssetResponse : IApiResponse, IDataConvertableFrom<ApiGameAs
             OriginalUploader = ApiGameUserResponse.FromOld(old.OriginalUploader),
             UploadDate = old.UploadDate,
             AssetType = old.AssetType,
+            Dependencies = old.Dependencies,
         };
     }
 

Refresh.GameServer/Importing/AssetImporter.cs

@@ -19,63 +19,62 @@ public AssetImporter(Logger? logger = null, IDateTimeProvider? timeProvider = nu
         this._timeProvider = timeProvider;
     }
 
-    public void ImportFromDataStoreCli(GameDatabaseContext context, IDataStore dataStore)
-    {
-        Console.WriteLine("This tool will scan and manually import existing assets into Refresh's database.");
-        Console.WriteLine("This will wipe all existing asset metadata in the database. Are you sure you want to follow through with this operation?");
-        Console.WriteLine();
-        Console.Write("Are you sure? [y/N] ");
-
-        char key = char.ToLower(Console.ReadKey().KeyChar);
-        Console.WriteLine();
-        if(key != 'y')
-        {
-            if(key != 'n') Console.WriteLine("Unsure what you mean, assuming no.");
-            Environment.Exit(0);
-            return;
-        }
-
-        this.ImportFromDataStore(context, dataStore);
-    }
-
-    public void ImportFromDataStore(GameDatabaseContext context, IDataStore dataStore)
+    public void ImportFromDataStore(GameDatabaseContext database, IDataStore dataStore)
     {
+        int updatedAssets = 0;
+        int newAssets = 0;
         this.Stopwatch.Start();
 
-        context.DeleteAllAssetMetadata();
-        this.Info("Deleted all asset metadata");
-
-        List<string> assetHashes = dataStore.GetKeysFromStore()
-            .Where(key => !key.Contains('/'))
-            .ToList();
+        IEnumerable<string> assetHashes = dataStore.GetKeysFromStore()
+            .Where(key => !key.Contains('/'));
 
         List<GameAsset> assets = new();
         foreach (string hash in assetHashes)
         {
             byte[] data = dataStore.GetDataFromStore(hash);
 
-            GameAsset? asset = this.ReadAndVerifyAsset(hash, data, null);
-            if (asset == null) continue;
+            GameAsset? newAsset = this.ReadAndVerifyAsset(hash, data, null);
+            if (newAsset == null) continue;
+
+            GameAsset? oldAsset = database.GetAssetFromHash(hash);
+
+            if (oldAsset != null)
+            {
+                newAsset.OriginalUploader = oldAsset.OriginalUploader;
+                newAsset.UploadDate = oldAsset.UploadDate;
+                updatedAssets++;
+            }
+            else
+            {
+                newAssets++;
+            }
 
-            assets.Add(asset);
-            this.Info($"Processed {asset.AssetType} asset {hash} ({AssetSafetyLevelExtensions.FromAssetType(asset.AssetType)})");
+            assets.Add(newAsset);
+            this.Info($"Processed {newAsset.AssetType} asset {hash} ({AssetSafetyLevelExtensions.FromAssetType(newAsset.AssetType)})");
         }
 
-        context.AddAssetsToDatabase(assets);
+        database.AddOrUpdateAssetsInDatabase(assets);
+
+        int hashCount = newAssets + updatedAssets;
 
-        this.Info($"Successfully imported {assets.Count}/{assetHashes.Count} assets into database");
-        if (assets.Count < assetHashes.Count)
+        this.Info($"Successfully imported {assets.Count}/{hashCount} assets ({newAssets} new, {updatedAssets} updated) into database");
+        if (assets.Count < hashCount)
         {
-            this.Warn($"{assetHashes.Count - assets.Count} assets were not imported");
+            this.Warn($"{hashCount - assets.Count} assets were not imported");
         }
     }
 
-    [Pure]
-    public GameAsset? ReadAndVerifyAsset(string hash, byte[] data, TokenPlatform? platform)
+    private static string GetHashOfShaBytes(byte[] data)
     {
-        string checkedHash = BitConverter.ToString(SHA1.HashData(data))
+        return BitConverter.ToString(data)
             .Replace("-", "")
             .ToLower();
+    }
+
+    [Pure]
+    public GameAsset? ReadAndVerifyAsset(string hash, byte[] data, TokenPlatform? platform)
+    {
+        string checkedHash = GetHashOfShaBytes(SHA1.HashData(data));
 
         if (checkedHash != hash)
         {
@@ -90,8 +89,77 @@ public void ImportFromDataStore(GameDatabaseContext context, IDataStore dataStor
             AssetHash = hash,
             AssetType = this.DetermineAssetType(data, platform),
             IsPSP = platform == TokenPlatform.PSP,
+            SizeInBytes = data.Length,
         };
+
+        if (AssetTypeHasDependencyTree(asset.AssetType, data))
+        {
+            try
+            {
+                List<string> dependencies = this.ParseDependencyTree(data);
+                foreach (string dependency in dependencies)
+                {
+                    asset.Dependencies.Add(dependency);
+                }
+            }
+            catch (Exception e)
+            {
+                this.Warn($"Could not parse dependency tree for {hash}: {e}");
+            }
+        }
 
         return asset;
     }
+
+    [Pure]
+    private static bool AssetTypeHasDependencyTree(GameAssetType type, byte[] data)
+    {
+        if (type is GameAssetType.Jpeg
+            or GameAssetType.Png
+            or GameAssetType.Tga
+            or GameAssetType.Texture
+            or GameAssetType.GameDataTexture
+            or GameAssetType.Unknown)
+        {
+            return false;
+        }
+
+        #if DEBUG
+        char typeChar = (char)data[3];
+        if (typeChar != 'b') throw new Exception($"Asset type {type} is not binary (char was '{typeChar}')");
+        #endif
+
+        return true;
+    }
+
+    private List<string> ParseDependencyTree(byte[] data)
+    {
+        List<string> dependencies = new();
+
+        // Parse dependency table
+        MemoryStream ms = new(data);
+        BEBinaryReader reader = new(ms);
+
+        ms.Seek(8, SeekOrigin.Begin);
+        uint dependencyTableOffset = reader.ReadUInt32();
+
+        ms.Seek(dependencyTableOffset, SeekOrigin.Begin);
+        uint dependencyCount = reader.ReadUInt32();
+
+        this.Debug($"Dependency count offset: {dependencyTableOffset}, count: {dependencyCount}");
+        for (int i = 0; i < dependencyCount; i++)
+        {
+            byte flags = reader.ReadByte();
+            if ((flags & 0x1) != 0) // UGC/SHA1
+            {
+                byte[] hashBytes = reader.ReadBytes(20);
+                dependencies.Add(GetHashOfShaBytes(hashBytes));
+            }
+            else if ((flags & 0x2) != 0) reader.ReadUInt32(); // Skip GUID
+
+            reader.ReadUInt32();
+        }
+
+        return dependencies;
+    }
 }

Refresh.GameServer/Importing/Importer.cs

@@ -36,6 +36,11 @@ protected void Warn(string message)
     {
         this._logger.LogWarning(BunkumCategory.UserContent, $"[{this.Stopwatch.ElapsedMilliseconds}ms] {message}");
     }
+
+    protected void Debug(string message)
+    {
+        this._logger.LogDebug(BunkumCategory.UserContent, $"[{this.Stopwatch.ElapsedMilliseconds}ms] {message}");
+    }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     private static bool MatchesMagic(ReadOnlySpan<byte> data, ReadOnlySpan<byte> magic)

Refresh.GameServer/RefreshGameServer.cs

@@ -59,7 +59,7 @@
         this._databaseProvider = databaseProvider.Invoke();
         this._dataStore = dataStore;
 
         this._server = listener == null ? new BunkumHttpServer(logConfig, sinks) : new BunkumHttpServer(listener, configuration: logConfig, sinks);
         this.Logger.LogDebug(RefreshContext.Startup, "Successfully initialized " + this.GetType().Name);
 
         this._server.Initialize = _ =>
@@ -216,14 +216,7 @@
         using GameDatabaseContext context = this.InitializeDatabase();
 
         AssetImporter importer = new();
-        if (!force)
-        {
-            importer.ImportFromDataStoreCli(context, this._dataStore);
-        }
-        else
-        {
-            importer.ImportFromDataStore(context, this._dataStore);
-        }
+        importer.ImportFromDataStore(context, this._dataStore);
     }
 
     public void ImportImages()

Refresh.GameServer/Types/Assets/GameAsset.cs

@@ -3,14 +3,14 @@
 
 namespace Refresh.GameServer.Types.Assets;
 
-[JsonObject(MemberSerialization.OptIn)]
 public partial class GameAsset : IRealmObject
 {
-    [PrimaryKey] [JsonProperty] public string AssetHash { get; set; } = string.Empty;
-    [JsonProperty] public GameUser? OriginalUploader { get; set; }
-    [JsonProperty] public DateTimeOffset UploadDate { get; set; }
-    [JsonProperty] public bool IsPSP { get; set; }
-    [JsonProperty] [Ignored] public GameAssetType AssetType
+    [PrimaryKey] public string AssetHash { get; set; } = string.Empty;
+    public GameUser? OriginalUploader { get; set; }
+    public DateTimeOffset UploadDate { get; set; }
+    public bool IsPSP { get; set; }
+    public int SizeInBytes { get; set; }
+    [Ignored] public GameAssetType AssetType
     {
         get => (GameAssetType)this._AssetType;
         set => this._AssetType = (int)value;
@@ -19,7 +19,7 @@ [JsonProperty] [Ignored] public GameAssetType AssetType
     // ReSharper disable once InconsistentNaming
     internal int _AssetType { get; set; }
 
-    [JsonProperty] public IList<GameAsset> Dependencies { get; } = null!;
+    public IList<string> Dependencies { get; } = null!;
 
-    [JsonProperty] [Ignored] public AssetSafetyLevel SafetyLevel => AssetSafetyLevelExtensions.FromAssetType(this.AssetType);
+    [Ignored] public AssetSafetyLevel SafetyLevel => AssetSafetyLevelExtensions.FromAssetType(this.AssetType);
 }