Rewrite asset detection + add many more assets to our detection.

Refresh.GameServer/Configuration/ConfigAssetFlags.cs

@@ -0,0 +1,28 @@
+using Refresh.GameServer.Types.Assets;
+
+namespace Refresh.GameServer.Configuration;
+
+public class ConfigAssetFlags
+{
+    /// <summary>
+    /// This asset can be dangerous to end users.
+    /// </summary>
+    public bool Dangerous { get; set; }
+
+    /// <summary>
+    /// This asset is a media-type asset, e.g. a PNG or TEX.
+    /// </summary>
+    public bool Media { get; set; }
+
+    /// <summary>
+    /// This asset will only ever be created by mods.
+    /// </summary>
+    public bool Modded { get; set; }
+
+    public AssetFlags ToAssetFlags()
+    {
+        return (this.Dangerous ? AssetFlags.Dangerous : AssetFlags.None) |
+               (this.Modded ? AssetFlags.Modded : AssetFlags.None) |
+               (this.Media ? AssetFlags.Media : AssetFlags.None);
+    }
+}

Refresh.GameServer/Configuration/GameServerConfig.cs

@@ -1,6 +1,6 @@
 using System.Diagnostics.CodeAnalysis;
 using Bunkum.Core.Configuration;
-using Refresh.GameServer.Types.Assets;
+using Microsoft.CSharp.RuntimeBinder;
 using Refresh.GameServer.Types.Roles;
 
 namespace Refresh.GameServer.Configuration;
@@ -9,16 +9,52 @@ namespace Refresh.GameServer.Configuration;
 [SuppressMessage("ReSharper", "RedundantDefaultMemberInitializer")]
 public class GameServerConfig : Config
 {
-    public override int CurrentConfigVersion => 15;
+    public override int CurrentConfigVersion => 16;
     public override int Version { get; set; } = 0;
-
-    protected override void Migrate(int oldVer, dynamic oldConfig) {}
+
+    protected override void Migrate(int oldVer, dynamic oldConfig)
+    {
+        if (oldVer < 16)
+        {
+            int oldSafetyLevel = (int)oldConfig.MaximumAssetSafetyLevel;
+            this.BlockedAssetFlags = new ConfigAssetFlags
+            {
+                Dangerous = oldSafetyLevel < 3,
+                Modded = oldSafetyLevel < 2,
+                Media = oldSafetyLevel < 1,
+            };
+
+            // There was no version bump for trusted users being added, so we just have to catch this error :/
+            try
+            {
+                int oldTrustedSafetyLevel = (int)oldConfig.MaximumAssetSafetyLevelForTrustedUsers;
+                this.BlockedAssetFlagsForTrustedUsers = new ConfigAssetFlags
+                {
+                    Dangerous = oldTrustedSafetyLevel < 3,
+                    Modded = oldTrustedSafetyLevel < 2,
+                    Media = oldTrustedSafetyLevel < 1,
+                };
+            }
+            catch (RuntimeBinderException)
+            {
+                this.BlockedAssetFlagsForTrustedUsers = this.BlockedAssetFlags;
+            }
+        }
+    }
 
     public string LicenseText { get; set; } = "Welcome to Refresh!";
-
-    public AssetSafetyLevel MaximumAssetSafetyLevel { get; set; } = AssetSafetyLevel.SafeMedia;
+
+    public ConfigAssetFlags BlockedAssetFlags { get; set; } = new()
+    {
+        Dangerous = true,
+        Modded = true,
+    };
     /// <seealso cref="GameUserRole.Trusted"/>
-    public AssetSafetyLevel MaximumAssetSafetyLevelForTrustedUsers { get; set; } = AssetSafetyLevel.SafeMedia;
+    public ConfigAssetFlags BlockedAssetFlagsForTrustedUsers { get; set; } = new()
+    {
+        Dangerous = true,
+        Modded = true,
+    };
     public bool AllowUsersToUseIpAuthentication { get; set; } = false;
     public bool UseTicketVerification { get; set; } = true;
     public bool RegistrationEnabled { get; set; } = true;

Refresh.GameServer/Database/GameDatabaseProvider.cs

@@ -34,7 +34,7 @@ protected GameDatabaseProvider(IDateTimeProvider time)
         this._time = time;
     }
 
-    protected override ulong SchemaVersion => 124;
+    protected override ulong SchemaVersion => 125;
 
     protected override string Filename => "refreshGameServer.realm";
 
@@ -349,6 +349,65 @@ protected override void Migrate(Migration migration, ulong oldVersion)
                 // and PSP is the only game to upload TGA files.
                 newAsset.IsPSP = newAsset.AssetType == GameAssetType.Tga;
             }
+
+            // In version 122, we started tracking asset types differently, so we need to convert to the new system
+            if (oldVersion < 125)
+            {
+                newAsset.AssetType = (int)oldAsset._AssetType switch
+                {
+                    -1 => GameAssetType.Unknown,
+                    0 => GameAssetType.Level,
+                    1 => GameAssetType.Plan,
+                    2 => GameAssetType.Texture,
+                    3 => GameAssetType.Jpeg,
+                    4 => GameAssetType.Png,
+                    5 => GameAssetType.GfxMaterial,
+                    6 => GameAssetType.Mesh,
+                    7 => GameAssetType.GameDataTexture,
+                    8 => GameAssetType.Palette,
+                    9 => GameAssetType.Script,
+                    10 => GameAssetType.ThingRecording,
+                    11 => GameAssetType.VoiceRecording,
+                    12 => GameAssetType.Painting,
+                    13 => GameAssetType.Tga,
+                    14 => GameAssetType.SyncedProfile,
+                    15 => GameAssetType.Mip,
+                    16 => GameAssetType.GriefSongState,
+                    17 => GameAssetType.Material,
+                    18 => GameAssetType.SoftPhysicsSettings,
+                    19 => GameAssetType.Bevel,
+                    20 => GameAssetType.StreamingLevelChunk,
+                    21 => GameAssetType.Animation,
+                    _ => throw new Exception("Invalid asset type " + oldAsset._AssetType),
+                };
+                newAsset.AssetFormat = (int)oldAsset._AssetType switch
+                {
+                    -1 => GameAssetFormat.Unknown,
+                    0 => GameAssetFormat.Binary,
+                    1 => GameAssetFormat.Binary,
+                    2 => GameAssetFormat.CompressedTexture,
+                    3 => GameAssetFormat.Unknown,
+                    4 => GameAssetFormat.Unknown,
+                    5 => GameAssetFormat.Binary,
+                    6 => GameAssetFormat.Binary,
+                    7 => GameAssetFormat.CompressedTexture,
+                    8 => GameAssetFormat.Binary,
+                    9 => GameAssetFormat.Binary,
+                    10 => GameAssetFormat.Binary,
+                    11 => GameAssetFormat.Binary,
+                    12 => GameAssetFormat.Binary,
+                    13 => GameAssetFormat.Unknown,
+                    14 => GameAssetFormat.Binary,
+                    15 => GameAssetFormat.Unknown,
+                    16 => GameAssetFormat.Unknown,
+                    17 => GameAssetFormat.Binary,
+                    18 => GameAssetFormat.Text,
+                    19 => GameAssetFormat.Binary,
+                    20 => GameAssetFormat.Binary,
+                    21 => GameAssetFormat.Binary,
+                    _ => throw new Exception("Invalid asset type " + oldAsset._AssetType),
+                };
+            }
         }
 
         // Remove all scores with a null level, as in version 92 we started tracking story leaderboards differently

Refresh.GameServer/Endpoints/ApiV3/DataTypes/Response/ApiInstanceResponse.cs

@@ -39,8 +39,8 @@ public class ApiInstanceResponse : IApiResponse
     public required string SoftwareLicenseUrl { get; set; }
 
     public required bool RegistrationEnabled { get; set; }
-    public required AssetSafetyLevel MaximumAssetSafetyLevel { get; set; }
-    public required AssetSafetyLevel MaximumAssetSafetyLevelForTrustedUsers { get; set; }
+    public required AssetFlags BlockedAssetFlags { get; set; }
+    public required AssetFlags BlockedAssetFlagsForTrustedUsers { get; set; }
 
     public required IEnumerable<ApiGameAnnouncementResponse> Announcements { get; set; }
     public required ApiRichPresenceConfigurationResponse RichPresenceConfiguration { get; set; }

Refresh.GameServer/Endpoints/ApiV3/InstanceApiEndpoints.cs

@@ -57,8 +57,8 @@ public ApiResponse<ApiInstanceResponse> GetInstanceInformation(RequestContext co
             SoftwareSourceUrl = "https://github.com/LittleBigRefresh/Refresh",
             SoftwareLicenseName = "AGPL-3.0",
             SoftwareLicenseUrl = "https://www.gnu.org/licenses/agpl-3.0.txt",
-            MaximumAssetSafetyLevel = gameConfig.MaximumAssetSafetyLevel,
-            MaximumAssetSafetyLevelForTrustedUsers = gameConfig.MaximumAssetSafetyLevelForTrustedUsers,
+            BlockedAssetFlags = gameConfig.BlockedAssetFlags.ToAssetFlags(),
+            BlockedAssetFlagsForTrustedUsers = gameConfig.BlockedAssetFlagsForTrustedUsers.ToAssetFlags(),
             Announcements = ApiGameAnnouncementResponse.FromOldList(database.GetAnnouncements(), dataContext),
             MaintenanceModeEnabled = gameConfig.MaintenanceMode,
             RichPresenceConfiguration = ApiRichPresenceConfigurationResponse.FromOld(RichPresenceConfiguration.Create(

Refresh.GameServer/Endpoints/Game/ResourceEndpoints.cs

@@ -67,21 +67,21 @@ public Response UploadAsset(RequestContext context, string hash, string type, by
 
         gameAsset.UploadDate = DateTimeOffset.FromUnixTimeSeconds(Math.Clamp(gameAsset.UploadDate.ToUnixTimeSeconds(), timeProvider.EarliestDate, timeProvider.TimestampSeconds));
 
-        AssetSafetyLevel safetyLevel = config.MaximumAssetSafetyLevel;
+        AssetFlags blockedAssetFlags = config.BlockedAssetFlags.ToAssetFlags();
         if (user.Role >= GameUserRole.Trusted)
-            safetyLevel = config.MaximumAssetSafetyLevelForTrustedUsers;
+            blockedAssetFlags = config.BlockedAssetFlagsForTrustedUsers.ToAssetFlags();
 
-        // Dont block any assets uploaded from PSP, and block any unwanted assets,
-        // for example, if asset safety level is Dangerous (2) and maximum is configured as Safe (0), return 401
-        // if asset safety is Safe (0), and maximum is configured as Safe (0), proceed
-        if (gameAsset.SafetyLevel > safetyLevel && !isPSP)
+        // Don't block any assets uploaded from PSP, else block any unwanted assets,
+        // For example, if the "blocked asset flags" has the "Media" bit set, and so does the asset,
+        // then that bit will be set after the AND operation, and we know to block it.
+        if ((gameAsset.AssetFlags & blockedAssetFlags) != 0 && !isPSP)
         {
             context.Logger.LogWarning(BunkumCategory.UserContent, $"{gameAsset.AssetType} {hash} by {user} is above configured safety limit " +
-                                                                 $"({gameAsset.SafetyLevel} > {safetyLevel})");
+                                                                 $"({gameAsset.AssetFlags} is blocked by {blockedAssetFlags})");
             return Unauthorized;
         }
 
-        if (isPSP && gameAsset.SafetyLevel == AssetSafetyLevel.SafeMedia && safetyLevel < AssetSafetyLevel.SafeMedia)
+        if (isPSP && gameAsset.AssetFlags.HasFlag(AssetFlags.Media) && blockedAssetFlags.HasFlag(AssetFlags.Media))
         {
             context.Logger.LogWarning(BunkumCategory.UserContent, $"{gameAsset.AssetType} {hash} by {user} cannot be uploaded because media is disabled");
             return Unauthorized;

Refresh.GameServer/Importing/AssetImporter.cs

@@ -55,7 +55,7 @@ public void ImportFromDataStore(GameDatabaseContext database, IDataStore dataSto
             }
 
             assets.Add(newAsset);
-            this.Info($"Processed {newAsset.AssetType} asset {hash} ({AssetSafetyLevelExtensions.FromAssetType(newAsset.AssetType)})");
+            this.Info($"Processed {newAsset.AssetType} asset {hash} ({AssetSafetyLevelExtensions.FromAssetType(newAsset.AssetType, newAsset.AssetFormat)})");
         }
 
         database.AddOrUpdateAssetsInDatabase(assets);
@@ -99,22 +99,25 @@ static char GetHexChar(int value)
             this.Warn($"{hash} is actually hashed as {checkedHash} - this asset is likely corrupt.");
             return null;
         }
-
+
+        (GameAssetType assetType, GameAssetFormat assetFormat) = this.DetermineAssetType(data, platform);
+
         GameAsset asset = new()
         {
             UploadDate = this._timeProvider.Now,
             OriginalUploader = null,
             AssetHash = hash,
-            AssetType = this.DetermineAssetType(data, platform),
+            AssetType = assetType,
+            AssetFormat = assetFormat,
             IsPSP = platform == TokenPlatform.PSP,
             SizeInBytes = data.Length,
         };
 
-        if (AssetTypeHasDependencyTree(asset.AssetType, data))
+        if (asset.AssetFormat.HasDependencyTree())
         {
             try
             {
-                List<string> dependencies = this.ParseDependencyTree(data);
+                List<string> dependencies = this.ParseAssetDependencies(data);
                 foreach (string dependency in dependencies)
                 {
                     asset.Dependencies.Add(dependency);
@@ -129,43 +132,30 @@ static char GetHexChar(int value)
         return asset;
     }
 
-    [Pure]
-    private static bool AssetTypeHasDependencyTree(GameAssetType type, byte[] data)
-    {
-        if (type is GameAssetType.Jpeg
-            or GameAssetType.Png
-            or GameAssetType.Tga
-            or GameAssetType.Texture
-            or GameAssetType.GameDataTexture
-            or GameAssetType.Mip
-            or GameAssetType.Unknown)
-        {
-            return false;
-        }
-
-        #if DEBUG
-        char typeChar = (char)data[3];
-        if (typeChar != 'b') throw new Exception($"Asset type {type} is not binary (char was '{typeChar}')");
-        #endif
-
-        return true;
-    }
-
-    private List<string> ParseDependencyTree(byte[] data)
+    // See toolkit's source code for this: https://github.com/ennuo/toolkit/blob/15342e1afca2d5ac1de49e207922099e7aacef86/lib/cwlib/src/main/java/cwlib/types/SerializedResource.java#L113
+    private List<string> ParseAssetDependencies(byte[] data)
     {
-        List<string> dependencies = new();
-
-        // Parse dependency table
         MemoryStream ms = new(data);
         BEBinaryReader reader = new(ms);
+
+        // Read magic from the asset
+        reader.ReadUInt32();
+
+        // Read the head revision of the asset 
+        uint head = reader.ReadUInt32();
+
+        // Dependency lists were only added in revision 0x109, so if we are less than that, then just skip trying to parse out the dependency tree
+        if (head < 0x109) 
+            return [];
 
-        ms.Seek(8, SeekOrigin.Begin);
         uint dependencyTableOffset = reader.ReadUInt32();
 
         ms.Seek(dependencyTableOffset, SeekOrigin.Begin);
         uint dependencyCount = reader.ReadUInt32();
 
-        this.Debug($"Dependency count offset: {dependencyTableOffset}, count: {dependencyCount}");
+        this.Debug($"Dependency table offset: {dependencyTableOffset}, count: {dependencyCount}");
+
+        List<string> dependencies = [];
 
         Span<byte> hashBuffer = stackalloc byte[20];
         for (int i = 0; i < dependencyCount; i++)

Refresh.GameServer/Importing/BEBinaryReader.cs

@@ -7,11 +7,41 @@ public class BEBinaryReader : BinaryReader
     public BEBinaryReader(Stream input) : base(input)
     {}
 
-    public override short ReadInt16() => BinaryPrimitives.ReadInt16BigEndian(this.ReadBytes(2));
-    public override int ReadInt32() => BinaryPrimitives.ReadInt32BigEndian(this.ReadBytes(4));
-    public override long ReadInt64() => BinaryPrimitives.ReadInt64BigEndian(this.ReadBytes(8));
-
-    public override ushort ReadUInt16() => BinaryPrimitives.ReadUInt16BigEndian(this.ReadBytes(2));
-    public override uint ReadUInt32() => BinaryPrimitives.ReadUInt32BigEndian(this.ReadBytes(4));
-    public override ulong ReadUInt64() => BinaryPrimitives.ReadUInt64BigEndian(this.ReadBytes(8));
+    public override short ReadInt16()
+    {
+        Span<byte> buf = stackalloc byte[sizeof(short)];
+        this.BaseStream.ReadExactly(buf);
+        return BinaryPrimitives.ReadInt16BigEndian(buf);
+    }
+    public override int ReadInt32()
+    {
+        Span<byte> buf = stackalloc byte[sizeof(int)];
+        this.BaseStream.ReadExactly(buf);
+        return BinaryPrimitives.ReadInt32BigEndian(buf);
+    }
+    public override long ReadInt64()
+    {
+        Span<byte> buf = stackalloc byte[sizeof(long)];
+        this.BaseStream.ReadExactly(buf);
+        return BinaryPrimitives.ReadInt64BigEndian(buf);
+    }
+
+    public override ushort ReadUInt16()
+    {
+        Span<byte> buf = stackalloc byte[sizeof(ushort)];
+        this.BaseStream.ReadExactly(buf);
+        return BinaryPrimitives.ReadUInt16BigEndian(buf);
+    }
+    public override uint ReadUInt32()
+    {
+        Span<byte> buf = stackalloc byte[sizeof(uint)];
+        this.BaseStream.ReadExactly(buf);
+        return BinaryPrimitives.ReadUInt32BigEndian(buf);
+    }
+    public override ulong ReadUInt64()
+    {
+        Span<byte> buf = stackalloc byte[sizeof(ulong)];
+        this.BaseStream.ReadExactly(buf);
+        return BinaryPrimitives.ReadUInt64BigEndian(buf);
+    }
 }