Allow conversion of assets not in the database to images

Refresh.GameServer/Endpoints/ApiV3/ResourceApiEndpoints.cs

@@ -61,7 +61,8 @@ public Response DownloadGameAsset(RequestContext context, IDataStore dataStore,
     [DocError(typeof(ApiValidationError), ApiValidationError.HashMissingErrorWhen)]
     [RateLimitSettings(RequestTimeoutDuration, MaxRequestAmount, RequestBlockDuration, BucketName)]
     public Response DownloadPspGameAsset(RequestContext context, IDataStore dataStore,
-        [DocSummary("The SHA1 hash of the asset")] string hash) => DownloadGameAsset(context, dataStore, $"psp/{hash}");
+        [DocSummary("The SHA1 hash of the asset")] string hash)
+        => this.DownloadGameAsset(context, dataStore, $"psp/{hash}");
 
     [ApiV3Endpoint("assets/{hash}/image", ContentType.Png), Authentication(false)]
     [ClientCacheResponse(9204111)] // 1 week, data may or may not change
@@ -70,7 +71,7 @@ public Response DownloadPspGameAsset(RequestContext context, IDataStore dataStor
     [DocError(typeof(ApiInternalError), ApiInternalError.CouldNotGetAssetErrorWhen)]
     [DocError(typeof(ApiValidationError), ApiValidationError.HashMissingErrorWhen)]
     public Response DownloadGameAssetAsImage(RequestContext context, IDataStore dataStore, GameDatabaseContext database,
-        [DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter importer)
+        [DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter imageImport, AssetImporter assetImport)
     {
         bool isPspAsset = hash.StartsWith("psp/");
 
@@ -87,16 +88,14 @@ public Response DownloadGameAssetAsImage(RequestContext context, IDataStore data
             if (convertedType != null)
             {
                 //Import the converted hash from the data store
-                importer.ImportAsset(realHash, isPspAsset, convertedType.Value, dataStore);
+                imageImport.ImportAsset(realHash, isPspAsset, convertedType.Value, dataStore);
             }
             //If not,
             else
             {
                 //Import the asset as normal
                 GameAsset? asset = database.GetAssetFromHash(realHash);
-                if (asset == null) return ApiInternalError.CouldNotGetAssetDatabaseError;
-
-                importer.ImportAsset(asset.AssetHash, asset.IsPSP, asset.AssetType, dataStore);
+                imageImport.ImportAsset(realHash, isPspAsset, asset?.AssetType, dataStore);
             }
         }
 
@@ -113,7 +112,8 @@ public Response DownloadGameAssetAsImage(RequestContext context, IDataStore data
     [DocError(typeof(ApiInternalError), ApiInternalError.CouldNotGetAssetErrorWhen)]
     [DocError(typeof(ApiValidationError), ApiValidationError.HashMissingErrorWhen)]
     public Response DownloadPspGameAssetAsImage(RequestContext context, IDataStore dataStore, GameDatabaseContext database,
-        [DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter importer) => this.DownloadGameAssetAsImage(context, dataStore, database, $"psp/{hash}", importer);
+        [DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter imageImport, AssetImporter assetImport) 
+        => this.DownloadGameAssetAsImage(context, dataStore, database, $"psp/{hash}", imageImport, assetImport);
 
     [ApiV3Endpoint("assets/{hash}"), Authentication(false)]
     [DocSummary("Gets information from the database about a particular hash. Includes user who uploaded, dependencies, timestamps, etc.")]

Refresh.GameServer/Importing/ImageImporter.cs

@@ -1,6 +1,7 @@
 using System.Collections.Concurrent;
 using Bunkum.Core.Storage;
 using NotEnoughLogs;
+using Refresh.GameServer.Authentication;
 using Refresh.GameServer.Database;
 using Refresh.GameServer.Extensions;
 using Refresh.GameServer.Resources;
@@ -68,9 +69,19 @@ private void ThreadTask(ConcurrentQueue<GameAsset> assetQueue, IDataStore dataSt
         this._runningCount--;
     }
 
-    public void ImportAsset(string hash, bool isPsp, GameAssetType type, IDataStore dataStore)
+    public void ImportAsset(string hash, bool isPsp, GameAssetType? type, IDataStore dataStore)
     {
         string dataStorePath = isPsp ? $"psp/{hash}" : hash;
+
+        if (type == null)
+        {
+            using Stream typeStream = dataStore.GetStreamFromStore(dataStorePath);
+            Span<byte> span = stackalloc byte[16];
+            _ = typeStream.Read(span);
+
+            (GameAssetType type, GameAssetFormat) assetType = this.DetermineAssetType(span, isPsp ? TokenPlatform.PSP : TokenPlatform.Website);
+            type = assetType.type;
+        }
 
         using Stream stream = dataStore.GetStreamFromStore(dataStorePath);
         using Stream writeStream = dataStore.OpenWriteStream($"png/{hash}");

Refresh.GameServer/Importing/Importer.cs

@@ -4,6 +4,7 @@
 using System.Security.Cryptography;
 using System.Text;
 using Bunkum.Core;
+using JetBrains.Annotations;
 using NotEnoughLogs;
 using Refresh.GameServer.Authentication;
 using Refresh.GameServer.Resources;
@@ -181,6 +182,7 @@ private bool IsMip(Span<byte> rawData)
         return true;
     }
 
+    [Pure]
     protected (GameAssetType, GameAssetFormat) DetermineAssetType(Span<byte> data, TokenPlatform? tokenPlatform)
     {
         GameAssetType? lbpAssetType = GameAssetTypeExtensions.LbpMagicToGameAssetType(data.Slice(0, 3));

Refresh.GameServer/RefreshGameServer.cs

@@ -59,6 +59,9 @@ public RefreshGameServer(
         DryArchiveConfig dryConfig = Config.LoadFromJsonFile<DryArchiveConfig>("dry.json", this.Logger);
         if (dryConfig.Enabled)
             this._dataStore = new AggregateDataStore(dataStore, new DryDataStore(dryConfig));
+
+        // Uncomment if you want to use production refresh as a source for assets
+        // this._dataStore = new AggregateDataStore(dataStore, new RemoteRefreshDataStore());
 
         this.SetupInitializer(() =>
         {

Refresh.GameServer/Storage/RemoteRefreshDataStore.cs

@@ -0,0 +1,70 @@
+using Bunkum.Core.Storage;
+using JetBrains.Annotations;
+
+namespace Refresh.GameServer.Storage;
+
+/// <summary>
+/// A datastore that uses production Refresh as a data source.
+/// Intended for testing purposes only; performance is not considered.
+/// </summary>
+public class RemoteRefreshDataStore : IDataStore
+{
+    private const string SourceUrl = "https://lbp.littlebigrefresh.com/api/v3/"; 
+    private const string WriteError = $"{nameof(RemoteRefreshDataStore)} is a read-only source, and cannot be written to.";
+    private const bool HideImages = true;
+
+    private readonly HttpClient _client = new()
+    {
+        BaseAddress = new Uri(SourceUrl),
+    };
+
+    [Pure]
+    private string? GetPath(ReadOnlySpan<char> key)
+    {
+        if (key.StartsWith("png/"))
+            return $"assets/{key[4..]}/image";
+
+        return $"assets/{key}/download";
+    }
+
+    private HttpResponseMessage Get(string endpoint)
+    {
+        HttpResponseMessage message = this._client.GetAsync(endpoint).Result;
+        return message;
+    }
+
+    public bool ExistsInStore(string key)
+    {
+        if (HideImages && key.StartsWith("png/"))
+            return false;
+
+        string? path = this.GetPath(key);
+
+        if (path == null)
+            throw new FormatException("The key was invalid.");
+
+        HttpResponseMessage resp = this.Get(path);
+        return resp.IsSuccessStatusCode;
+    }
+
+    public bool WriteToStore(string key, byte[] data) => throw new InvalidOperationException(WriteError);
+
+    public byte[] GetDataFromStore(string key)
+    {
+        string? path = this.GetPath(key);
+
+        if (path == null)
+            throw new FormatException("The key was invalid.");
+
+        HttpResponseMessage resp = this.Get(path);
+        resp.EnsureSuccessStatusCode();
+
+        return resp.Content.ReadAsByteArrayAsync().Result;
+    }
+
+    public bool RemoveFromStore(string key) => throw new InvalidOperationException(WriteError);
+    public string[] GetKeysFromStore() => []; // TODO: Implement when we want to store these as GameAssets
+    public bool WriteToStoreFromStream(string key, Stream data) => throw new InvalidOperationException(WriteError);
+    public Stream GetStreamFromStore(string key) => new MemoryStream(this.GetDataFromStore(key));
+    public Stream OpenWriteStream(string key) => throw new InvalidOperationException(WriteError);
+}