From d8ece9669fe763a92ac0fcc84424b97c9c93972b Mon Sep 17 00:00:00 2001 From: jvyden Date: Wed, 7 Aug 2024 16:23:16 -0400 Subject: [PATCH 1/2] Implement RemoteRefreshDataStore for testing --- Refresh.GameServer/RefreshGameServer.cs | 3 + .../Storage/RemoteRefreshDataStore.cs | 70 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 Refresh.GameServer/Storage/RemoteRefreshDataStore.cs diff --git a/Refresh.GameServer/RefreshGameServer.cs b/Refresh.GameServer/RefreshGameServer.cs index 62b6222b..157ea8d6 100644 --- a/Refresh.GameServer/RefreshGameServer.cs +++ b/Refresh.GameServer/RefreshGameServer.cs @@ -59,6 +59,9 @@ public RefreshGameServer( DryArchiveConfig dryConfig = Config.LoadFromJsonFile("dry.json", this.Logger); if (dryConfig.Enabled) this._dataStore = new AggregateDataStore(dataStore, new DryDataStore(dryConfig)); + + // Uncomment if you want to use production refresh as a source for assets + // this._dataStore = new AggregateDataStore(dataStore, new RemoteRefreshDataStore()); this.SetupInitializer(() => { diff --git a/Refresh.GameServer/Storage/RemoteRefreshDataStore.cs b/Refresh.GameServer/Storage/RemoteRefreshDataStore.cs new file mode 100644 index 00000000..ded465e7 --- /dev/null +++ b/Refresh.GameServer/Storage/RemoteRefreshDataStore.cs @@ -0,0 +1,70 @@ +using Bunkum.Core.Storage; +using JetBrains.Annotations; + +namespace Refresh.GameServer.Storage; + +/// +/// A datastore that uses production Refresh as a data source. +/// Intended for testing purposes only; performance is not considered. +/// +public class RemoteRefreshDataStore : IDataStore +{ + private const string SourceUrl = "https://lbp.littlebigrefresh.com/api/v3/"; + private const string WriteError = $"{nameof(RemoteRefreshDataStore)} is a read-only source, and cannot be written to."; + private const bool HideImages = true; + + private readonly HttpClient _client = new() + { + BaseAddress = new Uri(SourceUrl), + }; + + [Pure] + private string? GetPath(ReadOnlySpan key) + { + if (key.StartsWith("png/")) + return $"assets/{key[4..]}/image"; + + return $"assets/{key}/download"; + } + + private HttpResponseMessage Get(string endpoint) + { + HttpResponseMessage message = this._client.GetAsync(endpoint).Result; + return message; + } + + public bool ExistsInStore(string key) + { + if (HideImages && key.StartsWith("png/")) + return false; + + string? path = this.GetPath(key); + + if (path == null) + throw new FormatException("The key was invalid."); + + HttpResponseMessage resp = this.Get(path); + return resp.IsSuccessStatusCode; + } + + public bool WriteToStore(string key, byte[] data) => throw new InvalidOperationException(WriteError); + + public byte[] GetDataFromStore(string key) + { + string? path = this.GetPath(key); + + if (path == null) + throw new FormatException("The key was invalid."); + + HttpResponseMessage resp = this.Get(path); + resp.EnsureSuccessStatusCode(); + + return resp.Content.ReadAsByteArrayAsync().Result; + } + + public bool RemoveFromStore(string key) => throw new InvalidOperationException(WriteError); + public string[] GetKeysFromStore() => []; // TODO: Implement when we want to store these as GameAssets + public bool WriteToStoreFromStream(string key, Stream data) => throw new InvalidOperationException(WriteError); + public Stream GetStreamFromStore(string key) => new MemoryStream(this.GetDataFromStore(key)); + public Stream OpenWriteStream(string key) => throw new InvalidOperationException(WriteError); +} \ No newline at end of file From 50c103c205541876159e77668350da52c5126e11 Mon Sep 17 00:00:00 2001 From: jvyden Date: Wed, 7 Aug 2024 16:26:20 -0400 Subject: [PATCH 2/2] Allow conversion of assets not in the database to images --- .../Endpoints/ApiV3/ResourceApiEndpoints.cs | 14 +++++++------- Refresh.GameServer/Importing/ImageImporter.cs | 13 ++++++++++++- Refresh.GameServer/Importing/Importer.cs | 2 ++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/Refresh.GameServer/Endpoints/ApiV3/ResourceApiEndpoints.cs b/Refresh.GameServer/Endpoints/ApiV3/ResourceApiEndpoints.cs index a2ecb7eb..ce8abe27 100644 --- a/Refresh.GameServer/Endpoints/ApiV3/ResourceApiEndpoints.cs +++ b/Refresh.GameServer/Endpoints/ApiV3/ResourceApiEndpoints.cs @@ -61,7 +61,8 @@ public Response DownloadGameAsset(RequestContext context, IDataStore dataStore, [DocError(typeof(ApiValidationError), ApiValidationError.HashMissingErrorWhen)] [RateLimitSettings(RequestTimeoutDuration, MaxRequestAmount, RequestBlockDuration, BucketName)] public Response DownloadPspGameAsset(RequestContext context, IDataStore dataStore, - [DocSummary("The SHA1 hash of the asset")] string hash) => DownloadGameAsset(context, dataStore, $"psp/{hash}"); + [DocSummary("The SHA1 hash of the asset")] string hash) + => this.DownloadGameAsset(context, dataStore, $"psp/{hash}"); [ApiV3Endpoint("assets/{hash}/image", ContentType.Png), Authentication(false)] [ClientCacheResponse(9204111)] // 1 week, data may or may not change @@ -70,7 +71,7 @@ public Response DownloadPspGameAsset(RequestContext context, IDataStore dataStor [DocError(typeof(ApiInternalError), ApiInternalError.CouldNotGetAssetErrorWhen)] [DocError(typeof(ApiValidationError), ApiValidationError.HashMissingErrorWhen)] public Response DownloadGameAssetAsImage(RequestContext context, IDataStore dataStore, GameDatabaseContext database, - [DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter importer) + [DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter imageImport, AssetImporter assetImport) { bool isPspAsset = hash.StartsWith("psp/"); @@ -87,16 +88,14 @@ public Response DownloadGameAssetAsImage(RequestContext context, IDataStore data if (convertedType != null) { //Import the converted hash from the data store - importer.ImportAsset(realHash, isPspAsset, convertedType.Value, dataStore); + imageImport.ImportAsset(realHash, isPspAsset, convertedType.Value, dataStore); } //If not, else { //Import the asset as normal GameAsset? asset = database.GetAssetFromHash(realHash); - if (asset == null) return ApiInternalError.CouldNotGetAssetDatabaseError; - - importer.ImportAsset(asset.AssetHash, asset.IsPSP, asset.AssetType, dataStore); + imageImport.ImportAsset(realHash, isPspAsset, asset?.AssetType, dataStore); } } @@ -113,7 +112,8 @@ public Response DownloadGameAssetAsImage(RequestContext context, IDataStore data [DocError(typeof(ApiInternalError), ApiInternalError.CouldNotGetAssetErrorWhen)] [DocError(typeof(ApiValidationError), ApiValidationError.HashMissingErrorWhen)] public Response DownloadPspGameAssetAsImage(RequestContext context, IDataStore dataStore, GameDatabaseContext database, - [DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter importer) => this.DownloadGameAssetAsImage(context, dataStore, database, $"psp/{hash}", importer); + [DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter imageImport, AssetImporter assetImport) + => this.DownloadGameAssetAsImage(context, dataStore, database, $"psp/{hash}", imageImport, assetImport); [ApiV3Endpoint("assets/{hash}"), Authentication(false)] [DocSummary("Gets information from the database about a particular hash. Includes user who uploaded, dependencies, timestamps, etc.")] diff --git a/Refresh.GameServer/Importing/ImageImporter.cs b/Refresh.GameServer/Importing/ImageImporter.cs index 9df3fcef..05121bce 100644 --- a/Refresh.GameServer/Importing/ImageImporter.cs +++ b/Refresh.GameServer/Importing/ImageImporter.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using Bunkum.Core.Storage; using NotEnoughLogs; +using Refresh.GameServer.Authentication; using Refresh.GameServer.Database; using Refresh.GameServer.Extensions; using Refresh.GameServer.Resources; @@ -68,9 +69,19 @@ private void ThreadTask(ConcurrentQueue assetQueue, IDataStore dataSt this._runningCount--; } - public void ImportAsset(string hash, bool isPsp, GameAssetType type, IDataStore dataStore) + public void ImportAsset(string hash, bool isPsp, GameAssetType? type, IDataStore dataStore) { string dataStorePath = isPsp ? $"psp/{hash}" : hash; + + if (type == null) + { + using Stream typeStream = dataStore.GetStreamFromStore(dataStorePath); + Span span = stackalloc byte[16]; + _ = typeStream.Read(span); + + (GameAssetType type, GameAssetFormat) assetType = this.DetermineAssetType(span, isPsp ? TokenPlatform.PSP : TokenPlatform.Website); + type = assetType.type; + } using Stream stream = dataStore.GetStreamFromStore(dataStorePath); using Stream writeStream = dataStore.OpenWriteStream($"png/{hash}"); diff --git a/Refresh.GameServer/Importing/Importer.cs b/Refresh.GameServer/Importing/Importer.cs index 546642c1..db12c8ac 100644 --- a/Refresh.GameServer/Importing/Importer.cs +++ b/Refresh.GameServer/Importing/Importer.cs @@ -4,6 +4,7 @@ using System.Security.Cryptography; using System.Text; using Bunkum.Core; +using JetBrains.Annotations; using NotEnoughLogs; using Refresh.GameServer.Authentication; using Refresh.GameServer.Resources; @@ -181,6 +182,7 @@ private bool IsMip(Span rawData) return true; } + [Pure] protected (GameAssetType, GameAssetFormat) DetermineAssetType(Span data, TokenPlatform? tokenPlatform) { GameAssetType? lbpAssetType = GameAssetTypeExtensions.LbpMagicToGameAssetType(data.Slice(0, 3));