Skip to content

Commit

Permalink
Allow conversion of assets not in the database to images (#623)
Browse files Browse the repository at this point in the history
  • Loading branch information
Beyley authored Aug 7, 2024
2 parents e16a956 + 0ac4319 commit fb5afda
Show file tree
Hide file tree
Showing 5 changed files with 94 additions and 8 deletions.
14 changes: 7 additions & 7 deletions Refresh.GameServer/Endpoints/ApiV3/ResourceApiEndpoints.cs
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ public Response DownloadGameAsset(RequestContext context, IDataStore dataStore,
[DocError(typeof(ApiValidationError), ApiValidationError.HashMissingErrorWhen)]
[RateLimitSettings(RequestTimeoutDuration, MaxRequestAmount, RequestBlockDuration, BucketName)]
public Response DownloadPspGameAsset(RequestContext context, IDataStore dataStore,
[DocSummary("The SHA1 hash of the asset")] string hash) => DownloadGameAsset(context, dataStore, $"psp/{hash}");
[DocSummary("The SHA1 hash of the asset")] string hash)
=> this.DownloadGameAsset(context, dataStore, $"psp/{hash}");

[ApiV3Endpoint("assets/{hash}/image", ContentType.Png), Authentication(false)]
[ClientCacheResponse(9204111)] // 1 week, data may or may not change
Expand All @@ -70,7 +71,7 @@ public Response DownloadPspGameAsset(RequestContext context, IDataStore dataStor
[DocError(typeof(ApiInternalError), ApiInternalError.CouldNotGetAssetErrorWhen)]
[DocError(typeof(ApiValidationError), ApiValidationError.HashMissingErrorWhen)]
public Response DownloadGameAssetAsImage(RequestContext context, IDataStore dataStore, GameDatabaseContext database,
[DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter importer)
[DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter imageImport, AssetImporter assetImport)
{
bool isPspAsset = hash.StartsWith("psp/");

Expand All @@ -87,16 +88,14 @@ public Response DownloadGameAssetAsImage(RequestContext context, IDataStore data
if (convertedType != null)
{
//Import the converted hash from the data store
importer.ImportAsset(realHash, isPspAsset, convertedType.Value, dataStore);
imageImport.ImportAsset(realHash, isPspAsset, convertedType.Value, dataStore);
}
//If not,
else
{
//Import the asset as normal
GameAsset? asset = database.GetAssetFromHash(realHash);
if (asset == null) return ApiInternalError.CouldNotGetAssetDatabaseError;

importer.ImportAsset(asset.AssetHash, asset.IsPSP, asset.AssetType, dataStore);
imageImport.ImportAsset(realHash, isPspAsset, asset?.AssetType, dataStore);
}
}

Expand All @@ -113,7 +112,8 @@ public Response DownloadGameAssetAsImage(RequestContext context, IDataStore data
[DocError(typeof(ApiInternalError), ApiInternalError.CouldNotGetAssetErrorWhen)]
[DocError(typeof(ApiValidationError), ApiValidationError.HashMissingErrorWhen)]
public Response DownloadPspGameAssetAsImage(RequestContext context, IDataStore dataStore, GameDatabaseContext database,
[DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter importer) => this.DownloadGameAssetAsImage(context, dataStore, database, $"psp/{hash}", importer);
[DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter imageImport, AssetImporter assetImport)
=> this.DownloadGameAssetAsImage(context, dataStore, database, $"psp/{hash}", imageImport, assetImport);

[ApiV3Endpoint("assets/{hash}"), Authentication(false)]
[DocSummary("Gets information from the database about a particular hash. Includes user who uploaded, dependencies, timestamps, etc.")]
Expand Down
13 changes: 12 additions & 1 deletion Refresh.GameServer/Importing/ImageImporter.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System.Collections.Concurrent;
using Bunkum.Core.Storage;
using NotEnoughLogs;
using Refresh.GameServer.Authentication;
using Refresh.GameServer.Database;
using Refresh.GameServer.Extensions;
using Refresh.GameServer.Resources;
Expand Down Expand Up @@ -68,9 +69,19 @@ private void ThreadTask(ConcurrentQueue<GameAsset> assetQueue, IDataStore dataSt
this._runningCount--;
}

public void ImportAsset(string hash, bool isPsp, GameAssetType type, IDataStore dataStore)
public void ImportAsset(string hash, bool isPsp, GameAssetType? type, IDataStore dataStore)
{
string dataStorePath = isPsp ? $"psp/{hash}" : hash;

if (type == null)
{
using Stream typeStream = dataStore.GetStreamFromStore(dataStorePath);
Span<byte> span = stackalloc byte[16];
_ = typeStream.Read(span);

(GameAssetType type, GameAssetFormat) assetType = this.DetermineAssetType(span, isPsp ? TokenPlatform.PSP : TokenPlatform.Website);
type = assetType.type;
}

using Stream stream = dataStore.GetStreamFromStore(dataStorePath);
using Stream writeStream = dataStore.OpenWriteStream($"png/{hash}");
Expand Down
2 changes: 2 additions & 0 deletions Refresh.GameServer/Importing/Importer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Security.Cryptography;
using System.Text;
using Bunkum.Core;
using JetBrains.Annotations;
using NotEnoughLogs;
using Refresh.GameServer.Authentication;
using Refresh.GameServer.Resources;
Expand Down Expand Up @@ -181,6 +182,7 @@ private bool IsMip(Span<byte> rawData)
return true;
}

[Pure]
protected (GameAssetType, GameAssetFormat) DetermineAssetType(Span<byte> data, TokenPlatform? tokenPlatform)
{
GameAssetType? lbpAssetType = GameAssetTypeExtensions.LbpMagicToGameAssetType(data.Slice(0, 3));
Expand Down
3 changes: 3 additions & 0 deletions Refresh.GameServer/RefreshGameServer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ public RefreshGameServer(
DryArchiveConfig dryConfig = Config.LoadFromJsonFile<DryArchiveConfig>("dry.json", this.Logger);
if (dryConfig.Enabled)
this._dataStore = new AggregateDataStore(dataStore, new DryDataStore(dryConfig));

// Uncomment if you want to use production refresh as a source for assets
// this._dataStore = new AggregateDataStore(dataStore, new RemoteRefreshDataStore());

this.SetupInitializer(() =>
{
Expand Down
70 changes: 70 additions & 0 deletions Refresh.GameServer/Storage/RemoteRefreshDataStore.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
using Bunkum.Core.Storage;
using JetBrains.Annotations;

namespace Refresh.GameServer.Storage;

/// <summary>
/// A datastore that uses production Refresh as a data source.
/// Intended for testing purposes only; performance is not considered.
/// </summary>
public class RemoteRefreshDataStore : IDataStore
{
private const string SourceUrl = "https://lbp.littlebigrefresh.com/api/v3/";
private const string WriteError = $"{nameof(RemoteRefreshDataStore)} is a read-only source, and cannot be written to.";
private const bool HideImages = true;

private readonly HttpClient _client = new()
{
BaseAddress = new Uri(SourceUrl),
};

[Pure]
private string? GetPath(ReadOnlySpan<char> key)
{
if (key.StartsWith("png/"))
return $"assets/{key[4..]}/image";

return $"assets/{key}/download";
}

private HttpResponseMessage Get(string endpoint)
{
HttpResponseMessage message = this._client.GetAsync(endpoint).Result;
return message;
}

public bool ExistsInStore(string key)
{
if (HideImages && key.StartsWith("png/"))
return false;

string? path = this.GetPath(key);

if (path == null)
throw new FormatException("The key was invalid.");

HttpResponseMessage resp = this.Get(path);
return resp.IsSuccessStatusCode;
}

public bool WriteToStore(string key, byte[] data) => throw new InvalidOperationException(WriteError);

public byte[] GetDataFromStore(string key)
{
string? path = this.GetPath(key);

if (path == null)
throw new FormatException("The key was invalid.");

HttpResponseMessage resp = this.Get(path);
resp.EnsureSuccessStatusCode();

return resp.Content.ReadAsByteArrayAsync().Result;
}

public bool RemoveFromStore(string key) => throw new InvalidOperationException(WriteError);
public string[] GetKeysFromStore() => []; // TODO: Implement when we want to store these as GameAssets
public bool WriteToStoreFromStream(string key, Stream data) => throw new InvalidOperationException(WriteError);
public Stream GetStreamFromStore(string key) => new MemoryStream(this.GetDataFromStore(key));
public Stream OpenWriteStream(string key) => throw new InvalidOperationException(WriteError);
}

0 comments on commit fb5afda

Please sign in to comment.