diff --git a/Refresh.GameServer/Endpoints/ApiV3/DataTypes/Response/ApiGameLevelResponse.cs b/Refresh.GameServer/Endpoints/ApiV3/DataTypes/Response/ApiGameLevelResponse.cs index bcad45d4..3dcf090e 100644 --- a/Refresh.GameServer/Endpoints/ApiV3/DataTypes/Response/ApiGameLevelResponse.cs +++ b/Refresh.GameServer/Endpoints/ApiV3/DataTypes/Response/ApiGameLevelResponse.cs @@ -49,7 +49,7 @@ public class ApiGameLevelResponse : IApiResponse, IDataConvertableFrom this.DownloadGameAssetAsImage(context, dataStore, database, $"psp/{hash}"); + [DocSummary("The SHA1 hash of the asset")] string hash, ImageImporter importer) => this.DownloadGameAssetAsImage(context, dataStore, database, $"psp/{hash}", importer); [ApiV3Endpoint("assets/{hash}"), Authentication(false)] [DocSummary("Gets information from the database about a particular hash. Includes user who uploaded, dependencies, timestamps, etc.")] diff --git a/Refresh.GameServer/Importing/AssetImporter.cs b/Refresh.GameServer/Importing/AssetImporter.cs index fcafc5fc..0aab7476 100644 --- a/Refresh.GameServer/Importing/AssetImporter.cs +++ b/Refresh.GameServer/Importing/AssetImporter.cs @@ -25,15 +25,20 @@ public void ImportFromDataStore(GameDatabaseContext database, IDataStore dataSto int newAssets = 0; this.Stopwatch.Start(); - IEnumerable assetHashes = dataStore.GetKeysFromStore() - .Where(key => !key.Contains('/')); + IEnumerable assetHashes = dataStore.GetKeysFromStore(); - List assets = new(); - foreach (string hash in assetHashes) + List assets = new(assetHashes.Count()); + foreach (string path in assetHashes) { - byte[] data = dataStore.GetDataFromStore(hash); + bool isPsp = path.StartsWith("psp/"); + //If the hash has a `/` and it doesnt start with `psp/`, then its an invalid asset + if (path.Contains('/') && !isPsp) continue; + + string hash = isPsp ? path[4..] : path; + + byte[] data = dataStore.GetDataFromStore(path); - GameAsset? newAsset = this.ReadAndVerifyAsset(hash, data, null); + GameAsset? newAsset = this.ReadAndVerifyAsset(hash, data, isPsp ? TokenPlatform.PSP : null); if (newAsset == null) continue; GameAsset? oldAsset = database.GetAssetFromHash(hash); @@ -132,6 +137,7 @@ or GameAssetType.Png or GameAssetType.Tga or GameAssetType.Texture or GameAssetType.GameDataTexture + or GameAssetType.Mip or GameAssetType.Unknown) { return false; diff --git a/Refresh.GameServer/Importing/ImageImporter.Conversions.cs b/Refresh.GameServer/Importing/ImageImporter.Conversions.cs index 13af2ba5..0b9e86fd 100644 --- a/Refresh.GameServer/Importing/ImageImporter.Conversions.cs +++ b/Refresh.GameServer/Importing/ImageImporter.Conversions.cs @@ -1,6 +1,7 @@ using ICSharpCode.SharpZipLib.Zip.Compression; using Pfim; using Refresh.GameServer.Importing.Gtf; +using Refresh.GameServer.Importing.Mip; using SixLabors.ImageSharp.Formats; namespace Refresh.GameServer.Importing; @@ -33,6 +34,12 @@ private static void GtfToPng(Stream stream, Stream writeStream) using Image image = new GtfDecoder().Decode(new DecoderOptions(), stream); image.SaveAsPng(writeStream); } + + private static void MipToPng(Stream stream, Stream writeStream) + { + using Image image = new MipDecoder().Decode(new DecoderOptions(), stream); + image.SaveAsPng(writeStream); + } private static readonly PfimConfig Config = new(); diff --git a/Refresh.GameServer/Importing/ImageImporter.cs b/Refresh.GameServer/Importing/ImageImporter.cs index 579da99a..b30eaae9 100644 --- a/Refresh.GameServer/Importing/ImageImporter.cs +++ b/Refresh.GameServer/Importing/ImageImporter.cs @@ -3,6 +3,7 @@ using NotEnoughLogs; using Refresh.GameServer.Database; using Refresh.GameServer.Extensions; +using Refresh.GameServer.Resources; using Refresh.GameServer.Types.Assets; namespace Refresh.GameServer.Importing; @@ -22,6 +23,7 @@ public void ImportFromDataStore(GameDatabaseContext context, IDataStore dataStor assets.AddRange(context.GetAssetsByType(GameAssetType.GameDataTexture)); assets.AddRange(context.GetAssetsByType(GameAssetType.Jpeg)); assets.AddRange(context.GetAssetsByType(GameAssetType.Png)); + assets.AddRange(context.GetAssetsByType(GameAssetType.Mip)); this.Info("Acquired all other assets"); @@ -66,7 +68,7 @@ private void ThreadTask(ConcurrentQueue assetQueue, IDataStore dataSt this._runningCount--; } - public static void ImportAsset(GameAsset asset, IDataStore dataStore) + public void ImportAsset(GameAsset asset, IDataStore dataStore) { using Stream stream = dataStore.GetStreamFromStore(asset.IsPSP ? "psp/" + asset.AssetHash : asset.AssetHash); using Stream writeStream = dataStore.OpenWriteStream("png/" + asset.AssetHash); @@ -76,6 +78,15 @@ public static void ImportAsset(GameAsset asset, IDataStore dataStore) case GameAssetType.GameDataTexture: GtfToPng(stream, writeStream); break; + case GameAssetType.Mip: { + byte[] rawData = dataStore.GetDataFromStore(asset.IsPSP ? "psp/" + asset.AssetHash : asset.AssetHash); + byte[] data = ResourceHelper.PspDecrypt(rawData, this.PSPKey.Value); + + using MemoryStream dataStream = new(data); + + MipToPng(dataStream, writeStream); + break; + } case GameAssetType.Texture: TextureToPng(stream, writeStream); break; diff --git a/Refresh.GameServer/Importing/Importer.cs b/Refresh.GameServer/Importing/Importer.cs index 73985853..d6e7680e 100644 --- a/Refresh.GameServer/Importing/Importer.cs +++ b/Refresh.GameServer/Importing/Importer.cs @@ -1,10 +1,12 @@ using System.Buffers.Binary; using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Security.Cryptography; using System.Text; using Bunkum.Core; using NotEnoughLogs; using Refresh.GameServer.Authentication; +using Refresh.GameServer.Resources; using Refresh.GameServer.Types.Assets; namespace Refresh.GameServer.Importing; @@ -13,6 +15,7 @@ public abstract class Importer { private readonly Logger _logger; protected readonly Stopwatch Stopwatch; + protected readonly Lazy PSPKey; protected Importer(Logger? logger = null) { @@ -23,6 +26,36 @@ protected Importer(Logger? logger = null) this._logger = logger; this.Stopwatch = new Stopwatch(); + + this.PSPKey = new(() => + { + try + { + //Read the key + byte[] key = File.ReadAllBytes("keys/psp"); + + //If the hash matches, return the read key + if (SHA1.HashData(key).AsSpan().SequenceEqual(new byte[] { 0x12, 0xb5, 0xa8, 0xb5, 0x91, 0x55, 0x24, 0x96, 0x00, 0xdf, 0x0e, 0x33, 0xf9, 0xc5, 0xa8, 0x76, 0xc1, 0x85, 0x43, 0xfe })) + return key; + + //If the hash does not match, log an error and return null + this._logger.LogError(BunkumCategory.Digest, "PSP key failed to validate! Correct hash is 12b5a8b59155249600df0e33f9c5a876c18543fe"); + return null; + } + catch(Exception e) + { + if (e.GetType() == typeof(FileNotFoundException) || e.GetType() == typeof(DirectoryNotFoundException)) + { + this._logger.LogWarning(BunkumCategory.Digest, "PSP key file not found, encrypting/decrypting of PSP images will not work."); + } + else + { + this._logger.LogError(BunkumCategory.Digest, "Unknown error while loading PSP key! err: {0}", e.ToString()); + } + + return null; + } + }, LazyThreadSafetyMode.ExecutionAndPublication); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -99,8 +132,55 @@ private static bool IsPspTga(ReadOnlySpan data) return true; } - - protected GameAssetType DetermineAssetType(ReadOnlySpan data, TokenPlatform? tokenPlatform) + + private bool IsMip(Span rawData) + { + //If we dont have a key, then we cant determine the data type + if (this.PSPKey.Value == null) return false; + + //Data less than this size isn't encrypted(?) and all Mip files uploaded to the server will be encrypted + //See https://github.com/ennuo/lbparc/blob/16ad36aa7f4eae2f7b406829e604082750f16fe1/tools/toggle.js#L33 + if (rawData.Length < 0x19) return false; + + try + { + //Decrypt the data + ReadOnlySpan data = ResourceHelper.PspDecrypt(rawData, this.PSPKey.Value); + + uint clutOffset = BinaryPrimitives.ReadUInt32LittleEndian(data[..4]); + uint width = BinaryPrimitives.ReadUInt32LittleEndian(data[4..8]); + uint height = BinaryPrimitives.ReadUInt32LittleEndian(data[8..12]); + byte bpp = data[12]; + byte numBlocks = data[13]; + byte texMode = data[14]; + byte alpha = data[15]; + uint dataOffset = BinaryPrimitives.ReadUInt32LittleEndian(data[16..20]); + + //Its unlikely that any mip textures are ever gonna be this big + if (width > 512 || height > 512) return false; + + //We only support MIP files which have a bpp of 4 and 8 + if (bpp is not 8 and 4) return false; + + //Alpha can only be 0 or 1 + if (alpha > 1) return false; + + //If the data offset is past the end of the file, its not a MIP + if (dataOffset > data.Length || clutOffset > data.Length) return false; + + //If the size of the image is too big for the data passed in, its not a MIP + if (width * height * bpp / 8 > data.Length - dataOffset) return false; + } + catch + { + //If the data is invalid an invalid encrypted file, its not a MIP + return false; + } + + return true; + } + + protected GameAssetType DetermineAssetType(Span data, TokenPlatform? tokenPlatform) { // LBP assets if (MatchesMagic(data, "TEX "u8)) return GameAssetType.Texture; @@ -121,8 +201,10 @@ protected GameAssetType DetermineAssetType(ReadOnlySpan data, TokenPlatfor if (MatchesMagic(data, 0xFFD8FFE0)) return GameAssetType.Jpeg; if (MatchesMagic(data, 0x89504E470D0A1A0A)) return GameAssetType.Png; + // ReSharper disable once ConvertIfStatementToSwitchStatement if (tokenPlatform is null or TokenPlatform.PSP && IsPspTga(data)) return GameAssetType.Tga; - + if (tokenPlatform is null or TokenPlatform.PSP && this.IsMip(data)) return GameAssetType.Mip; + this.Warn($"Unknown asset header [0x{Convert.ToHexString(data[..4])}] [str: {Encoding.ASCII.GetString(data[..4])}]"); return GameAssetType.Unknown; diff --git a/Refresh.GameServer/Importing/Mip/MipDecoder.cs b/Refresh.GameServer/Importing/Mip/MipDecoder.cs new file mode 100644 index 00000000..385385d5 --- /dev/null +++ b/Refresh.GameServer/Importing/Mip/MipDecoder.cs @@ -0,0 +1,126 @@ +using SixLabors.ImageSharp.Formats; +using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.Metadata; + +namespace Refresh.GameServer.Importing.Mip; + +public class MipDecoder : ImageDecoder +{ + protected override Image Decode(DecoderOptions options, Stream stream, CancellationToken cancellationToken) + { + MipHeader header = MipHeader.Read(stream); + + Image image = new(options.Configuration, (int)header.Width, (int)header.Height); + Buffer2D pixels = image.Frames.RootFrame.PixelBuffer; + + this.ProcessPixels(header, stream, pixels); + + return image; + } + + private void ProcessPixels(MipHeader header, Stream stream, Buffer2D pixels) where TPixel : unmanaged, IPixel + { + stream.Seek(header.DataOffset, SeekOrigin.Begin); + + BinaryReader reader = new(stream); + + const int blockWidth = 16; + const int blockHeight = 8; + + int x = 0; + int y = 0; + int xStart = 0; + int xTarget = blockWidth; + int yStart = 0; + int yTarget = blockHeight; + for (int i = 0; i < header.Width * header.Height; i++) + { + #region hack to get swizzled coordinates + + if (x == xTarget && y == yTarget - 1) + { + xStart += blockWidth; + xTarget += blockWidth; + + if (xStart == header.Width) + { + xStart = 0; + xTarget = blockWidth; + yStart += blockHeight; + yTarget += blockHeight; + } + + x = xStart; + y = yStart; + } + else + { + if (x == xTarget) + { + y += 1; + x = xStart; + } + + if (y == yTarget) + { + xStart += blockWidth; + xTarget += blockWidth; + x = xStart; + y = yStart; + } + } + + #endregion + + switch (header.Bpp) + { + case 8: { + TPixel pixel = new(); + pixel.FromRgba32(header.Clut[reader.ReadByte()]); + pixels[x, (int)(header.Height - y - 1)] = pixel; + + x++; + break; + } + case 4: { + TPixel pixel = new(); + + byte data = reader.ReadByte(); + + pixel.FromRgba32(header.Clut[data & 0x0f]); + pixels[x, (int)(header.Height - y - 1)] = pixel; + + pixel.FromRgba32(header.Clut[data >> 4]); + pixels[x + 1, (int)(header.Height - y - 1)] = pixel; + + x += 2; + break; + } + default: + throw new Exception("Unknown BPP"); + } + + } + } + + protected override Image Decode(DecoderOptions options, Stream stream, CancellationToken cancellationToken) + { + return this.Decode(options, stream, cancellationToken); + } + + protected override ImageInfo Identify(DecoderOptions options, Stream stream, CancellationToken cancellationToken) + { + MipHeader header = MipHeader.Read(stream); + + return new ImageInfo( + new PixelTypeInfo(32), + new Size((int)header.Width, (int)header.Height), + new ImageMetadata + { + HorizontalResolution = header.Width, + VerticalResolution = header.Height, + ResolutionUnits = PixelResolutionUnit.AspectRatio, + } + ); + } +} \ No newline at end of file diff --git a/Refresh.GameServer/Importing/Mip/MipFormat.cs b/Refresh.GameServer/Importing/Mip/MipFormat.cs new file mode 100644 index 00000000..e4f3bcb4 --- /dev/null +++ b/Refresh.GameServer/Importing/Mip/MipFormat.cs @@ -0,0 +1,16 @@ +using Refresh.GameServer.Importing.Gtf; +using SixLabors.ImageSharp.Formats; + +namespace Refresh.GameServer.Importing.Mip; + +public class MipFormat : IImageFormat +{ + public MipMetadata CreateDefaultFormatMetadata() => new(); + + public string Name => "MIP"; + public string DefaultMimeType => ""; + public IEnumerable MimeTypes => new string[] { }; + public IEnumerable FileExtensions => new[] { "MIP" }; + + public static IImageFormat Instance { get; } = new MipFormat(); +} \ No newline at end of file diff --git a/Refresh.GameServer/Importing/Mip/MipHeader.cs b/Refresh.GameServer/Importing/Mip/MipHeader.cs new file mode 100644 index 00000000..9a9b5914 --- /dev/null +++ b/Refresh.GameServer/Importing/Mip/MipHeader.cs @@ -0,0 +1,43 @@ +using Rgba32 = SixLabors.ImageSharp.PixelFormats.Rgba32; + +namespace Refresh.GameServer.Importing.Mip; + +public class MipHeader +{ + public uint ClutOffset; + public uint Width; + public uint Height; + public byte Bpp; + public byte NumBlocks; + public byte TexMode; + public bool Alpha; + public uint DataOffset; + + public Rgba32[] Clut; + + public static MipHeader Read(Stream stream) + { + BinaryReader reader = new(stream); + + MipHeader header = new MipHeader + { + ClutOffset = reader.ReadUInt32(), + Width = reader.ReadUInt32(), + Height = reader.ReadUInt32(), + Bpp = reader.ReadByte(), + NumBlocks = reader.ReadByte(), + TexMode = reader.ReadByte(), + Alpha = reader.ReadByte() == 1, + DataOffset = reader.ReadUInt32(), + Clut = new Rgba32[256], + }; + + stream.Seek(header.ClutOffset, SeekOrigin.Begin); + for (int i = 0; i < header.Clut.Length; i++) + { + header.Clut[i] = new Rgba32(reader.ReadByte(), reader.ReadByte(), reader.ReadByte(), reader.ReadByte()); + } + + return header; + } +} \ No newline at end of file diff --git a/Refresh.GameServer/Importing/Mip/MipMetadata.cs b/Refresh.GameServer/Importing/Mip/MipMetadata.cs new file mode 100644 index 00000000..05418a03 --- /dev/null +++ b/Refresh.GameServer/Importing/Mip/MipMetadata.cs @@ -0,0 +1,6 @@ +namespace Refresh.GameServer.Importing.Mip; + +public class MipMetadata +{ + +} \ No newline at end of file diff --git a/Refresh.GameServer/Refresh.GameServer.csproj b/Refresh.GameServer/Refresh.GameServer.csproj index bf05fe60..5339b2d7 100644 --- a/Refresh.GameServer/Refresh.GameServer.csproj +++ b/Refresh.GameServer/Refresh.GameServer.csproj @@ -20,7 +20,7 @@ true - + ..\..\Bunkum\Bunkum.Core\bin\Debug\net8.0\Bunkum.Core.dll @@ -65,6 +65,8 @@ + + diff --git a/Refresh.GameServer/Resources/ResourceHelper.cs b/Refresh.GameServer/Resources/ResourceHelper.cs index 2e50094a..71f89ad3 100644 --- a/Refresh.GameServer/Resources/ResourceHelper.cs +++ b/Refresh.GameServer/Resources/ResourceHelper.cs @@ -1,4 +1,9 @@ +using System.Buffers; +using System.Buffers.Binary; using System.Reflection; +using System.Security.Cryptography; +using FastAes; +using IronCompress; namespace Refresh.GameServer.Resources; @@ -8,5 +13,49 @@ public static Stream StreamFromResource(string name) { Assembly assembly = Assembly.GetExecutingAssembly(); return assembly.GetManifestResourceStream(name)!; - } + } + + private static readonly ThreadLocal Iron = new(() => new Iron(ArrayPool.Shared)); + + private static void Encrypt(ReadOnlySpan data, ReadOnlySpan key) + { + throw new NotImplementedException(); //TODO + } + + /// + /// Decrypt a PSP asset + /// + /// The data to decrypt + /// The decryption key + /// The decrypted data. + public static byte[] PspDecrypt(Span data, ReadOnlySpan key) + { + byte[] initialCounter = new byte[16]; + key[..8].CopyTo(initialCounter); + + //Decrypt the data + AesCtr ctr = new(key, initialCounter); + ctr.DecryptBytes(data, data); + + //Get the data buffer + Span buf = data[..^0x19]; + //Get the info of the file + Span info = data[^0x19..]; + + uint len = BinaryPrimitives.ReadUInt32LittleEndian(info[..4]); + + ReadOnlySpan signature = info[^0x10..]; + + //If the hash in the file does not match the data contained, the file is likely corrupt. + if (!signature.SequenceEqual(MD5.HashData(data[..^0x10]))) throw new InvalidDataException("Encrypted PSP asset hash does not match signature, likely corrupt"); + + //If the data is not compressed, just return a copy of the raw buffer + if (info[0x4] != 1) return buf.ToArray(); + + //Decompress the data + using IronCompressResult decompressed = Iron.Value!.Decompress(Codec.LZO, buf, (int)len); + + //Return a copy of the decompressed data + return decompressed.AsSpan().ToArray(); + } } \ No newline at end of file diff --git a/Refresh.GameServer/Types/Assets/AssetSafetyLevel.cs b/Refresh.GameServer/Types/Assets/AssetSafetyLevel.cs index f398be8c..7da9837b 100644 --- a/Refresh.GameServer/Types/Assets/AssetSafetyLevel.cs +++ b/Refresh.GameServer/Types/Assets/AssetSafetyLevel.cs @@ -32,6 +32,7 @@ public static AssetSafetyLevel FromAssetType(GameAssetType type) GameAssetType.VoiceRecording => AssetSafetyLevel.Safe, GameAssetType.Painting => AssetSafetyLevel.Safe, GameAssetType.SyncedProfile => AssetSafetyLevel.Safe, + GameAssetType.Mip => AssetSafetyLevel.Safe, GameAssetType.Material => AssetSafetyLevel.PotentiallyUnwanted, GameAssetType.Mesh => AssetSafetyLevel.PotentiallyUnwanted, diff --git a/Refresh.GameServer/Types/Assets/GameAssetType.cs b/Refresh.GameServer/Types/Assets/GameAssetType.cs index 66e9dc13..e7c15bae 100644 --- a/Refresh.GameServer/Types/Assets/GameAssetType.cs +++ b/Refresh.GameServer/Types/Assets/GameAssetType.cs @@ -111,4 +111,10 @@ public enum GameAssetType /// This is bundled with Cross-Controller levels since the uploading logic for those levels isn't great. /// SyncedProfile = 14, + /// + /// A PSP MIP image file. While this file type has no magic, we do some heuristics on the file to detect it. + /// This image is uploaded by LBP PSP for new levels, and is what it loads for level badges. + /// + /// + Mip = 15, } \ No newline at end of file