From cfaae5cd713bfd485e86b08b68f004456c602cec Mon Sep 17 00:00:00 2001 From: Leonid Date: Mon, 25 Mar 2024 15:03:59 +0700 Subject: [PATCH 1/5] Add fast loading bundles via stream --- .../Helpers/HashingHelper.cs | 17 +++++++- .../Managers/AssetBundleManager.cs | 16 ++++++- BepInExFasterLoadAssetBundles/Patcher.cs | 42 ++++++++++++++++--- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/BepInExFasterLoadAssetBundles/Helpers/HashingHelper.cs b/BepInExFasterLoadAssetBundles/Helpers/HashingHelper.cs index ba4f9ba..5cb3a4d 100644 --- a/BepInExFasterLoadAssetBundles/Helpers/HashingHelper.cs +++ b/BepInExFasterLoadAssetBundles/Helpers/HashingHelper.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.Globalization; using System.IO; using System.Security.Cryptography; @@ -8,9 +9,21 @@ internal class HashingHelper { public static byte[] HashFile(string path) { - using var fileStream = File.OpenRead(path); + const int c_BufferSize = 81920; + + using var fileStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None, c_BufferSize, FileOptions.SequentialScan); using var sha1 = new SHA1Managed(); - return sha1.ComputeHash(fileStream); + + var array = ArrayPool.Shared.Rent(c_BufferSize); + int readBytes; + while ((readBytes = fileStream.Read(array, 0, c_BufferSize)) > 0) + { + sha1.TransformBlock(array, 0, readBytes, array, 0); + } + + sha1.TransformFinalBlock([], 0, 0); + + return sha1.Hash; } public static string HashToString(byte[] hash) diff --git a/BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs b/BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs index 63e6342..816b947 100644 --- a/BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs +++ b/BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs @@ -22,7 +22,7 @@ public AssetBundleManager(string cachePath) public bool TryRecompressAssetBundle(ref string path) { var originalFileName = Path.GetFileNameWithoutExtension(path); - byte[] hash = HashingHelper.HashFile(path); + var hash = HashingHelper.HashFile(path); var metadata = Patcher.MetadataManager.FindMetadataByHash(hash); if (metadata != null) @@ -88,6 +88,20 @@ public bool TryRecompressAssetBundle(ref string path) return true; } + public FileStream? TryRecompressAssetBundle(FileStream stream) + { + var path = string.Copy(stream.Name); + + stream.Dispose(); + + if (TryRecompressAssetBundle(ref path)) + { + return File.OpenRead(path); + } + + return null; + } + public void DeleteCachedAssetBundle(string path) { FileHelper.TryDeleteFile(path, out var fileException); diff --git a/BepInExFasterLoadAssetBundles/Patcher.cs b/BepInExFasterLoadAssetBundles/Patcher.cs index 27fc321..4856e4e 100644 --- a/BepInExFasterLoadAssetBundles/Patcher.cs +++ b/BepInExFasterLoadAssetBundles/Patcher.cs @@ -1,10 +1,8 @@ using System; using System.IO; -using System.Threading; using BepInEx; using BepInEx.Bootstrap; using BepInEx.Logging; -using BepInExFasterLoadAssetBundles.Helpers; using BepInExFasterLoadAssetBundles.Managers; using HarmonyLib; using UnityEngine; @@ -37,12 +35,20 @@ public static void ChainloaderInitialized() var thisType = typeof(Patcher); var harmony = BepInExFasterLoadAssetBundlesPatcher.Harmony; + // file harmony.Patch(AccessTools.Method(typeof(AssetBundle), nameof(AssetBundle.LoadFromFile_Internal)), prefix: new(thisType.GetMethod(nameof(LoadAssetBundleFromFileFast)))); - // todo - /*harmony.Patch(AccessTools.Method(typeof(AssetBundle), nameof(AssetBundle.LoadFromStreamInternal)), - prefix: new(thisType.GetMethod(nameof(LoadAssetBundleFromStreamFast))));*/ + harmony.Patch(AccessTools.Method(typeof(AssetBundle), nameof(AssetBundle.LoadFromFileAsync_Internal)), + prefix: new(thisType.GetMethod(nameof(LoadAssetBundleFromFileFast)))); + + + // streams + harmony.Patch(AccessTools.Method(typeof(AssetBundle), nameof(AssetBundle.LoadFromStreamInternal)), + prefix: new(thisType.GetMethod(nameof(LoadAssetBundleFromStreamFast)))); + + harmony.Patch(AccessTools.Method(typeof(AssetBundle), nameof(AssetBundle.LoadFromStreamAsyncInternal)), + prefix: new(thisType.GetMethod(nameof(LoadAssetBundleFromStreamFast)))); } public static void LoadAssetBundleFromFileFast(ref string path) @@ -69,4 +75,30 @@ public static void LoadAssetBundleFromFileFast(ref string path) path = tempPath; } } + + public static void LoadAssetBundleFromStreamFast(ref Stream stream) + { + if (stream is not FileStream fileStream) + { + return; + } + + var tempFilePath = fileStream.Name; + + try + { + var decompressedStream = AssetBundleManager.TryRecompressAssetBundle(fileStream); + if (decompressedStream != null) + { + stream = decompressedStream; + } + } + catch (Exception ex) + { + Logger.LogError($"Failed to decompress assetbundle\n{ex}"); + } + + // stream is disposed, returning new stream + stream = new FileStream(tempFilePath, FileMode.Open, FileAccess.Read, FileShare.None); + } } From 21887648c9e6580ef8db5f93c8e3ade6be317f44 Mon Sep 17 00:00:00 2001 From: Leonid Date: Mon, 25 Mar 2024 18:49:47 +0700 Subject: [PATCH 2/5] Do recompress bundle in background --- .../Helpers/AsyncHelper.cs | 22 ++++ .../Helpers/AsyncOperationHelper.cs | 49 ++++++++- .../Helpers/HashingHelper.cs | 13 ++- .../Managers/AssetBundleManager.cs | 101 +++++++++++------- .../Managers/MetadataManager.cs | 19 ++-- BepInExFasterLoadAssetBundles/Patcher.cs | 7 +- 6 files changed, 154 insertions(+), 57 deletions(-) create mode 100644 BepInExFasterLoadAssetBundles/Helpers/AsyncHelper.cs diff --git a/BepInExFasterLoadAssetBundles/Helpers/AsyncHelper.cs b/BepInExFasterLoadAssetBundles/Helpers/AsyncHelper.cs new file mode 100644 index 0000000..030e28b --- /dev/null +++ b/BepInExFasterLoadAssetBundles/Helpers/AsyncHelper.cs @@ -0,0 +1,22 @@ +using System; +using System.Threading.Tasks; + +namespace BepInExFasterLoadAssetBundles.Helpers; +internal static class AsyncHelper +{ + public static void Schedule(Func func) + { + _ = Task.Run(async () => + { + try + { + await func(); + } + + catch (Exception ex) + { + Patcher.Logger.LogError(ex); + } + }); + } +} diff --git a/BepInExFasterLoadAssetBundles/Helpers/AsyncOperationHelper.cs b/BepInExFasterLoadAssetBundles/Helpers/AsyncOperationHelper.cs index 296301e..adb9569 100644 --- a/BepInExFasterLoadAssetBundles/Helpers/AsyncOperationHelper.cs +++ b/BepInExFasterLoadAssetBundles/Helpers/AsyncOperationHelper.cs @@ -1,4 +1,6 @@ -using System.Threading; +using System; +using System.Runtime.CompilerServices; +using System.Threading; using UnityEngine; namespace BepInExFasterLoadAssetBundles.Helpers; @@ -11,4 +13,49 @@ public static void WaitUntilOperationComplete(T op) where T : AsyncOperation Thread.Sleep(100); } } + + public static AsyncOperationAwaiter WaitCompletionAsync(this T op) where T : AsyncOperation + { + return new AsyncOperationAwaiter(op); + } + + public struct AsyncOperationAwaiter : ICriticalNotifyCompletion + { + private AsyncOperation? m_AsyncOperation; + private Action? m_ContinuationAction; + + public AsyncOperationAwaiter(AsyncOperation asyncOperation) + { + m_AsyncOperation = asyncOperation; + m_ContinuationAction = null; + } + + public readonly AsyncOperationAwaiter GetAwaiter() => this; + public readonly bool IsCompleted => m_AsyncOperation!.isDone; + + public void GetResult() + { + if (m_AsyncOperation != null) + m_AsyncOperation.completed -= OnCompleted; + + m_AsyncOperation = null; + m_ContinuationAction = null; + } + + public void OnCompleted(Action continuation) + { + UnsafeOnCompleted(continuation); + } + + public void UnsafeOnCompleted(Action continuation) + { + m_ContinuationAction = continuation; + m_AsyncOperation!.completed += OnCompleted; + } + + private readonly void OnCompleted(AsyncOperation _) + { + m_ContinuationAction?.Invoke(); + } + } } diff --git a/BepInExFasterLoadAssetBundles/Helpers/HashingHelper.cs b/BepInExFasterLoadAssetBundles/Helpers/HashingHelper.cs index 5cb3a4d..28f8089 100644 --- a/BepInExFasterLoadAssetBundles/Helpers/HashingHelper.cs +++ b/BepInExFasterLoadAssetBundles/Helpers/HashingHelper.cs @@ -7,16 +7,21 @@ namespace BepInExFasterLoadAssetBundles.Helpers; internal class HashingHelper { + private const int c_BufferSize = 81920; + public static byte[] HashFile(string path) { - const int c_BufferSize = 81920; - using var fileStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None, c_BufferSize, FileOptions.SequentialScan); + return HashStream(fileStream); + } + + public static byte[] HashStream(Stream stream) + { using var sha1 = new SHA1Managed(); var array = ArrayPool.Shared.Rent(c_BufferSize); int readBytes; - while ((readBytes = fileStream.Read(array, 0, c_BufferSize)) > 0) + while ((readBytes = stream.Read(array, 0, c_BufferSize)) > 0) { sha1.TransformBlock(array, 0, readBytes, array, 0); } @@ -33,7 +38,7 @@ public static string HashToString(byte[] hash) for (var i = 0; i < hash.Length; i++) { var b = hash[i]; - b.TryFormat(chars[(i * 2)..], out var charsWritten, "X2"); + b.TryFormat(chars[(i * 2)..], out _, "X2"); } return chars.ToString(); diff --git a/BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs b/BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs index 816b947..7e19ba9 100644 --- a/BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs +++ b/BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs @@ -1,6 +1,8 @@ using System; using System.IO; +using System.Threading.Tasks; using BepInExFasterLoadAssetBundles.Helpers; +using BepInExFasterLoadAssetBundles.Models; using UnityEngine; namespace BepInExFasterLoadAssetBundles.Managers; @@ -21,8 +23,24 @@ public AssetBundleManager(string cachePath) public bool TryRecompressAssetBundle(ref string path) { - var originalFileName = Path.GetFileNameWithoutExtension(path); - var hash = HashingHelper.HashFile(path); + return TryRecompressAssetBundleInternal(ref path, null); + } + + public FileStream? TryRecompressAssetBundle(FileStream stream) + { + var path = string.Copy(stream.Name); + + if (TryRecompressAssetBundleInternal(ref path, HashingHelper.HashStream(stream))) + { + return File.OpenRead(path); + } + + return null; + } + + public bool TryRecompressAssetBundleInternal(ref string path, byte[]? hash) + { + hash ??= HashingHelper.HashFile(path); var metadata = Patcher.MetadataManager.FindMetadataByHash(hash); if (metadata != null) @@ -35,8 +53,7 @@ public bool TryRecompressAssetBundle(ref string path) var newPath = Path.Combine(CachePath, metadata.UncompressedAssetBundleName); if (File.Exists(newPath)) { - Patcher.Logger.LogDebug( - $"Found uncompressed bundle {metadata.UncompressedAssetBundleName}, loading it instead of {originalFileName}"); + Patcher.Logger.LogDebug($"Loading uncompressed bundle \"{metadata.UncompressedAssetBundleName}\""); path = newPath; metadata.LastAccessTime = DateTime.Now; @@ -45,69 +62,73 @@ public bool TryRecompressAssetBundle(ref string path) return true; } - Patcher.Logger.LogWarning($"Failed to find decompressed assetbundle at {newPath}. Probably it was deleted?"); + Patcher.Logger.LogWarning($"Failed to find decompressed assetbundle at \"{newPath}\". Probably it was deleted?"); + } + + var nonRefPath = path; + AsyncHelper.Schedule(() => DecompressAssetBundleAsync(nonRefPath, hash)); + + return false; + } + + public void DeleteCachedAssetBundle(string path) + { + FileHelper.TryDeleteFile(path, out var fileException); + if (fileException != null) + { + Patcher.Logger.LogError($"Failed to delete uncompressed assetbundle\n{fileException}"); } + } - metadata = new() + private async Task DecompressAssetBundleAsync(string path, byte[] hash) + { + var metadata = new Metadata() { OriginalAssetBundleHash = HashingHelper.HashToString(hash), LastAccessTime = DateTime.Now, }; + var originalFileName = Path.GetFileNameWithoutExtension(path); var outputName = originalFileName + '_' + metadata.GetHashCode() + ".assetbundle"; var outputPath = Path.Combine(CachePath, outputName); + // when loading assetbundle async via stream, the file can be still in use. Wait a bit for that + var tries = 5; + while (--tries > 0) + { + try + { + using var tempStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None); + } + catch (IOException) + { + await Task.Delay(100); + } + } + var op = AssetBundle.RecompressAssetBundleAsync(path, outputPath, BuildCompression.UncompressedRuntime, 0, ThreadPriority.High); - AsyncOperationHelper.WaitUntilOperationComplete(op); + + await op.WaitCompletionAsync(); if (op.result is not AssetBundleLoadResult.Success) { - Patcher.Logger.LogWarning($"Failed to decompress a assetbundle at {path}\n{op.humanReadableResult}"); - return false; + Patcher.Logger.LogWarning($"Failed to decompress a assetbundle at \"{path}\"\n{op.humanReadableResult}"); + return; } // check if unity returned the same assetbundle (means that assetbundle is already decompressed) if (hash.AsSpan().SequenceEqual(HashingHelper.HashFile(outputPath))) { - Patcher.Logger.LogDebug($"Assetbundle {originalFileName} is already uncompressed, adding to ignore list"); + Patcher.Logger.LogDebug($"Assetbundle \"{originalFileName}\" is already uncompressed, adding to ignore list"); metadata.ShouldNotDecompress = true; Patcher.MetadataManager.SaveMetadata(metadata); DeleteCachedAssetBundle(outputPath); - return false; + return; } - path = outputPath; - metadata.UncompressedAssetBundleName = outputName; Patcher.MetadataManager.SaveMetadata(metadata); - - Patcher.Logger.LogDebug($"Loading uncompressed bundle {outputName} instead of {originalFileName}"); - - return true; - } - - public FileStream? TryRecompressAssetBundle(FileStream stream) - { - var path = string.Copy(stream.Name); - - stream.Dispose(); - - if (TryRecompressAssetBundle(ref path)) - { - return File.OpenRead(path); - } - - return null; - } - - public void DeleteCachedAssetBundle(string path) - { - FileHelper.TryDeleteFile(path, out var fileException); - if (fileException != null) - { - Patcher.Logger.LogError($"Failed to delete uncompressed assetbundle\n{fileException}"); - } } } diff --git a/BepInExFasterLoadAssetBundles/Managers/MetadataManager.cs b/BepInExFasterLoadAssetBundles/Managers/MetadataManager.cs index 569dd5c..2c7c0a8 100644 --- a/BepInExFasterLoadAssetBundles/Managers/MetadataManager.cs +++ b/BepInExFasterLoadAssetBundles/Managers/MetadataManager.cs @@ -40,15 +40,18 @@ public MetadataManager(string metadataFile) public void SaveMetadata(Metadata metadata) { - var index = m_Metadata.FindIndex(m => m.OriginalAssetBundleHash == metadata.OriginalAssetBundleHash); - - if (index == -1) - { - m_Metadata.Add(metadata); - } - else + lock (m_Lock) { - m_Metadata[index] = metadata; + var index = m_Metadata.FindIndex(m => m.OriginalAssetBundleHash == metadata.OriginalAssetBundleHash); + + if (index == -1) + { + m_Metadata.Add(metadata); + } + else + { + m_Metadata[index] = metadata; + } } SaveFile(); diff --git a/BepInExFasterLoadAssetBundles/Patcher.cs b/BepInExFasterLoadAssetBundles/Patcher.cs index 4856e4e..6301211 100644 --- a/BepInExFasterLoadAssetBundles/Patcher.cs +++ b/BepInExFasterLoadAssetBundles/Patcher.cs @@ -83,7 +83,7 @@ public static void LoadAssetBundleFromStreamFast(ref Stream stream) return; } - var tempFilePath = fileStream.Name; + var previousPosition = fileStream.Position; try { @@ -96,9 +96,8 @@ public static void LoadAssetBundleFromStreamFast(ref Stream stream) catch (Exception ex) { Logger.LogError($"Failed to decompress assetbundle\n{ex}"); - } - // stream is disposed, returning new stream - stream = new FileStream(tempFilePath, FileMode.Open, FileAccess.Read, FileShare.None); + fileStream.Position = previousPosition; + } } } From 054d2e14d2d0431db3ac726919ec9da614939b98 Mon Sep 17 00:00:00 2001 From: Leonid Date: Wed, 27 Mar 2024 15:31:50 +0700 Subject: [PATCH 3/5] Make decompression less laggy --- BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs b/BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs index 7e19ba9..ed9ee4a 100644 --- a/BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs +++ b/BepInExFasterLoadAssetBundles/Managers/AssetBundleManager.cs @@ -106,7 +106,7 @@ private async Task DecompressAssetBundleAsync(string path, byte[] hash) } var op = AssetBundle.RecompressAssetBundleAsync(path, outputPath, - BuildCompression.UncompressedRuntime, 0, ThreadPriority.High); + BuildCompression.UncompressedRuntime, 0, ThreadPriority.Normal); await op.WaitCompletionAsync(); From f0a79ab75809d30656d055d0817682e73d9e563e Mon Sep 17 00:00:00 2001 From: Leonid Date: Wed, 27 Mar 2024 15:32:17 +0700 Subject: [PATCH 4/5] Fix original bundles are not loaded correctly --- BepInExFasterLoadAssetBundles/Patcher.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/BepInExFasterLoadAssetBundles/Patcher.cs b/BepInExFasterLoadAssetBundles/Patcher.cs index 6301211..456abb4 100644 --- a/BepInExFasterLoadAssetBundles/Patcher.cs +++ b/BepInExFasterLoadAssetBundles/Patcher.cs @@ -91,13 +91,14 @@ public static void LoadAssetBundleFromStreamFast(ref Stream stream) if (decompressedStream != null) { stream = decompressedStream; + return; } } catch (Exception ex) { Logger.LogError($"Failed to decompress assetbundle\n{ex}"); - - fileStream.Position = previousPosition; } + + fileStream.Position = previousPosition; } } From 03809cde0f6432b61ea89cd9d4881dd710875681 Mon Sep 17 00:00:00 2001 From: Leonid Date: Thu, 28 Mar 2024 03:16:11 +0700 Subject: [PATCH 5/5] Updated changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dad446d..d90452b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.2.0] 2024-03-28 +### Changed +- Decompression is now happens in background. +- Decompression thread priority is set to `Normal` instead of `High`. +- AssetBundle loaded via `FileStream` will be now cached. + ## [0.1.0] 2024-03-25 ### Added - Debug log when decompressed assetbundle is loaded instead.