From 9fbac85500e7fe9e678e3063fa4bbb04f0951940 Mon Sep 17 00:00:00 2001 From: Diogo Trindade Date: Thu, 16 Jan 2025 00:16:22 +0000 Subject: [PATCH] datacore database optimizations --- src/StarBreaker.DataCore/DataCoreBinary.cs | 2 +- src/StarBreaker.DataCore/DataCoreDatabase.cs | 154 ++++++++++--------- src/StarBreaker.DataCore/DataForge.cs | 2 +- src/StarBreaker.Sandbox/Crc32CSandbox.cs | 4 +- src/StarBreaker.Sandbox/DataCoreSandbox.cs | 2 +- 5 files changed, 87 insertions(+), 77 deletions(-) diff --git a/src/StarBreaker.DataCore/DataCoreBinary.cs b/src/StarBreaker.DataCore/DataCoreBinary.cs index 89bc172..74c4050 100644 --- a/src/StarBreaker.DataCore/DataCoreBinary.cs +++ b/src/StarBreaker.DataCore/DataCoreBinary.cs @@ -171,7 +171,7 @@ public XElement GetFromMainRecord(DataCoreRecord record, DataCoreExtractionConte private XElement GetFromInstance(int structIndex, int instanceIndex, DataCoreExtractionContext context) { - var reader = Database.GetReader(Database.Offsets[structIndex][instanceIndex]); + var reader = Database.GetReader(structIndex, instanceIndex); var element = GetFromStruct(structIndex, ref reader, context); context.Elements[(structIndex, instanceIndex)] = element; diff --git a/src/StarBreaker.DataCore/DataCoreDatabase.cs b/src/StarBreaker.DataCore/DataCoreDatabase.cs index 9bd6b0e..267181a 100644 --- a/src/StarBreaker.DataCore/DataCoreDatabase.cs +++ b/src/StarBreaker.DataCore/DataCoreDatabase.cs @@ -1,4 +1,3 @@ -using System.Collections.Concurrent; using System.Collections.Frozen; using System.Runtime.CompilerServices; using System.Text; @@ -6,11 +5,8 @@ namespace StarBreaker.DataCore; -public class DataCoreDatabase +public sealed class DataCoreDatabase { - public uint Version { get; } - - private readonly ConcurrentDictionary _propertiesCache = new(); private readonly int DataSectionOffset; private readonly byte[] DataSection; @@ -44,22 +40,22 @@ public class DataCoreDatabase public readonly DataCoreReference[] ReferenceValues; public readonly DataCoreStringId2[] EnumOptions; - - public readonly FrozenDictionary Offsets; - - public readonly FrozenDictionary CachedStrings; - public readonly FrozenDictionary CachedStrings2; - public readonly FrozenDictionary RecordMap; public readonly FrozenSet MainRecords; + private readonly FrozenDictionary Offsets; + private readonly DataCorePropertyDefinition[][] Properties; + private readonly FrozenDictionary CachedStrings; + private readonly FrozenDictionary CachedStrings2; + private readonly FrozenDictionary RecordMap; + public DataCoreDatabase(Stream fs) { using var reader = new BinaryReader(fs); _ = reader.ReadUInt32(); - Version = reader.ReadUInt32(); - if (Version is < 5 or > 6) - throw new Exception($"Unsupported file version: {Version}"); + var version = reader.ReadUInt32(); + if (version is < 5 or > 6) + throw new Exception($"Unsupported file version: {version}"); _ = reader.ReadUInt32(); _ = reader.ReadUInt32(); @@ -121,26 +117,23 @@ public DataCoreDatabase(Stream fs) EnumOptions = reader.BaseStream.ReadArray(enumOptionCount); CachedStrings = ReadStringTable(reader.ReadBytes((int)textLength).AsSpan()); - if (Version >= 6) + if (version >= 6) CachedStrings2 = ReadStringTable(reader.ReadBytes((int)textLength2).AsSpan()); else - CachedStrings2 = FrozenDictionary.Empty; + CachedStrings2 = CachedStrings; var bytesRead = (int)fs.Position; + Properties = ReadProperties(); Offsets = ReadOffsets(bytesRead, DataMappings); DataSectionOffset = bytesRead; - DataSection = new byte[fs.Length - bytesRead]; - if (reader.Read(DataSection, 0, DataSection.Length) != DataSection.Length) - throw new Exception("Failed to read data section"); + DataSection = reader.ReadBytes((int)(fs.Length - bytesRead)); RecordMap = RecordDefinitions.ToFrozenDictionary(x => x.Id); var mainRecords = new Dictionary(); foreach (var record in RecordDefinitions) - { mainRecords[record.GetFileName(this)] = record; - } MainRecords = mainRecords.Values.Select(x => x.Id).ToFrozenSet(); @@ -149,17 +142,17 @@ public DataCoreDatabase(Stream fs) #endif } - public SpanReader GetReader(int offset) => new(DataSection, offset - DataSectionOffset); - public string GetString(DataCoreStringId id) => CachedStrings[id.Id]; - public string GetString2(DataCoreStringId2 id) + public SpanReader GetReader(int structIndex, int instanceIndex) { - if (Version < 6) - return CachedStrings[id.Id]; - - return CachedStrings2[id.Id]; + var info = Offsets[structIndex]; + var offset = info.Offset + info.Size * instanceIndex; + return new SpanReader(DataSection, offset - DataSectionOffset); } + public string GetString(DataCoreStringId id) => CachedStrings[id.Id]; + public string GetString2(DataCoreStringId2 id) => CachedStrings2[id.Id]; public DataCoreRecord GetRecord(CigGuid guid) => RecordMap[guid]; + public DataCorePropertyDefinition[] GetProperties(int structIndex) => Properties[structIndex]; private static FrozenDictionary ReadStringTable(ReadOnlySpan span) { @@ -178,68 +171,73 @@ private static FrozenDictionary ReadStringTable(ReadOnlySpan return strings.ToFrozenDictionary(); } - private FrozenDictionary ReadOffsets(int initialOffset, Span mappings) + private FrozenDictionary ReadOffsets(int initialOffset, ReadOnlySpan mappings) { - var instances = new Dictionary(mappings.Length); + var instances = new Dictionary(); + var offset = initialOffset; foreach (var mapping in mappings) { - var arr = new int[mapping.StructCount]; - - for (var i = 0; i < mapping.StructCount; i++) - { - arr[i] = initialOffset; - initialOffset += CalculateStructSize(mapping.StructIndex); - } - - instances.Add(mapping.StructIndex, arr); + var size = CalculateStructSize(mapping.StructIndex); + instances[mapping.StructIndex] = new StructOffsetAndSize(offset, size); + offset += (int)(size * mapping.StructCount); } return instances.ToFrozenDictionary(); } - public DataCorePropertyDefinition[] GetProperties(int structIndex) => - _propertiesCache.GetOrAdd(structIndex, static (index, db) => + private DataCorePropertyDefinition[][] ReadProperties() + { + var result = new DataCorePropertyDefinition[StructDefinitions.Length][]; + + for (var i = 0; i < StructDefinitions.Length; i++) { - var @this = db.StructDefinitions[index]; - var structs = db.StructDefinitions.AsSpan(); - var properties = db.PropertyDefinitions.AsSpan(); + result[i] = GetStructProperties(i, this); + } - if (@this is { AttributeCount: 0, ParentTypeIndex: -1 }) - return []; + return result; + } - // Calculate total property count to avoid resizing - int totalPropertyCount = @this.AttributeCount; - var baseStruct = @this; - while (baseStruct.ParentTypeIndex != -1) - { - baseStruct = structs[baseStruct.ParentTypeIndex]; - totalPropertyCount += baseStruct.AttributeCount; - } + private static DataCorePropertyDefinition[] GetStructProperties(int index, DataCoreDatabase db) + { + var @this = db.StructDefinitions[index]; + var structs = db.StructDefinitions.AsSpan(); + var properties = db.PropertyDefinitions.AsSpan(); + + if (@this is { AttributeCount: 0, ParentTypeIndex: -1 }) return []; - // Pre-allocate array with exact size needed - var result = new DataCorePropertyDefinition[totalPropertyCount]; + // Calculate total property count to avoid resizing + int totalPropertyCount = @this.AttributeCount; + var baseStruct = @this; + while (baseStruct.ParentTypeIndex != -1) + { + baseStruct = structs[baseStruct.ParentTypeIndex]; + totalPropertyCount += baseStruct.AttributeCount; + } - // Reset to start struct for actual property copying - baseStruct = @this; - var currentPosition = totalPropertyCount; + // Pre-allocate array with exact size needed + var result = new DataCorePropertyDefinition[totalPropertyCount]; - // Copy properties in reverse order to avoid InsertRange - do - { - int count = baseStruct.AttributeCount; - currentPosition -= count; - properties.Slice(baseStruct.FirstAttributeIndex, count) - .CopyTo(result.AsSpan(currentPosition, count)); + // Reset to start struct for actual property copying + baseStruct = @this; + var currentPosition = totalPropertyCount; + + // Copy properties in reverse order to avoid InsertRange + do + { + int count = baseStruct.AttributeCount; + currentPosition -= count; + properties.Slice(baseStruct.FirstAttributeIndex, count) + .CopyTo(result.AsSpan(currentPosition, count)); - if (baseStruct.ParentTypeIndex == -1) break; - baseStruct = structs[baseStruct.ParentTypeIndex]; - } while (true); + if (baseStruct.ParentTypeIndex == -1) break; + baseStruct = structs[baseStruct.ParentTypeIndex]; + } while (true); - return result; - }, this); + return result; + } - public int CalculateStructSize(int structIndex) + private int CalculateStructSize(int structIndex) { var size = 0; @@ -279,4 +277,16 @@ public int CalculateStructSize(int structIndex) return size; } + + public readonly struct StructOffsetAndSize + { + public readonly int Offset; + public readonly int Size; + + public StructOffsetAndSize(int offset, int size) + { + Offset = offset; + Size = size; + } + } } \ No newline at end of file diff --git a/src/StarBreaker.DataCore/DataForge.cs b/src/StarBreaker.DataCore/DataForge.cs index 08ed13e..2f02a15 100644 --- a/src/StarBreaker.DataCore/DataForge.cs +++ b/src/StarBreaker.DataCore/DataForge.cs @@ -17,7 +17,7 @@ public Dictionary GetRecordsByFileName(string? fileNameF var structsPerFileName = new Dictionary(); foreach (var recordId in DataCore.Database.MainRecords) { - var record = DataCore.Database.RecordMap[recordId]; + var record = DataCore.Database.GetRecord(recordId); var fileName = record.GetFileName(DataCore.Database); if (fileNameFilter != null && !FileSystemName.MatchesSimpleExpression(fileNameFilter, fileName)) diff --git a/src/StarBreaker.Sandbox/Crc32CSandbox.cs b/src/StarBreaker.Sandbox/Crc32CSandbox.cs index 024d3ff..a5f480c 100644 --- a/src/StarBreaker.Sandbox/Crc32CSandbox.cs +++ b/src/StarBreaker.Sandbox/Crc32CSandbox.cs @@ -23,8 +23,8 @@ public static void Run() IEnumerable haystack = []; haystack = new List() - .Concat(EnumeratePaths(dcb.DataCore.Database.CachedStrings.Values, '/')) - .Concat(EnumeratePaths(dcb.DataCore.Database.CachedStrings2.Values, '/')) + //.Concat(EnumeratePaths(dcb.DataCore.Database.CachedStrings.Values, '/')) + //.Concat(EnumeratePaths(dcb.DataCore.Database.CachedStrings2.Values, '/')) //.Concat(["head_eyedetail"]) //.Concat(StreamLines("strings.txt")) .Concat(StreamLines("mats.txt")) diff --git a/src/StarBreaker.Sandbox/DataCoreSandbox.cs b/src/StarBreaker.Sandbox/DataCoreSandbox.cs index 617d297..e58d117 100644 --- a/src/StarBreaker.Sandbox/DataCoreSandbox.cs +++ b/src/StarBreaker.Sandbox/DataCoreSandbox.cs @@ -16,7 +16,7 @@ private static void ExtractAll() var dcb = new DataForge(new MemoryStream(File.ReadAllBytes(@"D:\StarCitizen\P4k\Data\Game2.dcb"))); timer.LogReset("Loaded DataForge"); Directory.CreateDirectory(@"D:\StarCitizen\DataCore\Sandbox"); - dcb.ExtractAll(@"D:\StarCitizen\DataCore\Sandbox"); + dcb.ExtractAllParallel(@"D:\StarCitizen\DataCore\Sandbox"); timer.LogReset("Extracted all records."); }