get type gen'd reading working

diogotr7 · Jan 26, 2025 · 7f96814 · 7f96814
1 parent 5302102
commit 7f96814
Show file tree

Hide file tree

Showing 14 changed files with 209 additions and 101 deletions.
diff --git a/src/StarBreaker.Cli/DataCoreCommands/DataCoreExtractCommand.cs b/src/StarBreaker.Cli/DataCoreCommands/DataCoreExtractCommand.cs
@@ -12,7 +12,7 @@ namespace StarBreaker.Cli;
 [Command("dcb-extract", Description = "Extracts a DataCore binary file into separate xml files")]
 public class DataCoreExtractCommand : ICommand
 {
-    [CommandOption("p4k", 'p', Description = "Path to the Game.p4k")]
+    [CommandOption("p4k", 'p', Description = "Path to the Data.p4k")]
     public string? P4kFile { get; init; }
 
     [CommandOption("dcb", 'd', Description = "Path to the Game.dcb")]

diff --git a/src/StarBreaker.Cli/DataCoreCommands/DataCoreTypeGeneratorCommand.cs b/src/StarBreaker.Cli/DataCoreCommands/DataCoreTypeGeneratorCommand.cs
@@ -0,0 +1,74 @@
+using CliFx;
+using CliFx.Attributes;
+using CliFx.Infrastructure;
+using StarBreaker.Common;
+using StarBreaker.DataCore;
+using StarBreaker.DataCore.TypeGenerator;
+using StarBreaker.P4k;
+
+namespace StarBreaker.Cli;
+
+[Command("dcb-generate", Description = "Generates C# types for DataCore structs and enums. Allows typesafe access to DataCore records.")]
+public class DataCoreTypeGeneratorCommand : ICommand
+{
+    [CommandOption("p4k", 'p', Description = "Path to the Data.p4k")]
+    public string? P4kFile { get; init; }
+
+    [CommandOption("dcb", 'd', Description = "Path to the Game.dcb")]
+    public string? DcbFile { get; init; }
+
+    [CommandOption("output", 'o', Description = "Path to the output directory")]
+    public required string OutputDirectory { get; init; }
+
+    public ValueTask ExecuteAsync(IConsole console)
+    {
+        if (P4kFile == null && DcbFile == null)
+        {
+            console.Output.WriteLine("P4k and DCB files are required.");
+            return default;
+        }
+
+        if (!string.IsNullOrEmpty(P4kFile) && !string.IsNullOrEmpty(DcbFile))
+        {
+            console.Output.WriteLine("Only one of P4k and DCB files can be specified.");
+            return default;
+        }
+
+        Stream? dcbStream = null;
+        if (!string.IsNullOrEmpty(DcbFile))
+        {
+            dcbStream = File.OpenRead(DcbFile);
+            console.Output.WriteLine("DCB loaded.");
+        }
+        else if (!string.IsNullOrEmpty(P4kFile))
+        {
+            var p4k = new P4kFileSystem(P4k.P4kFile.FromFile(P4kFile));
+            console.Output.WriteLine("P4k loaded.");
+            foreach (var file in DataCoreUtils.KnownPaths)
+            {
+                if (!p4k.FileExists(file)) continue;
+
+                dcbStream = p4k.OpenRead(file);
+                console.Output.WriteLine($"{file} found");
+                break;
+            }
+        }
+
+        if (dcbStream == null)
+        {
+            console.Output.WriteLine("DataCore not found.");
+            return default;
+        }
+
+
+        console.Output.WriteLine("Generating DataCore types...");
+        var dcr = new DataCoreTypeGenerator(new DataCoreDatabase(dcbStream));
+
+        console.Output.WriteLine("Writing DataCore types...");
+        dcr.Generate(OutputDirectory);
+
+        console.Output.WriteLine("Done.");
+
+        return default;
+    }
+}
diff --git a/src/StarBreaker.Cli/Program.cs b/src/StarBreaker.Cli/Program.cs
@@ -12,6 +12,7 @@
     .AddCommand<WatchExportCommand>()
     .AddCommand<WatchImportCommand>()
     .AddCommand<DataCoreExtractCommand>()
+    .AddCommand<DataCoreTypeGeneratorCommand>()
     .AddCommand<ExtractP4kCommand>()
     .AddCommand<DumpP4kCommand>()
     .AddCommand<ExtractProtobufsCommand>()

diff --git a/src/StarBreaker.Cli/Properties/launchSettings.json b/src/StarBreaker.Cli/Properties/launchSettings.json
@@ -56,6 +56,10 @@
       "commandName": "Project",
       "commandLineArgs": "dcb-extract -p \"\\\\UNRAID\\backups\\StarCitizen\\Data.p4k\" -o \"C:\\Development\\StarCitizen\\DataCoreExport\""
     },
+    "dcb-generate": {
+      "commandName": "Project",
+      "commandLineArgs": "dcb-generate -p \"C:\\Program Files\\Roberts Space Industries\\StarCitizen\\PTU\\Data.p4k\" -o \"C:\\Development\\StarCitizen\\StarBreaker\\src\\StarBreaker.DataCore.Generated\\Generated\""
+    },
     "dds-merge": {
       "commandName": "Project",
       "commandLineArgs": "dds-merge -i \"D:\\StarCitizen\\P4k\\Data\\Textures\\bubble_ddna.dds\" -o \"D:\\StarCitizen\\DDS\\bubble_ddna.dds\""

diff --git a/src/StarBreaker.Cli/StarBreaker.Cli.csproj b/src/StarBreaker.Cli/StarBreaker.Cli.csproj
@@ -10,6 +10,7 @@
   <ItemGroup>
     <ProjectReference Include="..\StarBreaker.Chf\StarBreaker.Chf.csproj" />
     <ProjectReference Include="..\StarBreaker.CryChunkFile\StarBreaker.CryChunkFile.csproj" />
+    <ProjectReference Include="..\StarBreaker.DataCore.TypeGenerator\StarBreaker.DataCore.TypeGenerator.csproj" />
     <ProjectReference Include="..\StarBreaker.DataCore\StarBreaker.DataCore.csproj" />
     <ProjectReference Include="..\StarBreaker.Dds\StarBreaker.Dds.csproj" />
     <ProjectReference Include="..\StarBreaker.P4k\StarBreaker.P4k.csproj" />

diff --git a/src/StarBreaker.Common/Utilities/TimeLogger.cs b/src/StarBreaker.Common/Utilities/TimeLogger.cs
@@ -11,13 +11,15 @@ public TimeLogger()
         timestamp = Stopwatch.GetTimestamp();
     }
 
+    [Conditional("DEBUG")]
     public void LogReset(string message)
     {
         var elapsed = Stopwatch.GetElapsedTime(timestamp);
         Console.WriteLine($"{message}: {elapsed.TotalMilliseconds}ms");
         timestamp = Stopwatch.GetTimestamp();
     }
-
+
+    [Conditional("DEBUG")]
     public void LogContinue(string message)
     {
         var elapsed = Stopwatch.GetElapsedTime(timestamp);

diff --git a/src/StarBreaker.DataCore.Generated/DataCoreBinaryGenerated.cs b/src/StarBreaker.DataCore.Generated/DataCoreBinaryGenerated.cs
@@ -1,11 +1,24 @@
-namespace StarBreaker.DataCore;
+using StarBreaker.DataCoreGenerated;
+
+namespace StarBreaker.DataCore;
 
 public sealed class DataCoreBinaryGenerated : IDataCoreBinary<IDataCoreReadable>
 {
     public DataCoreDatabase Database { get; }
+
+    public DataCoreBinaryGenerated(DataCoreDatabase database)
+    {
+        Database = database;
+    }
+
     public IDataCoreReadable GetFromMainRecord(DataCoreRecord record)
     {
-        throw new NotImplementedException();
+        var data = TypeMap.ReadFromRecord(Database, record.StructIndex, record.InstanceIndex);
+
+        if (data == null)
+            throw new InvalidOperationException($"Failed to read data from record {record}");
+
+        return data;
     }
 
     public void SaveToFile(DataCoreRecord record, string path)

diff --git a/src/StarBreaker.DataCore.Generated/DataCoreHelper.cs b/src/StarBreaker.DataCore.Generated/DataCoreHelper.cs
@@ -28,7 +28,7 @@ public static class DataCoreHelper
             return null;
 
         var reader = db.GetReader(structIndex, instanceIndex);
-        return T.Read(db, db.StructDefinitions[structIndex], ref reader);
+        return T.Read(db, ref reader);
     }
 
     public static T EnumParse<T>(string value, T unknown) where T : struct, Enum

diff --git a/src/StarBreaker.DataCore.Generated/IDataCoreReadable.cs b/src/StarBreaker.DataCore.Generated/IDataCoreReadable.cs
@@ -6,5 +6,5 @@ public interface IDataCoreReadable;
 
 public interface IDataCoreReadable<out T> : IDataCoreReadable where T : class, IDataCoreReadable<T>
 {
-    static abstract T Read(DataCoreDatabase db, DataCoreStructDefinition structDefinition, ref SpanReader reader);
+    static abstract T Read(DataCoreDatabase db, ref SpanReader reader);
 }
diff --git a/...Breaker.DataCore/DataCoreCodeGenerator.cs → ...re.TypeGenerator/DataCoreCodeGenerator.cs b/...Breaker.DataCore/DataCoreCodeGenerator.cs → ...re.TypeGenerator/DataCoreCodeGenerator.cs
@@ -1,12 +1,12 @@
 using System.Text;
 
-namespace StarBreaker.DataCore;
+namespace StarBreaker.DataCore.TypeGenerator;
 
-public class DataCoreCodeGenerator
+public class DataCoreTypeGenerator
 {
     private readonly DataCoreDatabase Database;
 
-    public DataCoreCodeGenerator(DataCoreDatabase database)
+    public DataCoreTypeGenerator(DataCoreDatabase database)
     {
         Database = database;
     }
@@ -256,16 +256,14 @@ private string GetPropertyType(DataCorePropertyDefinition property)
     private void WriteSpecialConstructor(StringBuilder sb, DataCoreStructDefinition structDefinition, int structIndex)
     {
         if (structDefinition.ParentTypeIndex != -1)
-            sb.AppendLine($"    public new static {structDefinition.GetName(Database)} Read(DataCoreDatabase db, DataCoreStructDefinition structDefinition, ref SpanReader reader)");
+            sb.AppendLine($"    public new static {structDefinition.GetName(Database)} Read(DataCoreDatabase db, ref SpanReader reader)");
         else
-            sb.AppendLine($"    public static {structDefinition.GetName(Database)} Read(DataCoreDatabase db, DataCoreStructDefinition structDefinition, ref SpanReader reader)");
+            sb.AppendLine($"    public static {structDefinition.GetName(Database)} Read(DataCoreDatabase db, ref SpanReader reader)");
 
         var allprops = Database.GetProperties(structIndex).AsSpan();
 
         //for now we ignore parent types
         sb.AppendLine("    {");
-        sb.AppendLine($"        return new {structDefinition.GetName(Database)}");
-        sb.AppendLine("        {");
 
         foreach (var property in allprops)
         {
@@ -275,6 +273,17 @@ private void WriteSpecialConstructor(StringBuilder sb, DataCoreStructDefinition
                 WriteArrayRead(sb, property);
         }
 
+        sb.AppendLine();
+        sb.AppendLine($"        return new {structDefinition.GetName(Database)}");
+        sb.AppendLine("        {");
+
+        foreach (var property in allprops)
+        {
+            var name = property.GetName(Database);
+            sb.AppendLine($"            @{name} = _{name},");
+        }
+
+
         sb.AppendLine("        };");
 
         sb.AppendLine("    }");
@@ -288,28 +297,41 @@ private void WriteSingleRead(StringBuilder sb, DataCorePropertyDefinition proper
         switch (property.DataType)
         {
             case DataType.Class:
-                sb.AppendLine($"            @{name} = {propertyType}.Read(db, structDefinition, ref reader),");
+                sb.AppendLine($"        var _{name} = {propertyType}.Read(db, ref reader);");
                 break;
             case DataType.EnumChoice:
-            {
                 var enumName = Database.EnumDefinitions[property.StructIndex].GetName(Database);
-                sb.AppendLine($"            @{name} = DataCoreHelper.EnumParse<{enumName}>(reader.Read<DataCoreStringId>().ToString(db), {enumName}.__Unknown),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.EnumParse<{enumName}>(reader.Read<DataCoreStringId>().ToString(db), {enumName}.__Unknown);");
                 break;
-            }
             case DataType.Reference:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadFromReference<{propertyType}>(db, reader.Read<DataCoreReference>()),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadFromReference<{propertyType}>(db, reader.Read<DataCoreReference>());");
                 break;
             case DataType.StrongPointer:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadFromPointer<{propertyType}>(db, reader.Read<DataCorePointer>()),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadFromPointer<{propertyType}>(db, reader.Read<DataCorePointer>());");
                 break;
             case DataType.WeakPointer:
                 //do as default. we probably should handle this, it's actually feasible now :D
-                sb.AppendLine($"            @{name} = reader.Read<{propertyType}>(),");
+                sb.AppendLine($"        var _{name} = reader.Read<{propertyType}>();");
                 break;
-            default:
+            case DataType.Guid:
+            case DataType.Locale:
+            case DataType.Double:
+            case DataType.Single:
+            case DataType.String:
+            case DataType.UInt64:
+            case DataType.UInt32:
+            case DataType.UInt16:
+            case DataType.Byte:
+            case DataType.Int64:
+            case DataType.Int32:
+            case DataType.Int16:
+            case DataType.SByte:
+            case DataType.Boolean:
                 //this one should be fine for everything else.
-                sb.AppendLine($"            @{name} = reader.Read<{propertyType}>(),");
+                sb.AppendLine($"        var _{name} = reader.Read<{propertyType}>();");
                 break;
+            default:
+                throw new ArgumentOutOfRangeException();
         }
     }
 
@@ -321,68 +343,64 @@ private void WriteArrayRead(StringBuilder sb, DataCorePropertyDefinition propert
         switch (property.DataType)
         {
             case DataType.Reference:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadReferenceArray<{propertyType}>(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadReferenceArray<{propertyType}>(db, ref reader);");
                 break;
             case DataType.StrongPointer:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadStrongPointerArray<{propertyType}>(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadStrongPointerArray<{propertyType}>(db, ref reader);");
                 break;
             case DataType.WeakPointer:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadWeakPointerArray<{propertyType}>(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadWeakPointerArray<{propertyType}>(db, ref reader);");
                 break;
             case DataType.Class:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadClassArray<{propertyType}>(db, ref reader, {property.StructIndex}),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadClassArray<{propertyType}>(db, ref reader, {property.StructIndex});");
                 break;
             case DataType.Boolean:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadBoolArray(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadBoolArray(db, ref reader);");
                 break;
             case DataType.Byte:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadByteArray(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadByteArray(db, ref reader);");
                 break;
             case DataType.SByte:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadSByteArray(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadSByteArray(db, ref reader);");
                 break;
             case DataType.Int16:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadInt16Array(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadInt16Array(db, ref reader);");
                 break;
             case DataType.UInt16:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadUInt16Array(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadUInt16Array(db, ref reader);");
                 break;
             case DataType.Int32:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadInt32Array(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadInt32Array(db, ref reader);");
                 break;
             case DataType.UInt32:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadUInt32Array(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadUInt32Array(db, ref reader);");
                 break;
             case DataType.Int64:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadInt64Array(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadInt64Array(db, ref reader);");
                 break;
             case DataType.UInt64:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadUInt64Array(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadUInt64Array(db, ref reader);");
                 break;
             case DataType.Single:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadSingleArray(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadSingleArray(db, ref reader);");
                 break;
             case DataType.Double:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadDoubleArray(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadDoubleArray(db, ref reader);");
                 break;
             case DataType.Guid:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadGuidArray(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadGuidArray(db, ref reader);");
                 break;
             case DataType.Locale:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadLocaleArray(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadLocaleArray(db, ref reader);");
                 break;
             case DataType.String:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadStringArray(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadStringArray(db, ref reader);");
                 break;
             case DataType.EnumChoice:
-            {
-                var enumName = Database.EnumDefinitions[property.StructIndex].GetName(Database);
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadEnumArray<{enumName}>(db, ref reader),");
+                sb.AppendLine($"        var _{name} = DataCoreHelper.ReadEnumArray<{Database.EnumDefinitions[property.StructIndex].GetName(Database)}>(db, ref reader);");
                 break;
-            }
             default:
-                sb.AppendLine($"            @{name} = DataCoreHelper.ReadDummyArray<{propertyType}>(db, ref reader),");
-                break;
+                throw new ArgumentOutOfRangeException();
         }
     }
 }
diff --git a/src/StarBreaker.DataCore.TypeGenerator/StarBreaker.DataCore.TypeGenerator.csproj b/src/StarBreaker.DataCore.TypeGenerator/StarBreaker.DataCore.TypeGenerator.csproj
@@ -0,0 +1,5 @@
+<Project Sdk="Microsoft.NET.Sdk">
+    <ItemGroup>
+      <ProjectReference Include="..\StarBreaker.DataCore\StarBreaker.DataCore.csproj" />
+    </ItemGroup>
+</Project>
diff --git a/src/StarBreaker.DataCore/DataForge.cs b/src/StarBreaker.DataCore/DataForge.cs
@@ -4,6 +4,24 @@
 
 namespace StarBreaker.DataCore;
 
+public static class DataForge
+{
+    public static DataForge<string> FromDcbJson(string path)
+    {
+        return new DataForge<string>(new DataCoreBinaryJson(new DataCoreDatabase(File.OpenRead(path))));
+    }
+
+    public static DataForge<string> FromDcbXml(string path)
+    {
+        return new DataForge<string>(new DataCoreBinaryXml(new DataCoreDatabase(File.OpenRead(path))));
+    }
+
+    public static DataForge<IDataCoreObject> FromDcbObjects(string path)
+    {
+        return new DataForge<IDataCoreObject>(new DataCoreBinaryObjects(new DataCoreDatabase(File.OpenRead(path))));
+    }
+}
+
 public class DataForge<T>
 {
     public IDataCoreBinary<T> DataCore { get;  }