Fix SeStringReadOnlySpan.ToString and add SeStringBuilder.AppendMacroString

src/Lumina.Tests/Lumina.Tests.csproj

@@ -4,6 +4,8 @@
         <TargetFramework>net8.0</TargetFramework>
 
         <IsPackable>false</IsPackable>
+
+        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     </PropertyGroup>
 
     <ItemGroup>

src/Lumina.Tests/SeStringBuilderTests.cs

@@ -1,8 +1,15 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
 using System.Text;
+using Lumina.Data;
+using Lumina.Data.Files.Excel;
+using Lumina.Data.Structs.Excel;
+using Lumina.Excel;
 using Lumina.Text;
 using Lumina.Text.Expressions;
+using Lumina.Text.Parse;
 using Lumina.Text.Payloads;
 using Lumina.Text.ReadOnly;
 using Xunit;
@@ -382,7 +389,7 @@ public void AddonIsParsedCorrectly()
                 .Clear()
                 .PushColorType( 508 )
                 .PushEdgeColorType( 509 )
-                .Append("Discard "u8)
+                .Append( "Discard "u8 )
                 .BeginMacro( MacroCode.If )
                 .BeginBinaryExpression( ExpressionType.Equal )
                 .AppendLocalNumberExpression( 2 )
@@ -401,7 +408,7 @@ public void AddonIsParsedCorrectly()
                 .BeginMacro( MacroCode.Num )
                 .AppendLocalNumberExpression( 2 )
                 .EndMacro()
-                .Append(" "  )
+                .Append( " " )
                 .BeginMacro( MacroCode.EnNoun )
                 .AppendStringExpression( "Item" )
                 .AppendIntExpression( 3 )
@@ -411,7 +418,7 @@ public void AddonIsParsedCorrectly()
                 .EndMacro()
                 .EndExpression()
                 .EndMacro()
-                .Append("?"  )
+                .Append( "?" )
                 .PopEdgeColorType()
                 .PopColorType()
                 .ToReadOnlySeString(),
@@ -421,7 +428,196 @@ public void AddonIsParsedCorrectly()
             var r = row.ReadColumn< SeString >( 0 ).AsReadOnly();
             _outputHelper.WriteLine( $"{row.RowId}\t{r.ExtractText()}\t{r}" );
             if( expected.TryGetValue( row.RowId, out var expectedSeString ) )
-                Assert.True( expectedSeString == r, $"{row.RowId} does not match; expected {expectedSeString}" );
+                Assert.StrictEqual( expectedSeString, r );
+        }
+    }
+
+    [Fact]
+    public unsafe void InterpolationHandlerTest1()
+    {
+        const string test = "asdf";
+        Assert.Equal(
+            "Left:1234    \nRight:    1234\nasdf\nint*: 0x0000000012345678",
+            new SeStringBuilder()
+                .Append( $"Left:{0x1234,-8:X}\nRight:{0x1234,8:X}\n{test}\nint*: 0x{(void*) 0x12345678:X16}" )
+                .ToReadOnlySeString()
+                .ToString() );
+    }
+
+    [Fact]
+    public void InterpolationHandlerTest2()
+    {
+        var boldHello = new SeStringBuilder().AppendBold( "Hello" ).ToReadOnlySeString();
+        Assert.Equal(
+            "|Left    |\n|   Right|\n<bold(1)>Hello<bold(0)>\nnull",
+            new SeStringBuilder()
+                .Append( $"|{"Left",-8}|\n|{"Right"u8,8}|\n{boldHello}\n{(object) null}" )
+                .ToReadOnlySeString()
+                .ToString() );
+    }
+
+    [Fact]
+    public void InterpolationHandlerTest3() =>
+        Assert.Equal(
+            "<italic(1)>test<italic(0)>",
+            new SeStringBuilder()
+                .Append( $"{"<italic(1)>test<italic(0)>":m}" )
+                .ToReadOnlySeString()
+                .ToString() );
+
+    [Fact]
+    public void ThrowsOnInvalidMacroStrings1() =>
+        Assert.Throws< MacroStringParseException >( () => new SeStringBuilder().AppendMacroString( "<bad_payload>"u8 ) );
+
+    [Fact]
+    public void ThrowsOnInvalidMacroStrings2() =>
+        Assert.Throws< MacroStringParseException >( () => new SeStringBuilder().AppendMacroString( "<if([a=b])>"u8 ) );
+
+    [Fact]
+    public void ThrowsOnInvalidMacroStrings3() =>
+        Assert.Throws< MacroStringParseException >( () => new SeStringBuilder().AppendMacroString( "<if(1,2,3>"u8 ) );
+
+    [Fact]
+    public void ThrowsOnInvalidMacroStrings4() =>
+        Assert.Throws< MacroStringParseException >( () => new SeStringBuilder().AppendMacroString( "<if,2,3>"u8 ) );
+
+    [Fact]
+    public void ThrowsOnInvalidMacroStrings5() =>
+        Assert.Throws< MacroStringParseException >( () => new SeStringBuilder().AppendMacroString( "<if(1,2,3)"u8 ) );
+
+    [Fact]
+    public void ThrowsOnInvalidMacroStrings6() =>
+        Assert.Throws< MacroStringParseException >( () => new SeStringBuilder().AppendMacroString( "<if(1,2,3"u8 ) );
+
+    [Fact]
+    public void ThrowsOnInvalidMacroStrings7() =>
+        Assert.Throws< MacroStringParseException >( () => new SeStringBuilder().AppendMacroString( "< asdf >"u8 ) );
+
+    [Fact]
+    public void PooledObjectsStateTest()
+    {
+        for( var i = 0; i < 64; i++ )
+        {
+            Assert.Equal(
+                $"{i}<string({i})>{i}<string(<string({i})>)>{i}",
+                ReadOnlySeString.FromMacroString( $"{i}<string({i})>{i}<string(<string({i})>)>{i}" ).ToString() );
+        }
+    }
+
+    [Fact]
+    public void ClearZeroBuffers()
+    {
+        var ssb = new SeStringBuilder();
+        ssb.AppendMacroString( "a<string(a,b,c,d,1,2,3,4,<string(asdfasdf)>)>aaaaaa" );
+        ssb.Clear();
+        var mssFree = (List< MemoryStream >)
+            typeof( SeStringBuilder )
+                .GetField( "_mssFree", BindingFlags.Instance | BindingFlags.NonPublic )!
+                .GetValue( ssb )!;
+        Assert.DoesNotContain( mssFree, x => x.GetBuffer().AsSpan().ContainsAnyExcept( (byte) 0 ) );
+    }
+
+    [Fact]
+    public void FriendlyErrorMessage()
+    {
+        try
+        {
+            const string dummy = "AAAA";
+            ReadOnlySeString.FromMacroString( $"{dummy}<string(bbbb<STRING(ccc)>>{dummy}" );
+        }
+        catch( MacroStringParseException e )
+        {
+            Assert.Equal( 34, e.ByteOffset );
+            Assert.Equal( 17, e.CodepointIndex );
+            Assert.False( e.BeforeError.StartsWith( "..." ) );
+            Assert.False( e.AfterError.EndsWith( "..." ) );
+        }
+
+        try
+        {
+            const string dummy =
+                "AAAABBBBCCCCDDDDAAAABBBBCCCCDDDDAAAABBBBCCCCDDDDAAAABBBBCCCCDDDD0000111122223333000011112222333300001111222233330000111122223333";
+            ReadOnlySeString.FromMacroString( $"{dummy}<string(bbbb<STRING(ccc)>>{dummy}" );
+        }
+        catch( MacroStringParseException e )
+        {
+            Assert.Equal( 282, e.ByteOffset );
+            Assert.Equal( 141, e.CodepointIndex );
+            Assert.StartsWith( "...", e.BeforeError );
+            Assert.EndsWith( "...", e.AfterError );
+        }
+
+        try
+        {
+            const string dummy =
+                "AAAABBBBCCCCDDDDAAAABBBBCCCCDDDDAAAABBBBCCCCDDDDAAAABBBBCCCCDDDD0000111122223333000011112222333300001111222233330000111122223333";
+            ReadOnlySeString.FromMacroString( $"{dummy}<string(bbbb<STRING(ccc)>>" );
+        }
+        catch( MacroStringParseException e )
+        {
+            Assert.Equal( 282, e.ByteOffset );
+            Assert.Equal( 141, e.CodepointIndex );
+            Assert.StartsWith( "...", e.BeforeError );
+            Assert.False( e.AfterError.EndsWith( "..." ) );
+        }
+
+        try
+        {
+            const string dummy =
+                "AAAABBBBCCCCDDDDAAAABBBBCCCCDDDDAAAABBBBCCCCDDDDAAAABBBBCCCCDDDD0000111122223333000011112222333300001111222233330000111122223333";
+            ReadOnlySeString.FromMacroString( $"<string(bbbb<STRING(ccc)>>{dummy}" );
+        }
+        catch( MacroStringParseException e )
+        {
+            Assert.Equal( 26, e.ByteOffset );
+            Assert.Equal( 13, e.CodepointIndex );
+            Assert.False( e.BeforeError.StartsWith( "..." ) );
+            Assert.EndsWith( "...", e.AfterError );
+        }
+    }
+
+    [RequiresGameInstallationFact]
+    public void AllSheetsTextColumnCodec()
+    {
+        var gameData = new GameData( @"C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\game\sqpack" );
+        var ssb = new SeStringBuilder();
+        foreach( var sheetName in gameData.Excel.GetSheetNames() )
+        {
+            var languages = gameData.GetFile< ExcelHeaderFile >( ExcelModule.BuildExcelHeaderPath( sheetName ) )?.Languages ?? [Language.None];
+            foreach( var language in languages )
+            {
+                if( gameData.Excel.GetSheetRaw( sheetName, language ) is not { } sheet )
+                    continue;
+
+                // CustomTalkDefineClient: it currently fails at reading string columns in sheets of subrow variant. 
+                if( sheet.Variant != ExcelVariant.Default )
+                    continue;
+
+                foreach( var row in sheet )
+                {
+                    for( var i = 0; i < sheet.Columns.Length; i++ )
+                    {
+                        if( sheet.Columns[ i ].Type != ExcelColumnDataType.String )
+                            continue;
+
+                        var test1 = row.ReadColumn< SeString >( i ).AsReadOnly();
+                        if( test1.Data.Span.IndexOf( "payload:"u8 ) != -1 )
+                            throw new( $"Unsupported payload at {sheetName}#{row.RowId}; {test1}" );
+
+                        ReadOnlySeString test2;
+                        try
+                        {
+                            test2 = ssb.Clear().AppendMacroString( test1.ToString() ).ToReadOnlySeString();
+                        }
+                        catch( Exception e )
+                        {
+                            throw new( $"Error at {sheetName}#{row.RowId}({language})", e );
+                        }
+
+                        Assert.True( test1.AsSpan().Data.SequenceEqual( test2.AsSpan().Data ), $"Parse-encode failure at {sheetName}#{row.RowId}({language})" );
+                    }
+                }
+            }
         }
     }
 }

src/Lumina/Misc/MemoryChunkStorage.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+namespace Lumina.Misc;
+
+/// <summary>Dummy struct for reserving a chunk of memory in stack at compile-time the bat without having to stackalloc.</summary>
+[StructLayout( LayoutKind.Explicit, Size = 256, Pack = 1 )]
+internal struct MemoryChunkStorage
+{
+    /// <summary>Reinterprets a reference of <see cref="MemoryChunkStorage"/> as a <see cref="Span{T}"/> of <typeparamref name="T"/> without zero-initializing.
+    /// </summary>
+    /// <param name="storage">Backing storage.</param>
+    /// <typeparam name="T">Element type.</typeparam>
+    /// <returns><see cref="Span{T}"/>.</returns>
+    /// <remarks>Multiple calls to this function with discard output may result in two instances sharing the memory buffer.</remarks>
+    [MethodImpl( MethodImplOptions.AggressiveInlining )]
+    [SuppressMessage( "ReSharper", "OutParameterValueIsAlwaysDiscarded.Local", Justification = "Stack reservation" )]
+    public static Span< T > AsSpanUninitialized< T >( out MemoryChunkStorage storage ) where T : struct
+    {
+        Unsafe.SkipInit( out storage );
+        return MemoryMarshal.Cast< MemoryChunkStorage, T >( MemoryMarshal.CreateSpan( ref storage, 1 ) );
+    }
+}

src/Lumina/Text/Expressions/BaseExpression.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Text;
 
 namespace Lumina.Text.Expressions;
 
@@ -24,6 +25,18 @@ public abstract class BaseExpression
     /// <param name="stream">Target to write this expression to.</param>
     public abstract void Encode( Stream stream );
 
+    /// <summary>Represent this expression as a part of macro string.</summary>
+    /// <param name="sb">Target string builder.</param>
+    public abstract void AppendMacroStringToStringBuilder( StringBuilder sb );
+
+    /// <inheritdoc/>
+    public override string ToString()
+    {
+        var sb = new StringBuilder();
+        AppendMacroStringToStringBuilder( sb );
+        return sb.ToString();
+    }
+
     /// <summary>
     /// Parse given Stream into an Expression.
     /// </summary>

src/Lumina/Text/Expressions/BinaryExpression.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Text;
 
 namespace Lumina.Text.Expressions;
 
@@ -37,18 +38,37 @@ public BinaryExpression( ExpressionType typeByte, BaseExpression operand1, BaseE
     public override ExpressionType ExpressionType { get; }
 
     /// <inheritdoc />
-    public override string ToString()
+    public override void AppendMacroStringToStringBuilder( StringBuilder sb )
     {
-        return ExpressionType switch
+        sb.Append( '[' );
+        Operand1.AppendMacroStringToStringBuilder( sb );
+
+        switch( ExpressionType )
         {
-            ExpressionType.GreaterThanOrEqualTo => $"[{Operand1}>={Operand2}]",
-            ExpressionType.GreaterThan => $"[{Operand1}>{Operand2}]",
-            ExpressionType.LessThanOrEqualTo => $"[{Operand1}<={Operand2}]",
-            ExpressionType.LessThan => $"[{Operand1}<{Operand2}]",
-            ExpressionType.Equal => $"[{Operand1}=={Operand2}]",
-            ExpressionType.NotEqual => $"[{Operand1}!={Operand2}]",
-            _ => throw new NotImplementedException() // cannot reach, as this instance is immutable and this field is filtered from constructor
-        };
+            case ExpressionType.GreaterThanOrEqualTo:
+                sb.Append( ">=" );
+                break;
+            case ExpressionType.GreaterThan:
+                sb.Append( '>' );
+                break;
+            case ExpressionType.LessThanOrEqualTo:
+                sb.Append( "<=" );
+                break;
+            case ExpressionType.LessThan:
+                sb.Append( '<' );
+                break;
+            case ExpressionType.Equal:
+                sb.Append( "==" );
+                break;
+            case ExpressionType.NotEqual:
+                sb.Append( "!=" );
+                break;
+            default:
+                throw new NotSupportedException(); // cannot reach, as this instance is immutable and this field is filtered from constructor
+        }
+
+        Operand2.AppendMacroStringToStringBuilder( sb );
+        sb.Append( ']' );
     }
 
     /// <summary>

src/Lumina/Text/Expressions/IntegerExpression.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Text;
 
 namespace Lumina.Text.Expressions;
 
@@ -94,6 +95,9 @@ public static int CalculateSize( uint value )
     /// <inheritdoc />
     public override string ToString() => ( (int)Value ).ToString();
 
+    /// <inheritdoc />
+    public override void AppendMacroStringToStringBuilder( StringBuilder sb ) => sb.Append( (int) Value );
+
     /// <summary>
     /// Parse given Stream into an IntegerExpression.
     /// </summary>