Update machine to 3.5.1

* Fix Async stream write bug for S3 buckets
sillsdev · Nov 26, 2024 · 9d6e4b8 · 9d6e4b8
1 parent 1df752c
commit 9d6e4b8
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 16 deletions.
diff --git a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj
@@ -36,9 +36,9 @@
 		<PackageReference Include="Hangfire.Mongo" Version="1.10.8" />
 		<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.8" />
 		<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.8" />
-		<PackageReference Include="SIL.Machine" Version="3.5.0" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
-		<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.5.0" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
-		<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.5.0" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
+		<PackageReference Include="SIL.Machine" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
+		<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
+		<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
 		<PackageReference Include="SIL.WritingSystems" Version="14.1.1" />
 		<PackageReference Include="System.Linq.Async" Version="6.0.1" />
 	</ItemGroup>

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs
@@ -93,16 +93,14 @@ CancellationToken cancellationToken
         JsonObject? buildOptionsObject = null;
         if (buildOptions is not null)
             buildOptionsObject = JsonSerializer.Deserialize<JsonObject>(buildOptions);
-        await using StreamWriter sourceTrainWriter =
-            new(await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken));
-        await using StreamWriter targetTrainWriter =
-            new(await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken));
 
-        await using Stream pretranslateStream = await _sharedFileService.OpenWriteAsync(
-            $"builds/{buildId}/pretranslate.src.json",
-            cancellationToken
-        );
-        await using Utf8JsonWriter pretranslateWriter = new(pretranslateStream, PretranslateWriterOptions);
+        using MemoryStream sourceStream = new();
+        using MemoryStream targetStream = new();
+        using MemoryStream pretranslationStream = new();
+
+        using StreamWriter targetTrainWriter = new(targetStream, Encoding.Default);
+        using StreamWriter sourceTrainWriter = new(sourceStream, Encoding.Default);
+        await using Utf8JsonWriter pretranslateWriter = new(pretranslationStream, PretranslateWriterOptions);
 
         int trainCount = 0;
         int pretranslateCount = 0;
@@ -113,8 +111,8 @@ CancellationToken cancellationToken
             {
                 if (row.SourceSegment.Length > 0 || row.TargetSegment.Length > 0)
                 {
-                    sourceTrainWriter.Write($"{row.SourceSegment}\n");
-                    targetTrainWriter.Write($"{row.TargetSegment}\n");
+                    sourceTrainWriter.WriteLine(row.SourceSegment);
+                    targetTrainWriter.WriteLine(row.TargetSegment);
                 }
                 if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0)
                     trainCount++;
@@ -140,6 +138,21 @@ CancellationToken cancellationToken
 
         pretranslateWriter.WriteEndArray();
 
+        await sourceTrainWriter.FlushAsync(cancellationToken);
+        await targetTrainWriter.FlushAsync(cancellationToken);
+
+        async Task WriteStreamAsync(MemoryStream stream, string path)
+        {
+            stream.Position = 0;
+            await using StreamWriter writer = new(await _sharedFileService.OpenWriteAsync(path, cancellationToken));
+            await writer.WriteAsync(Encoding.Default.GetString(stream.ToArray()));
+            await writer.FlushAsync(cancellationToken);
+        }
+
+        await WriteStreamAsync(sourceStream, $"builds/{buildId}/train.src.txt");
+        await WriteStreamAsync(targetStream, $"builds/{buildId}/train.trg.txt");
+        await WriteStreamAsync(pretranslationStream, $"builds/{buildId}/pretranslate.src.json");
+
         return (trainCount, pretranslateCount);
     }
 

diff --git a/src/Serval/src/Serval.Shared/Serval.Shared.csproj b/src/Serval/src/Serval.Shared/Serval.Shared.csproj
@@ -19,7 +19,7 @@
 		<PackageReference Include="Grpc.Core.Api" Version="2.65.0" />
 		<PackageReference Include="Grpc.HealthCheck" Version="2.65.0" />
 		<PackageReference Include="Grpc.Net.ClientFactory" Version="2.65.0" />
-		<PackageReference Include="SIL.Machine" Version="3.5.0" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
+		<PackageReference Include="SIL.Machine" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
 		<PackageReference Include="Microsoft.FeatureManagement.AspNetCore" Version="3.5.0" />
 	</ItemGroup>
 

diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj b/src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj
@@ -17,7 +17,7 @@
 	  <PackageReference Include="SIL.WritingSystems" Version="14.1.1" />
 	  <PackageReference Include="System.Text.RegularExpressions" Version="4.3.1" />
 	  <PackageReference Include="SIL.Scripture" Version="12.0.1"/>
-	  <PackageReference Include="SIL.Machine" Version="3.5.0" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
+	  <PackageReference Include="SIL.Machine" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
 	</ItemGroup>
 
 	<ItemGroup>