From 6c0c4a0eddb7bd303c9c0ce4f0907aaa56e315dd Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 28 Nov 2024 08:27:19 -0500 Subject: [PATCH] Tests pass --- .../src/Serval.Shared/Models/CorpusFile.cs | 7 +- .../TranslationEnginesController.cs | 6 - .../Models/ParallelCorpus.cs | 3 + .../Services/EngineService.cs | 5 +- .../Services/PretranslationService.cs | 5 +- .../Services/EngineServiceTests.cs | 107 +++++++++--------- .../Services/PretranslationServiceTests.cs | 5 +- .../test/Serval.Translation.Tests/Usings.cs | 1 + 8 files changed, 75 insertions(+), 64 deletions(-) diff --git a/src/Serval/src/Serval.Shared/Models/CorpusFile.cs b/src/Serval/src/Serval.Shared/Models/CorpusFile.cs index 277ce5e9..3b83c257 100644 --- a/src/Serval/src/Serval.Shared/Models/CorpusFile.cs +++ b/src/Serval/src/Serval.Shared/Models/CorpusFile.cs @@ -8,7 +8,7 @@ public record CorpusFile private string? _filename; - public async Task PopulateFilenameAsync( + public async Task PopulateFilenameAsync( IRequestClient getDataFileClient, string owner, CancellationToken cancellationToken @@ -22,7 +22,10 @@ CancellationToken cancellationToken { _filename = result.Message.Filename; } - throw new InvalidOperationException($"The data file {Id} cannot be found."); + else + { + throw new InvalidOperationException($"The data file {Id} cannot be found."); + } } public void SetFilename(string filename) diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs index 8fc5bac1..ab56c599 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -852,7 +852,6 @@ CancellationToken cancellationToken /// The corpus id or parallel corpus id /// The text id /// The source[s] of the data to populate the USFM file with. - /// The data file client /// /// The book in USFM format /// The specified book does not exist in the source or target corpus. @@ -879,13 +878,11 @@ public async Task GetPretranslatedUsfmAsync( [NotNull] string textId, [FromQuery(Name = "text-origin")] PretranslationUsfmTextOrigin? textOrigin, [FromQuery] PretranslationUsfmTemplate? template, - [FromServices] IRequestClient getDataFileClient, CancellationToken cancellationToken ) { Engine engine = await _engineService.GetAsync(id, cancellationToken); await AuthorizeAsync(engine); - await engine.PopulateFilenamesAsync(getDataFileClient, cancellationToken); if (!engine.Corpora.Any(c => c.Id == corpusId) && !engine.ParallelCorpora.Any(c => c.Id == corpusId)) return NotFound(); if (engine.ModelRevision == 0) @@ -1034,7 +1031,6 @@ CancellationToken cancellationToken /// /// The translation engine id /// The build config (see remarks) - /// The data file client /// /// The new build job /// The build configuration was invalid. @@ -1055,7 +1051,6 @@ CancellationToken cancellationToken public async Task> StartBuildAsync( [NotNull] string id, [FromBody] TranslationBuildConfigDto buildConfig, - [FromServices] IRequestClient getDataFileClient, CancellationToken cancellationToken ) { @@ -1063,7 +1058,6 @@ CancellationToken cancellationToken Engine engine = await _engineService.GetAsync(id, cancellationToken); await AuthorizeAsync(engine); - await engine.PopulateFilenamesAsync(getDataFileClient, cancellationToken); Build build = Map(engine, buildConfig, deploymentVersion); await _engineService.StartBuildAsync(build, cancellationToken); diff --git a/src/Serval/src/Serval.Translation/Models/ParallelCorpus.cs b/src/Serval/src/Serval.Translation/Models/ParallelCorpus.cs index e3ffad9d..a1a02ffd 100644 --- a/src/Serval/src/Serval.Translation/Models/ParallelCorpus.cs +++ b/src/Serval/src/Serval.Translation/Models/ParallelCorpus.cs @@ -15,5 +15,8 @@ CancellationToken cancellationToken await Task.WhenAll( SourceCorpora.Select(corpus => corpus.PopulateFilenamesAsync(getDataFileClient, owner, cancellationToken)) ); + await Task.WhenAll( + TargetCorpora.Select(corpus => corpus.PopulateFilenamesAsync(getDataFileClient, owner, cancellationToken)) + ); } } diff --git a/src/Serval/src/Serval.Translation/Services/EngineService.cs b/src/Serval/src/Serval.Translation/Services/EngineService.cs index c3ef933c..12978f3f 100644 --- a/src/Serval/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/EngineService.cs @@ -12,7 +12,8 @@ public class EngineService( IOptionsMonitor dataFileOptions, IDataAccessContext dataAccessContext, ILoggerFactory loggerFactory, - IScriptureDataFileService scriptureDataFileService + IScriptureDataFileService scriptureDataFileService, + IRequestClient getDataFileClient ) : OwnedEntityServiceBase(engines), IEngineService { private readonly IRepository _builds = builds; @@ -23,6 +24,7 @@ IScriptureDataFileService scriptureDataFileService private readonly IDataAccessContext _dataAccessContext = dataAccessContext; private readonly ILogger _logger = loggerFactory.CreateLogger(); private readonly IScriptureDataFileService _scriptureDataFileService = scriptureDataFileService; + private readonly IRequestClient _getDataFileClient = getDataFileClient; public async Task TranslateAsync( string engineId, @@ -217,6 +219,7 @@ private Dictionary> GetChapters(string fileLocation, string sc public async Task StartBuildAsync(Build build, CancellationToken cancellationToken = default) { Engine engine = await GetAsync(build.EngineRef, cancellationToken); + await engine.PopulateFilenamesAsync(_getDataFileClient, cancellationToken); await _builds.InsertAsync(build, cancellationToken); TranslationEngineApi.TranslationEngineApiClient client = diff --git a/src/Serval/src/Serval.Translation/Services/PretranslationService.cs b/src/Serval/src/Serval.Translation/Services/PretranslationService.cs index 4e0e6097..160c751f 100644 --- a/src/Serval/src/Serval.Translation/Services/PretranslationService.cs +++ b/src/Serval/src/Serval.Translation/Services/PretranslationService.cs @@ -5,11 +5,13 @@ namespace Serval.Translation.Services; public class PretranslationService( IRepository pretranslations, IRepository engines, - IScriptureDataFileService scriptureDataFileService + IScriptureDataFileService scriptureDataFileService, + IRequestClient getDataFileClient ) : EntityServiceBase(pretranslations), IPretranslationService { private readonly IRepository _engines = engines; private readonly IScriptureDataFileService _scriptureDataFileService = scriptureDataFileService; + private readonly IRequestClient _getDataFileClient = getDataFileClient; public async Task> GetAllAsync( string engineId, @@ -40,6 +42,7 @@ public async Task GetUsfmAsync( ) { Engine? engine = await _engines.GetAsync(engineId, cancellationToken); + await engine!.PopulateFilenamesAsync(_getDataFileClient, cancellationToken); Corpus? corpus = engine?.Corpora.SingleOrDefault(c => c.Id == corpusId); ParallelCorpus? parallelCorpus = engine?.ParallelCorpora.SingleOrDefault(c => c.Id == corpusId); diff --git a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs index 4417178d..9546414e 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs @@ -133,7 +133,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified() new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -154,7 +154,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified() new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -207,7 +207,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -229,7 +229,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -282,7 +282,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -304,7 +304,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -356,7 +356,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -377,7 +377,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -430,7 +430,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -451,7 +451,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -504,7 +504,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -525,7 +525,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -550,7 +550,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -571,7 +571,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "text1" } }, @@ -652,7 +652,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -684,7 +684,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -736,7 +736,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -757,7 +757,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -833,7 +833,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -849,7 +849,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MRK" } }, @@ -872,7 +872,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -888,7 +888,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MRK" } }, @@ -965,7 +965,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -988,7 +988,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -1065,7 +1065,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -1088,7 +1088,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MAT" } }, @@ -1114,7 +1114,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MRK" } }, @@ -1136,7 +1136,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.txt", - Format = FileFormat.Text, + Format = V1.FileFormat.Text, TextId = "MRK" } }, @@ -1212,7 +1212,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1228,7 +1228,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1251,7 +1251,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1267,7 +1267,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1363,7 +1363,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1379,7 +1379,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1412,7 +1412,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1428,7 +1428,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1495,7 +1495,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1522,7 +1522,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1552,7 +1552,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1579,7 +1579,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1640,7 +1640,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1656,7 +1656,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1675,7 +1675,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1691,7 +1691,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1734,7 +1734,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1750,7 +1750,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1769,7 +1769,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1785,7 +1785,7 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -1845,7 +1845,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file1.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file1.zip" } }, @@ -1872,7 +1872,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file3.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file3.zip" } }, @@ -1891,7 +1891,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file2.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file2.zip" } }, @@ -1907,7 +1907,7 @@ await env.Service.StartBuildAsync( new V1.CorpusFile { Location = "file4.zip", - Format = FileFormat.Paratext, + Format = V1.FileFormat.Paratext, TextId = "file4.zip" } }, @@ -2113,7 +2113,8 @@ public TestEnvironment() dataFileOptions, new MemoryDataAccessContext(), new LoggerFactory(), - scriptureDataFileService + scriptureDataFileService, + Substitute.For>() ); } diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs index 13973449..bc76727e 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PretranslationServiceTests.cs @@ -392,6 +392,7 @@ public TestEnvironment() ScriptureDataFileService = Substitute.For(); ScriptureDataFileService.GetParatextProjectSettings("file1.zip").Returns(CreateProjectSettings("SRC")); ScriptureDataFileService.GetParatextProjectSettings("file2.zip").Returns(CreateProjectSettings("TRG")); + GetDataFileClient = Substitute.For>(); var zipSubstituteSource = Substitute.For(); var zipSubstituteTarget = Substitute.For(); zipSubstituteSource @@ -425,13 +426,15 @@ Shared.Services.ZipParatextProjectTextUpdater GetTextUpdater(string type) } ScriptureDataFileService.GetZipParatextProjectTextUpdater("file1.zip").Returns(x => GetTextUpdater("SRC")); ScriptureDataFileService.GetZipParatextProjectTextUpdater("file2.zip").Returns(x => GetTextUpdater("TRG")); - Service = new PretranslationService(Pretranslations, Engines, ScriptureDataFileService); + + Service = new PretranslationService(Pretranslations, Engines, ScriptureDataFileService, GetDataFileClient); } public PretranslationService Service { get; } public MemoryRepository Pretranslations { get; } public MemoryRepository Engines { get; } public IScriptureDataFileService ScriptureDataFileService { get; } + public IRequestClient GetDataFileClient { get; } public IZipContainer TargetZipContainer { get; } public IList TextUpdaters { get; } diff --git a/src/Serval/test/Serval.Translation.Tests/Usings.cs b/src/Serval/test/Serval.Translation.Tests/Usings.cs index ef8a3ff7..9116c0c9 100644 --- a/src/Serval/test/Serval.Translation.Tests/Usings.cs +++ b/src/Serval/test/Serval.Translation.Tests/Usings.cs @@ -7,6 +7,7 @@ global using NSubstitute; global using NUnit.Framework; global using Serval.Shared.Configuration; +global using Serval.Shared.Contracts; global using Serval.Shared.Services; global using Serval.Shared.Utils; global using Serval.Translation.Contracts;