Passing API tests

sillsdev · Dec 16, 2024 · 5a4a2f9 · 5a4a2f9
1 parent 07f6ca4
commit 5a4a2f9
Show file tree

Hide file tree

Showing 14 changed files with 250 additions and 168 deletions.
diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs
@@ -10676,7 +10676,7 @@ public partial class WordAlignmentBuild
         public System.Collections.Generic.IList<TrainingCorpus2>? TrainOn { get; set; } = default!;
 
         [Newtonsoft.Json.JsonProperty("wordAlignOn", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
-        public System.Collections.Generic.IList<TrainingCorpus2>? WordAlignOn { get; set; } = default!;
+        public System.Collections.Generic.IList<WordAlignmentCorpus>? WordAlignOn { get; set; } = default!;
 
         [Newtonsoft.Json.JsonProperty("step", Required = Newtonsoft.Json.Required.Always)]
         public int Step { get; set; } = default!;
@@ -10738,6 +10738,20 @@ public partial class ParallelCorpusFilter2
 
     }
 
+    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
+    public partial class WordAlignmentCorpus
+    {
+        [Newtonsoft.Json.JsonProperty("parallelCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public ResourceLink? ParallelCorpus { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public System.Collections.Generic.IList<ParallelCorpusFilter2>? SourceFilters { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("targetFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public System.Collections.Generic.IList<ParallelCorpusFilter2>? TargetFilters { get; set; } = default!;
+
+    }
+
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
     public partial class WordAlignmentBuildConfig
     {
@@ -10748,7 +10762,7 @@ public partial class WordAlignmentBuildConfig
         public System.Collections.Generic.IList<TrainingCorpusConfig2>? TrainOn { get; set; } = default!;
 
         [Newtonsoft.Json.JsonProperty("wordAlignOn", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
-        public System.Collections.Generic.IList<TrainingCorpusConfig2>? WordAlignOn { get; set; } = default!;
+        public System.Collections.Generic.IList<WordAlignmentCorpusConfig>? WordAlignOn { get; set; } = default!;
 
         [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
         public object? Options { get; set; } = default!;
@@ -10784,6 +10798,20 @@ public partial class ParallelCorpusFilterConfig2
 
     }
 
+    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
+    public partial class WordAlignmentCorpusConfig
+    {
+        [Newtonsoft.Json.JsonProperty("parallelCorpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public string? ParallelCorpusId { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public System.Collections.Generic.IList<ParallelCorpusFilterConfig2>? SourceFilters { get; set; } = default!;
+
+        [Newtonsoft.Json.JsonProperty("targetFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public System.Collections.Generic.IList<ParallelCorpusFilterConfig2>? TargetFilters { get; set; } = default!;
+
+    }
+
     [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
     public partial class FileParameter
     {

diff --git a/src/Serval/src/Serval.Translation/Services/EngineService.cs b/src/Serval/src/Serval.Translation/Services/EngineService.cs
@@ -143,15 +143,11 @@ public override async Task<Engine> CreateAsync(Engine engine, CancellationToken
         {
             engine.DateCreated = DateTime.UtcNow;
             await Entities.InsertAsync(engine, cancellationToken);
-            TranslationEngineApi.TranslationEngineApiClient? client;
-            try
-            {
-                client = _grpcClientFactory.CreateClient<TranslationEngineApi.TranslationEngineApiClient>(engine.Type);
-            }
-            catch (InvalidOperationException)
-            {
+            TranslationEngineApi.TranslationEngineApiClient? client =
+                _grpcClientFactory.CreateClient<TranslationEngineApi.TranslationEngineApiClient>(engine.Type);
+            if (client is null)
                 throw new InvalidOperationException($"'{engine.Type}' is an invalid engine type.");
-            }
+
             var request = new CreateRequest
             {
                 EngineType = engine.Type,

diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildConfigDto.cs
@@ -4,7 +4,7 @@ public record WordAlignmentBuildConfigDto
 {
     public string? Name { get; init; }
     public IReadOnlyList<TrainingCorpusConfigDto>? TrainOn { get; init; }
-    public IReadOnlyList<TrainingCorpusConfigDto>? WordAlignOn { get; init; }
+    public IReadOnlyList<WordAlignmentCorpusConfigDto>? WordAlignOn { get; init; }
 
     /// <example>
     /// {

diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentBuildDto.cs
@@ -8,7 +8,7 @@ public record WordAlignmentBuildDto
     public string? Name { get; init; }
     public required ResourceLinkDto Engine { get; init; }
     public IReadOnlyList<TrainingCorpusDto>? TrainOn { get; init; }
-    public IReadOnlyList<TrainingCorpusDto>? WordAlignOn { get; init; }
+    public IReadOnlyList<WordAlignmentCorpusDto>? WordAlignOn { get; init; }
     public required int Step { get; init; }
     public double? PercentCompleted { get; init; }
     public string? Message { get; init; }

diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusConfigDto.cs
@@ -0,0 +1,8 @@
+namespace Serval.WordAlignment.Contracts;
+
+public record WordAlignmentCorpusConfigDto
+{
+    public string? ParallelCorpusId { get; init; }
+    public IReadOnlyList<ParallelCorpusFilterConfigDto>? SourceFilters { get; init; }
+    public IReadOnlyList<ParallelCorpusFilterConfigDto>? TargetFilters { get; init; }
+}
diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusDto.cs
@@ -0,0 +1,8 @@
+namespace Serval.WordAlignment.Contracts;
+
+public record WordAlignmentCorpusDto
+{
+    public ResourceLinkDto? ParallelCorpus { get; init; }
+    public IReadOnlyList<ParallelCorpusFilterDto>? SourceFilters { get; init; }
+    public IReadOnlyList<ParallelCorpusFilterDto>? TargetFilters { get; init; }
+}
diff --git a/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs b/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs
@@ -9,6 +9,7 @@ public class WordAlignmentEnginesController(
     IBuildService buildService,
     IWordAlignmentService wordAlignmentService,
     IOptionsMonitor<ApiOptions> apiOptions,
+    IConfiguration configuration,
     IUrlService urlService,
     ILogger<WordAlignmentEnginesController> logger
 ) : ServalControllerBase(authService)
@@ -22,6 +23,7 @@ ILogger<WordAlignmentEnginesController> logger
     private readonly IOptionsMonitor<ApiOptions> _apiOptions = apiOptions;
     private readonly IUrlService _urlService = urlService;
     private readonly ILogger<WordAlignmentEnginesController> _logger = logger;
+    private readonly IConfiguration _configuration = configuration;
 
     /// <summary>
     /// Get all word alignment engines
@@ -568,9 +570,11 @@ public async Task<ActionResult<WordAlignmentBuildDto>> StartBuildAsync(
         CancellationToken cancellationToken
     )
     {
+        string deploymentVersion = _configuration.GetValue<string>("deploymentVersion") ?? "Unknown";
+
         Engine engine = await _engineService.GetAsync(id, cancellationToken);
         await AuthorizeAsync(engine);
-        Build build = Map(engine, buildConfig);
+        Build build = Map(engine, buildConfig, deploymentVersion);
         await _engineService.StartBuildAsync(build, cancellationToken);
 
         WordAlignmentBuildDto dto = Map(build);
@@ -756,18 +760,64 @@ private WordAlignmentParallelCorpusDto Map(string engineId, ParallelCorpus sourc
         };
     }
 
-    private static Build Map(Engine engine, WordAlignmentBuildConfigDto source)
+    private static Build Map(Engine engine, WordAlignmentBuildConfigDto source, string deploymentVersion)
     {
         return new Build
         {
             EngineRef = engine.Id,
             Name = source.Name,
             WordAlignOn = Map(engine, source.WordAlignOn),
             TrainOn = Map(engine, source.TrainOn),
-            Options = Map(source.Options)
+            Options = Map(source.Options),
+            DeploymentVersion = deploymentVersion
         };
     }
 
+    private static List<WordAlignmentCorpus>? Map(Engine engine, IReadOnlyList<WordAlignmentCorpusConfigDto>? source)
+    {
+        if (source is null)
+            return null;
+
+        var corpusIds = new HashSet<string>(engine.ParallelCorpora.Select(c => c.Id));
+        var wordAlignmentCorpora = new List<WordAlignmentCorpus>();
+        foreach (WordAlignmentCorpusConfigDto cc in source)
+        {
+            if (cc.ParallelCorpusId == null)
+            {
+                throw new InvalidOperationException($"One of ParallelCorpusId and CorpusId must be set.");
+            }
+            if (!corpusIds.Contains(cc.ParallelCorpusId))
+            {
+                throw new InvalidOperationException(
+                    $"The parallel corpus {cc.ParallelCorpusId} is not valid: This parallel corpus does not exist for engine {engine.Id}."
+                );
+            }
+            if (
+                cc.SourceFilters != null
+                && cc.SourceFilters.Count > 0
+                && (
+                    cc.SourceFilters.Select(sf => sf.CorpusId).Distinct().Count() > 1
+                    || cc.SourceFilters[0].CorpusId
+                        != engine.ParallelCorpora.Where(pc => pc.Id == cc.ParallelCorpusId).First().SourceCorpora[0].Id
+                )
+            )
+            {
+                throw new InvalidOperationException(
+                    $"Only the first source corpus in a parallel corpus may be filtered for pretranslation."
+                );
+            }
+            wordAlignmentCorpora.Add(
+                new WordAlignmentCorpus
+                {
+                    ParallelCorpusRef = cc.ParallelCorpusId,
+                    SourceFilters = cc.SourceFilters?.Select(Map).ToList(),
+                    TargetFilters = cc.TargetFilters?.Select(Map).ToList()
+                }
+            );
+        }
+        return wordAlignmentCorpora;
+    }
+
     private static List<TrainingCorpus>? Map(Engine engine, IReadOnlyList<TrainingCorpusConfigDto>? source)
     {
         if (source is null)
@@ -893,6 +943,26 @@ private TrainingCorpusDto Map(string engineId, TrainingCorpus source)
         };
     }
 
+    private WordAlignmentCorpusDto Map(string engineId, WordAlignmentCorpus source)
+    {
+        return new WordAlignmentCorpusDto
+        {
+            ParallelCorpus =
+                source.ParallelCorpusRef != null
+                    ? new ResourceLinkDto
+                    {
+                        Id = source.ParallelCorpusRef,
+                        Url = _urlService.GetUrl(
+                            Endpoints.GetParallelTranslationCorpus,
+                            new { id = engineId, parallelCorpusId = source.ParallelCorpusRef }
+                        )
+                    }
+                    : null,
+            SourceFilters = source.SourceFilters?.Select(Map).ToList(),
+            TargetFilters = source.TargetFilters?.Select(Map).ToList()
+        };
+    }
+
     private ParallelCorpusFilterDto Map(ParallelCorpusFilter source)
     {
         return new ParallelCorpusFilterDto

diff --git a/src/Serval/src/Serval.WordAlignment/Models/Build.cs b/src/Serval/src/Serval.WordAlignment/Models/Build.cs
@@ -7,7 +7,7 @@ public record Build : IInitializableEntity
     public string? Name { get; init; }
     public required string EngineRef { get; init; }
     public IReadOnlyList<TrainingCorpus>? TrainOn { get; init; }
-    public IReadOnlyList<TrainingCorpus>? WordAlignOn { get; init; }
+    public IReadOnlyList<WordAlignmentCorpus>? WordAlignOn { get; init; }
     public int Step { get; init; }
     public double? PercentCompleted { get; init; }
     public string? Message { get; init; }

diff --git a/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentCorpus.cs b/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentCorpus.cs
@@ -0,0 +1,8 @@
+namespace Serval.WordAlignment.Models;
+
+public record WordAlignmentCorpus
+{
+    public string? ParallelCorpusRef { get; set; }
+    public IReadOnlyList<ParallelCorpusFilter>? SourceFilters { get; set; }
+    public IReadOnlyList<ParallelCorpusFilter>? TargetFilters { get; set; }
+}
diff --git a/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs b/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs
@@ -70,17 +70,11 @@ public override async Task<Engine> CreateAsync(Engine engine, CancellationToken
         {
             engine.DateCreated = DateTime.UtcNow;
             await Entities.InsertAsync(engine, cancellationToken);
-            WordAlignmentEngineApi.WordAlignmentEngineApiClient client;
-            try
-            {
-                client = _grpcClientFactory.CreateClient<WordAlignmentEngineApi.WordAlignmentEngineApiClient>(
-                    engine.Type
-                );
-            }
-            catch (InvalidOperationException)
-            {
+            WordAlignmentEngineApi.WordAlignmentEngineApiClient? client =
+                _grpcClientFactory.CreateClient<WordAlignmentEngineApi.WordAlignmentEngineApiClient>(engine.Type);
+            if (client is null)
                 throw new InvalidOperationException($"'{engine.Type}' is an invalid engine type.");
-            }
+
             var request = new CreateRequest
             {
                 EngineType = engine.Type,
@@ -92,6 +86,11 @@ public override async Task<Engine> CreateAsync(Engine engine, CancellationToken
             if (engine.Name is not null)
                 request.EngineName = engine.Name;
             await client.CreateAsync(request, cancellationToken: cancellationToken);
+            await Entities.UpdateAsync(
+                engine,
+                u => u.Set(e => e.IsInitialized, true),
+                cancellationToken: CancellationToken.None
+            );
         }
         catch (RpcException rpcex)
         {
@@ -165,7 +164,7 @@ public async Task StartBuildAsync(Build build, CancellationToken cancellationTok
         {
             StartBuildRequest request;
             Dictionary<string, TrainingCorpus>? trainOn = build.TrainOn?.ToDictionary(c => c.ParallelCorpusRef!);
-            Dictionary<string, TrainingCorpus>? wordAlignOn = build.WordAlignOn?.ToDictionary(c =>
+            Dictionary<string, WordAlignmentCorpus>? wordAlignOn = build.WordAlignOn?.ToDictionary(c =>
                 c.ParallelCorpusRef!
             );
             IReadOnlyList<Shared.Models.ParallelCorpus> parallelCorpora = engine
@@ -451,7 +450,7 @@ private Shared.Models.AlignedWordPair Map(V1.AlignedWordPair source)
     private V1.ParallelCorpus Map(
         Shared.Models.ParallelCorpus source,
         TrainingCorpus? trainingCorpus,
-        TrainingCorpus? wordAlignmentCorpus,
+        WordAlignmentCorpus? wordAlignmentCorpus,
         bool trainOnAllCorpora,
         bool wordAlignOnAllCorpora
     )