Use chapter-filtering for terms

src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs

@@ -116,7 +116,7 @@
         await env.RunBuildJobAsync(corpus1, useKeyTerms: true);
 
         (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync();
         Assert.Multiple(() =>
         {
             Assert.That(src1Count, Is.EqualTo(0));
             Assert.That(src2Count, Is.EqualTo(0));
@@ -238,7 +238,7 @@
         using TestEnvironment env = new();
         ParallelCorpus corpus1 = TestEnvironment.TextFileCorpus(sourceLanguage: "xxx", targetLanguage: "zzz");
 
         Assert.ThrowsAsync<InvalidOperationException>(async () =>
         {
             await env.RunBuildJobAsync(corpus1, engineId: "engine2");
         });
@@ -308,7 +308,7 @@
                     new DummyCorpus(new List<string>() { "LEV", "MRK", "MAT" }, new List<string>() { "MAT" })
                 }
             );
         Assert.DoesNotThrowAsync(async () =>
         {
             await env.RunBuildJobAsync(corpus);
         });
@@ -359,7 +359,7 @@
     }
 
     [Test]
-    public async Task ParallelCorpusLogic()
+    public async Task ParallelCorpusAsync()
     {
         using TestEnvironment env = new();
         var corpora = new List<ParallelCorpus>()
@@ -508,6 +508,246 @@
         });
     }
 
+    [Test]
+    public async Task ParallelCorpusAsync_UseKeyTerms()
+    {
+        using TestEnvironment env = new();
+        var corpora = new List<ParallelCorpus>()
+        {
+            new ParallelCorpus()
+            {
+                Id = "1",
+                SourceCorpora = new List<MonolingualCorpus>()
+                {
+                    new()
+                    {
+                        Id = "_1",
+                        Language = "en",
+                        Files = new List<CorpusFile> { env.ParatextFile("pt-source1") },
+                        TrainOnChapters = new()
+                        {
+                            {
+                                "MAT",
+                                new() { 1 }
+                            },
+                            {
+                                "LEV",
+                                new() { }
+                            }
+                        },
+                        PretranslateChapters = new()
+                        {
+                            {
+                                "1CH",
+                                new() { }
+                            }
+                        }
+                    },
+                    new()
+                    {
+                        Id = "_1",
+                        Language = "en",
+                        Files = new List<CorpusFile> { env.ParatextFile("pt-source2") },
+                        TrainOnChapters = new()
+                        {
+                            {
+                                "MAT",
+                                new() { 1 }
+                            },
+                            {
+                                "MRK",
+                                new() { }
+                            }
+                        },
+                        PretranslateChapters = new() { }
+                    },
+                },
+                TargetCorpora = new List<MonolingualCorpus>()
+                {
+                    new()
+                    {
+                        Id = "_1",
+                        Language = "en",
+                        Files = new List<CorpusFile> { env.ParatextFile("pt-target1") },
+                        TrainOnChapters = new()
+                        {
+                            {
+                                "MAT",
+                                new() { 1 }
+                            },
+                            {
+                                "MRK",
+                                new() { }
+                            }
+                        }
+                    },
+                    new()
+                    {
+                        Id = "_2",
+                        Language = "en",
+                        Files = new List<CorpusFile> { env.ParatextFile("pt-target2") },
+                        TrainOnChapters = new()
+                        {
+                            {
+                                "MAT",
+                                new() { 1 }
+                            },
+                            {
+                                "MRK",
+                                new() { }
+                            },
+                            {
+                                "LEV",
+                                new() { }
+                            }
+                        }
+                    }
+                }
+            }
+        };
+        await env.RunBuildJobAsync(corpora, useKeyTerms: true);
+        string source = await env.GetSourceExtractAsync();
+        string target = await env.GetTargetExtractAsync();
+        Assert.Multiple(() =>
+        {
+            StringAssert.StartsWith(
+                @"Source one, chapter fourteen, verse fifty-five. Segment b.
+Source one, chapter fourteen, verse fifty-six.
+Source two, chapter one, verse one.
+Source two, chapter one, verse two.
+Source two, chapter one, verse three.
+Source one, chapter one, verse four.
+Source two, chapter one, verse five. Source two, chapter one, verse six.
+Source one, chapter one, verse seven, eight, and nine. Source one, chapter one, verse ten.
+Source two, chapter one, verse one.
+",
+                source
+            );
+            StringAssert.StartsWith(
+                @"Target two, chapter fourteen, verse fifty-five.
+Target two, chapter fourteen, verse fifty-six.
+Target one, chapter one, verse one.
+Target one, chapter one, verse two.
+Target one, chapter one, verse three.
+
+Target one, chapter one, verse five and six.
+Target one, chapter one, verse seven and eight. Target one, chapter one, verse nine and ten.
+
+",
+                target
+            );
+            StringAssert.Contains("Abraham", source);
+            StringAssert.Contains("Abraham", target);
+            StringAssert.DoesNotContain("Zedekiah", source);
+            StringAssert.DoesNotContain("Zedekiah", target);
+        });
+        JsonArray? pretranslations = await env.GetPretranslationsAsync();
+        Assert.That(pretranslations, Is.Not.Null);
+        Assert.That(pretranslations!.Count, Is.EqualTo(7), pretranslations.ToJsonString());
+        Assert.That(
+            pretranslations[2]!["translation"]!.ToString(),
+            Is.EqualTo("Source one, chapter twelve, verse one.")
+        );
+    }
+
+    [Test]
+    public async Task ParallelCorpusAsync_UseKeyTerms_TextIds()
+    {
+        using TestEnvironment env = new();
+        var corpora = new List<ParallelCorpus>()
+        {
+            new ParallelCorpus()
+            {
+                Id = "1",
+                SourceCorpora = new List<MonolingualCorpus>()
+                {
+                    new()
+                    {
+                        Id = "_1",
+                        Language = "en",
+                        Files = new List<CorpusFile> { env.ParatextFile("pt-source1") },
+                        TrainOnTextIds = ["MAT", "LEV"],
+                        PretranslateTextIds = ["1CH"]
+                    },
+                    new()
+                    {
+                        Id = "_1",
+                        Language = "en",
+                        Files = new List<CorpusFile> { env.ParatextFile("pt-source2") },
+                        TrainOnTextIds = ["MAT", "MRK"],
+                        PretranslateTextIds = []
+                    },
+                },
+                TargetCorpora = new List<MonolingualCorpus>()
+                {
+                    new()
+                    {
+                        Id = "_1",
+                        Language = "en",
+                        Files = new List<CorpusFile> { env.ParatextFile("pt-target1") },
+                        TrainOnTextIds = ["MAT", "MRK"]
+                    },
+                    new()
+                    {
+                        Id = "_2",
+                        Language = "en",
+                        Files = new List<CorpusFile> { env.ParatextFile("pt-target2") },
+                        TrainOnTextIds = ["MAT", "MRK", "LEV"]
+                    }
+                }
+            }
+        };
+        await env.RunBuildJobAsync(corpora, useKeyTerms: true);
+        string source = await env.GetSourceExtractAsync();
+        string target = await env.GetTargetExtractAsync();
+        Assert.Multiple(() =>
+        {
+            StringAssert.StartsWith(
+                @"Source one, chapter fourteen, verse fifty-five. Segment b.
+Source one, chapter fourteen, verse fifty-six.
+Source two, chapter one, verse one.
+Source two, chapter one, verse two.
+Source two, chapter one, verse three.
+Source one, chapter one, verse four.
+Source two, chapter one, verse five. Source two, chapter one, verse six.
+Source one, chapter one, verse seven, eight, and nine. Source one, chapter one, verse ten.
+Source one, chapter two, verse one.
+Source one, chapter two, verse two.
+
+Source two, chapter one, verse one.
+",
+                source
+            );
+            StringAssert.StartsWith(
+                @"Target two, chapter fourteen, verse fifty-five.
+Target two, chapter fourteen, verse fifty-six.
+Target one, chapter one, verse one.
+Target one, chapter one, verse two.
+Target one, chapter one, verse three.
+
+Target one, chapter one, verse five and six.
+Target one, chapter one, verse seven and eight. Target one, chapter one, verse nine and ten.
+Target one, chapter two, verse one.
+
+Target one, chapter two, verse three.
+
+",
+                target
+            );
+            StringAssert.Contains("Abraham", source);
+            StringAssert.Contains("Abraham", target);
+            StringAssert.DoesNotContain("Zedekiah", source);
+            StringAssert.DoesNotContain("Zedekiah", target);
+        });
+        JsonArray? pretranslations = await env.GetPretranslationsAsync();
+        Assert.That(pretranslations, Is.Not.Null);
+        Assert.That(pretranslations!.Count, Is.EqualTo(7), pretranslations.ToJsonString());
+        Assert.That(
+            pretranslations[2]!["translation"]!.ToString(),
+            Is.EqualTo("Source one, chapter twelve, verse one.")
+        );
+    }
+
     private class TestEnvironment : DisposableBase
     {
         private static readonly string TestDataPath = Path.Combine(

src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml

@@ -6,4 +6,11 @@
     <Notes />
     <Denials />
   </TermRendering>
+  <TermRendering Id="צִדְקִיָּהוּ-2" Guess="false">
+    <Renderings>Zedekiah</Renderings>
+    <Glossary />
+    <Changes />
+    <Notes />
+    <Denials />
+  </TermRendering>
 </TermRenderingsList>

src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml

@@ -6,4 +6,11 @@
     <Notes />
     <Denials />
   </TermRendering>
+  <TermRendering Id="צִדְקִיָּהוּ-2" Guess="false">
+    <Renderings>Zedekiah</Renderings>
+    <Glossary />
+    <Changes />
+    <Notes />
+    <Denials />
+  </TermRendering>
 </TermRenderingsList>

src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CorpusService.cs

@@ -36,14 +36,16 @@ public IEnumerable<ITextCorpus> CreateTextCorpora(IReadOnlyList<CorpusFile> file
         return corpora;
     }
 
-    public IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> files)
+    public IEnumerable<ITextCorpus> CreateTermCorpora(
+        IReadOnlyList<(CorpusFile File, Dictionary<string, HashSet<int>> Chapters)> corpora
+    )
     {
-        foreach (CorpusFile file in files)
+        foreach ((CorpusFile file, Dictionary<string, HashSet<int>> chapters) in corpora)
         {
             switch (file.Format)
             {
                 case FileFormat.Paratext:
-                    yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]);
+                    yield return new ParatextBackupTermsCorpus(file.Location, ["PN"], chapters: chapters);
                     break;
             }
         }

src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ICorpusService.cs

@@ -3,5 +3,7 @@
 public interface ICorpusService
 {
     IEnumerable<ITextCorpus> CreateTextCorpora(IReadOnlyList<CorpusFile> files);
-    IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> files);
+    IEnumerable<ITextCorpus> CreateTermCorpora(
+        IReadOnlyList<(CorpusFile File, Dictionary<string, HashSet<int>> Chapters)> corpora
+    );
 }

src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs

@@ -82,15 +82,17 @@
 
             if (useKeyTerms)
             {
-                ITextCorpus? sourceTermCorpus = _corpusService
-                    .CreateTermCorpora(corpus.SourceCorpora.SelectMany(sc => sc.Files).ToList())
-                    .FirstOrDefault();
-                ITextCorpus? targetTermCorpus = _corpusService
-                    .CreateTermCorpora(corpus.TargetCorpora.SelectMany(tc => tc.Files).ToList())
-                    .FirstOrDefault();
-                if (sourceTermCorpus is not null && targetTermCorpus is not null)
+                ITextCorpus[]? sourceTermCorpora = _corpusService
+                    .CreateTermCorpora(corpus.SourceCorpora.SelectMany(GetChaptersPerFile).ToArray())
+                    .ToArray();
+                ITextCorpus[]? targetTermCorpora = _corpusService
+                    .CreateTermCorpora(corpus.TargetCorpora.SelectMany(GetChaptersPerFile).ToArray())
+                    .ToArray();
+                if (sourceTermCorpora is not null && targetTermCorpora is not null)
                 {
-                    IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus);
+                    IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpora
+                        .ChooseRandom(Seed)
+                        .AlignRows(targetTermCorpora.ChooseFirst());
                     foreach (ParallelTextRow row in parallelKeyTermsCorpus)
                     {
                         train(new Row(row.TextId, row.Refs, row.SourceText, row.TargetText, 1));
@@ -111,6 +113,19 @@
         }
     }
 
+    private static IEnumerable<(CorpusFile File, Dictionary<string, HashSet<int>> Chapters)> GetChaptersPerFile(
+        MonolingualCorpus mc
+    )
+    {
+        Dictionary<string, HashSet<int>>? chapters = mc.TrainOnChapters;
+        if (chapters is null && mc.TrainOnTextIds is not null)
+        {
+            chapters = mc.TrainOnTextIds.Select(tid => (tid, new HashSet<int> { })).ToDictionary();
+        }
+        chapters ??= [];
+        return mc.Files.Select(f => (f, chapters));
+    }
+
     private static ITextCorpus FilterPretranslateCorpora(MonolingualCorpus corpus, ITextCorpus textCorpus)
     {
         textCorpus = textCorpus.Transform(CleanSegment);