fixes from integration

sillsdev · Jun 7, 2024 · 054c9a6 · 054c9a6
1 parent 67f63b8
commit 054c9a6
Showing 1 changed file with 17 additions and 21 deletions.
diff --git a/src/SIL.Machine.AspNetCore/Services/NmtPreprocessBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/NmtPreprocessBuildJob.cs
@@ -149,7 +149,7 @@ CancellationToken cancellationToken
                     continue;
                 }
 
-                Row[] trainRows = rows.Where(r => IsIncluded(r, corpus.TrainOnChapters)).Cast<Row>().ToArray();
+                Row[] trainRows = rows.Where(row => IsInTrain(row, corpus)).Cast<Row>().ToArray();
                 if (trainRows.Length > 0)
                 {
                     Row row = trainRows[0];
@@ -187,7 +187,7 @@ CancellationToken cancellationToken
             foreach (Row row in AlignPretranslateCorpus(corpus, sourceTextCorpora[0], targetTextCorpus))
             {
                 if (
-                    IsIncluded(row, corpus.PretranslateChapters)
+                    IsInPretranslate(row, corpus)
                     && row.SourceSegment.Length > 0
                     && (row.TargetSegment.Length == 0 || !IsInTrain(row, corpus))
                 )
@@ -231,22 +231,24 @@ JobCompletionStatus completionStatus
         }
     }
 
-    private static bool IsInTrain(Row row, Corpus corpus)
+    private static bool IsInTrain(Row? row, Corpus corpus)
     {
         return IsIncluded(row, corpus.TrainOnTextIds, corpus.TrainOnChapters);
     }
 
-    private static bool IsInPretranslate(Row row, Corpus corpus)
+    private static bool IsInPretranslate(Row? row, Corpus corpus)
     {
         return IsIncluded(row, corpus.PretranslateTextIds, corpus.PretranslateChapters);
     }
 
     private static bool IsIncluded(
-        Row row,
+        Row? row,
         IReadOnlySet<string>? textIds,
         IReadOnlyDictionary<string, HashSet<int>>? chapters
     )
     {
+        if (row is null)
+            return false;
         if (chapters is not null)
         {
             return row.Refs.Any(r => IsInChapters(chapters, r));
@@ -272,14 +274,11 @@ private static bool IsInChapters(IReadOnlyDictionary<string, HashSet<int>> bookC
         ITextCorpus trgCorpus
     )
     {
-        if (!corpus.TrainOnAll)
-        {
-            IEnumerable<string> textIds = corpus.TrainOnChapters is not null
-                ? corpus.TrainOnChapters.Keys
-                : corpus.TrainOnTextIds;
-            srcCorpora = srcCorpora.Select(sc => sc.FilterTexts(textIds)).ToArray();
-            trgCorpus = trgCorpus.FilterTexts(textIds);
-        }
+        IEnumerable<string>? textIds = corpus.TrainOnChapters is not null
+            ? corpus.TrainOnChapters.Keys
+            : corpus.TrainOnTextIds;
+        srcCorpora = srcCorpora.Select(sc => sc.FilterTexts(textIds)).ToArray();
+        trgCorpus = trgCorpus.FilterTexts(textIds);
 
         if (trgCorpus.IsScripture())
         {
@@ -396,14 +395,11 @@ ITextCorpus trgCorpus
 
     private static IEnumerable<Row> AlignPretranslateCorpus(Corpus corpus, ITextCorpus srcCorpus, ITextCorpus trgCorpus)
     {
-        if (!corpus.PretranslateAll)
-        {
-            IEnumerable<string> textIds = corpus.PretranslateChapters is not null
-                ? corpus.PretranslateChapters.Keys
-                : corpus.PretranslateTextIds;
-            srcCorpus = srcCorpus.FilterTexts(textIds);
-            trgCorpus = trgCorpus.FilterTexts(textIds);
-        }
+        IEnumerable<string>? textIds = corpus.PretranslateChapters is not null
+            ? corpus.PretranslateChapters.Keys
+            : corpus.PretranslateTextIds;
+        srcCorpus = srcCorpus.FilterTexts(textIds);
+        trgCorpus = trgCorpus.FilterTexts(textIds);
 
         int rowCount = 0;
         StringBuilder srcSegBuffer = new();