Deduplicate kbts

sillsdev · Dec 2, 2024 · 7b9c4a8 · 7b9c4a8
1 parent 03e4716
commit 7b9c4a8
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs
@@ -121,7 +121,7 @@ public async Task RunAsync_EnableKeyTerms()
             Assert.That(src1Count, Is.EqualTo(14));
             Assert.That(src2Count, Is.EqualTo(0));
             Assert.That(trgCount, Is.EqualTo(1));
-            Assert.That(termCount, Is.EqualTo(166));
+            Assert.That(termCount, Is.EqualTo(144));
         });
     }
 

diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs
@@ -101,7 +101,9 @@ public async Task Preprocess(
                     IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpora
                         .ChooseRandom(Seed)
                         .AlignRows(targetTermCorpora.ChooseFirst());
-                    foreach (ParallelTextRow row in parallelKeyTermsCorpus)
+                    foreach (
+                        ParallelTextRow row in parallelKeyTermsCorpus.DistinctBy(row => row.SourceText + row.TargetText)
+                    )
                     {
                         await train(new Row(row.TextId, row.Refs, row.SourceText, row.TargetText, 1));
                     }