Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Filter key terms by book/chapter #508

Closed
wants to merge 2 commits into from
Closed

Filter key terms by book/chapter #508

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2acb423
Filter key terms by book/chapter
Enkidu93 Oct 11, 2024
ff2f3a0
Add test
Enkidu93 Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,16 @@
return corpora;
}

public IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> files)
public IEnumerable<ITextCorpus> CreateTermCorpora(
IReadOnlyList<(CorpusFile File, Dictionary<string, HashSet<int>>? Chapters)> corpora
)
{
foreach (CorpusFile file in files)
foreach ((CorpusFile file, Dictionary<string, HashSet<int>>? chapters) in corpora)
{
switch (file.Format)
{
case FileFormat.Paratext:
yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]);
yield return new ParatextBackupTermsCorpus(file.Location, ["PN"], chapters: chapters);

Check failure on line 48 in src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs

View workflow job for this annotation

GitHub Actions / Build 

The best overload for 'ParatextBackupTermsCorpus' does not have a parameter named 'chapters'

Check failure on line 48 in src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs

View workflow job for this annotation

GitHub Actions / Build 

The best overload for 'ParatextBackupTermsCorpus' does not have a parameter named 'chapters'
break;
}
}
Expand Down
4 changes: 3 additions & 1 deletion src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,7 @@
public interface ICorpusService
{
IEnumerable<ITextCorpus> CreateTextCorpora(IReadOnlyList<CorpusFile> files);
IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> files);
IEnumerable<ITextCorpus> CreateTermCorpora(
IReadOnlyList<(CorpusFile File, Dictionary<string, HashSet<int>>? Chapters)> corpora
);
}
16 changes: 10 additions & 6 deletions src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,15 +221,19 @@ row.Ref is not ScriptureRef sr

if ((bool?)buildOptionsObject?["use_key_terms"] ?? true)
{
ITextCorpus? sourceTermCorpus = _corpusService
.CreateTermCorpora(corpus.SourceCorpora.SelectMany(sc => sc.Files).ToList())
ITextCorpus? sourceTermCorpora = _corpusService
.CreateTermCorpora(
corpus.SourceCorpora.SelectMany(sc => sc.Files.Select(f => (f, sc.TrainOnChapters))).ToList()
)
.FirstOrDefault();
ITextCorpus? targetTermCorpus = _corpusService
.CreateTermCorpora(corpus.TargetCorpora.SelectMany(tc => tc.Files).ToList())
ITextCorpus? targetTermCorpora = _corpusService
.CreateTermCorpora(
corpus.TargetCorpora.SelectMany(tc => tc.Files.Select(f => (f, tc.TrainOnChapters))).ToList()
)
.FirstOrDefault();
if (sourceTermCorpus is not null && targetTermCorpus is not null)
if (sourceTermCorpora is not null && targetTermCorpora is not null)
{
IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus);
IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpora.AlignRows(targetTermCorpora);
foreach (ParallelTextRow row in parallelKeyTermsCorpus)
{
await sourceTrainWriter.WriteAsync($"{row.SourceText}\n");
Expand Down
145 changes: 144 additions & 1 deletion src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@ public void RunAsync_OnlyParseSelectedBooks_PretranslateOnBadBook()
}

[Test]
public async Task ParallelCorpusLogic()
public async Task ParallelCorpusAsync()
{
using TestEnvironment env = new();
var corpora = new List<ParallelCorpus>()
Expand Down Expand Up @@ -478,6 +478,149 @@ await env.GetTargetExtractAsync(),
);
}

[Test]
public async Task ParallelCorpusAsync_UseKeyTerms()
{
using TestEnvironment env = new();
var corpora = new List<ParallelCorpus>()
{
new ParallelCorpus()
{
Id = "1",
SourceCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source1") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"LEV",
new() { }
}
},
PretranslateChapters = new()
{
{
"1CH",
new() { }
}
}
},
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source2") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"MRK",
new() { }
}
},
},
},
TargetCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target1") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"MRK",
new() { }
}
}
},
new()
{
Id = "_2",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target2") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"MRK",
new() { }
},
{
"LEV",
new() { }
}
}
}
}
}
};
await env.RunBuildJobAsync(corpora, useKeyTerms: true);
string source = await env.GetSourceExtractAsync();
string target = await env.GetTargetExtractAsync();
Assert.Multiple(() =>
{
StringAssert.StartsWith(
@"Source one, chapter fourteen, verse fifty-five. Segment b.
Source one, chapter fourteen, verse fifty-six.
Source one, chapter one, verse one.
Source two, chapter one, verse two.
Source two, chapter one, verse three.
Source two, chapter one, verse four.
Source two, chapter one, verse five. Source two, chapter one, verse six.
Source two, chapter one, verse seven. Source two, chapter one, verse eight.
Source two, chapter one, verse nine. Source two, chapter one, verse ten.
Source two, chapter one, verse one.
",
source
);
StringAssert.StartsWith(
@"Target two, chapter fourteen, verse fifty-five.
Target two, chapter fourteen, verse fifty-six.
Target one, chapter one, verse one.
Target one, chapter one, verse two.
Target one, chapter one, verse three.

Target one, chapter one, verse five and six.
Target one, chapter one, verse seven and eight.
Target one, chapter one, verse nine and ten.

",
target
);
StringAssert.Contains("Abraham", source);
StringAssert.Contains("Abraham", target);
StringAssert.DoesNotContain("Zedekiah", source);
StringAssert.DoesNotContain("Zedekiah", target);
});
JsonArray? pretranslations = await env.GetPretranslationsAsync();
Assert.That(pretranslations, Is.Not.Null);
Assert.That(pretranslations!.Count, Is.EqualTo(37), pretranslations.ToJsonString());
Assert.That(
pretranslations[2]!["translation"]!.ToString(),
Is.EqualTo("Source one, chapter twelve, verse one.")
);
}

private class TestEnvironment : DisposableBase
{
private static readonly string TestDataPath = Path.Combine(
Expand Down
7 changes: 7 additions & 0 deletions src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,11 @@
<Notes />
<Denials />
</TermRendering>
<TermRendering Id="צִדְקִיָּהוּ-2" Guess="false">
<Renderings>Zedekiah</Renderings>
<Glossary />
<Changes />
<Notes />
<Denials />
</TermRendering>
</TermRenderingsList>
7 changes: 7 additions & 0 deletions src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,11 @@
<Notes />
<Denials />
</TermRendering>
<TermRendering Id="צִדְקִיָּהוּ-2" Guess="false">
<Renderings>Zedekiah</Renderings>
<Glossary />
<Changes />
<Notes />
<Denials />
</TermRendering>
</TermRenderingsList>
Loading