Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use chapter-filtering for terms #545

Merged
merged 14 commits into from
Nov 27, 2024
Merged

Use chapter-filtering for terms #545

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
4f37c23
Use chapter-filtering for terms
Enkidu93 Nov 26, 2024
e5ead88
Use chapter-filtering for terms
Enkidu93 Nov 26, 2024
8f25c6b
Add support for key term filtering by textId
Enkidu93 Nov 26, 2024
e84ba85
Merge branch 'filter_kbt_by_chapter' of https://github.com/sillsdev/s…
Enkidu93 Nov 26, 2024
e02ad7c
Change function name
Enkidu93 Nov 26, 2024
069dc40
Use latest machine version
Enkidu93 Nov 27, 2024
788beb9
When no filter is specified, only include kbts that are associated wi…
Enkidu93 Nov 27, 2024
50387ec
Use chapter-filtering for terms
Enkidu93 Nov 26, 2024
2d73488
Add support for key term filtering by textId
Enkidu93 Nov 26, 2024
69d6d52
Change function name
Enkidu93 Nov 26, 2024
551e6cd
When no filter is specified, only include kbts that are associated wi…
Enkidu93 Nov 27, 2024
768cbea
Merge branch 'filter_kbt_by_chapter' of https://github.com/sillsdev/s…
Enkidu93 Nov 27, 2024
dd7f620
Update to new machine version
Enkidu93 Nov 27, 2024
bff3084
Remove accidentally added ?
Enkidu93 Nov 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@
<PackageReference Include="Hangfire.Mongo" Version="1.10.8" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.8" />
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.8" />
<PackageReference Include="SIL.Machine" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
<PackageReference Include="SIL.Machine" Version="3.5.2" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.5.2" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.5.2" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
<PackageReference Include="SIL.WritingSystems" Version="14.1.1" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
Expand Down
258 changes: 249 additions & 9 deletions src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,10 +118,10 @@ public async Task RunAsync_EnableKeyTerms()
(int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync();
Assert.Multiple(() =>
{
Assert.That(src1Count, Is.EqualTo(0));
Assert.That(src1Count, Is.EqualTo(14));
Assert.That(src2Count, Is.EqualTo(0));
Assert.That(trgCount, Is.EqualTo(0));
Assert.That(termCount, Is.EqualTo(5726));
Assert.That(trgCount, Is.EqualTo(1));
Assert.That(termCount, Is.EqualTo(166));
});
}

Expand All @@ -136,9 +136,9 @@ public async Task RunAsync_DisableKeyTerms()
(int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync();
Assert.Multiple(() =>
{
Assert.That(src1Count, Is.EqualTo(0));
Assert.That(src1Count, Is.EqualTo(14));
Assert.That(src2Count, Is.EqualTo(0));
Assert.That(trgCount, Is.EqualTo(0));
Assert.That(trgCount, Is.EqualTo(1));
Assert.That(termCount, Is.EqualTo(0));
});
}
Expand Down Expand Up @@ -359,7 +359,7 @@ public void RunAsync_OnlyParseSelectedBooks_PretranslateOnBadBook()
}

[Test]
public async Task ParallelCorpusLogic()
public async Task ParallelCorpusAsync()
{
using TestEnvironment env = new();
var corpora = new List<ParallelCorpus>()
Expand Down Expand Up @@ -508,6 +508,246 @@ public async Task ParallelCorpusLogic()
});
}

[Test]
public async Task ParallelCorpusAsync_UseKeyTerms()
{
using TestEnvironment env = new();
var corpora = new List<ParallelCorpus>()
{
new ParallelCorpus()
{
Id = "1",
SourceCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source1") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"LEV",
new() { }
}
},
PretranslateChapters = new()
{
{
"1CH",
new() { }
}
}
},
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source2") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"MRK",
new() { }
}
},
PretranslateChapters = new() { }
},
},
TargetCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target1") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"MRK",
new() { }
}
}
},
new()
{
Id = "_2",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target2") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"MRK",
new() { }
},
{
"LEV",
new() { }
}
}
}
}
}
};
await env.RunBuildJobAsync(corpora, useKeyTerms: true);
string source = await env.GetSourceExtractAsync();
string target = await env.GetTargetExtractAsync();
Assert.Multiple(() =>
{
StringAssert.StartsWith(
@"Source one, chapter fourteen, verse fifty-five. Segment b.
Source one, chapter fourteen, verse fifty-six.
Source two, chapter one, verse one.
Source two, chapter one, verse two.
Source two, chapter one, verse three.
Source one, chapter one, verse four.
Source two, chapter one, verse five. Source two, chapter one, verse six.
Source one, chapter one, verse seven, eight, and nine. Source one, chapter one, verse ten.
Source two, chapter one, verse one.
",
source
);
StringAssert.StartsWith(
@"Target two, chapter fourteen, verse fifty-five.
Target two, chapter fourteen, verse fifty-six.
Target one, chapter one, verse one.
Target one, chapter one, verse two.
Target one, chapter one, verse three.

Target one, chapter one, verse five and six.
Target one, chapter one, verse seven and eight. Target one, chapter one, verse nine and ten.

",
target
);
StringAssert.Contains("Abraham", source);
StringAssert.Contains("Abraham", target);
StringAssert.DoesNotContain("Zedekiah", source);
StringAssert.DoesNotContain("Zedekiah", target);
});
JsonArray? pretranslations = await env.GetPretranslationsAsync();
Assert.That(pretranslations, Is.Not.Null);
Assert.That(pretranslations!.Count, Is.EqualTo(7), pretranslations.ToJsonString());
Assert.That(
pretranslations[2]!["translation"]!.ToString(),
Is.EqualTo("Source one, chapter twelve, verse one.")
);
}

[Test]
public async Task ParallelCorpusAsync_UseKeyTerms_TextIds()
{
using TestEnvironment env = new();
var corpora = new List<ParallelCorpus>()
{
new ParallelCorpus()
{
Id = "1",
SourceCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source1") },
TrainOnTextIds = ["MAT", "LEV"],
PretranslateTextIds = ["1CH"]
},
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source2") },
TrainOnTextIds = ["MAT", "MRK"],
PretranslateTextIds = []
},
},
TargetCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target1") },
TrainOnTextIds = ["MAT", "MRK"]
},
new()
{
Id = "_2",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target2") },
TrainOnTextIds = ["MAT", "MRK", "LEV"]
}
}
}
};
await env.RunBuildJobAsync(corpora, useKeyTerms: true);
string source = await env.GetSourceExtractAsync();
string target = await env.GetTargetExtractAsync();
Assert.Multiple(() =>
{
StringAssert.StartsWith(
@"Source one, chapter fourteen, verse fifty-five. Segment b.
Source one, chapter fourteen, verse fifty-six.
Source two, chapter one, verse one.
Source two, chapter one, verse two.
Source two, chapter one, verse three.
Source one, chapter one, verse four.
Source two, chapter one, verse five. Source two, chapter one, verse six.
Source one, chapter one, verse seven, eight, and nine. Source one, chapter one, verse ten.
Source one, chapter two, verse one.
Source one, chapter two, verse two.

Source two, chapter one, verse one.
",
source
);
StringAssert.StartsWith(
@"Target two, chapter fourteen, verse fifty-five.
Target two, chapter fourteen, verse fifty-six.
Target one, chapter one, verse one.
Target one, chapter one, verse two.
Target one, chapter one, verse three.

Target one, chapter one, verse five and six.
Target one, chapter one, verse seven and eight. Target one, chapter one, verse nine and ten.
Target one, chapter two, verse one.

Target one, chapter two, verse three.

",
target
);
StringAssert.Contains("Abraham", source);
StringAssert.Contains("Abraham", target);
StringAssert.DoesNotContain("Zedekiah", source);
StringAssert.DoesNotContain("Zedekiah", target);
});
JsonArray? pretranslations = await env.GetPretranslationsAsync();
Assert.That(pretranslations, Is.Not.Null);
Assert.That(pretranslations!.Count, Is.EqualTo(7), pretranslations.ToJsonString());
Assert.That(
pretranslations[2]!["translation"]!.ToString(),
Is.EqualTo("Source one, chapter twelve, verse one.")
);
}

private class TestEnvironment : DisposableBase
{
private static readonly string TestDataPath = Path.Combine(
Expand Down Expand Up @@ -613,8 +853,8 @@ public TestEnvironment()
Id = "src_1",
Language = "es",
Files = [ParatextFile("pt-source1")],
TrainOnTextIds = [],
PretranslateTextIds = []
TrainOnTextIds = null,
PretranslateTextIds = null
}
},
TargetCorpora = new List<MonolingualCorpus>()
Expand All @@ -624,7 +864,7 @@ public TestEnvironment()
Id = "trg_1",
Language = "en",
Files = [ParatextFile("pt-target1")],
TrainOnTextIds = []
TrainOnTextIds = null
}
}
};
Expand Down
7 changes: 7 additions & 0 deletions src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,11 @@
<Notes />
<Denials />
</TermRendering>
<TermRendering Id="צִדְקִיָּהוּ-2" Guess="false">
<Renderings>Zedekiah</Renderings>
<Glossary />
<Changes />
<Notes />
<Denials />
</TermRendering>
</TermRenderingsList>
7 changes: 7 additions & 0 deletions src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,11 @@
<Notes />
<Denials />
</TermRendering>
<TermRendering Id="צִדְקִיָּהוּ-2" Guess="false">
<Renderings>Zedekiah</Renderings>
<Glossary />
<Changes />
<Notes />
<Denials />
</TermRendering>
</TermRenderingsList>
2 changes: 1 addition & 1 deletion src/Serval/src/Serval.Shared/Serval.Shared.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<PackageReference Include="Grpc.Core.Api" Version="2.65.0" />
<PackageReference Include="Grpc.HealthCheck" Version="2.65.0" />
<PackageReference Include="Grpc.Net.ClientFactory" Version="2.65.0" />
<PackageReference Include="SIL.Machine" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="SIL.Machine" Version="3.5.2" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="Microsoft.FeatureManagement.AspNetCore" Version="3.5.0" />
</ItemGroup>

Expand Down
2 changes: 1 addition & 1 deletion src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
<PackageReference Include="SIL.WritingSystems" Version="14.1.1" />
<PackageReference Include="System.Text.RegularExpressions" Version="4.3.1" />
<PackageReference Include="SIL.Scripture" Version="12.0.1"/>
<PackageReference Include="SIL.Machine" Version="3.5.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
<PackageReference Include="SIL.Machine" Version="3.5.2" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
</ItemGroup>

<ItemGroup>
Expand Down
8 changes: 5 additions & 3 deletions src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CorpusService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,16 @@ public IEnumerable<ITextCorpus> CreateTextCorpora(IReadOnlyList<CorpusFile> file
return corpora;
}

public IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> files)
public IEnumerable<ITextCorpus> CreateTermCorpora(
IReadOnlyList<(CorpusFile File, Dictionary<string, HashSet<int>> Chapters)> corpora
)
{
foreach (CorpusFile file in files)
foreach ((CorpusFile file, Dictionary<string, HashSet<int>> chapters) in corpora)
{
switch (file.Format)
{
case FileFormat.Paratext:
yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]);
yield return new ParatextBackupTermsCorpus(file.Location, ["PN"], chapters: chapters);
break;
}
}
Expand Down
4 changes: 3 additions & 1 deletion src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ICorpusService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,7 @@
public interface ICorpusService
{
IEnumerable<ITextCorpus> CreateTextCorpora(IReadOnlyList<CorpusFile> files);
IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> files);
IEnumerable<ITextCorpus> CreateTermCorpora(
IReadOnlyList<(CorpusFile File, Dictionary<string, HashSet<int>> Chapters)> corpora
);
}
Loading