From 0af5ec77641a47befc042ad21e4188f9c78d554c Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 26 Aug 2024 15:11:08 +0700 Subject: [PATCH] End-to-end S/R tests with Lexbox and Mongo (#342) This end-to-end test runs against a real Lexbox instance, writing data to a real MongoDB instance rather than a MongoConnectionDouble. It creates a new project in Lexbox (using the existing sena-3 project as the initial data), and clones that project into both a LfProject object and a FwProject object. The FwProject object uses liblcm code to modify the data, then pushes that modified data to Lexbox. The LfProject then makes its own changes and goes through the LfMerge Send/Receive process; because the LfProject's changes are the "ours" in Chorus, the changes from the LfProject "win" and overwrite the FwProject's changes. The "real" MongoDB is provided by using a Docker image of Mongo version 6 (via the `mongo:6` tag). This requires a working copy of Docker in the environment running the tests. Although there is only one test created here, the SRTestEnvironment class contains several helper methods that would be useful in many other end-to-end test scenarios. --- src/LfMerge.Core.Tests/E2E/E2ETestBase.cs | 234 +++++++++++++++++ .../E2E/LexboxSendReceiveTests.cs | 61 +++++ src/LfMerge.Core.Tests/LcmTestHelper.cs | 83 ++++++ src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs | 44 ++++ .../LfMerge.Core.Tests.csproj | 3 + src/LfMerge.Core.Tests/MercurialTestHelper.cs | 26 +- src/LfMerge.Core.Tests/SRTestEnvironment.cs | 240 ++++++++++++++++++ src/LfMerge.Core.Tests/TestDoubles.cs | 5 + src/LfMerge.Core.Tests/TestEnvironment.cs | 6 +- .../ConvertLcmToMongoComments.cs | 21 +- .../ConvertMongoToLcmComments.cs | 25 +- src/LfMerge.Core/LanguageForgeProject.cs | 4 +- src/LfMerge.Core/MagicStrings.cs | 1 + src/LfMerge.Core/Settings/LfMergeSettings.cs | 6 +- 14 files changed, 742 insertions(+), 17 deletions(-) create mode 100644 src/LfMerge.Core.Tests/E2E/E2ETestBase.cs create mode 100644 src/LfMerge.Core.Tests/E2E/LexboxSendReceiveTests.cs create mode 100644 src/LfMerge.Core.Tests/LcmTestHelper.cs create mode 100644 src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs create mode 100644 src/LfMerge.Core.Tests/SRTestEnvironment.cs diff --git a/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs new file mode 100644 index 00000000..3d11a912 --- /dev/null +++ b/src/LfMerge.Core.Tests/E2E/E2ETestBase.cs @@ -0,0 +1,234 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Autofac; +using LfMerge.Core.Actions; +using LfMerge.Core.FieldWorks; +using LfMerge.Core.MongoConnector; +using NUnit.Framework; +using NUnit.Framework.Interfaces; +using SIL.LCModel; +using SIL.TestUtilities; + +namespace LfMerge.Core.Tests.E2E +{ + [TestFixture] + [Category("LongRunning")] + [Category("IntegrationTests")] + public class E2ETestBase + { + public LfMerge.Core.Logging.ILogger Logger => MainClass.Logger; + public TemporaryFolder TempFolderForClass { get; set; } + public TemporaryFolder TempFolderForTest { get; set; } + public string Sena3ZipPath { get; set; } + private readonly HashSet ProjectIdsToDelete = []; + public SRTestEnvironment TestEnv { get; set; } + + public MongoConnectionDouble _mongoConnection; + public MongoProjectRecordFactory _recordFactory; + + public E2ETestBase() + { + } + + private static string TestName => TestContext.CurrentContext.Test.Name; + private static string TestNameForPath => string.Concat(TestName.Split(Path.GetInvalidPathChars())); // Easiest way to strip out all invalid chars + + [OneTimeSetUp] + public async Task FixtureSetup() + { + // Ensure top-level /tmp/LfMergeSRTests folder and subfolders exist + var tempPath = Path.Join(Path.GetTempPath(), "LfMergeSRTests"); + Directory.CreateDirectory(tempPath); + var testDataPath = Path.Join(tempPath, "data"); + Directory.CreateDirectory(testDataPath); + var lcmDataPath = Path.Join(tempPath, "lcm-common"); + Directory.CreateDirectory(lcmDataPath); + Environment.SetEnvironmentVariable("FW_CommonAppData", lcmDataPath); + + var derivedClassName = this.GetType().Name; + TempFolderForClass = new TemporaryFolder(Path.Join(tempPath, derivedClassName)); + + // Ensure sena-3.zip is available to all tests as a starting point + Sena3ZipPath = Path.Join(testDataPath, "sena-3.zip"); + if (!File.Exists(Sena3ZipPath)) { + using var testEnv = new SRTestEnvironment(TempFolderForTest); + if (!await testEnv.IsLexBoxAvailable()) { + Assert.Ignore("Can't run E2E tests without a copy of LexBox to test against. Please either launch LexBox on localhost port 80, or set the appropriate environment variables to point to a running copy of LexBox."); + } + await testEnv.Login(); + await testEnv.DownloadProjectBackup("sena-3", Sena3ZipPath); + } + } + + [OneTimeTearDown] + public void FixtureTeardown() + { + Environment.SetEnvironmentVariable("FW_CommonAppData", null); + var result = TestContext.CurrentContext.Result; + var nonSuccess = result.FailCount + result.InconclusiveCount + result.WarningCount; + // Only delete class temp folder if we passed or skipped all tests + if (nonSuccess == 0) TempFolderForClass.Dispose(); + } + + [SetUp] + public async Task TestSetup() + { + TempFolderForTest = new TemporaryFolder(TempFolderForClass, TestNameForPath); + TestEnv = new SRTestEnvironment(TempFolderForTest); + if (!await TestEnv.IsLexBoxAvailable()) { + Assert.Ignore("Can't run E2E tests without a copy of LexBox to test against. Please either launch LexBox on localhost port 80, or set the appropriate environment variables to point to a running copy of LexBox."); + } + await TestEnv.Login(); + + MagicStrings.SetMinimalModelVersion(LcmCache.ModelVersion); + _mongoConnection = MainClass.Container.Resolve() as MongoConnectionDouble; + if (_mongoConnection == null) + throw new AssertionException("E2E tests need a mock MongoConnection that stores data in order to work."); + _recordFactory = MainClass.Container.Resolve() as MongoProjectRecordFactoryDouble; + if (_recordFactory == null) + throw new AssertionException("E2E tests need a mock MongoProjectRecordFactory in order to work."); + } + + [TearDown] + public async Task TestTeardown() + { + var outcome = TestContext.CurrentContext.Result.Outcome; + var success = outcome == ResultState.Success || outcome == ResultState.Ignored; + // Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation + TestEnv.DeleteTempFolderDuringCleanup = success; + // On failure, also leave LexBox project(s) in place for post-test investigation, even though this might tend to clutter things up a little + if (success) { + foreach (var projId in ProjectIdsToDelete) { + await TestEnv.DeleteLexBoxProject(projId); + } + } + ProjectIdsToDelete.Clear(); + TestEnv.Dispose(); + } + + public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForTest.Path, "webwork", projectCode); + public string FwDataPathForProject(string projectCode) => Path.Join(TestFolderForProject(projectCode), $"{projectCode}.fwdata"); + + public string CloneRepoFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null) + { + var projUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(code); + newCode ??= code; + var dest = TestFolderForProject(newCode); + if (waitTime is null) { + MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest); + } else { + var start = DateTime.UtcNow; + var success = false; + while (!success) { + try { + MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest); + } catch { + if (DateTime.UtcNow > start + waitTime) { + throw; // Give up + } + System.Threading.Thread.Sleep(250); + continue; + } + // If we got this far, no exception so we succeeded + success = true; + } + } + return dest; + } + + public FwProject CloneFwProjectFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null) + { + var dest = CloneRepoFromLexbox(code, newCode, waitTime); + var dirInfo = new DirectoryInfo(dest); + if (!dirInfo.Exists) throw new InvalidOperationException($"Failed to clone {code} from lexbox, cannot create FwProject"); + var dirname = dirInfo.Name; + var fwdataPath = Path.Join(dest, $"{dirname}.fwdata"); + MercurialTestHelper.ChangeBranch(dest, "tip"); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(TestEnv.NullProgress, false, fwdataPath); + var settings = new LfMergeSettingsDouble(TempFolderForTest.Path); + return new FwProject(settings, dirname); + } + + public async Task CreateEmptyFlexProjectInLexbox() + { + var randomGuid = Guid.NewGuid(); + var testCode = $"sr-{randomGuid}"; + var testPath = TestFolderForProject(testCode); + MercurialTestHelper.InitializeHgRepo(testPath); + MercurialTestHelper.CreateFlexRepo(testPath); + // Now create project in LexBox + var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); + var pushUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(testCode).AbsoluteUri; + MercurialTestHelper.HgPush(testPath, pushUrl); + if (result.Id.HasValue) ProjectIdsToDelete.Add(result.Id.Value); + return testCode; + } + + public async Task CreateNewProjectFromTemplate(string origZipPath) + { + var randomGuid = Guid.NewGuid(); + var testCode = $"sr-{randomGuid}"; + var testPath = TestFolderForProject(testCode); + // Now create project in LexBox + var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid); + await TestEnv.ResetAndUploadZip(testCode, origZipPath); + if (result.Id.HasValue) ProjectIdsToDelete.Add(result.Id.Value); + return testCode; + } + + public Task CreateNewProjectFromSena3() => CreateNewProjectFromTemplate(Sena3ZipPath); + + public void CommitAndPush(FwProject project, string code, string? localCode = null, string? commitMsg = null) + { + TestEnv.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg); + } + + public async Task CreateLfProjectFromSena3() + { + var projCode = await CreateNewProjectFromSena3(); + var projPath = CloneRepoFromLexbox(projCode, waitTime:TimeSpan.FromSeconds(5)); + MercurialTestHelper.ChangeBranch(projPath, "tip"); + var fwdataPath = Path.Join(projPath, $"{projCode}.fwdata"); + LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(TestEnv.NullProgress, false, fwdataPath); + + // Do an initial clone from LexBox to the mock LF + var lfProject = LanguageForgeProject.Create(projCode, TestEnv.Settings); + lfProject.IsInitialClone = true; + var transferLcmToMongo = new TransferLcmToMongoAction(TestEnv.Settings, TestEnv.NullLogger, _mongoConnection, _recordFactory); + transferLcmToMongo.Run(lfProject); + + return lfProject; + } + + public void SendReceiveToLexbox(LanguageForgeProject lfProject) + { + TestEnv.Settings.LanguageDepotRepoUri = SRTestEnvironment.LexboxUrlForProjectWithAuth(lfProject.ProjectCode).AbsoluteUri; + var syncAction = new SynchronizeAction(TestEnv.Settings, TestEnv.Logger); + syncAction.Run(lfProject); + } + + public (string, DateTime, DateTime) UpdateFwGloss(FwProject project, Guid entryId, Func textConverter) + { + var fwEntry = LcmTestHelper.GetEntry(project, entryId); + Assert.That(fwEntry, Is.Not.Null); + var origModifiedDate = fwEntry.DateModified; + var unchangedGloss = LcmTestHelper.UpdateAnalysisText(project, fwEntry.SensesOS[0].Gloss, textConverter); + return (unchangedGloss, origModifiedDate, fwEntry.DateModified); + } + + public (string, DateTime, DateTime) UpdateLfGloss(LanguageForgeProject lfProject, Guid entryId, string wsId, Func textConverter) + { + var lfEntry = _mongoConnection.GetLfLexEntryByGuid(entryId); + Assert.That(lfEntry, Is.Not.Null); + var unchangedGloss = lfEntry.Senses[0].Gloss[wsId].Value; + lfEntry.Senses[0].Gloss["pt"].Value = textConverter(unchangedGloss); + // Remember that in LfMerge it's AuthorInfo that corresponds to the Lcm modified date + DateTime origModifiedDate = lfEntry.AuthorInfo.ModifiedDate; + lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow; + _mongoConnection.UpdateRecord(lfProject, lfEntry); + return (unchangedGloss, origModifiedDate, lfEntry.AuthorInfo.ModifiedDate); + } + } +} diff --git a/src/LfMerge.Core.Tests/E2E/LexboxSendReceiveTests.cs b/src/LfMerge.Core.Tests/E2E/LexboxSendReceiveTests.cs new file mode 100644 index 00000000..358f83a6 --- /dev/null +++ b/src/LfMerge.Core.Tests/E2E/LexboxSendReceiveTests.cs @@ -0,0 +1,61 @@ +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using System.Text.RegularExpressions; + +namespace LfMerge.Core.Tests.E2E +{ + [TestFixture] + [Category("LongRunning")] + [Category("IntegrationTests")] + public class LexboxSendReceiveTests : E2ETestBase + { + // This test will often trigger a race condition in LexBox that causes the *next* test to fail + // when `hg clone` returns 404. This is because of hgweb's directory cache, which only refreshes + // if it hasn't been refreshed more than N seconds ago (default 20, LexBox currently uses 5). + // Which means that even though CreateLfProjectFromSena3 has created the LexBox project, LexBox's + // copy of hgweb doesn't see it yet so it can't be cloned. + // + // The solution will be to adjust CloneRepoFromLexbox to take a parameter that is the number of + // seconds to wait for the project to become visible, and retry 404's until that much time has elapsed. + // Then only throw an exception if hgweb is still returning 404 after its dir cache should be refreshed. + [Test] + public async Task E2E_CheckFwProjectCreation() + { + var code = await CreateEmptyFlexProjectInLexbox(); + Console.WriteLine($"Created new project {code}"); + } + + [Test] + public async Task E2E_LFDataChangedLDDataChanged_LFWins() + { + // Setup + + var lfProject = await CreateLfProjectFromSena3(); + var fwProjectCode = Regex.Replace(lfProject.ProjectCode, "^sr-", "fw-"); + var fwProject = CloneFwProjectFromLexbox(lfProject.ProjectCode, fwProjectCode); + + // Modify FW data first, then push to Lexbox + Guid entryId = LcmTestHelper.GetFirstEntry(fwProject).Guid; + var (unchangedGloss, origFwDateModified, fwDateModified) = UpdateFwGloss(fwProject, entryId, text => text + " - changed in FW"); + CommitAndPush(fwProject, lfProject.ProjectCode, fwProjectCode, "Modified gloss in FW"); + + // Modify LF data second + var (_, origLfDateModified, _) = UpdateLfGloss(lfProject, entryId, "pt", text => text + " - changed in LF"); + + // Exercise + + SendReceiveToLexbox(lfProject); + + // Verify + + // LF side should win conflict since its modified date was later + var lfEntryAfterSR = _mongoConnection.GetLfLexEntryByGuid(entryId); + Assert.That(lfEntryAfterSR?.Senses?[0]?.Gloss?["pt"]?.Value, Is.EqualTo(unchangedGloss + " - changed in LF")); + // LF's modified dates should have been updated by the sync action + Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(origLfDateModified)); + // Remember that FieldWorks's DateModified is stored in local time for some incomprehensible reason... + Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(fwDateModified.ToUniversalTime())); + } + } +} diff --git a/src/LfMerge.Core.Tests/LcmTestHelper.cs b/src/LfMerge.Core.Tests/LcmTestHelper.cs new file mode 100644 index 00000000..d8dc1215 --- /dev/null +++ b/src/LfMerge.Core.Tests/LcmTestHelper.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using LfMerge.Core.FieldWorks; +using SIL.LCModel; +using SIL.LCModel.Infrastructure; + +namespace LfMerge.Core.Tests +{ + public static class LcmTestHelper + { + public static IEnumerable GetEntries(FwProject project) + { + return project?.ServiceLocator?.LanguageProject?.LexDbOA?.Entries ?? []; + } + + public static int CountEntries(FwProject project) + { + var repo = project?.ServiceLocator?.GetInstance(); + return repo.Count; + } + + public static ILexEntry GetEntry(FwProject project, Guid guid) + { + var repo = project?.ServiceLocator?.GetInstance(); + return repo.GetObject(guid); + } + + public static ILexEntry GetFirstEntry(FwProject project) + { + var repo = project?.ServiceLocator?.GetInstance(); + return repo.AllInstances().First(); + } + + public static string? GetVernacularText(IMultiUnicode field) + { + return field.BestVernacularAlternative?.Text; + } + + public static string? GetAnalysisText(IMultiUnicode field) + { + return field.BestAnalysisAlternative?.Text; + } + + public static void SetVernacularText(FwProject project, IMultiUnicode field, string newText) + { + var accessor = project.Cache.ActionHandlerAccessor; + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", accessor, () => { + field.SetVernacularDefaultWritingSystem(newText); + }); + } + + public static void SetAnalysisText(FwProject project, IMultiUnicode field, string newText) + { + var accessor = project.Cache.ActionHandlerAccessor; + UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", accessor, () => { + field.SetAnalysisDefaultWritingSystem(newText); + }); + } + + public static string UpdateVernacularText(FwProject project, IMultiUnicode field, Func textConverter) + { + var oldText = field.BestVernacularAlternative?.Text; + if (oldText != null) + { + var newText = textConverter(oldText); + SetVernacularText(project, field, newText); + } + return oldText; + } + + public static string UpdateAnalysisText(FwProject project, IMultiUnicode field, Func textConverter) + { + var oldText = field.BestAnalysisAlternative?.Text; + if (oldText != null) + { + var newText = textConverter(oldText); + SetAnalysisText(project, field, newText); + } + return oldText; + } + } +} diff --git a/src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs b/src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs new file mode 100644 index 00000000..17d8ed91 --- /dev/null +++ b/src/LfMerge.Core.Tests/LexboxGraphQLTypes.cs @@ -0,0 +1,44 @@ +using System; + +namespace LfMerge.Core.Tests.LexboxGraphQLTypes +{ + public enum ProjectType + { + Unknown = 0, + FLEx = 1, + WeSay = 2, + OneStoryEditor = 3, + OurWord = 4, + AdaptIt = 5, + } + public enum RetentionPolicy + { + Unknown = 0, + Verified = 1, + Test = 2, + Dev = 3, + Training = 4, + } + + public record CreateProjectInput( + Guid? Id, + string Name, + string Description, + string Code, + ProjectType Type, + RetentionPolicy RetentionPolicy, + bool IsConfidential, + Guid? ProjectManagerId, + Guid? OrgId + ); + + public enum CreateProjectResult + { + Created, + Requested + } + + public record CreateProjectResponse(Guid? Id, CreateProjectResult Result); + public record CreateProject(CreateProjectResponse CreateProjectResponse); + public record CreateProjectGqlResponse(CreateProject CreateProject); +} diff --git a/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj b/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj index e016ab04..daa93089 100644 --- a/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj +++ b/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj @@ -32,6 +32,8 @@ See full changelog at https://github.com/sillsdev/LfMerge/blob/develop/CHANGELOG + + @@ -44,6 +46,7 @@ See full changelog at https://github.com/sillsdev/LfMerge/blob/develop/CHANGELOG + diff --git a/src/LfMerge.Core.Tests/MercurialTestHelper.cs b/src/LfMerge.Core.Tests/MercurialTestHelper.cs index 5ae2f89a..148bdf31 100644 --- a/src/LfMerge.Core.Tests/MercurialTestHelper.cs +++ b/src/LfMerge.Core.Tests/MercurialTestHelper.cs @@ -19,9 +19,10 @@ public static class MercurialTestHelper private static string RunHgCommand(string repoPath, string args) { var result = CommandLineRunner.Run(HgCommand, args, repoPath, 120, new NullProgress()); - Assert.That(result.ExitCode, Is.EqualTo(0), + if (result.ExitCode == 0) return result.StandardOutput; + throw new System.Exception( $"hg {args} failed.\nStdOut: {result.StandardOutput}\nStdErr: {result.StandardError}"); - return result.StandardOutput; + } public static void InitializeHgRepo(string projectFolderPath) @@ -53,6 +54,11 @@ public static void HgAddFile(string repoPath, string file) RunHgCommand(repoPath, $"add {file}"); } + public static void HgClean(string repoPath) + { + RunHgCommand(repoPath, $"purge --no-confirm"); + } + public static void HgCommit(string repoPath, string message) { RunHgCommand(repoPath, $"commit -A -u dummyUser -m \"{message}\""); @@ -63,6 +69,11 @@ public static void HgCreateBranch(string repoPath, int branchName) RunHgCommand(repoPath, $"branch -f \"{branchName}\""); } + public static void HgPush(string repoPath, string remoteUri) + { + RunHgCommand(repoPath, $"push {remoteUri}"); + } + public static void CreateFlexRepo(string lDProjectFolderPath, int modelVersion = 0) { if (modelVersion <= 0) @@ -81,6 +92,17 @@ public static void CloneRepo(string sourceRepo, string destinationRepo) RunHgCommand(destinationRepo, $"clone {sourceRepo} ."); } + public static void CloneRepoAtRev(string sourceRepo, string destinationRepo, string rev) + { + Directory.CreateDirectory(destinationRepo); + RunHgCommand(destinationRepo, $"clone {sourceRepo} -U -r {rev} ."); + } + + public static void CloneRepoAtRevnum(string sourceRepo, string destinationRepo, int revnum) + { + CloneRepoAtRev(sourceRepo, destinationRepo, revnum.ToString()); + } + public static void ChangeBranch(string repoPath, string newBranch) { RunHgCommand(repoPath, $"update {newBranch}"); diff --git a/src/LfMerge.Core.Tests/SRTestEnvironment.cs b/src/LfMerge.Core.Tests/SRTestEnvironment.cs new file mode 100644 index 00000000..350321ed --- /dev/null +++ b/src/LfMerge.Core.Tests/SRTestEnvironment.cs @@ -0,0 +1,240 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading.Tasks; +using BirdMessenger; +using BirdMessenger.Collections; +using GraphQL; +using GraphQL.Client.Http; +using GraphQL.Client.Serializer.SystemTextJson; +using LfMerge.Core.FieldWorks; +using LfMerge.Core.Logging; +using NUnit.Framework; +using SIL.TestUtilities; + +namespace LfMerge.Core.Tests +{ + /// + /// Test environment for end-to-end testing, i.e. Send/Receive with a real LexBox instance + /// + public class SRTestEnvironment : TestEnvironment + { + public static readonly string LexboxHostname = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotPublicHostname) ?? "localhost"; + public static readonly string LexboxProtocol = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_LanguageDepotUriProtocol) ?? "http"; + public static readonly string LexboxPort = LexboxProtocol == "http" ? "80" : "443"; + public static readonly string LexboxUsername = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_HgUsername) ?? "admin"; + public static readonly string LexboxPassword = Environment.GetEnvironmentVariable(MagicStrings.EnvVar_TrustToken) ?? "pass"; + public static readonly Uri LexboxUrl = new($"{LexboxProtocol}://{LexboxHostname}:{LexboxPort}"); + public static readonly Uri LexboxUrlBasicAuth = new($"{LexboxProtocol}://{WebUtility.UrlEncode(LexboxUsername)}:{WebUtility.UrlEncode(LexboxPassword)}@{LexboxHostname}:{LexboxPort}"); + public static Cookie AdminLoginCookie { get; set; } + public readonly CookieContainer Cookies = new(); + private HttpClientHandler Handler { get; init; } + private Lazy LazyHttp { get; init; } + public HttpClient Http => LazyHttp.Value; + public readonly SIL.Progress.IProgress NullProgress = new SIL.Progress.NullProgress(); + public readonly ILogger NullLogger = new NullLogger(); + private bool AlreadyLoggedIn = false; + private TemporaryFolder TempFolder { get; init; } + private Lazy LazyGqlClient { get; init; } + public GraphQLHttpClient GqlClient => LazyGqlClient.Value; + + public SRTestEnvironment(TemporaryFolder? tempFolder = null) + : base(true, true, true, tempFolder ?? new TemporaryFolder(TestName + Path.GetRandomFileName())) + { + Handler = new() { CookieContainer = Cookies }; + LazyHttp = new(() => new HttpClient(Handler)); + LazyGqlClient = new(() => new GraphQLHttpClient(new Uri(LexboxUrl, "/api/graphql"), new SystemTextJsonSerializer(), Http)); + TempFolder = _languageForgeServerFolder; // Better name for what E2E tests use it for + Settings.CommitWhenDone = true; // For SR tests specifically, we *do* want changes to .fwdata files to be persisted + } + + public async Task Login() + { + if (AlreadyLoggedIn) return; + if (AdminLoginCookie is null) { + await LoginAs(LexboxUsername, LexboxPassword); + } else { + Cookies.Add(AdminLoginCookie); + AlreadyLoggedIn = true; + } + } + + public async Task LoginAs(string lexboxUsername, string lexboxPassword) + { + if (AlreadyLoggedIn) return; + var loginResult = await Http.PostAsync(new Uri(LexboxUrl, "api/login"), JsonContent.Create(new { EmailOrUsername=lexboxUsername, Password=lexboxPassword })); + var cookies = Cookies.GetCookies(LexboxUrl); + AdminLoginCookie = cookies[".LexBoxAuth"]; + AlreadyLoggedIn = true; + // Http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", Jwt); + // Bearer auth on LexBox requires logging in to LexBox via their OAuth flow. For now we'll let the cookie container handle it. + } + + public async Task IsLexBoxAvailable() + { + try { + var httpResponse = await Http.GetAsync(new Uri(LexboxUrl, "api/healthz")); + return httpResponse.IsSuccessStatusCode && httpResponse.Headers.TryGetValues("lexbox-version", out var _ignore); + } catch { return false; } + } + + public static Uri LexboxUrlForProject(string code) => new Uri(LexboxUrl, $"hg/{code}"); + public static Uri LexboxUrlForProjectWithAuth(string code) => new Uri(LexboxUrlBasicAuth, $"hg/{code}"); + + public async Task CreateLexBoxProject(string code, Guid? projId = null, string? name = null, string? description = null, Guid? managerId = null, Guid? orgId = null) + { + projId ??= Guid.NewGuid(); + name ??= code; + description ??= $"Auto-created project for test {TestName}"; + var mutation = """ + mutation createProject($input: CreateProjectInput!) { + createProject(input: $input) { + createProjectResponse { + id + result + } + errors { + ... on DbError { + code + } + } + } + } + """; + var input = new LexboxGraphQLTypes.CreateProjectInput(projId, name, description, code, LexboxGraphQLTypes.ProjectType.FLEx, LexboxGraphQLTypes.RetentionPolicy.Dev, false, managerId, orgId); + var request = new GraphQLRequest { + Query = mutation, + Variables = new { input }, + }; + var gqlResponse = await GqlClient.SendMutationAsync(request); + Assert.That(gqlResponse.Errors, Is.Null.Or.Empty, () => string.Join("\n", gqlResponse.Errors.Select(error => error.Message))); + var response = gqlResponse.Data.CreateProject.CreateProjectResponse; + Assert.That(response.Result, Is.EqualTo(LexboxGraphQLTypes.CreateProjectResult.Created)); + Assert.That(response.Id, Is.EqualTo(projId)); + return response; + } + + public async Task DeleteLexBoxProject(Guid projectId) + { + var mutation = """ + mutation SoftDeleteProject($input: SoftDeleteProjectInput!) { + softDeleteProject(input: $input) { + project { + id + } + errors { + ... on Error { + message + } + } + } + } + """; + var input = new { projectId }; + var request = new GraphQLRequest { + Query = mutation, + Variables = new { input }, + }; + var response = await GqlClient.SendMutationAsync(request); + Assert.That(response.Errors, Is.Null.Or.Empty, () => string.Join("\n", response.Errors.Select(error => error.Message))); + } + + public static void InitRepo(string code, string dest) + { + var sourceUrl = LexboxUrlForProjectWithAuth(code); + MercurialTestHelper.CloneRepo(sourceUrl.AbsoluteUri, dest); + } + + public void InitRepo(string code) => InitRepo(code, Path.Join(TempFolder.Path, code)); + + public async Task ResetAndUploadZip(string code, string zipPath) + { + var resetUrl = new Uri(LexboxUrl, $"api/project/resetProject/{code}"); + await Http.PostAsync(resetUrl, null); + await UploadZip(code, zipPath); + } + + public async Task ResetToEmpty(string code) + { + var resetUrl = new Uri(LexboxUrl, $"api/project/resetProject/{code}"); + await Http.PostAsync(resetUrl, null); + var finishResetUrl = new Uri(LexboxUrl, $"api/project/finishResetProject/{code}"); + await Http.PostAsync(finishResetUrl, null); + } + + public async Task TusUpload(Uri tusEndpoint, string path, string mimeType) + { + var file = new FileInfo(path); + if (!file.Exists) return; + var metadata = new MetadataCollection { { "filetype", mimeType } }; + var createOpts = new TusCreateRequestOption { + Endpoint = tusEndpoint, + UploadLength = file.Length, + Metadata = metadata, + }; + var createResponse = await Http.TusCreateAsync(createOpts); + + // That doesn't actually upload the file; TusPatchAsync does the actual upload + using var fileStream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); + var patchOpts = new TusPatchRequestOption { + FileLocation = createResponse.FileLocation, + Stream = fileStream, + }; + await Http.TusPatchAsync(patchOpts); + } + + public Task UploadZip(string code, string zipPath) + { + var sourceUrl = new Uri(LexboxUrl, $"/api/project/upload-zip/{code}"); + return TusUpload(sourceUrl, zipPath, "application/zip"); + } + + public async Task DownloadProjectBackup(string code, string destZipPath) + { + var backupUrl = new Uri(LexboxUrl, $"api/project/backupProject/{code}"); + var result = await Http.GetAsync(backupUrl); + var filename = result.Content.Headers.ContentDisposition?.FileName; + using (var outStream = File.Create(destZipPath)) + { + await result.Content.CopyToAsync(outStream); + } + } + + public void CommitAndPush(FwProject project, string code, string baseDir, string? localCode = null, string? commitMsg = null) + { + project.Cache.ActionHandlerAccessor.Commit(); + if (!project.IsDisposed) project.Dispose(); + CommitAndPush(code, baseDir, localCode, commitMsg); + } + + public void CommitAndPush(string code, string baseDir, string? localCode = null, string? commitMsg = null) + { + localCode ??= code; + var projUrl = new Uri(LexboxUrl, $"/hg/{code}"); + var withAuth = new UriBuilder(projUrl) { UserName = "admin", Password = "pass" }; + commitMsg ??= "Auto-commit"; + var projectDir = Path.Combine(baseDir, "webwork", localCode); + var fwdataPath = Path.Join(projectDir, $"{localCode}.fwdata"); + LfMergeBridge.LfMergeBridge.DisassembleFwdataFile(NullProgress, false, fwdataPath); + MercurialTestHelper.HgClean(projectDir); // Ensure ConfigurationSettings, etc., don't get committed + MercurialTestHelper.HgCommit(projectDir, commitMsg); + MercurialTestHelper.HgPush(projectDir, withAuth.Uri.AbsoluteUri); + } + + private static string TestName + { + get + { + var testName = TestContext.CurrentContext.Test.Name; + var firstInvalidChar = testName.IndexOfAny(Path.GetInvalidPathChars()); + if (firstInvalidChar >= 0) + testName = testName.Substring(0, firstInvalidChar); + return testName; + } + } + + } +} diff --git a/src/LfMerge.Core.Tests/TestDoubles.cs b/src/LfMerge.Core.Tests/TestDoubles.cs index 01a9a079..86eec1fb 100644 --- a/src/LfMerge.Core.Tests/TestDoubles.cs +++ b/src/LfMerge.Core.Tests/TestDoubles.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Linq.Expressions; +using Autofac; using Bugsnag.Payload; using LfMergeBridge.LfMergeModel; using IniParser.Model; @@ -417,6 +418,10 @@ class ChorusHelperDouble: ChorusHelper { public override string GetSyncUri(ILfProject project) { + var settings = MainClass.Container.Resolve(); + // Allow tests to override LanguageDepotRepoUri if necessary (e.g., the E2E tests which need ChorusHelperDouble but use a "real" LexBox instance) + if (!string.IsNullOrEmpty(settings.LanguageDepotRepoUri)) + return settings.LanguageDepotRepoUri; var server = LanguageDepotMock.Server; return server != null && server.IsStarted ? server.Url : LanguageDepotMock.ProjectFolderPath; } diff --git a/src/LfMerge.Core.Tests/TestEnvironment.cs b/src/LfMerge.Core.Tests/TestEnvironment.cs index cae38056..4fbff2ca 100644 --- a/src/LfMerge.Core.Tests/TestEnvironment.cs +++ b/src/LfMerge.Core.Tests/TestEnvironment.cs @@ -22,9 +22,10 @@ namespace LfMerge.Core.Tests { public class TestEnvironment : IDisposable { - private readonly TemporaryFolder _languageForgeServerFolder; + protected readonly TemporaryFolder _languageForgeServerFolder; private readonly bool _resetLfProjectsDuringCleanup; private readonly bool _releaseSingletons; + public bool DeleteTempFolderDuringCleanup { get; set; } = true; public LfMergeSettings Settings; private readonly MongoConnectionDouble _mongoConnection; public ILogger Logger => MainClass.Logger; @@ -107,7 +108,8 @@ public void Dispose() MainClass.Container = null; if (_resetLfProjectsDuringCleanup) LanguageForgeProjectAccessor.Reset(); - _languageForgeServerFolder?.Dispose(); + if (DeleteTempFolderDuringCleanup) + _languageForgeServerFolder?.Dispose(); Settings = null; if (_releaseSingletons) SingletonsContainer.Release(); diff --git a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoComments.cs b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoComments.cs index b8e35015..eca44e4a 100644 --- a/src/LfMerge.Core/DataConverters/ConvertLcmToMongoComments.cs +++ b/src/LfMerge.Core/DataConverters/ConvertLcmToMongoComments.cs @@ -288,25 +288,38 @@ public ILexEntry OwningEntry(Guid guidOfUnknownLcmObject) } private GetChorusNotesResponse CallLfMergeBridge(List comments, out string bridgeOutput) + { + return CallLfMergeBridge(comments, out bridgeOutput, _project.FwDataPath, _progress, _logger); + } + + public static GetChorusNotesResponse CallLfMergeBridge(List comments, out string bridgeOutput, string fwDataPath, IProgress progress, ILogger logger) { // Call into LF Bridge to do the work. bridgeOutput = string.Empty; var bridgeInput = new LfMergeBridge.GetChorusNotesInput { LfComments = comments }; var options = new Dictionary { - {"-p", _project.FwDataPath}, + {"-p", fwDataPath}, }; LfMergeBridge.LfMergeBridge.ExtraInputData.Add(options, bridgeInput); - if (!LfMergeBridge.LfMergeBridge.Execute("Language_Forge_Get_Chorus_Notes", _progress, + if (!LfMergeBridge.LfMergeBridge.Execute("Language_Forge_Get_Chorus_Notes", progress, options, out bridgeOutput)) { - _logger.Error("Got an error from Language_Forge_Get_Chorus_Notes: {0}", bridgeOutput); + logger.Error("Got an error from Language_Forge_Get_Chorus_Notes: {0}", bridgeOutput); return null; } else { var success = LfMergeBridge.LfMergeBridge.ExtraOutputData.TryGetValue(options, out var outputObject); - return outputObject as GetChorusNotesResponse; + if (success) + { + return outputObject as GetChorusNotesResponse; + } + else + { + logger.Error("Language_Forge_Get_Chorus_Notes failed to return any data. Its output was: {0}", bridgeOutput); + return null; + } } } } diff --git a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmComments.cs b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmComments.cs index 8f70fb4e..47d6f89a 100644 --- a/src/LfMerge.Core/DataConverters/ConvertMongoToLcmComments.cs +++ b/src/LfMerge.Core/DataConverters/ConvertMongoToLcmComments.cs @@ -102,31 +102,44 @@ private LfLexEntry GetLexEntry(ObjectId idOfEntry) } private WriteToChorusNotesResponse CallLfMergeBridge(List> lfComments, out string bridgeOutput) + { + return CallLfMergeBridge(lfComments, out bridgeOutput, _project.FwDataPath, _progress, _logger); + } + + public static WriteToChorusNotesResponse CallLfMergeBridge(List> lfComments, out string bridgeOutput, string fwDataPath, IProgress progress, ILogger logger) { bridgeOutput = string.Empty; var options = new Dictionary { - {"-p", _project.FwDataPath}, + {"-p", fwDataPath}, }; try { var bridgeInput = new WriteToChorusNotesInput { LfComments = lfComments }; LfMergeBridge.LfMergeBridge.ExtraInputData.Add(options, bridgeInput); - if (!LfMergeBridge.LfMergeBridge.Execute("Language_Forge_Write_To_Chorus_Notes", _progress, + if (!LfMergeBridge.LfMergeBridge.Execute("Language_Forge_Write_To_Chorus_Notes", progress, options, out bridgeOutput)) { - _logger.Error("Got an error from Language_Forge_Write_To_Chorus_Notes: {0}", bridgeOutput); + logger.Error("Got an error from Language_Forge_Write_To_Chorus_Notes: {0}", bridgeOutput); return null; } else { var success = LfMergeBridge.LfMergeBridge.ExtraOutputData.TryGetValue(options, out var outputObject); - return outputObject as WriteToChorusNotesResponse; + if (success) + { + return outputObject as WriteToChorusNotesResponse; + } + else + { + logger.Error("Language_Forge_Write_To_Chorus_Notes failed to return any data. Its output was: {0}", bridgeOutput); + return null; + } } } catch (NullReferenceException) { - _logger.Debug("Got an exception. Before rethrowing it, here is what LfMergeBridge sent:"); - _logger.Debug("{0}", bridgeOutput); + logger.Debug("Got an exception. Before rethrowing it, here is what LfMergeBridge sent:"); + logger.Debug("{0}", bridgeOutput); throw; } } diff --git a/src/LfMerge.Core/LanguageForgeProject.cs b/src/LfMerge.Core/LanguageForgeProject.cs index 8f2dec12..6c3991e9 100644 --- a/src/LfMerge.Core/LanguageForgeProject.cs +++ b/src/LfMerge.Core/LanguageForgeProject.cs @@ -20,13 +20,13 @@ public class LanguageForgeProject: ILfProject private readonly string _projectCode; private ILanguageDepotProject _languageDepotProject; - public static LanguageForgeProject Create(string projectCode) + public static LanguageForgeProject Create(string projectCode, LfMergeSettings settings = null) { LanguageForgeProject project; if (CachedProjects.TryGetValue(projectCode, out project)) return project; - project = new LanguageForgeProject(projectCode); + project = new LanguageForgeProject(projectCode, settings); CachedProjects.Add(projectCode, project); return project; } diff --git a/src/LfMerge.Core/MagicStrings.cs b/src/LfMerge.Core/MagicStrings.cs index 25f9cbc6..f0966513 100644 --- a/src/LfMerge.Core/MagicStrings.cs +++ b/src/LfMerge.Core/MagicStrings.cs @@ -30,6 +30,7 @@ static MagicStrings() public const string EnvVar_LanguageDepotPublicHostname = "LFMERGE_LANGUAGE_DEPOT_HG_PUBLIC_HOSTNAME"; public const string EnvVar_LanguageDepotPrivateHostname = "LFMERGE_LANGUAGE_DEPOT_HG_PRIVATE_HOSTNAME"; public const string EnvVar_LanguageDepotUriProtocol = "LFMERGE_LANGUAGE_DEPOT_HG_PROTOCOL"; + public const string EnvVar_LanguageDepotUriPort = "LFMERGE_LANGUAGE_DEPOT_HG_PORT"; public const string EnvVar_TrustToken = "LANGUAGE_DEPOT_TRUST_TOKEN"; public const string EnvVar_HgUsername = "LANGUAGE_DEPOT_HG_USERNAME"; diff --git a/src/LfMerge.Core/Settings/LfMergeSettings.cs b/src/LfMerge.Core/Settings/LfMergeSettings.cs index ac399145..9f253ce3 100644 --- a/src/LfMerge.Core/Settings/LfMergeSettings.cs +++ b/src/LfMerge.Core/Settings/LfMergeSettings.cs @@ -64,7 +64,11 @@ public bool VerboseProgress { } } - public string LanguageDepotRepoUri => Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri) ?? DefaultLfMergeSettings.LanguageDepotRepoUri; + private string _languageDepotRepoUri; + public string LanguageDepotRepoUri { + get => _languageDepotRepoUri ?? Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri) ?? DefaultLfMergeSettings.LanguageDepotRepoUri; + set => _languageDepotRepoUri = value; + } // Settings calculated at runtime from sources other than environment variables