End-to-end S/R tests with Lexbox and Mongo (#342)

This end-to-end test runs against a real Lexbox instance, writing data
to a real MongoDB instance rather than a MongoConnectionDouble. It
creates a new project in Lexbox (using the existing sena-3 project as
the initial data), and clones that project into both a LfProject object
and a FwProject object. The FwProject object uses liblcm code to modify
the data, then pushes that modified data to Lexbox. The LfProject then
makes its own changes and goes through the LfMerge Send/Receive process;
because the LfProject's changes are the "ours" in Chorus, the changes
from the LfProject "win" and overwrite the FwProject's changes.

The "real" MongoDB is provided by using a Docker image of Mongo version
6 (via the `mongo:6` tag). This requires a working copy of Docker in the
environment running the tests.

Although there is only one test created here, the SRTestEnvironment
class contains several helper methods that would be useful in many other
end-to-end test scenarios.

src/LfMerge.Core.Tests/E2E/E2ETestBase.cs

@@ -0,0 +1,234 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading.Tasks;
+using Autofac;
+using LfMerge.Core.Actions;
+using LfMerge.Core.FieldWorks;
+using LfMerge.Core.MongoConnector;
+using NUnit.Framework;
+using NUnit.Framework.Interfaces;
+using SIL.LCModel;
+using SIL.TestUtilities;
+
+namespace LfMerge.Core.Tests.E2E
+{
+	[TestFixture]
+	[Category("LongRunning")]
+	[Category("IntegrationTests")]
+	public class E2ETestBase
+	{
+		public LfMerge.Core.Logging.ILogger Logger => MainClass.Logger;
+		public TemporaryFolder TempFolderForClass { get; set; }
+		public TemporaryFolder TempFolderForTest { get; set; }
+		public string Sena3ZipPath { get; set; }
+		private readonly HashSet<Guid> ProjectIdsToDelete = [];
+		public SRTestEnvironment TestEnv { get; set; }
+
+		public MongoConnectionDouble _mongoConnection;
+		public MongoProjectRecordFactory _recordFactory;
+
+		public E2ETestBase()
+		{
+		}
+
+		private static string TestName => TestContext.CurrentContext.Test.Name;
+		private static string TestNameForPath => string.Concat(TestName.Split(Path.GetInvalidPathChars())); // Easiest way to strip out all invalid chars
+
+		[OneTimeSetUp]
+		public async Task FixtureSetup()
+		{
+			// Ensure top-level /tmp/LfMergeSRTests folder and subfolders exist
+			var tempPath = Path.Join(Path.GetTempPath(), "LfMergeSRTests");
+			Directory.CreateDirectory(tempPath);
+			var testDataPath = Path.Join(tempPath, "data");
+			Directory.CreateDirectory(testDataPath);
+			var lcmDataPath = Path.Join(tempPath, "lcm-common");
+			Directory.CreateDirectory(lcmDataPath);
+			Environment.SetEnvironmentVariable("FW_CommonAppData", lcmDataPath);
+
+			var derivedClassName = this.GetType().Name;
+			TempFolderForClass = new TemporaryFolder(Path.Join(tempPath, derivedClassName));
+
+			// Ensure sena-3.zip is available to all tests as a starting point
+			Sena3ZipPath = Path.Join(testDataPath, "sena-3.zip");
+			if (!File.Exists(Sena3ZipPath)) {
+				using var testEnv = new SRTestEnvironment(TempFolderForTest);
+				if (!await testEnv.IsLexBoxAvailable()) {
+					Assert.Ignore("Can't run E2E tests without a copy of LexBox to test against. Please either launch LexBox on localhost port 80, or set the appropriate environment variables to point to a running copy of LexBox.");
+				}
+				await testEnv.Login();
+				await testEnv.DownloadProjectBackup("sena-3", Sena3ZipPath);
+			}
+		}
+
+		[OneTimeTearDown]
+		public void FixtureTeardown()
+		{
+			Environment.SetEnvironmentVariable("FW_CommonAppData", null);
+			var result = TestContext.CurrentContext.Result;
+			var nonSuccess = result.FailCount + result.InconclusiveCount + result.WarningCount;
+			// Only delete class temp folder if we passed or skipped all tests
+			if (nonSuccess == 0) TempFolderForClass.Dispose();
+		}
+
+		[SetUp]
+		public async Task TestSetup()
+		{
+			TempFolderForTest = new TemporaryFolder(TempFolderForClass, TestNameForPath);
+			TestEnv = new SRTestEnvironment(TempFolderForTest);
+			if (!await TestEnv.IsLexBoxAvailable()) {
+				Assert.Ignore("Can't run E2E tests without a copy of LexBox to test against. Please either launch LexBox on localhost port 80, or set the appropriate environment variables to point to a running copy of LexBox.");
+			}
+			await TestEnv.Login();
+
+			MagicStrings.SetMinimalModelVersion(LcmCache.ModelVersion);
+			_mongoConnection = MainClass.Container.Resolve<IMongoConnection>() as MongoConnectionDouble;
+			if (_mongoConnection == null)
+				throw new AssertionException("E2E tests need a mock MongoConnection that stores data in order to work.");
+			_recordFactory = MainClass.Container.Resolve<MongoProjectRecordFactory>() as MongoProjectRecordFactoryDouble;
+			if (_recordFactory == null)
+				throw new AssertionException("E2E tests need a mock MongoProjectRecordFactory in order to work.");
+		}
+
+		[TearDown]
+		public async Task TestTeardown()
+		{
+			var outcome = TestContext.CurrentContext.Result.Outcome;
+			var success = outcome == ResultState.Success || outcome == ResultState.Ignored;
+			// Only delete temp folder if test passed, otherwise we'll want to leave it in place for post-test investigation
+			TestEnv.DeleteTempFolderDuringCleanup = success;
+			// On failure, also leave LexBox project(s) in place for post-test investigation, even though this might tend to clutter things up a little
+			if (success) {
+				foreach (var projId in ProjectIdsToDelete) {
+					await TestEnv.DeleteLexBoxProject(projId);
+				}
+			}
+			ProjectIdsToDelete.Clear();
+			TestEnv.Dispose();
+		}
+
+		public string TestFolderForProject(string projectCode) => Path.Join(TempFolderForTest.Path, "webwork", projectCode);
+		public string FwDataPathForProject(string projectCode) => Path.Join(TestFolderForProject(projectCode), $"{projectCode}.fwdata");
+
+		public string CloneRepoFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null)
+		{
+			var projUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(code);
+			newCode ??= code;
+			var dest = TestFolderForProject(newCode);
+			if (waitTime is null) {
+				MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest);
+			} else {
+				var start = DateTime.UtcNow;
+				var success = false;
+				while (!success) {
+					try {
+						MercurialTestHelper.CloneRepo(projUrl.AbsoluteUri, dest);
+					} catch {
+						if (DateTime.UtcNow > start + waitTime) {
+							throw; // Give up
+						}
+						System.Threading.Thread.Sleep(250);
+						continue;
+					}
+					// If we got this far, no exception so we succeeded
+					success = true;
+				}
+			}
+			return dest;
+		}
+
+		public FwProject CloneFwProjectFromLexbox(string code, string? newCode = null, TimeSpan? waitTime = null)
+		{
+			var dest = CloneRepoFromLexbox(code, newCode, waitTime);
+			var dirInfo = new DirectoryInfo(dest);
+			if (!dirInfo.Exists) throw new InvalidOperationException($"Failed to clone {code} from lexbox, cannot create FwProject");
+			var dirname = dirInfo.Name;
+			var fwdataPath = Path.Join(dest, $"{dirname}.fwdata");
+			MercurialTestHelper.ChangeBranch(dest, "tip");
+			LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(TestEnv.NullProgress, false, fwdataPath);
+			var settings = new LfMergeSettingsDouble(TempFolderForTest.Path);
+			return new FwProject(settings, dirname);
+		}
+
+		public async Task<string> CreateEmptyFlexProjectInLexbox()
+		{
+			var randomGuid = Guid.NewGuid();
+			var testCode = $"sr-{randomGuid}";
+			var testPath = TestFolderForProject(testCode);
+			MercurialTestHelper.InitializeHgRepo(testPath);
+			MercurialTestHelper.CreateFlexRepo(testPath);
+			// Now create project in LexBox
+			var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid);
+			var pushUrl = SRTestEnvironment.LexboxUrlForProjectWithAuth(testCode).AbsoluteUri;
+			MercurialTestHelper.HgPush(testPath, pushUrl);
+			if (result.Id.HasValue) ProjectIdsToDelete.Add(result.Id.Value);
+			return testCode;
+		}
+
+		public async Task<string> CreateNewProjectFromTemplate(string origZipPath)
+		{
+			var randomGuid = Guid.NewGuid();
+			var testCode = $"sr-{randomGuid}";
+			var testPath = TestFolderForProject(testCode);
+			// Now create project in LexBox
+			var result = await TestEnv.CreateLexBoxProject(testCode, randomGuid);
+			await TestEnv.ResetAndUploadZip(testCode, origZipPath);
+			if (result.Id.HasValue) ProjectIdsToDelete.Add(result.Id.Value);
+			return testCode;
+		}
+
+		public Task<string> CreateNewProjectFromSena3() => CreateNewProjectFromTemplate(Sena3ZipPath);
+
+		public void CommitAndPush(FwProject project, string code, string? localCode = null, string? commitMsg = null)
+		{
+			TestEnv.CommitAndPush(project, code, TempFolderForTest.Path, localCode, commitMsg);
+		}
+
+		public async Task<LanguageForgeProject> CreateLfProjectFromSena3()
+		{
+			var projCode = await CreateNewProjectFromSena3();
+			var projPath = CloneRepoFromLexbox(projCode, waitTime:TimeSpan.FromSeconds(5));
+			MercurialTestHelper.ChangeBranch(projPath, "tip");
+			var fwdataPath = Path.Join(projPath, $"{projCode}.fwdata");
+			LfMergeBridge.LfMergeBridge.ReassembleFwdataFile(TestEnv.NullProgress, false, fwdataPath);
+
+			// Do an initial clone from LexBox to the mock LF
+			var lfProject = LanguageForgeProject.Create(projCode, TestEnv.Settings);
+			lfProject.IsInitialClone = true;
+			var transferLcmToMongo = new TransferLcmToMongoAction(TestEnv.Settings, TestEnv.NullLogger, _mongoConnection, _recordFactory);
+			transferLcmToMongo.Run(lfProject);
+
+			return lfProject;
+		}
+
+		public void SendReceiveToLexbox(LanguageForgeProject lfProject)
+		{
+			TestEnv.Settings.LanguageDepotRepoUri = SRTestEnvironment.LexboxUrlForProjectWithAuth(lfProject.ProjectCode).AbsoluteUri;
+			var syncAction = new SynchronizeAction(TestEnv.Settings, TestEnv.Logger);
+			syncAction.Run(lfProject);
+		}
+
+		public (string, DateTime, DateTime) UpdateFwGloss(FwProject project, Guid entryId, Func<string, string> textConverter)
+		{
+			var fwEntry = LcmTestHelper.GetEntry(project, entryId);
+			Assert.That(fwEntry, Is.Not.Null);
+			var origModifiedDate = fwEntry.DateModified;
+			var unchangedGloss = LcmTestHelper.UpdateAnalysisText(project, fwEntry.SensesOS[0].Gloss, textConverter);
+			return (unchangedGloss, origModifiedDate, fwEntry.DateModified);
+		}
+
+		public (string, DateTime, DateTime) UpdateLfGloss(LanguageForgeProject lfProject, Guid entryId, string wsId, Func<string, string> textConverter)
+		{
+			var lfEntry = _mongoConnection.GetLfLexEntryByGuid(entryId);
+			Assert.That(lfEntry, Is.Not.Null);
+			var unchangedGloss = lfEntry.Senses[0].Gloss[wsId].Value;
+			lfEntry.Senses[0].Gloss["pt"].Value = textConverter(unchangedGloss);
+			// Remember that in LfMerge it's AuthorInfo that corresponds to the Lcm modified date
+			DateTime origModifiedDate = lfEntry.AuthorInfo.ModifiedDate;
+			lfEntry.AuthorInfo.ModifiedDate = DateTime.UtcNow;
+			_mongoConnection.UpdateRecord(lfProject, lfEntry);
+			return (unchangedGloss, origModifiedDate, lfEntry.AuthorInfo.ModifiedDate);
+		}
+	}
+}

src/LfMerge.Core.Tests/E2E/LexboxSendReceiveTests.cs

@@ -0,0 +1,61 @@
+using System;
+using System.Threading.Tasks;
+using NUnit.Framework;
+using System.Text.RegularExpressions;
+
+namespace LfMerge.Core.Tests.E2E
+{
+	[TestFixture]
+	[Category("LongRunning")]
+	[Category("IntegrationTests")]
+	public class LexboxSendReceiveTests : E2ETestBase
+	{
+		// This test will often trigger a race condition in LexBox that causes the *next* test to fail
+		// when `hg clone` returns 404. This is because of hgweb's directory cache, which only refreshes
+		// if it hasn't been refreshed more than N seconds ago (default 20, LexBox currently uses 5).
+		// Which means that even though CreateLfProjectFromSena3 has created the LexBox project, LexBox's
+		// copy of hgweb doesn't see it yet so it can't be cloned.
+		//
+		// The solution will be to adjust CloneRepoFromLexbox to take a parameter that is the number of
+		// seconds to wait for the project to become visible, and retry 404's until that much time has elapsed.
+		// Then only throw an exception if hgweb is still returning 404 after its dir cache should be refreshed.
+		[Test]
+		public async Task E2E_CheckFwProjectCreation()
+		{
+			var code = await CreateEmptyFlexProjectInLexbox();
+			Console.WriteLine($"Created new project {code}");
+		}
+
+		[Test]
+		public async Task E2E_LFDataChangedLDDataChanged_LFWins()
+		{
+			// Setup
+
+			var lfProject = await CreateLfProjectFromSena3();
+			var fwProjectCode = Regex.Replace(lfProject.ProjectCode, "^sr-", "fw-");
+			var fwProject = CloneFwProjectFromLexbox(lfProject.ProjectCode, fwProjectCode);
+
+			// Modify FW data first, then push to Lexbox
+			Guid entryId = LcmTestHelper.GetFirstEntry(fwProject).Guid;
+			var (unchangedGloss, origFwDateModified, fwDateModified) = UpdateFwGloss(fwProject, entryId, text => text + " - changed in FW");
+			CommitAndPush(fwProject, lfProject.ProjectCode, fwProjectCode, "Modified gloss in FW");
+
+			// Modify LF data second
+			var (_, origLfDateModified, _) = UpdateLfGloss(lfProject, entryId, "pt", text => text + " - changed in LF");
+
+			// Exercise
+
+			SendReceiveToLexbox(lfProject);
+
+			// Verify
+
+			// LF side should win conflict since its modified date was later
+			var lfEntryAfterSR = _mongoConnection.GetLfLexEntryByGuid(entryId);
+			Assert.That(lfEntryAfterSR?.Senses?[0]?.Gloss?["pt"]?.Value, Is.EqualTo(unchangedGloss + " - changed in LF"));
+			// LF's modified dates should have been updated by the sync action
+			Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(origLfDateModified));
+			// Remember that FieldWorks's DateModified is stored in local time for some incomprehensible reason...
+			Assert.That(lfEntryAfterSR.AuthorInfo.ModifiedDate, Is.GreaterThan(fwDateModified.ToUniversalTime()));
+		}
+	}
+}

src/LfMerge.Core.Tests/LcmTestHelper.cs

@@ -0,0 +1,83 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using LfMerge.Core.FieldWorks;
+using SIL.LCModel;
+using SIL.LCModel.Infrastructure;
+
+namespace LfMerge.Core.Tests
+{
+	public static class LcmTestHelper
+	{
+		public static IEnumerable<ILexEntry> GetEntries(FwProject project)
+		{
+			return project?.ServiceLocator?.LanguageProject?.LexDbOA?.Entries ?? [];
+		}
+
+		public static int CountEntries(FwProject project)
+		{
+			var repo = project?.ServiceLocator?.GetInstance<ILexEntryRepository>();
+			return repo.Count;
+		}
+
+		public static ILexEntry GetEntry(FwProject project, Guid guid)
+		{
+			var repo = project?.ServiceLocator?.GetInstance<ILexEntryRepository>();
+			return repo.GetObject(guid);
+		}
+
+		public static ILexEntry GetFirstEntry(FwProject project)
+		{
+			var repo = project?.ServiceLocator?.GetInstance<ILexEntryRepository>();
+			return repo.AllInstances().First();
+		}
+
+		public static string? GetVernacularText(IMultiUnicode field)
+		{
+			return field.BestVernacularAlternative?.Text;
+		}
+
+		public static string? GetAnalysisText(IMultiUnicode field)
+		{
+			return field.BestAnalysisAlternative?.Text;
+		}
+
+		public static void SetVernacularText(FwProject project, IMultiUnicode field, string newText)
+		{
+			var accessor = project.Cache.ActionHandlerAccessor;
+			UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", accessor, () => {
+				field.SetVernacularDefaultWritingSystem(newText);
+			});
+		}
+
+		public static void SetAnalysisText(FwProject project, IMultiUnicode field, string newText)
+		{
+			var accessor = project.Cache.ActionHandlerAccessor;
+			UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("undo", "redo", accessor, () => {
+				field.SetAnalysisDefaultWritingSystem(newText);
+			});
+		}
+
+		public static string UpdateVernacularText(FwProject project, IMultiUnicode field, Func<string, string> textConverter)
+		{
+			var oldText = field.BestVernacularAlternative?.Text;
+			if (oldText != null)
+			{
+				var newText = textConverter(oldText);
+				SetVernacularText(project, field, newText);
+			}
+			return oldText;
+		}
+
+		public static string UpdateAnalysisText(FwProject project, IMultiUnicode field, Func<string, string> textConverter)
+		{
+			var oldText = field.BestAnalysisAlternative?.Text;
+			if (oldText != null)
+			{
+				var newText = textConverter(oldText);
+				SetAnalysisText(project, field, newText);
+			}
+			return oldText;
+		}
+	}
+}