From f72f3e092dbe51a30191a59ac0522ac71f2bc00f Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 28 Oct 2024 13:19:28 +0700 Subject: [PATCH] Standardize FW Lite ProjectData.Id (#1170) * standardize on ProjectData.Id being the lexbox Id, add a property to track the FieldWorks project Id. when doing a sync the fw project ids must match otherwise there's an error * use OpenCrdtProject helper in SyncFixture isolate SyncFailsWithMismatchedProjectIds test to not break other tests * Update expected data model in snapshot tests --------- Co-authored-by: Robin Munn --- backend/CrdtMerge/Program.cs | 2 +- .../Fixtures/SyncFixture.cs | 20 +++++++++-------- .../SyncFixtureTests.cs | 2 +- .../FwLiteProjectSync.Tests/SyncTests.cs | 22 +++++++++++++++++++ .../CrdtFwdataProjectSyncService.cs | 4 ++++ backend/FwLite/FwLiteProjectSync/Program.cs | 2 +- ...elSnapshotTests.VerifyDbModel.verified.txt | 1 + backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs | 2 +- backend/FwLite/LcmCrdt/CrdtProject.cs | 10 ++++++++- .../FwLite/LcmCrdt/CurrentProjectService.cs | 3 ++- backend/FwLite/LcmCrdt/ProjectsService.cs | 5 +++-- .../Services/ImportFwdataService.cs | 4 ++-- 12 files changed, 58 insertions(+), 19 deletions(-) diff --git a/backend/CrdtMerge/Program.cs b/backend/CrdtMerge/Program.cs index 674aa6dfd..030692c48 100644 --- a/backend/CrdtMerge/Program.cs +++ b/backend/CrdtMerge/Program.cs @@ -74,7 +74,7 @@ // var crdtProject = projectsService.GetProject(crdtProjectName); var crdtProject = File.Exists(crdtFile) ? new CrdtProject(projectCode, crdtFile) : // TODO: use projectName (once we have it) instead of projectCode here - await projectsService.CreateProject(new(projectCode, fwdataApi.ProjectId, SeedNewProjectData: false, Path: projectFolder)); + await projectsService.CreateProject(new(projectCode, SeedNewProjectData: false, Path: projectFolder, FwProjectId: fwdataApi.ProjectId)); var miniLcmApi = await services.OpenCrdtProject(crdtProject); var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun); logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges); diff --git a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs index e658e760f..944de1834 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs @@ -1,4 +1,5 @@ -using FwDataMiniLcmBridge; +using System.Runtime.CompilerServices; +using FwDataMiniLcmBridge; using FwDataMiniLcmBridge.Api; using FwDataMiniLcmBridge.LcmUtils; using FwDataMiniLcmBridge.Tests.Fixtures; @@ -17,15 +18,18 @@ public class SyncFixture : IAsyncLifetime public CrdtFwdataProjectSyncService SyncService => _services.ServiceProvider.GetRequiredService(); + public IServiceProvider Services => _services.ServiceProvider; private readonly string _projectName; + private readonly MockProjectContext _projectContext = new(null); - public static SyncFixture Create(string projectName) => new(projectName); + public static SyncFixture Create([CallerMemberName] string projectName = "") => new(projectName); private SyncFixture(string projectName) { _projectName = projectName; var crdtServices = new ServiceCollection() .AddLcmCrdtClient() + .AddSingleton(_projectContext) .AddTestFwDataBridge() .AddFwLiteProjectSync() .Configure(c => c.ProjectsFolder = Path.Combine(".", _projectName, "FwData")) @@ -45,9 +49,8 @@ public async Task InitializeAsync() .ProjectsFolder; if (Path.Exists(projectsFolder)) Directory.Delete(projectsFolder, true); Directory.CreateDirectory(projectsFolder); - var lcmCache = _services.ServiceProvider.GetRequiredService() + _services.ServiceProvider.GetRequiredService() .NewProject(new FwDataProject(_projectName, projectsFolder), "en", "fr"); - var projectGuid = lcmCache.LanguageProject.Guid; FwDataApi = _services.ServiceProvider.GetRequiredService().GetFwDataMiniLcmApi(_projectName, false); var crdtProjectsFolder = @@ -55,9 +58,8 @@ public async Task InitializeAsync() if (Path.Exists(crdtProjectsFolder)) Directory.Delete(crdtProjectsFolder, true); Directory.CreateDirectory(crdtProjectsFolder); var crdtProject = await _services.ServiceProvider.GetRequiredService() - .CreateProject(new(_projectName, projectGuid)); - _services.ServiceProvider.GetRequiredService().Project = crdtProject; - CrdtApi = _services.ServiceProvider.GetRequiredService(); + .CreateProject(new(_projectName, FwProjectId: FwDataApi.ProjectId)); + CrdtApi = (CrdtMiniLcmApi) await _services.ServiceProvider.OpenCrdtProject(crdtProject); } public async Task DisposeAsync() @@ -65,11 +67,11 @@ public async Task DisposeAsync() await _services.DisposeAsync(); } - public IMiniLcmApi CrdtApi { get; set; } = null!; + public CrdtMiniLcmApi CrdtApi { get; set; } = null!; public FwDataMiniLcmApi FwDataApi { get; set; } = null!; } -public class MockProjectContext(CrdtProject project) : ProjectContext +public class MockProjectContext(CrdtProject? project) : ProjectContext { public override CrdtProject? Project { get; set; } = project; } diff --git a/backend/FwLite/FwLiteProjectSync.Tests/SyncFixtureTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/SyncFixtureTests.cs index 1edc7d1dd..f4515dc6c 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/SyncFixtureTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/SyncFixtureTests.cs @@ -7,7 +7,7 @@ public class SyncFixtureTests [Fact] public async Task CanStart() { - var fixture = SyncFixture.Create("test-sync-fixture"); + var fixture = SyncFixture.Create(); await fixture.InitializeAsync(); await fixture.DisposeAsync(); } diff --git a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs index c51f1022d..7bb3a08da 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs @@ -1,4 +1,7 @@ using FwLiteProjectSync.Tests.Fixtures; +using LcmCrdt; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; using MiniLcm; using MiniLcm.Models; using SystemTextJsonPatch; @@ -83,6 +86,25 @@ public async Task FirstSyncJustDoesAnImport() .For(e => e.ComplexForms).Exclude(c => c.Id)); } + [Fact] + public static async Task SyncFailsWithMismatchedProjectIds() + { + var fixture = SyncFixture.Create(); + await fixture.InitializeAsync(); + var crdtApi = fixture.CrdtApi; + var fwdataApi = fixture.FwDataApi; + await fixture.SyncService.Sync(crdtApi, fwdataApi); + + var newFwProjectId = Guid.NewGuid(); + await fixture.Services.GetRequiredService().ProjectData. + ExecuteUpdateAsync(updates => updates.SetProperty(p => p.FwProjectId, newFwProjectId)); + await fixture.Services.GetRequiredService().PopulateProjectDataCache(force: true); + + Func syncTask = async () => await fixture.SyncService.Sync(crdtApi, fwdataApi); + await syncTask.Should().ThrowAsync(); + await fixture.DisposeAsync(); + } + [Fact] public async Task CreatingAnEntryInEachProjectSyncsAcrossBoth() { diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index ca8decd69..8b7278591 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -16,6 +16,10 @@ public record SyncResult(int CrdtChanges, int FwdataChanges); public async Task Sync(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataApi, bool dryRun = false) { + if (crdtApi is CrdtMiniLcmApi crdt && crdt.ProjectData.FwProjectId != fwdataApi.ProjectId) + { + throw new InvalidOperationException($"Project id mismatch, CRDT Id: {crdt.ProjectData.FwProjectId}, FWData Id: {fwdataApi.ProjectId}"); + } var projectSnapshot = await GetProjectSnapshot(fwdataApi.Project.Name, fwdataApi.Project.ProjectsPath); SyncResult result = await Sync(crdtApi, fwdataApi, dryRun, fwdataApi.EntryCount, projectSnapshot); diff --git a/backend/FwLite/FwLiteProjectSync/Program.cs b/backend/FwLite/FwLiteProjectSync/Program.cs index 18e7dee48..e59799d28 100644 --- a/backend/FwLite/FwLiteProjectSync/Program.cs +++ b/backend/FwLite/FwLiteProjectSync/Program.cs @@ -56,7 +56,7 @@ public static Task Main(string[] args) var crdtProject = projectsService.GetProject(crdtProjectName); if (crdtProject is null) { - crdtProject = await projectsService.CreateProject(new(crdtProjectName, fwdataApi.ProjectId, SeedNewProjectData: false)); + crdtProject = await projectsService.CreateProject(new(crdtProjectName, FwProjectId:fwdataApi.ProjectId, SeedNewProjectData: false)); } var syncService = services.GetRequiredService(); diff --git a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt index 18a0d6aca..a7b2cc8c5 100644 --- a/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt +++ b/backend/FwLite/LcmCrdt.Tests/DataModelSnapshotTests.VerifyDbModel.verified.txt @@ -3,6 +3,7 @@ Properties: Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd ClientId (Guid) Required + FwProjectId (Guid?) Name (string) Required OriginDomain (string) Keys: diff --git a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs index 2a4b69461..0bc0c8ead 100644 --- a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs +++ b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs @@ -13,7 +13,7 @@ namespace LcmCrdt; public class CrdtMiniLcmApi(DataModel dataModel, CurrentProjectService projectService) : IMiniLcmApi { private Guid ClientId { get; } = projectService.ProjectData.ClientId; - + public ProjectData ProjectData => projectService.ProjectData; private IQueryable Entries => dataModel.GetLatestObjects(); private IQueryable ComplexFormComponents => dataModel.GetLatestObjects(); diff --git a/backend/FwLite/LcmCrdt/CrdtProject.cs b/backend/FwLite/LcmCrdt/CrdtProject.cs index 736c0df5f..1dce69d9b 100644 --- a/backend/FwLite/LcmCrdt/CrdtProject.cs +++ b/backend/FwLite/LcmCrdt/CrdtProject.cs @@ -8,7 +8,15 @@ public class CrdtProject(string name, string dbPath) : IProjectIdentifier public ProjectData? Data { get; set; } } -public record ProjectData(string Name, Guid Id, string? OriginDomain, Guid ClientId) +/// +/// +/// +/// Name of the project +/// Id, consistent across all clients, matches the project Id in Lexbox +/// Server to sync with, null if not synced +/// Unique id for this client machine +/// FieldWorks project id, aka LangProjectId +public record ProjectData(string Name, Guid Id, string? OriginDomain, Guid ClientId, Guid? FwProjectId = null) { public static string? GetOriginDomain(Uri? uri) { diff --git a/backend/FwLite/LcmCrdt/CurrentProjectService.cs b/backend/FwLite/LcmCrdt/CurrentProjectService.cs index 1a00a5f8d..336ad465c 100644 --- a/backend/FwLite/LcmCrdt/CurrentProjectService.cs +++ b/backend/FwLite/LcmCrdt/CurrentProjectService.cs @@ -45,8 +45,9 @@ private static string CacheKey(Guid projectId) return memoryCache.Get(CacheKey(projectId)); } - public async ValueTask PopulateProjectDataCache() + public async ValueTask PopulateProjectDataCache(bool force = false) { + if (force) RemoveProjectDataCache(); var projectData = await GetProjectData(); return projectData; } diff --git a/backend/FwLite/LcmCrdt/ProjectsService.cs b/backend/FwLite/LcmCrdt/ProjectsService.cs index c92802c1b..4a9950e8b 100644 --- a/backend/FwLite/LcmCrdt/ProjectsService.cs +++ b/backend/FwLite/LcmCrdt/ProjectsService.cs @@ -39,7 +39,8 @@ public record CreateProjectRequest( Uri? Domain = null, Func? AfterCreate = null, bool SeedNewProjectData = true, - string? Path = null); + string? Path = null, + Guid? FwProjectId = null); public async Task CreateProject(CreateProjectRequest request) { @@ -55,7 +56,7 @@ public async Task CreateProject(CreateProjectRequest request) var projectData = new ProjectData(name, request.Id ?? Guid.NewGuid(), ProjectData.GetOriginDomain(request.Domain), - Guid.NewGuid()); + Guid.NewGuid(), request.FwProjectId); await InitProjectDb(db, projectData); await serviceScope.ServiceProvider.GetRequiredService().PopulateProjectDataCache(); if (request.SeedNewProjectData) diff --git a/backend/FwLite/LocalWebApp/Services/ImportFwdataService.cs b/backend/FwLite/LocalWebApp/Services/ImportFwdataService.cs index 35511a3c4..20f6ab04c 100644 --- a/backend/FwLite/LocalWebApp/Services/ImportFwdataService.cs +++ b/backend/FwLite/LocalWebApp/Services/ImportFwdataService.cs @@ -25,18 +25,18 @@ public async Task Import(string projectName) } try { + using var fwDataApi = fwDataFactory.GetFwDataMiniLcmApi(fwDataProject, false); var project = await projectsService.CreateProject(new(fwDataProject.Name, SeedNewProjectData: false, + FwProjectId: fwDataApi.ProjectId, AfterCreate: async (provider, project) => { - using var fwDataApi = fwDataFactory.GetFwDataMiniLcmApi(fwDataProject, false); var crdtApi = provider.GetRequiredService(); await miniLcmImport.ImportProject(crdtApi, fwDataApi, fwDataApi.EntryCount); })); var timeSpent = Stopwatch.GetElapsedTime(startTime); logger.LogInformation("Import of {ProjectName} complete, took {TimeSpend}", fwDataProject.Name, timeSpent.Humanize(2)); return project; - } catch {