CRDT sync handles semantic domains (#1217)

* Add failing sync test for semantic domains * Implement semantic domain sync - Add GetSemanticDomain() to read API - CreateSemanticDomain now returns the created object - Add UpdateSemanticDomain() to write API - Add DeleteSemanticDomain() to write API - Add UpdateSemanticDomainProxy class - Add SemanticDomainSync class - Sync semantic domains in CRDT-FW sync service * Get sync test passing * Add new test for semdoms syncing in entries * Fix Predefined inconsistency for semantic domains --------- Co-authored-by: Kevin Hahn <[email protected]>
sillsdev · Nov 11, 2024 · 85f9555 · 85f9555
1 parent fd07169
commit 85f9555
Show file tree

Hide file tree

Showing 13 changed files with 325 additions and 41 deletions.
diff --git a/backend/FwHeadless/FwHeadless.csproj b/backend/FwHeadless/FwHeadless.csproj
@@ -24,6 +24,8 @@
   </ItemGroup>
 
   <ItemGroup>
+      <!-- json files get imported by default because this is a web project, so exclude those here so they aren't imported again below -->
+      <Content Remove="Mercurial\contrib\asv.conf.json"/>
       <Content Include="Mercurial\**" CopyToOutputDirectory="Always" Watch="false" />
       <Content Include="MercurialExtensions\**" CopyToOutputDirectory="Always" Watch="false" />
   </ItemGroup>

diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs
@@ -1,4 +1,4 @@
-using System.Collections.Frozen;
+using System.Collections.Frozen;
 using System.Reflection;
 using System.Text;
 using FwDataMiniLcmBridge.Api.UpdateProxy;
@@ -246,22 +246,34 @@ public Task DeletePartOfSpeech(Guid id)
         return Task.CompletedTask;
     }
 
+    internal SemanticDomain FromLcmSemanticDomain(ICmSemanticDomain semanticDomain)
+    {
+        return new SemanticDomain
+        {
+            Id = semanticDomain.Guid,
+            Name = FromLcmMultiString(semanticDomain.Name),
+            Code = semanticDomain.Abbreviation.UiString ?? "",
+            Predefined = true, // TODO: Look up in a GUID list of predefined data
+        };
+    }
+
     public IAsyncEnumerable<SemanticDomain> GetSemanticDomains()
     {
         return
             SemanticDomainRepository
             .AllInstances()
             .OrderBy(p => p.Abbreviation.UiString)
             .ToAsyncEnumerable()
-            .Select(semanticDomain => new SemanticDomain
-            {
-                Id = semanticDomain.Guid,
-                Name = FromLcmMultiString(semanticDomain.Name),
-                Code = semanticDomain.Abbreviation.UiString ?? ""
-            });
+            .Select(FromLcmSemanticDomain);
     }
 
-    public Task CreateSemanticDomain(SemanticDomain semanticDomain)
+    public Task<SemanticDomain?> GetSemanticDomain(Guid id)
+    {
+        var semDom = GetLcmSemanticDomain(id);
+        return Task.FromResult(semDom is null ? null : FromLcmSemanticDomain(semDom));
+    }
+
+    public async Task<SemanticDomain> CreateSemanticDomain(SemanticDomain semanticDomain)
     {
         if (semanticDomain.Id == Guid.Empty) semanticDomain.Id = Guid.NewGuid();
         UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Create Semantic Domain",
@@ -273,8 +285,35 @@ public Task CreateSemanticDomain(SemanticDomain semanticDomain)
                     .Create(semanticDomain.Id, Cache.LangProject.SemanticDomainListOA);
                 lcmSemanticDomain.OcmCodes = semanticDomain.Code;
                 UpdateLcmMultiString(lcmSemanticDomain.Name, semanticDomain.Name);
+                // TODO: Find out if semantic domains are guaranteed to have an "en" writing system, or if we should use lcmCache.DefautlAnalWs instead
                 UpdateLcmMultiString(lcmSemanticDomain.Abbreviation, new MultiString(){{"en", semanticDomain.Code}});
             });
+        return await GetSemanticDomain(semanticDomain.Id) ?? throw new InvalidOperationException("Semantic domain was not created");
+    }
+
+    public Task<SemanticDomain> UpdateSemanticDomain(Guid id, UpdateObjectInput<SemanticDomain> update)
+    {
+        var lcmSemanticDomain = SemanticDomainRepository.GetObject(id);
+        UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Update Semantic Domain",
+            "Revert Semantic Domain",
+            Cache.ServiceLocator.ActionHandler,
+            () =>
+            {
+                var updateProxy = new UpdateSemanticDomainProxy(lcmSemanticDomain, this);
+                update.Apply(updateProxy);
+            });
+        return Task.FromResult(FromLcmSemanticDomain(lcmSemanticDomain));
+    }
+
+    public Task DeleteSemanticDomain(Guid id)
+    {
+        UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Delete Semantic Domain",
+            "Revert delete",
+            Cache.ServiceLocator.ActionHandler,
+            () =>
+            {
+                SemanticDomainRepository.GetObject(id).Delete();
+            });
         return Task.CompletedTask;
     }
 
@@ -428,12 +467,7 @@ private Sense FromLexSense(ILexSense sense)
             Definition = FromLcmMultiString(sense.Definition),
             PartOfSpeech = sense.MorphoSyntaxAnalysisRA?.GetPartOfSpeech()?.Name.get_String(enWs).Text ?? "",
             PartOfSpeechId = sense.MorphoSyntaxAnalysisRA?.GetPartOfSpeech()?.Guid,
-            SemanticDomains = sense.SemanticDomainsRC.Select(s => new SemanticDomain
-            {
-                Id = s.Guid,
-                Name = FromLcmMultiString(s.Name),
-                Code = s.OcmCodes
-            }).ToList(),
+            SemanticDomains = sense.SemanticDomainsRC.Select(FromLcmSemanticDomain).ToList(),
             ExampleSentences = sense.ExamplesOS.Select(sentence => FromLexExampleSentence(sense.Guid, sentence)).ToList()
         };
         return s;

diff --git a/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateSemanticDomainProxy.cs b/backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateSemanticDomainProxy.cs
@@ -0,0 +1,30 @@
+using System.Diagnostics.CodeAnalysis;
+using MiniLcm.Models;
+using SIL.LCModel;
+
+namespace FwDataMiniLcmBridge.Api.UpdateProxy;
+
+public class UpdateSemanticDomainProxy : SemanticDomain
+{
+    private readonly ICmSemanticDomain _lcmSemanticDomain;
+    private readonly FwDataMiniLcmApi _lexboxLcmApi;
+
+    public UpdateSemanticDomainProxy(ICmSemanticDomain lcmSemanticDomain, FwDataMiniLcmApi lexboxLcmApi)
+    {
+        _lcmSemanticDomain = lcmSemanticDomain;
+        Id = lcmSemanticDomain.Guid;
+        _lexboxLcmApi = lexboxLcmApi;
+    }
+
+    public override MultiString Name
+    {
+        get => new UpdateMultiStringProxy(_lcmSemanticDomain.Name, _lexboxLcmApi);
+        set => throw new NotImplementedException();
+    }
+
+    public override string Code
+    {
+        get => _lcmSemanticDomain.Abbreviation.BestAnalysisVernacularAlternative.Text;
+        set => throw new NotImplementedException();
+    }
+}
diff --git a/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/SyncTests.cs
@@ -213,6 +213,85 @@ await crdtApi.CreateEntry(new Entry()
                 .For(e => e.ComplexForms).Exclude(c => c.Id));
     }
 
+    [Fact]
+    public async Task SemanticDomainsSyncBothWays()
+    {
+        var crdtApi = _fixture.CrdtApi;
+        var fwdataApi = _fixture.FwDataApi;
+        await _syncService.Sync(crdtApi, fwdataApi);
+
+        var semdom3 = new SemanticDomain()
+        {
+            Id = new Guid("f4491f9b-3c5e-42ab-afc0-f22e19d0fff5"),
+            Name = new MultiString() { { "en", "Language and thought" } },
+            Code = "3",
+            Predefined = true,
+        };
+        await fwdataApi.CreateSemanticDomain(semdom3);
+
+        var semdom4 = new SemanticDomain()
+        {
+            Id = new Guid("62b4ae33-f3c2-447a-9ef7-7e41805b6a02"),
+            Name = new MultiString() { { "en", "Social behavior" } },
+            Code = "4",
+            Predefined = true,
+        };
+        await crdtApi.CreateSemanticDomain(semdom4);
+
+        await _syncService.Sync(crdtApi, fwdataApi);
+
+        var crdtSemanticDomains = await crdtApi.GetSemanticDomains().ToArrayAsync();
+        var fwdataSemanticDomains = await fwdataApi.GetSemanticDomains().ToArrayAsync();
+        crdtSemanticDomains.Should().ContainEquivalentOf(semdom3);
+        crdtSemanticDomains.Should().ContainEquivalentOf(semdom4);
+        fwdataSemanticDomains.Should().ContainEquivalentOf(semdom3);
+        fwdataSemanticDomains.Should().ContainEquivalentOf(semdom4);
+
+        crdtSemanticDomains.Should().BeEquivalentTo(fwdataSemanticDomains);
+    }
+
+    [Fact]
+    public async Task SemanticDomainsSyncInEntries()
+    {
+        var crdtApi = _fixture.CrdtApi;
+        var fwdataApi = _fixture.FwDataApi;
+        await _syncService.Sync(crdtApi, fwdataApi);
+
+        var semdom3 = new SemanticDomain()
+        {
+            Id = new Guid("f4491f9b-3c5e-42ab-afc0-f22e19d0fff5"),
+            Name = new MultiString() { { "en", "Language and thought" } },
+            Code = "3",
+            Predefined = true,
+        };
+        await fwdataApi.CreateSemanticDomain(semdom3);
+        // Note we do *not* call crdtApi.CreateSemanticDomain(semdom3);
+
+        await fwdataApi.CreateEntry(new Entry()
+        {
+            LexemeForm = { { "en", "Pear" } },
+            Senses =
+            [
+                new Sense() { Gloss = { { "en", "Pear" } }, SemanticDomains = [ semdom3 ] }
+            ]
+        });
+        await crdtApi.CreateEntry(new Entry()
+        {
+            LexemeForm = { { "en", "Banana" } },
+            Senses =
+            [
+                new Sense() { Gloss = { { "en", "Banana" } }, SemanticDomains = [ semdom3 ] }
+            ]
+        });
+        await _syncService.Sync(crdtApi, fwdataApi);
+
+        var crdtEntries = await crdtApi.GetEntries().ToArrayAsync();
+        var fwdataEntries = await fwdataApi.GetEntries().ToArrayAsync();
+        crdtEntries.Should().BeEquivalentTo(fwdataEntries,
+            options => options.For(e => e.Components).Exclude(c => c.Id)
+                .For(e => e.ComplexForms).Exclude(c => c.Id));
+    }
+
     [Fact]
     public async Task UpdatingAnEntryInEachProjectSyncsAcrossBoth()
     {

diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs
@@ -28,7 +28,10 @@ public async Task<SyncResult> Sync(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataA
         if (!dryRun)
         {
             await SaveProjectSnapshot(fwdataApi.Project.Name, fwdataApi.Project.ProjectsPath,
-                new ProjectSnapshot(await fwdataApi.GetEntries().ToArrayAsync(), await fwdataApi.GetPartsOfSpeech().ToArrayAsync()));
+                new ProjectSnapshot(
+                    await fwdataApi.GetEntries().ToArrayAsync(),
+                    await fwdataApi.GetPartsOfSpeech().ToArrayAsync(),
+                    await fwdataApi.GetSemanticDomains().ToArrayAsync()));
         }
         return result;
     }
@@ -48,12 +51,16 @@ private async Task<SyncResult> Sync(IMiniLcmApi crdtApi, IMiniLcmApi fwdataApi,
             return new SyncResult(entryCount, 0);
         }
 
-        //todo sync complex form types, parts of speech, semantic domains, writing systems
+        //todo sync complex form types, writing systems
 
         var currentFwDataPartsOfSpeech = await fwdataApi.GetPartsOfSpeech().ToArrayAsync();
         var crdtChanges = await PartOfSpeechSync.Sync(currentFwDataPartsOfSpeech, projectSnapshot.PartsOfSpeech, crdtApi);
         var fwdataChanges = await PartOfSpeechSync.Sync(await crdtApi.GetPartsOfSpeech().ToArrayAsync(), currentFwDataPartsOfSpeech, fwdataApi);
 
+        var currentFwDataSemanticDomains = await fwdataApi.GetSemanticDomains().ToArrayAsync();
+        crdtChanges += await SemanticDomainSync.Sync(currentFwDataSemanticDomains, projectSnapshot.SemanticDomains, crdtApi);
+        fwdataChanges += await SemanticDomainSync.Sync(await crdtApi.GetSemanticDomains().ToArrayAsync(), currentFwDataSemanticDomains, fwdataApi);
+
         var currentFwDataEntries = await fwdataApi.GetEntries().ToArrayAsync();
         crdtChanges += await EntrySync.Sync(currentFwDataEntries, projectSnapshot.Entries, crdtApi);
         LogDryRun(crdtApi, "crdt");
@@ -77,7 +84,7 @@ private void LogDryRun(IMiniLcmApi api, string type)
         logger.LogInformation($"Dry run {type} changes: {dryRunApi.DryRunRecords.Count}");
     }
 
-    public record ProjectSnapshot(Entry[] Entries, PartOfSpeech[] PartsOfSpeech);
+    public record ProjectSnapshot(Entry[] Entries, PartOfSpeech[] PartsOfSpeech, SemanticDomain[] SemanticDomains);
 
     private async Task<ProjectSnapshot?> GetProjectSnapshot(string projectName, string? projectPath)
     {

diff --git a/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs b/backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs
@@ -1,4 +1,4 @@
-using MiniLcm;
+using MiniLcm;
 using MiniLcm.Models;
 
 namespace FwLiteProjectSync;
@@ -49,6 +49,7 @@ public Task<PartOfSpeech> CreatePartOfSpeech(PartOfSpeech partOfSpeech)
         DryRunRecords.Add(new DryRunRecord(nameof(CreatePartOfSpeech), $"Create part of speech {partOfSpeech.Name}"));
         return Task.FromResult(partOfSpeech); // Since this is a dry run, api.GetPartOfSpeech would return null
     }
+
     public Task<PartOfSpeech> UpdatePartOfSpeech(Guid id, UpdateObjectInput<PartOfSpeech> update)
     {
         DryRunRecords.Add(new DryRunRecord(nameof(UpdatePartOfSpeech), $"Update part of speech {id}"));
@@ -66,10 +67,27 @@ public IAsyncEnumerable<SemanticDomain> GetSemanticDomains()
         return api.GetSemanticDomains();
     }
 
-    public Task CreateSemanticDomain(SemanticDomain semanticDomain)
+    public Task<SemanticDomain?> GetSemanticDomain(Guid id)
+    {
+        return api.GetSemanticDomain(id);
+    }
+
+    public Task<SemanticDomain> CreateSemanticDomain(SemanticDomain semanticDomain)
     {
         DryRunRecords.Add(new DryRunRecord(nameof(CreateSemanticDomain),
             $"Create semantic domain {semanticDomain.Name}"));
+        return Task.FromResult(semanticDomain);
+    }
+
+    public Task<SemanticDomain> UpdateSemanticDomain(Guid id, UpdateObjectInput<SemanticDomain> update)
+    {
+        DryRunRecords.Add(new DryRunRecord(nameof(UpdateSemanticDomain), $"Update part of speech {id}"));
+        return GetSemanticDomain(id)!;
+    }
+
+    public Task DeleteSemanticDomain(Guid id)
+    {
+        DryRunRecords.Add(new DryRunRecord(nameof(DeleteSemanticDomain), $"Delete part of speech {id}"));
         return Task.CompletedTask;
     }
 

diff --git a/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs b/backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs
@@ -1,4 +1,4 @@
-using System.Linq.Expressions;
+using System.Linq.Expressions;
 using SIL.Harmony;
 using SIL.Harmony.Changes;
 using LcmCrdt.Changes;
@@ -107,9 +107,29 @@ public async Task DeletePartOfSpeech(Guid id)
         return SemanticDomains.AsAsyncEnumerable();
     }
 
-    public async Task CreateSemanticDomain(MiniLcm.Models.SemanticDomain semanticDomain)
+    public Task<MiniLcm.Models.SemanticDomain?> GetSemanticDomain(Guid id)
     {
-        await dataModel.AddChange(ClientId, new CreateSemanticDomainChange(semanticDomain.Id, semanticDomain.Name, semanticDomain.Code));
+        return SemanticDomains.FirstOrDefaultAsync(semdom => semdom.Id == id);
+    }
+
+    public async Task<MiniLcm.Models.SemanticDomain> CreateSemanticDomain(MiniLcm.Models.SemanticDomain semanticDomain)
+    {
+        await dataModel.AddChange(ClientId, new CreateSemanticDomainChange(semanticDomain.Id, semanticDomain.Name, semanticDomain.Code, semanticDomain.Predefined));
+        return await GetSemanticDomain(semanticDomain.Id) ?? throw new NullReferenceException();
+    }
+
+    public async Task<SemanticDomain> UpdateSemanticDomain(Guid id, UpdateObjectInput<SemanticDomain> update)
+    {
+        var semDom = await GetSemanticDomain(id);
+        if (semDom is null) throw new NullReferenceException($"unable to find semantic domain with id {id}");
+
+        await dataModel.AddChanges(ClientId, [..semDom.ToChanges(update.Patch)]);
+        return await GetSemanticDomain(id) ?? throw new NullReferenceException();
+    }
+
+    public async Task DeleteSemanticDomain(Guid id)
+    {
+        await dataModel.AddChange(ClientId, new DeleteChange<SemanticDomain>(id));
     }
 
     public async Task BulkImportSemanticDomains(IEnumerable<MiniLcm.Models.SemanticDomain> semanticDomains)

diff --git a/backend/FwLite/LcmCrdt/Objects/JsonPatchChangeExtractor.cs b/backend/FwLite/LcmCrdt/Objects/JsonPatchChangeExtractor.cs
@@ -1,4 +1,4 @@
-using LcmCrdt.Changes;
+using LcmCrdt.Changes;
 using LcmCrdt.Changes.Entries;
 using LcmCrdt.Utils;
 using SIL.Harmony.Changes;
@@ -154,4 +154,10 @@ public static IEnumerable<IChange> ToChanges(this PartOfSpeech pos, JsonPatchDoc
         if (patch.Operations.Count > 0)
             yield return new JsonPatchChange<PartOfSpeech>(pos.Id, patch);
     }
+
+    public static IEnumerable<IChange> ToChanges(this SemanticDomain semDom, JsonPatchDocument<SemanticDomain> patch)
+    {
+        if (patch.Operations.Count > 0)
+            yield return new JsonPatchChange<SemanticDomain>(semDom.Id, patch);
+    }
 }
diff --git a/backend/FwLite/MiniLcm/IMiniLcmReadApi.cs b/backend/FwLite/MiniLcm/IMiniLcmReadApi.cs
@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using MiniLcm.Models;
 
 namespace MiniLcm;
@@ -13,6 +13,7 @@ public interface IMiniLcmReadApi
     IAsyncEnumerable<Entry> SearchEntries(string query, QueryOptions? options = null);
     Task<Entry?> GetEntry(Guid id);
     Task<PartOfSpeech?> GetPartOfSpeech(Guid id);
+    Task<SemanticDomain?> GetSemanticDomain(Guid id);
 }
 
 public record QueryOptions(