Merge branch 'develop' into bug/fix-last-commit-date-race

.github/workflows/codeql.yml

@@ -5,6 +5,11 @@ on:
     branches: [ "develop", "main" ]
   pull_request:
     branches: [ "develop", "main" ]
+    paths:
+      - "**/*.cs"
+      - "**/*.csproj"
+      - "**/*.ts"
+      - "**/*.js"
   schedule:
     - cron: '34 21 * * 2'
 

.github/workflows/develop-api.yaml

@@ -51,15 +51,24 @@ jobs:
       k8s-environment: develop
       deploy-domain: lexbox.dev.languagetechnology.org
 
-  integration-tests:
-    name: Integration tests
-    concurrency: develop
-    uses: ./.github/workflows/integration-test.yaml
-    permissions:
-      checks: write
-    secrets: inherit
-    needs: deploy-api
+  integration-test-gha:
+    name: Self hosted integration tests
+    needs: [build-api, set-version]
+    uses: ./.github/workflows/integration-test-gha.yaml
     with:
-      environment: develop
-      runs-on: self-hosted
-      hg-version: 6
+      lexbox-api-tag: ${{ needs.set-version.outputs.version }}
+
+
+      # for now disabling integration tests on self hosted since they're flaky, depend on tests in gha above
+#   integration-tests:
+#     name: Integration tests
+#     concurrency: develop
+#     uses: ./.github/workflows/integration-test.yaml
+#     permissions:
+#       checks: write
+#     secrets: inherit
+#     needs: deploy-api
+#     with:
+#       environment: develop
+#       runs-on: self-hosted
+#       hg-version: 6

.github/workflows/fw-lite.yaml

@@ -23,8 +23,6 @@ jobs:
     name: Build FW Lite and run tests
     timeout-minutes: 20
     runs-on: windows-latest
-    env:
-      NuGetPackageSourceCredentials_github: ${{ secrets.GH_NUGET_PACKAGE_CREDS }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -40,7 +38,6 @@ jobs:
       - name: Dotnet build
         working-directory: backend/FwLite/FwLiteDesktop
         run: |
-          dotnet nuget enable source github
           dotnet build --configuration Release
 
       - name: Dotnet test
@@ -54,6 +51,50 @@ jobs:
           pnpm install
           pnpm run build-app
 
+  publish-mac:
+    name: Publish FW Lite app for Mac
+    needs: build-and-test
+    timeout-minutes: 30
+    runs-on: macos-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '8.x'
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: './frontend/package.json'
+
+      - name: Build viewer
+        working-directory: frontend/viewer
+        run: |
+          corepack enable
+          pnpm install
+          pnpm run build-app
+
+      - name: Dotnet build
+        working-directory: backend/FwLite/LocalWebApp
+        run: dotnet build --configuration Release
+
+      - name: Publish OSX
+        working-directory: backend/FwLite/LocalWebApp
+        run: dotnet publish -r osx-x64 --artifacts-path ../artifacts
+
+      - name: Publish OSX ARM
+        working-directory: backend/FwLite/LocalWebApp
+        run: dotnet publish -r osx-arm64 --artifacts-path ../artifacts
+
+      - name: Upload local web app artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: fw-lite-local-web-app-mac
+          if-no-files-found: error
+          path: backend/FwLite/artifacts/publish/LocalWebApp/*
+
+
   publish-app:
     name: Publish FW Lite app
 
@@ -95,14 +136,6 @@ jobs:
         working-directory: backend/FwLite/LocalWebApp
         run: dotnet publish -r linux-x64 --artifacts-path ../artifacts
 
-      - name: Publish OSX
-        working-directory: backend/FwLite/LocalWebApp
-        run: dotnet publish -r osx-x64 --artifacts-path ../artifacts
-
-      - name: Publish OSX ARM
-        working-directory: backend/FwLite/LocalWebApp
-        run: dotnet publish -r osx-arm64 --artifacts-path ../artifacts
-
       - name: Publish Windows
         working-directory: backend/FwLite/LocalWebApp
         run: dotnet publish -r win-x64 --artifacts-path ../artifacts

.github/workflows/integration-test-gha.yaml

@@ -0,0 +1,70 @@
+name: Self contained integration tests
+on:
+  workflow_dispatch:
+    inputs:
+      lexbox-api-tag:
+        description: 'The version of lexbox-api to test'
+        default: 'develop'
+        required: true
+  workflow_call:
+    inputs:
+      lexbox-api-tag:
+        description: 'The version of lexbox-api to test'
+        default: 'develop'
+        type: string
+        required: true
+
+jobs:
+  execute:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Install Task
+        uses: arduino/setup-task@v2
+      - run: task setup-local-env
+      - name: setup k8s
+        uses: helm/[email protected]
+        with:
+          config: deployment/gha/kind.yaml
+      - name: Verify k8s
+        run: |
+          kubectl cluster-info
+          kubectl get nodes
+      - name: Update image lexbox-api version
+        uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2
+        with:
+          cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml"
+      - name: deploy
+        run: |
+          kubectl create namespace languagedepot
+          kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.0/cert-manager.yaml
+          kubectl wait --for=condition=Ready --timeout=90s pod -l 'app in (cert-manager, webhook)' -n cert-manager
+          kubectl apply -k ./deployment/gha
+          kubectl wait --for=condition=Ready --timeout=120s pod -l 'app.kubernetes.io/component=controller' -n languagedepot
+          kubectl wait --for=condition=Ready --timeout=120s pod -l 'app in (lexbox, ui, hg, db)' -n languagedepot
+      - name: status
+        if: failure()
+        run: |
+          kubectl describe pods -l 'app in (lexbox, ui, hg, db)' -n languagedepot
+          echo "========== LOGS =========="
+          kubectl logs -l 'app in (lexbox, ui, hg, db)' -n languagedepot --prefix --all-containers --tail=50
+          echo "========== INGRESS =========="
+          kubectl logs -l 'app.kubernetes.io/name=ingress-nginx' -n languagedepot --prefix --all-containers --tail=50
+      - name: forward ingress
+        run: kubectl port-forward service/ingress-nginx-controller 6579:80 -n languagedepot &
+      - name: verify ingress
+        run: curl -v http://localhost:6579
+      - name: build
+        run: dotnet restore LexBoxOnly.slnf && dotnet build --no-restore LexBoxOnly.slnf
+      - name: Dotnet test
+        env:
+          TEST_SERVER_HOSTNAME: 'localhost:6579'
+          TEST_STANDARD_HG_HOSTNAME: 'hg.localhost:6579'
+          TEST_RESUMABLE_HG_HOSTNAME: 'resumable.localhost:6579'
+          TEST_PROJECT_CODE: 'sena-3'
+          TEST_DEFAULT_PASSWORD: 'pass'
+        run: dotnet test LexBoxOnly.slnf --logger GitHubActions --filter "Category=Integration|Category=FlakyIntegration" --blame-hang-timeout 40m
+

README.md

@@ -101,6 +101,7 @@ The SvelteKit UI will be available at http://localhost:3000.
 * http://localhost:5158/api/graphql/ui - GraphQL UI
 * http://localhost:8088/hg - hg web UI (add the project code and use the url in FLEx to clone)
 * http://localhost:1080 - maildev UI
+* http://localhost:4810 - pgadmin UI (username [email protected], password pass)
 * http://localhost:18888 - [aspire dashboard](https://learn.microsoft.com/en-us/dotnet/aspire/fundamentals/dashboard) (OTEL traces)
 
 ### Seeded data

Taskfile.yml

@@ -31,28 +31,32 @@ includes:
 
 tasks:
   setup:
-    deps: [ setup-win, setup-unix ]
+    deps: [ setup-win, setup-unix, setup-local-env ]
     cmds:
       - git config blame.ignoreRevsFile .git-blame-ignore-revs
-      - echo "HONEYCOMB_API_KEY=__REPLACE__" >> deployment/local-dev/local.env
-      - echo "#OTEL_SDK_DISABLED=true" >> deployment/local-dev/local.env
-      - echo "GOOGLE_OAUTH_CLIENT_ID=__REPLACE__.apps.googleusercontent.com" >> deployment/local-dev/local.env
-      - echo "GOOGLE_OAUTH_CLIENT_SECRET=__REPLACE__" >> deployment/local-dev/local.env
       - kubectl --context=docker-desktop apply -f deployment/setup/namespace.yaml
       - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.0/cert-manager.yaml
       - docker build -t local-dev-init data/
   setup-win:
     platforms: [ windows ]
     cmds:
+      - powershell "if (!(test-path deployment/local-dev/local.env)) { cp deployment/local-dev/local.env.template deployment/local-dev/local.env }"
       - powershell -File download.ps1 sena-3 'https://drive.google.com/uc?export=download&id=1I-hwc0RHoQqW774gbS5qR-GHa1E7BlsS' 'BEC5131799DB07BF8D84D8FC1F3169FB2574F2A1F4C37F6898EAB563A4AE95B8'
       - powershell -File download.ps1 empty 'https://drive.google.com/uc?export=download&id=1p73u-AGdSwNkg_5KEv9-4iLRuN-1V-LD' 'F4EB48D2C7B3294DCA93965F14F058E56D797F38D562B86CF0372F774E1B486B'
       - powershell -File download.ps1 elawa 'https://drive.usercontent.google.com/download?export=download&id=1Jk-eSDho8ATBMS-Kmfatwi-MWQth26ro&confirm=t' "E3608F1E3188CE5FDB166FBF9D5AAD06558DB68EFA079FB453881572B50CB8E3"
   setup-unix:
     platforms: [ linux, darwin ]
     cmds:
+      - "test -f deployment/local-dev/local.env || cp deployment/local-dev/local.env.template deployment/local-dev/local.env"
       - wget -c -O {{.DATA_DIR}}/sena-3.zip 'https://drive.google.com/uc?export=download&id=1I-hwc0RHoQqW774gbS5qR-GHa1E7BlsS'
       - wget -c -O {{.DATA_DIR}}/empty.zip 'https://drive.google.com/uc?export=download&id=1p73u-AGdSwNkg_5KEv9-4iLRuN-1V-LD'
       - wget -c -O {{.DATA_DIR}}/elawa.zip 'https://drive.usercontent.google.com/download?export=download&id=1Jk-eSDho8ATBMS-Kmfatwi-MWQth26ro&confirm=t'
+  setup-local-env:
+    cmds:
+      - echo "HONEYCOMB_API_KEY=__REPLACE__" > deployment/local-dev/local.env
+      - echo "#OTEL_SDK_DISABLED=true" >> deployment/local-dev/local.env
+      - echo "GOOGLE_OAUTH_CLIENT_ID=__REPLACE__.apps.googleusercontent.com" >> deployment/local-dev/local.env
+      - echo "GOOGLE_OAUTH_CLIENT_SECRET=__REPLACE__" >> deployment/local-dev/local.env
 
   # k8s
   up:

backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs

@@ -98,7 +98,7 @@ public Task<WritingSystems> GetWritingSystems()
             .Select(ws => ws.Id).ToHashSet();
         var writingSystems = new WritingSystems
         {
-            Vernacular = Cache.ServiceLocator.WritingSystems.VernacularWritingSystems.Select(ws => new WritingSystem
+            Vernacular = WritingSystemContainer.CurrentVernacularWritingSystems.Select(ws => new WritingSystem
             {
                 //todo determine current and create a property for that.
                 Id = ws.Id,
@@ -107,7 +107,7 @@ public Task<WritingSystems> GetWritingSystems()
                 Font = ws.DefaultFontName,
                 Exemplars = ws.CharacterSets.FirstOrDefault(s => s.Type == "index")?.Characters.ToArray() ?? []
             }).ToArray(),
-            Analysis = Cache.ServiceLocator.WritingSystems.AnalysisWritingSystems.Select(ws => new WritingSystem
+            Analysis = WritingSystemContainer.CurrentAnalysisWritingSystems.Select(ws => new WritingSystem
             {
                 Id = ws.Id,
                 Name = ws.LanguageTag,
@@ -164,13 +164,13 @@ public async Task CreatePartOfSpeech(PartOfSpeech partOfSpeech)
 
     public async IAsyncEnumerable<SemanticDomain> GetSemanticDomains()
     {
-        foreach (var semanticDomain in SemanticDomainRepository.AllInstances().OrderBy(p => p.Name.BestAnalysisAlternative.Text))
+        foreach (var semanticDomain in SemanticDomainRepository.AllInstances().OrderBy(p => p.Abbreviation.UiString))
         {
             yield return new SemanticDomain
             {
                 Id = semanticDomain.Guid,
                 Name = FromLcmMultiString(semanticDomain.Name),
-                Code = semanticDomain.OcmCodes ?? ""
+                Code = semanticDomain.Abbreviation.UiString ?? ""
             };
         }
     }

backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateSenseProxy.cs

@@ -68,17 +68,32 @@ public override Guid? PartOfSpeechId
             {
                 if (semanticDomain.Id != default) sense.SemanticDomainsRC.Remove(sense.SemanticDomainsRC.First(sd => sd.Guid == semanticDomain.Id));
             },
-            i => new UpdateProxySemanticDomain { Id = sense.SemanticDomainsRC.ElementAt(i).Guid },
+            i => new UpdateProxySemanticDomain(sense.SemanticDomainsRC, sense.SemanticDomainsRC.ElementAt(i).Guid, lexboxLcmApi),
             sense.SemanticDomainsRC.Count
         );
         set => throw new NotImplementedException();
     }
 
-    public class UpdateProxySemanticDomain
+    public class UpdateProxySemanticDomain(
+        ILcmReferenceCollection<ICmSemanticDomain> senseSemanticDomainsRc,
+        Guid id,
+        FwDataMiniLcmApi lexboxLcmApi)
     {
-        public Guid Id { get; set; }
+        public Guid Id
+        {
+            get => id;
+            set
+            {
+                if (value == id) return;
+                if (value == default) throw new ArgumentException("Cannot set to default");
+                senseSemanticDomainsRc.Remove(senseSemanticDomainsRc.First(sd => sd.Guid == id));
+                senseSemanticDomainsRc.Add(lexboxLcmApi.GetLcmSemanticDomain(value));
+                id = value;
+            }
+        }
+
         public string? Code { get; set; }
-        public MultiString? Name { get; set; }
+        public MultiString? Name { get; set; } = new();
     }
 
     public override IList<ExampleSentence> ExampleSentences

backend/FwLite/FwDataMiniLcmBridge/FieldWorksProjectList.cs

@@ -7,6 +7,7 @@ public class FieldWorksProjectList
 {
     public static IEnumerable<IProjectIdentifier> EnumerateProjects()
     {
+        if (!Directory.Exists(ProjectLoader.ProjectFolder)) Directory.CreateDirectory(ProjectLoader.ProjectFolder);
         foreach (var directory in Directory.EnumerateDirectories(ProjectLoader.ProjectFolder))
         {
             var projectName = Path.GetFileName(directory);

backend/FwLite/FwDataMiniLcmBridge/FwDataMiniLcmBridge.csproj

@@ -11,6 +11,7 @@
         <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
         <PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
+        <PackageReference Include="SIL.Core" Version="14.2.0-beta0009" />
         <PackageReference Include="Microsoft.ICU.ICU4C.Runtime" Version="72.1.0.3" Condition="$([MSBuild]::IsOsPlatform('Windows'))" />
         <PackageReference Include="SIL.LCModel" Version="11.0.0-beta0100 " />
         <PackageReference Include="structuremap.patched" Version="4.7.3" />

backend/FwLite/LcmCrdt/Changes/CreateSemanticDomainChange.cs

@@ -1,13 +1,15 @@
-using SIL.Harmony;
+using System.Text.Json.Serialization;
+using SIL.Harmony;
 using SIL.Harmony.Changes;
 using SIL.Harmony.Entities;
 using MiniLcm;
 using SemanticDomain = LcmCrdt.Objects.SemanticDomain;
 
 namespace LcmCrdt.Changes;
 
-public class CreateSemanticDomainChange(Guid semanticDomainId, MultiString name, string code, bool predefined = false)
-    : CreateChange<SemanticDomain>(semanticDomainId), ISelfNamedType<CreateSemanticDomainChange>
+// must use the name `entityId` to support json deserialization as it must match the name of the property
+public class CreateSemanticDomainChange(Guid entityId, MultiString name, string code, bool predefined = false)
+    : CreateChange<SemanticDomain>(entityId), ISelfNamedType<CreateSemanticDomainChange>
 {
     public MultiString Name { get; } = name;
     public bool Predefined { get; } = predefined;

backend/FwLite/LocalWebApp/LocalWebApp.csproj

@@ -47,4 +47,24 @@
         <EmbeddedResource Include="..\..\..\frontend\viewer\dist\**\*" />
     </ItemGroup>
 
+    <ItemGroup Condition="$([MSBuild]::IsOsPlatform('macOS'))">
+        <!--
+          This assumes that icu4c libs are installed under "/opt/local/lib" on macOS. This is the default
+          location for libs that MacPorts installs. If we need to start bundling icu4c libs from
+          somewhere other than "/opt/local/lib", this path and some other paths in the build scripts will
+          need to be updated. We have to bundle the icu4c libs somehow unless macOS starts to include them
+          by default.
+          -->
+        <Content Include="/opt/local/lib/libicu*.??.dylib" Condition="Exists('/opt/local/lib/')">
+            <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        </Content>
+        <!--
+        Homebrew installation of icu4c libraries typically resides in /usr/local/lib.
+        The following line includes these libraries and copies them to the output directory if the OS is macOS.
+        -->
+        <Content Include="/opt/homebrew/Cellar/icu4c/*/lib/libicu*.??.dylib" Condition="Exists('/opt/homebrew/Cellar/icu4c/74.2/lib/')">
+            <Link>%(Filename)%(Extension)</Link>
+            <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        </Content>
+    </ItemGroup>
 </Project>

backend/FwLite/LocalWebApp/LocalWebAppServer.cs

@@ -52,8 +52,16 @@ public static WebApplication SetupAppServer(string[] args, Action<WebApplication
         }
 
 //configure dotnet to serve static files from the embedded resources
-        var sharedOptions =
-            new SharedOptions() { FileProvider = new ManifestEmbeddedFileProvider(typeof(Program).Assembly) };
+        SharedOptions sharedOptions;
+        try
+        {
+            sharedOptions = new SharedOptions() { FileProvider = new ManifestEmbeddedFileProvider(typeof(Program).Assembly) };
+        }
+        catch (InvalidOperationException e)
+        {
+            throw new Exception(
+                "Unable to load embedded files, this is likely due to the viewer app not being built, run 'pnpm run build-app' from the viewer folder", e);
+        }
         app.UseDefaultFiles(new DefaultFilesOptions(sharedOptions));
         var staticFileOptions = new StaticFileOptions(sharedOptions);
         app.UseStaticFiles(staticFileOptions);