attempt to run k8s in gha

.github/workflows/codeql.yml

@@ -5,6 +5,11 @@ on:
     branches: [ "develop", "main" ]
   pull_request:
     branches: [ "develop", "main" ]
+    paths:
+      - "**/*.cs"
+      - "**/*.csproj"
+      - "**/*.ts"
+      - "**/*.js"
   schedule:
     - cron: '34 21 * * 2'
 

.github/workflows/develop-api.yaml

@@ -51,15 +51,24 @@ jobs:
       k8s-environment: develop
       deploy-domain: lexbox.dev.languagetechnology.org
 
-  integration-tests:
-    name: Integration tests
-    concurrency: develop
-    uses: ./.github/workflows/integration-test.yaml
-    permissions:
-      checks: write
-    secrets: inherit
-    needs: deploy-api
+  integration-test-gha:
+    name: Self hosted integration tests
+    needs: [build-api, set-version]
+    uses: ./.github/workflows/integration-test-gha.yaml
     with:
-      environment: develop
-      runs-on: self-hosted
-      hg-version: 6
+      lexbox-api-tag: ${{ needs.set-version.outputs.version }}
+
+
+      # for now disabling integration tests on self hosted since they're flaky, depend on tests in gha above
+#   integration-tests:
+#     name: Integration tests
+#     concurrency: develop
+#     uses: ./.github/workflows/integration-test.yaml
+#     permissions:
+#       checks: write
+#     secrets: inherit
+#     needs: deploy-api
+#     with:
+#       environment: develop
+#       runs-on: self-hosted
+#       hg-version: 6

.github/workflows/integration-test-gha.yaml

@@ -0,0 +1,70 @@
+name: Self contained integration tests
+on:
+  workflow_dispatch:
+    inputs:
+      lexbox-api-tag:
+        description: 'The version of lexbox-api to test'
+        default: 'develop'
+        required: true
+  workflow_call:
+    inputs:
+      lexbox-api-tag:
+        description: 'The version of lexbox-api to test'
+        default: 'develop'
+        type: string
+        required: true
+
+jobs:
+  execute:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Install Task
+        uses: arduino/setup-task@v2
+      - run: task setup-local-env
+      - name: setup k8s
+        uses: helm/[email protected]
+        with:
+          config: deployment/gha/kind.yaml
+      - name: Verify k8s
+        run: |
+          kubectl cluster-info
+          kubectl get nodes
+      - name: Update image lexbox-api version
+        uses: mikefarah/yq@0b34c9a00de1c575a34eea05af1d956a525c4fc1 # v4.34.2
+        with:
+          cmd: yq eval -i '(.images.[] | select(.name == "ghcr.io/sillsdev/lexbox-api").newTag) = "${{ inputs.lexbox-api-tag }}"' "./deployment/gha/kustomization.yaml"
+      - name: deploy
+        run: |
+          kubectl create namespace languagedepot
+          kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.0/cert-manager.yaml
+          kubectl wait --for=condition=Ready --timeout=90s pod -l 'app in (cert-manager, webhook)' -n cert-manager
+          kubectl apply -k ./deployment/gha
+          kubectl wait --for=condition=Ready --timeout=120s pod -l 'app.kubernetes.io/component=controller' -n languagedepot
+          kubectl wait --for=condition=Ready --timeout=120s pod -l 'app in (lexbox, ui, hg, db)' -n languagedepot
+      - name: status
+        if: failure()
+        run: |
+          kubectl describe pods -l 'app in (lexbox, ui, hg, db)' -n languagedepot
+          echo "========== LOGS =========="
+          kubectl logs -l 'app in (lexbox, ui, hg, db)' -n languagedepot --prefix --all-containers --tail=50
+          echo "========== INGRESS =========="
+          kubectl logs -l 'app.kubernetes.io/name=ingress-nginx' -n languagedepot --prefix --all-containers --tail=50
+      - name: forward ingress
+        run: kubectl port-forward service/ingress-nginx-controller 6579:80 -n languagedepot &
+      - name: verify ingress
+        run: curl -v http://localhost:6579
+      - name: build
+        run: dotnet restore LexBoxOnly.slnf && dotnet build --no-restore LexBoxOnly.slnf
+      - name: Dotnet test
+        env:
+          TEST_SERVER_HOSTNAME: 'localhost:6579'
+          TEST_STANDARD_HG_HOSTNAME: 'hg.localhost:6579'
+          TEST_RESUMABLE_HG_HOSTNAME: 'resumable.localhost:6579'
+          TEST_PROJECT_CODE: 'sena-3'
+          TEST_DEFAULT_PASSWORD: 'pass'
+        run: dotnet test LexBoxOnly.slnf --logger GitHubActions --filter "Category=Integration|Category=FlakyIntegration" --blame-hang-timeout 40m
+

Taskfile.yml

@@ -31,7 +31,7 @@ includes:
 
 tasks:
   setup:
-    deps: [ setup-win, setup-unix ]
+    deps: [ setup-win, setup-unix, setup-local-env ]
     cmds:
       - git config blame.ignoreRevsFile .git-blame-ignore-revs
       - kubectl --context=docker-desktop apply -f deployment/setup/namespace.yaml
@@ -51,6 +51,12 @@ tasks:
       - wget -c -O {{.DATA_DIR}}/sena-3.zip 'https://drive.google.com/uc?export=download&id=1I-hwc0RHoQqW774gbS5qR-GHa1E7BlsS'
       - wget -c -O {{.DATA_DIR}}/empty.zip 'https://drive.google.com/uc?export=download&id=1p73u-AGdSwNkg_5KEv9-4iLRuN-1V-LD'
       - wget -c -O {{.DATA_DIR}}/elawa.zip 'https://drive.usercontent.google.com/download?export=download&id=1Jk-eSDho8ATBMS-Kmfatwi-MWQth26ro&confirm=t'
+  setup-local-env:
+    cmds:
+      - echo "HONEYCOMB_API_KEY=__REPLACE__" > deployment/local-dev/local.env
+      - echo "#OTEL_SDK_DISABLED=true" >> deployment/local-dev/local.env
+      - echo "GOOGLE_OAUTH_CLIENT_ID=__REPLACE__.apps.googleusercontent.com" >> deployment/local-dev/local.env
+      - echo "GOOGLE_OAUTH_CLIENT_SECRET=__REPLACE__" >> deployment/local-dev/local.env
 
   # k8s
   up:

backend/Testing/SyncReverseProxy/SendReceiveServiceTests.cs

@@ -133,14 +133,8 @@ public async Task SendReceiveAfterProjectReset(HgProtocol protocol)
         var srResult = _sendReceiveService.SendReceiveProject(sendReceiveParams, AdminAuth);
 
         // First, save the current value of `hg tip` from the original project
-        var tipUri = new UriBuilder
-        {
-            Scheme = TestingEnvironmentVariables.HttpScheme,
-            Host = TestingEnvironmentVariables.ServerHostname,
-            Path = $"hg/{projectConfig.Code}/tags",
-            Query = "?style=json"
-        };
-        var response = await _adminApiTester.HttpClient.GetAsync(tipUri.Uri);
+        var tipUri = $"/hg/{projectConfig.Code}/tags?style=json";
+        var response = await _adminApiTester.HttpClient.GetAsync(tipUri);
         var jsonResult = await response.Content.ReadFromJsonAsync<JsonObject>();
         var originalTip = jsonResult?["node"]?.AsValue()?.ToString();
         originalTip.ShouldNotBeNull();
@@ -155,7 +149,7 @@ public async Task SendReceiveAfterProjectReset(HgProtocol protocol)
         await _adminApiTester.HttpClient.PostAsync($"{_adminApiTester.BaseUrl}/api/project/finishResetProject/{projectConfig.Code}", null);
 
         // Step 2: verify project is now empty, i.e. tip is "0000000..."
-        response = await _adminApiTester.HttpClient.GetAsync(tipUri.Uri);
+        response = await _adminApiTester.HttpClient.GetAsync(tipUri);
         jsonResult = await response.Content.ReadFromJsonAsync<JsonObject>();
         var emptyTip = jsonResult?["node"]?.AsValue()?.ToString();
         emptyTip.ShouldNotBeNull();
@@ -178,7 +172,7 @@ public async Task SendReceiveAfterProjectReset(HgProtocol protocol)
         _output.WriteLine(srResultStep3);
 
         // Step 4: verify project tip is same hash as original project tip
-        response = await _adminApiTester.HttpClient.GetAsync(tipUri.Uri);
+        response = await _adminApiTester.HttpClient.GetAsync(tipUri);
         jsonResult = await response.Content.ReadFromJsonAsync<JsonObject>();
         var postSRTip = jsonResult?["node"]?.AsValue()?.ToString();
         postSRTip.ShouldNotBeNull();

backend/Testing/Testing.csproj

@@ -15,7 +15,7 @@
     <Choose>
         <When Condition=" '$(MercurialVersion)' == '6' ">
             <ItemGroup>
-                <PackageReference Include="SIL.Chorus.LibChorus" Version="6.0.0-beta0048" />
+                <PackageReference Include="SIL.Chorus.LibChorus" Version="6.0.0-beta0049" />
                 <PackageReference Include="SIL.Chorus.Mercurial" Version="6.*" />
             </ItemGroup>
         </When>
@@ -48,6 +48,10 @@
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             <PrivateAssets>all</PrivateAssets>
         </PackageReference>
+        <PackageReference Include="GitHubActionsTestLogger" Version="2.4.1">
+          <PrivateAssets>all</PrivateAssets>
+          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
         <PackageReference Include="XunitXml.TestLogger" Version="3.1.17" />
     </ItemGroup>
 

deployment/gha/app-config.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+data:
+  environment-name: "Development"

deployment/gha/change-storage-class.patch.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: hg-repos
+  namespace: languagedepot
+spec:
+  storageClassName: standard # Because kind only supports the standard storage class

deployment/gha/kind.yaml

@@ -0,0 +1,2 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4

deployment/gha/kustomization.yaml

@@ -0,0 +1,24 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: languagedepot
+
+resources:
+- ../local-dev/
+
+patches:
+  - path: lexbox.patch.yaml
+  - target:
+      version: v1
+      kind: PersistentVolumeClaim
+    path: change-storage-class.patch.yaml
+  - path: app-config.yaml
+
+images:
+  - name: local-dev-init #revert change made by local-dev patch
+    newName: busybox
+  - name: ghcr.io/sillsdev/lexbox-api
+    newTag: develop #will be replaced by workflow
+  - name: ghcr.io/sillsdev/lexbox-ui
+    newTag: develop
+  - name: ghcr.io/sillsdev/lexbox-hgweb
+    newTag: latest

deployment/gha/lexbox.patch.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: lexbox
+  namespace: languagedepot
+spec:
+  template:
+    spec:
+      containers:
+        - name: lexbox-api
+          volumeMounts:
+            - mountPath: /frontend
+              name: gql-schema
+            - mountPath: /var/www
+              name: www
+        - name: otel-collector
+          env: #don't try to export to honeycomb
+            - name: COLLECTOR_CONFIG_OVERRIDE
+              value: |
+                exporters:
+                  otlp/aspire:
+                    endpoint: localhost:18889
+                    tls:
+                      insecure: true
+                service:
+                  pipelines:
+                    traces:
+                      exporters: [otlp/aspire]
+                    metrics:
+                      exporters: [otlp/aspire]
+                    logs:
+                      exporters: [otlp/aspire]
+      volumes:
+        - name: gql-schema
+          emptyDir: {}
+        - name: www
+          emptyDir: {}