From ec0fbb896c3ec7dc6cbc394e444a869cfeb29774 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 22 Mar 2024 14:49:48 -0400 Subject: [PATCH] Use MongoDB Atlas for internal QA (#348) Fix E2E Tests - issues with setup/teardown and TrainOn Preserve all engines for later inspection when E2E testing. Fix tests - TrainOn should be init to null. Fix for inodes on serval-claim --- .vscode/settings.json | 1 - deploy/qa-int-values.yaml | 1 + .../templates/machine-engine-deployment.yaml | 9 +- .../templates/machine-job-deployment.yaml | 9 +- deploy/serval/templates/secrets.yaml | 9 + .../templates/serval-api-deployment.yaml | 9 +- deploy/values.yaml | 4 +- docker-compose.withatlas.yml | 157 ++++++++++++++++++ tests/Serval.E2ETests/MissingServicesTests.cs | 16 +- tests/Serval.E2ETests/ServalApiSlowTests.cs | 16 +- tests/Serval.E2ETests/ServalApiTests.cs | 22 ++- tests/Serval.E2ETests/ServalClientHelper.cs | 45 +++-- 12 files changed, 255 insertions(+), 43 deletions(-) create mode 100644 docker-compose.withatlas.yml diff --git a/.vscode/settings.json b/.vscode/settings.json index 30d3ef56..dd52926b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,4 @@ { - "dotnet-test-explorer.testProjectPath": "**/*Tests.csproj", "editor.formatOnSave": true, "[csharp]": { "editor.defaultFormatter": "csharpier.csharpier-vscode", diff --git a/deploy/qa-int-values.yaml b/deploy/qa-int-values.yaml index fc78815f..6e1bbf36 100644 --- a/deploy/qa-int-values.yaml +++ b/deploy/qa-int-values.yaml @@ -12,6 +12,7 @@ servalImage: ghcr.io/sillsdev/serval:1.2.0 machineImage: ghcr.io/sillsdev/machine:3.7.0 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:1.2.0 ClearMLQueue: lambert_24gb +MongoConnectionPrefix: qa_int_ SharedFileLocation: s3://aqua-ml-data/int-qa/ servalClaimSize: 1Gi machineClaimSize: 2Gi diff --git a/deploy/serval/templates/machine-engine-deployment.yaml b/deploy/serval/templates/machine-engine-deployment.yaml index 478d555e..2b2ff14b 100644 --- a/deploy/serval/templates/machine-engine-deployment.yaml +++ b/deploy/serval/templates/machine-engine-deployment.yaml @@ -29,10 +29,11 @@ spec: value: http://*:80 - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Protocols value: Http2 + {{- template "secrets.mongodb" . }} - name: ASPNETCORE_ConnectionStrings__Hangfire - value: mongodb://mongo:27017/machine_jobs?replicaSet=myRS + value: $(mongodb_connection)machine_jobs - name: ASPNETCORE_ConnectionStrings__Mongo - value: mongodb://mongo:27017/machine?replicaSet=myRS + value: $(mongodb_connection)machine - name: ASPNETCORE_ConnectionStrings__Serval value: http://serval-api:81 {{- template "secrets.clearml" . }} @@ -58,6 +59,10 @@ spec: - mountPath: /root/.aspnet/DataProtection-Keys name: server-keys hostname: machine-engine + dnsConfig: + nameservers: + - 8.8.8.8 + - 208.67.222.222 restartPolicy: Always volumes: - name: machine-mount diff --git a/deploy/serval/templates/machine-job-deployment.yaml b/deploy/serval/templates/machine-job-deployment.yaml index 8c3c238b..6e2355b3 100644 --- a/deploy/serval/templates/machine-job-deployment.yaml +++ b/deploy/serval/templates/machine-job-deployment.yaml @@ -29,10 +29,11 @@ spec: value: http://*:80 - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Protocols value: Http2 + {{- template "secrets.mongodb" . }} - name: ASPNETCORE_ConnectionStrings__Hangfire - value: mongodb://mongo:27017/machine_jobs?replicaSet=myRS + value: $(mongodb_connection)machine_jobs - name: ASPNETCORE_ConnectionStrings__Mongo - value: mongodb://mongo:27017/machine?replicaSet=myRS + value: $(mongodb_connection)machine - name: ASPNETCORE_ConnectionStrings__Serval value: http://serval-api:81 {{- template "secrets.clearml" . }} @@ -56,6 +57,10 @@ spec: name: serval-mount readOnly: true hostname: machine-job + dnsConfig: + nameservers: + - 8.8.8.8 + - 208.67.222.222 restartPolicy: Always volumes: - name: machine-mount diff --git a/deploy/serval/templates/secrets.yaml b/deploy/serval/templates/secrets.yaml index dccb52f7..ba4c82ad 100644 --- a/deploy/serval/templates/secrets.yaml +++ b/deploy/serval/templates/secrets.yaml @@ -18,6 +18,15 @@ name: aqua-ml-data key: ClearML_SecretKey {{- end }} +{{- define "secrets.mongodb" }} + - name: mongodb_connection_part + valueFrom: + secretKeyRef: + name: aqua-ml-data + key: mongodb_connection + - name: mongodb_connection + value: $(mongodb_connection_part){{ .Values.MongoConnectionPrefix }} +{{- end }} {{- define "secrets.s3" }} - name: SharedFile__Uri value: {{ .Values.SharedFileLocation}} diff --git a/deploy/serval/templates/serval-api-deployment.yaml b/deploy/serval/templates/serval-api-deployment.yaml index 7e084c7a..ec275e0d 100644 --- a/deploy/serval/templates/serval-api-deployment.yaml +++ b/deploy/serval/templates/serval-api-deployment.yaml @@ -35,10 +35,11 @@ spec: value: http://*:81 - name: ASPNETCORE_Kestrel__Endpoints__Http2__Protocols value: Http2 + {{- template "secrets.mongodb" . }} - name: ASPNETCORE_ConnectionStrings__Hangfire - value: mongodb://mongo:27017/serval_jobs?replicaSet=myRS + value: $(mongodb_connection)serval_jobs - name: ASPNETCORE_ConnectionStrings__Mongo - value: mongodb://mongo:27017/serval?replicaSet=myRS + value: $(mongodb_connection)serval - name: ASPNETCORE_Translation__Engines__0__Type value: Echo - name: ASPNETCORE_Translation__Engines__0__Address @@ -70,6 +71,10 @@ spec: - mountPath: /root/.aspnet/DataProtection-Keys name: server-keys hostname: serval-api + dnsConfig: + nameservers: + - 8.8.8.8 + - 208.67.222.222 restartPolicy: Always volumes: - name: serval-mount diff --git a/deploy/values.yaml b/deploy/values.yaml index 696940ca..835d091d 100644 --- a/deploy/values.yaml +++ b/deploy/values.yaml @@ -13,6 +13,6 @@ machineImage: ghcr.io/sillsdev/machine:3.7.0 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:1.2.0 ClearMLQueue: production SharedFileLocation: s3://aqua-ml-data/production/ -servalClaimSize: 5Gi -machineClaimSize: 40Gi +servalClaimSize: 10Gi +machineClaimSize: 41Gi enableEcho: true \ No newline at end of file diff --git a/docker-compose.withatlas.yml b/docker-compose.withatlas.yml new file mode 100644 index 00000000..34a792b7 --- /dev/null +++ b/docker-compose.withatlas.yml @@ -0,0 +1,157 @@ +version: "3" +services: + serval-api: + hostname: serval-api + container_name: serval_cntr + build: + context: . + dockerfile: dockerfile.development + environment: + - ASPNETCORE_ENVIRONMENT=Staging + - ASPNETCORE_DeploymentVersion=docker-compose + - Auth__Domain=sil-appbuilder.auth0.com + - Auth__Audience=https://serval-api.org/ + - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80 + - ASPNETCORE_Kestrel__Endpoints__Http2__Url=http://*:81 + - ASPNETCORE_Kestrel__Endpoints__Http2__Protocols=Http2 + - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING:?connection string needed}serval_jobs + - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING:?connection string needed}serval + - ASPNETCORE_Translation__Engines__0__Type=Echo + - ASPNETCORE_Translation__Engines__0__Address=http://echo + - ASPNETCORE_Translation__Engines__1__Type=SmtTransfer + - ASPNETCORE_Translation__Engines__1__Address=http://machine-engine + - ASPNETCORE_Translation__Engines__2__Type=Nmt + - ASPNETCORE_Translation__Engines__2__Address=http://machine-engine + expose: + - 80 + - 81 + ports: + - 80:80 + volumes: + - .:/app:ro + - ~/.nuget/packages:/root/.nuget/packages:ro + - /var/lib/serval:/var/lib/serval + working_dir: '/app/src/Serval.ApiServer' + entrypoint: + - dotnet + - run + - --no-build + - --no-launch-profile + - --additionalProbingPath + - /root/.nuget/packages + + echo: + hostname: echo + container_name: echo_cntr + build: + context: . + dockerfile: dockerfile.development + environment: + - ASPNETCORE_ENVIRONMENT=Staging + - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80 + - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2 + - ASPNETCORE_ConnectionStrings__TranslationPlatformApi=http://serval-api:81 + expose: + - 80 + ports: + - 81:80 + depends_on: + - serval-api + volumes: + - .:/app:ro + - ~/.nuget/packages:/root/.nuget/packages:ro + - /var/lib/serval:/var/lib/serval + working_dir: '/app/samples/EchoTranslationEngine' + entrypoint: + - dotnet + - run + - --no-build + - --no-launch-profile + - --additionalProbingPath + - /root/.nuget/packages + + machine-engine: + hostname: machine-engine + container_name: machine-engine-cntr + build: + context: ${MACHINE_TESTING_DIR:-../machine} + dockerfile: ../machine/dockerfile.development + + environment: + - ASPNETCORE_ENVIRONMENT=Staging + - ASPNETCORE_Kestrel__Endpoints__Https__Url=http://*:80 + - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2 + - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING:?connection string needed}machine_jobs + - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING:?connection string needed}machine + - ASPNETCORE_ConnectionStrings__Serval=http://serval-api:81 + - ClearML__ApiServer=https://api.sil.hosted.allegro.ai + - ClearML__Queue=lambert_24gb + - ClearML__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest} + - ClearML__Project=docker-compose + - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}" + - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}" + - SharedFile__Uri=s3://aqua-ml-data/docker-compose/ + - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}" + - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}" + expose: + - 80 + ports: + - 82:80 + depends_on: + - serval-api + volumes: + - ${MACHINE_TESTING_DIR:-../machine}:/app:ro + - ~/.nuget/packages:/root/.nuget/packages:ro + - /var/lib/machine:/var/lib/machine + - /var/lib/serval:/var/lib/serval + working_dir: '/app/src/SIL.Machine.Serval.EngineServer' + entrypoint: + - dotnet + - run + - --no-build + - --no-launch-profile + - --additionalProbingPath + - /root/.nuget/packages + + machine-job-server: + hostname: machine-job-server + container_name: machine-job-cntr + build: + context: ${MACHINE_TESTING_DIR:-../machine} + dockerfile: ../machine/dockerfile.development + environment: + - ASPNETCORE_ENVIRONMENT=Staging + - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING:?connection string needed}machine_jobs + - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING:?connection string needed}machine + - ASPNETCORE_ConnectionStrings__Serval=http://serval-api:81 + - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80 + - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2 + - ClearML__ApiServer=https://api.sil.hosted.allegro.ai + - ClearML__Queue=lambert_24gb + - ClearML__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest} + - ClearML__Project=docker-compose + - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}" + - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}" + - SharedFile__Uri=s3://aqua-ml-data/docker-compose/ + - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}" + - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}" + expose: + - 80 + ports: + - 83:80 + depends_on: + - machine-engine + - serval-api + volumes: + - ${MACHINE_TESTING_DIR:-../machine}:/app:ro + - ~/.nuget/packages:/root/.nuget/packages:ro + - /var/lib/machine:/var/lib/machine + - /var/lib/serval:/var/lib/serval + working_dir: '/app/src/SIL.Machine.Serval.JobServer' + entrypoint: + - dotnet + - run + - --no-build + - --no-launch-profile + - --additionalProbingPath + - /root/.nuget/packages diff --git a/tests/Serval.E2ETests/MissingServicesTests.cs b/tests/Serval.E2ETests/MissingServicesTests.cs index c2efe209..f98da13b 100644 --- a/tests/Serval.E2ETests/MissingServicesTests.cs +++ b/tests/Serval.E2ETests/MissingServicesTests.cs @@ -6,13 +6,19 @@ public class MissingServicesTests { private ServalClientHelper _helperClient; - [SetUp] - public async Task Setup() + [OneTimeSetUp] + public async Task OneTimeSetup() { _helperClient = new ServalClientHelper("https://serval-api.org/", ignoreSSLErrors: true); await _helperClient.InitAsync(); } + [SetUp] + public void Setup() + { + _helperClient.Setup(); + } + [Test] [Category("MongoWorking")] public void UseMongoAndAuth0Async() @@ -96,6 +102,12 @@ public void UseMissingEngineServerAsync() [TearDown] public async Task TearDown() + { + await _helperClient.TearDown(); + } + + [OneTimeTearDown] + public async Task OneTimeTearDown() { await _helperClient.DisposeAsync(); } diff --git a/tests/Serval.E2ETests/ServalApiSlowTests.cs b/tests/Serval.E2ETests/ServalApiSlowTests.cs index 0324ab0b..0aa9943a 100644 --- a/tests/Serval.E2ETests/ServalApiSlowTests.cs +++ b/tests/Serval.E2ETests/ServalApiSlowTests.cs @@ -7,13 +7,19 @@ public class ServalApiSlowTests { private ServalClientHelper _helperClient; - [SetUp] - public async Task SetUp() + [OneTimeSetUp] + public async Task OneTimeSetup() { _helperClient = new ServalClientHelper("https://serval-api.org/", ignoreSSLErrors: true); await _helperClient.InitAsync(); } + [SetUp] + public void Setup() + { + _helperClient.Setup(); + } + [Test] public async Task GetSmtWholeBible() { @@ -26,6 +32,12 @@ public async Task GetSmtWholeBible() [TearDown] public async Task TearDown() + { + await _helperClient.TearDown(); + } + + [OneTimeTearDown] + public async Task OneTimeTearDown() { await _helperClient.DisposeAsync(); } diff --git a/tests/Serval.E2ETests/ServalApiTests.cs b/tests/Serval.E2ETests/ServalApiTests.cs index 20ebf8d2..49261532 100644 --- a/tests/Serval.E2ETests/ServalApiTests.cs +++ b/tests/Serval.E2ETests/ServalApiTests.cs @@ -13,6 +13,12 @@ public async Task OneTimeSetup() await _helperClient.InitAsync(); } + [SetUp] + public void Setup() + { + _helperClient.Setup(); + } + [Test] public async Task GetEchoSuggestion() { @@ -105,10 +111,7 @@ public async Task NmtBatch() string engineId = await _helperClient.CreateNewEngineAsync("Nmt", "es", "en", "NMT1"); string[] books = ["MAT.txt", "1JN.txt", "2JN.txt"]; string cId1 = await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); - _helperClient.TranslationBuildConfig.TrainOn = new List - { - new() { CorpusId = cId1, TextIds = ["1JN.txt"] } - }; + _helperClient.TranslationBuildConfig.TrainOn = [new() { CorpusId = cId1, TextIds = ["1JN.txt"] }]; string cId2 = await _helperClient.AddTextCorpusToEngineAsync(engineId, ["3JN.txt"], "es", "en", true); await _helperClient.BuildEngineAsync(engineId); await Task.Delay(1000); @@ -127,11 +130,7 @@ public async Task NmtQueueMultiple() string[] engineIds = new string[NUM_ENGINES]; for (int i = 0; i < NUM_ENGINES; i++) { - _helperClient.TranslationBuildConfig = new() - { - Pretranslate = new List(), - Options = "{\"max_steps\":10}" - }; + _helperClient.InitTranslationBuildConfig(); engineIds[i] = await _helperClient.CreateNewEngineAsync("Nmt", "es", "en", $"NMT1_{i}"); string engineId = engineIds[i]; string[] books = ["MAT.txt", "1JN.txt", "2JN.txt"]; @@ -192,8 +191,6 @@ public async Task NmtLargeBatchAndDownload() string[] books = ["bible_LARGEFILE.txt"]; await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); string cId = await _helperClient.AddTextCorpusToEngineAsync(engineId, ["3JN.txt"], "es", "en", true); - _helperClient.TranslationBuildConfig.Options = - "{\"max_steps\":10, \"train_params\": {\"per_device_train_batch_size\":4}}"; await _helperClient.BuildEngineAsync(engineId); await Task.Delay(1000); IList lTrans = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync( @@ -403,7 +400,8 @@ public async Task ParatextProjectNmtJobAsync() _helperClient.TranslationBuildConfig.Pretranslate!.Add( new PretranslateCorpusConfig { CorpusId = corpus.Id, ScriptureRange = "JHN" } ); - _helperClient.TranslationBuildConfig.Options = "{\"max_steps\":10, \"use_key_terms\":true}"; + _helperClient.TranslationBuildConfig.Options = + "{\"max_steps\":10, \"use_key_terms\":true, \"train_params\": {\"per_device_train_batch_size\":4}}"; await _helperClient.BuildEngineAsync(engineId); Assert.That( diff --git a/tests/Serval.E2ETests/ServalClientHelper.cs b/tests/Serval.E2ETests/ServalClientHelper.cs index 6f342181..abd6e55a 100644 --- a/tests/Serval.E2ETests/ServalClientHelper.cs +++ b/tests/Serval.E2ETests/ServalClientHelper.cs @@ -2,9 +2,14 @@ namespace Serval.E2ETests; public class ServalClientHelper : IAsyncDisposable { + public DataFilesClient DataFilesClient { get; } + public TranslationEnginesClient TranslationEnginesClient { get; } + public TranslationEngineTypesClient TranslationEngineTypesClient { get; } + + public TranslationBuildConfig TranslationBuildConfig { get; set; } + private string _authToken = ""; private readonly HttpClient _httpClient; - private readonly Dictionary _enginePerUser = []; private readonly string _prefix; private readonly string _audience; @@ -30,18 +35,14 @@ public ServalClientHelper(string audience, string prefix = "SCE_", bool ignoreSS TranslationEnginesClient = new TranslationEnginesClient(_httpClient); TranslationEngineTypesClient = new TranslationEngineTypesClient(_httpClient); _prefix = prefix; - TranslationBuildConfig = new TranslationBuildConfig - { - Pretranslate = new List(), - Options = "{\"max_steps\":10}" - }; + TranslationBuildConfig = InitTranslationBuildConfig(); } public async Task InitAsync() { string? authUrl = Environment.GetEnvironmentVariable("SERVAL_AUTH_URL"); if (authUrl is null) - throw new InvalidOperationException("The environment variable SERVAL_HOST_URL is not set."); + throw new InvalidOperationException("The environment variable SERVAL_AUTH_URL is not set."); string? clientId = Environment.GetEnvironmentVariable("SERVAL_CLIENT_ID"); if (clientId is null) throw new InvalidOperationException("The environment variable SERVAL_CLIENT_ID is not set."); @@ -50,28 +51,37 @@ public async Task InitAsync() throw new InvalidOperationException("The environment variable SERVAL_CLIENT_SECRET is not set."); if (string.IsNullOrEmpty(_authToken)) + { _authToken = await GetAuth0AuthenticationAsync(authUrl, _audience, clientId, clientSecret); - - _httpClient.DefaultRequestHeaders.Add("authorization", $"Bearer {_authToken}"); + _httpClient.DefaultRequestHeaders.Add("authorization", $"Bearer {_authToken}"); + } await ClearEnginesAsync(); } - public DataFilesClient DataFilesClient { get; } - public TranslationEnginesClient TranslationEnginesClient { get; } - public TranslationEngineTypesClient TranslationEngineTypesClient { get; } + public void Setup() + { + InitTranslationBuildConfig(); + } - public TranslationBuildConfig TranslationBuildConfig { get; set; } + public TranslationBuildConfig InitTranslationBuildConfig() + { + TranslationBuildConfig = new TranslationBuildConfig + { + Pretranslate = [], + TrainOn = null, + Options = "{\"max_steps\":10, \"train_params\": {\"per_device_train_batch_size\":4}}" + }; + return TranslationBuildConfig; + } - public async Task ClearEnginesAsync(string name = "") + public async Task ClearEnginesAsync() { IList existingTranslationEngines = await TranslationEnginesClient.GetAllAsync(); foreach (TranslationEngine translationEngine in existingTranslationEngines) { - if (translationEngine.Name?.Contains(_prefix + name) ?? false) + if (translationEngine.Name?.Contains(_prefix) ?? false) await TranslationEnginesClient.DeleteAsync(translationEngine.Id); } - TranslationBuildConfig.Pretranslate = new List(); - _enginePerUser.Clear(); } public async Task CreateNewEngineAsync( @@ -92,7 +102,6 @@ public async Task CreateNewEngineAsync( IsModelPersisted = isModelPersisted } ); - _enginePerUser.Add(name, engine.Id); return engine.Id; }