From 0812665e577eb61a19b9a737f215d2bfe818ddc3 Mon Sep 17 00:00:00 2001 From: Mudiaga Obriki Date: Tue, 26 Nov 2024 14:36:06 +0100 Subject: [PATCH] Initialized statistics to an empty array and made resulting changes --- .../Services/PostprocessBuildJob.cs | 2 - src/Serval/src/Serval.Client/Client.g.cs | 137 +----------------- .../Contracts/TranslationBuildDto.cs | 2 +- .../TranslationEnginesController.cs | 74 +--------- .../src/Serval.Translation/Models/Build.cs | 5 +- .../Models/TranslationBuildSummary.cs | 8 - .../Services/TranslationPlatformServiceV1.cs | 9 +- 7 files changed, 11 insertions(+), 226 deletions(-) delete mode 100644 src/Serval/src/Serval.Translation/Models/TranslationBuildSummary.cs diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs index d4836174..c9922e41 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs @@ -33,8 +33,6 @@ CancellationToken cancellationToken await PlatformService.InsertPretranslationsAsync(engineId, pretranslationsStream, cancellationToken); } - // here's where to add more data to send - int additionalCorpusSize = await SaveModelAsync(engineId, buildId); await DataAccessContext.WithTransactionAsync( async (ct) => diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 44bbe2f2..1b5d4721 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -4526,16 +4526,6 @@ public partial interface ITranslationEnginesClient /// A server side error occurred. System.Threading.Tasks.Task GetModelDownloadUrlAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); - /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - /// - /// Get a build summary for a given build ID. - /// - /// The translation engine id - /// The build job id - /// The build summary - /// A server side error occurred. - System.Threading.Tasks.Task GetBuildSummaryAsync(string id, string buildId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); - } [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] @@ -7862,116 +7852,6 @@ public string BaseUrl } } - /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - /// - /// Get a build summary for a given build ID. - /// - /// The translation engine id - /// The build job id - /// The build summary - /// A server side error occurred. - public virtual async System.Threading.Tasks.Task GetBuildSummaryAsync(string id, string buildId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) - { - if (id == null) - throw new System.ArgumentNullException("id"); - - if (buildId == null) - throw new System.ArgumentNullException("buildId"); - - var client_ = _httpClient; - var disposeClient_ = false; - try - { - using (var request_ = new System.Net.Http.HttpRequestMessage()) - { - request_.Method = new System.Net.Http.HttpMethod("GET"); - request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); - - var urlBuilder_ = new System.Text.StringBuilder(); - if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "translation/engines/{id}/builds/{buildId}/summary" - urlBuilder_.Append("translation/engines/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/builds/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(buildId, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/summary"); - - PrepareRequest(client_, request_, urlBuilder_); - - var url_ = urlBuilder_.ToString(); - request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); - - PrepareRequest(client_, request_, url_); - - var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - var disposeResponse_ = true; - try - { - var headers_ = new System.Collections.Generic.Dictionary>(); - foreach (var item_ in response_.Headers) - headers_[item_.Key] = item_.Value; - if (response_.Content != null && response_.Content.Headers != null) - { - foreach (var item_ in response_.Content.Headers) - headers_[item_.Key] = item_.Value; - } - - ProcessResponse(client_, response_); - - var status_ = (int)response_.StatusCode; - if (status_ == 200) - { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); - if (objectResponse_.Object == null) - { - throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); - } - return objectResponse_.Object; - } - else - if (status_ == 401) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null); - } - else - if (status_ == 403) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null); - } - else - if (status_ == 404) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or build does not exist.", status_, responseText_, headers_, null); - } - else - if (status_ == 503) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null); - } - else - { - var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); - } - } - finally - { - if (disposeResponse_) - response_.Dispose(); - } - } - } - finally - { - if (disposeClient_) - client_.Dispose(); - } - } - protected struct ObjectResponseResult { public ObjectResponseResult(T responseObject, string responseText) @@ -9940,7 +9820,7 @@ public partial class TranslationBuild public string? DeploymentVersion { get; set; } = default!; [Newtonsoft.Json.JsonProperty("statistics", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IDictionary? Statistics { get; set; } = default!; + public System.Collections.Generic.IList>? Statistics { get; set; } = default!; } @@ -10093,21 +9973,6 @@ public partial class ModelDownloadUrl } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class TranslationBuildSummary - { - [Newtonsoft.Json.JsonProperty("linesTrainedOn", Required = Newtonsoft.Json.Required.Always)] - public int LinesTrainedOn { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("linesPretranslated", Required = Newtonsoft.Json.Required.Always)] - public int LinesPretranslated { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("totalBuildTime", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public System.TimeSpan TotalBuildTime { get; set; } = default!; - - } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] public partial class Queue { diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs index 6d1c7047..d58339ca 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs +++ b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs @@ -28,5 +28,5 @@ public record TranslationBuildDto /// public object? Options { get; init; } public string? DeploymentVersion { get; init; } - public Dictionary? Statistics { get; set; } + public Dictionary[]? Statistics { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs index 42f6cd97..c0e3d13d 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -1177,79 +1177,6 @@ CancellationToken cancellationToken return Ok(Map(modelInfo)); } - /// - /// Get a build summary for a given build ID. - /// - /// The translation engine id - /// The build job id - /// - /// The build summary - /// The client is not authenticated. - /// The authenticated client does not own the translation engine. - /// The engine or build does not exist. - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.ReadTranslationEngines)] - [HttpGet("{id}/builds/{buildId}/summary")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> GetBuildSummaryAsync( - [NotNull] string id, - [NotNull] string buildId, - CancellationToken cancellationToken - ) - { - await AuthorizeAsync(id, cancellationToken); - - Build build = await _buildService.GetAsync(buildId, cancellationToken); - - // Calculate total time for build - TimeSpan totalBuildTime = build.DateFinished.HasValue - ? build.DateFinished.Value - build.DateStarted - : TimeSpan.Zero; - - var summary = new TranslationBuildSummary - { - LinesTrainedOn = 1, - LinesPretranslated = 2, - TotalBuildTime = totalBuildTime - }; - - return Ok(summary); - } - - // private async Task<(int TrainCount, int PretranslateCount)> GetBuildCountsFromLogAsync( - // string buildId, - // CancellationToken cancellationToken - // ) - // { - // // 1. Access the logs for the given buildId. - // // - This depends on how you store and access your logs (e.g., file system, database, logging service). - // // - Replace this with your actual log access logic. - // string logContent = await GetLogContentAsync(buildId, cancellationToken); - - // // 2. Parse the log content to find the relevant log entry. - // // - This assumes the log entry format you provided in the PreprocessBuildJob. - // // - Adjust the parsing logic if your log format is different. - // int trainCount = 0; - // int pretranslateCount = 0; - - // // Example parsing using regular expressions (adjust as needed): - // Match match = Regex.Match( - // logContent, - // @"{""Event"",""BuildPreprocess""},{""EngineId"",""\w+""},{""BuildId"",""\w+""},{""NumTrainRows"",(\d+)},{""NumPretranslateRows"",(\d+)}," - // ); - // if (match.Success) - // { - // trainCount = int.Parse(match.Groups[1].Value); - // pretranslateCount = int.Parse(match.Groups[2].Value); - // } - - // return (trainCount, pretranslateCount); - // } - private async Task AuthorizeAsync(string id, CancellationToken cancellationToken) { Engine engine = await _engineService.GetAsync(id, cancellationToken); @@ -1585,6 +1512,7 @@ private TranslationBuildDto Map(Build source) DateFinished = source.DateFinished, Options = source.Options, DeploymentVersion = source.DeploymentVersion, + Statistics = source.Statistics }; } diff --git a/src/Serval/src/Serval.Translation/Models/Build.cs b/src/Serval/src/Serval.Translation/Models/Build.cs index 9d92ce67..f1773797 100644 --- a/src/Serval/src/Serval.Translation/Models/Build.cs +++ b/src/Serval/src/Serval.Translation/Models/Build.cs @@ -13,10 +13,9 @@ public record Build : IEntity public string? Message { get; init; } public int? QueueDepth { get; init; } public JobState State { get; init; } = JobState.Pending; - public DateTime DateStarted { get; init; } = DateTime.UtcNow; + public DateTime DateCreated { get; init; } = DateTime.UtcNow; public DateTime? DateFinished { get; init; } public IReadOnlyDictionary? Options { get; init; } public string? DeploymentVersion { get; init; } - public Dictionary Statistics { get; init; } = - new Dictionary() { { "trainCount", "0" } }; + public Dictionary[] Statistics { get; init; } = []; } diff --git a/src/Serval/src/Serval.Translation/Models/TranslationBuildSummary.cs b/src/Serval/src/Serval.Translation/Models/TranslationBuildSummary.cs deleted file mode 100644 index 145583c5..00000000 --- a/src/Serval/src/Serval.Translation/Models/TranslationBuildSummary.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Translation.Models; - -public class TranslationBuildSummary -{ - public int LinesTrainedOn { get; set; } - public int LinesPretranslated { get; set; } - public TimeSpan TotalBuildTime { get; set; } -} diff --git a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs index a475cc21..a8f82edd 100644 --- a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs +++ b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs @@ -276,18 +276,21 @@ ServerCallContext context throw new RpcException(new Status(StatusCode.NotFound, "Build not found.")); } + var newStatistics = new Dictionary(); foreach (var entry in request.Statistics) { - build.Statistics[entry.Key] = entry.Value; + newStatistics[entry.Key] = entry.Value; } + var updatedStatistics = build.Statistics.Concat(new[] { newStatistics }).ToArray(); + await _builds.UpdateAsync( b => b.Id == request.BuildId, - u => u.Set(b => b.Statistics, build.Statistics), + u => u.Set(b => b.Statistics, updatedStatistics), cancellationToken: context.CancellationToken ); - return Empty; + return new Empty(); } public override async Task IncrementTranslationEngineCorpusSize(