Skip to content

Commit

Permalink
Initialized statistics to an empty array and made resulting changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@mudiagaobrikisil
mudiagaobrikisil committed Nov 26, 2024
1 parent a6fff2f commit 0812665
Show file tree
Hide file tree
Showing 7 changed files with 11 additions and 226 deletions.
2 changes: 0 additions & 2 deletions src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,6 @@ CancellationToken cancellationToken
await PlatformService.InsertPretranslationsAsync(engineId, pretranslationsStream, cancellationToken);
}

// here's where to add more data to send

int additionalCorpusSize = await SaveModelAsync(engineId, buildId);
await DataAccessContext.WithTransactionAsync(
async (ct) =>
Expand Down
137 changes: 1 addition & 136 deletions src/Serval/src/Serval.Client/Client.g.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4526,16 +4526,6 @@ public partial interface ITranslationEnginesClient
/// <exception cref="ServalApiException">A server side error occurred.</exception>
System.Threading.Tasks.Task<ModelDownloadUrl> GetModelDownloadUrlAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));

/// <param name="cancellationToken">A cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
/// <summary>
/// Get a build summary for a given build ID.
/// </summary>
/// <param name="id">The translation engine id</param>
/// <param name="buildId">The build job id</param>
/// <returns>The build summary</returns>
/// <exception cref="ServalApiException">A server side error occurred.</exception>
System.Threading.Tasks.Task<TranslationBuildSummary> GetBuildSummaryAsync(string id, string buildId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));

}

[System.CodeDom.Compiler.GeneratedCode("NSwag", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
Expand Down Expand Up @@ -7862,116 +7852,6 @@ public string BaseUrl
}
}

/// <param name="cancellationToken">A cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
/// <summary>
/// Get a build summary for a given build ID.
/// </summary>
/// <param name="id">The translation engine id</param>
/// <param name="buildId">The build job id</param>
/// <returns>The build summary</returns>
/// <exception cref="ServalApiException">A server side error occurred.</exception>
public virtual async System.Threading.Tasks.Task<TranslationBuildSummary> GetBuildSummaryAsync(string id, string buildId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken))
{
if (id == null)
throw new System.ArgumentNullException("id");

if (buildId == null)
throw new System.ArgumentNullException("buildId");

var client_ = _httpClient;
var disposeClient_ = false;
try
{
using (var request_ = new System.Net.Http.HttpRequestMessage())
{
request_.Method = new System.Net.Http.HttpMethod("GET");
request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json"));

var urlBuilder_ = new System.Text.StringBuilder();
if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl);
// Operation Path: "translation/engines/{id}/builds/{buildId}/summary"
urlBuilder_.Append("translation/engines/");
urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture)));
urlBuilder_.Append("/builds/");
urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(buildId, System.Globalization.CultureInfo.InvariantCulture)));
urlBuilder_.Append("/summary");

PrepareRequest(client_, request_, urlBuilder_);

var url_ = urlBuilder_.ToString();
request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute);

PrepareRequest(client_, request_, url_);

var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
var disposeResponse_ = true;
try
{
var headers_ = new System.Collections.Generic.Dictionary<string, System.Collections.Generic.IEnumerable<string>>();
foreach (var item_ in response_.Headers)
headers_[item_.Key] = item_.Value;
if (response_.Content != null && response_.Content.Headers != null)
{
foreach (var item_ in response_.Content.Headers)
headers_[item_.Key] = item_.Value;
}

ProcessResponse(client_, response_);

var status_ = (int)response_.StatusCode;
if (status_ == 200)
{
var objectResponse_ = await ReadObjectResponseAsync<TranslationBuildSummary>(response_, headers_, cancellationToken).ConfigureAwait(false);
if (objectResponse_.Object == null)
{
throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null);
}
return objectResponse_.Object;
}
else
if (status_ == 401)
{
string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
throw new ServalApiException("The client is not authenticated.", status_, responseText_, headers_, null);
}
else
if (status_ == 403)
{
string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
throw new ServalApiException("The authenticated client does not own the translation engine.", status_, responseText_, headers_, null);
}
else
if (status_ == 404)
{
string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
throw new ServalApiException("The engine or build does not exist.", status_, responseText_, headers_, null);
}
else
if (status_ == 503)
{
string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details.", status_, responseText_, headers_, null);
}
else
{
var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null);
}
}
finally
{
if (disposeResponse_)
response_.Dispose();
}
}
}
finally
{
if (disposeClient_)
client_.Dispose();
}
}

protected struct ObjectResponseResult<T>
{
public ObjectResponseResult(T responseObject, string responseText)
Expand Down Expand Up @@ -9940,7 +9820,7 @@ public partial class TranslationBuild
public string? DeploymentVersion { get; set; } = default!;

[Newtonsoft.Json.JsonProperty("statistics", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
public System.Collections.Generic.IDictionary<string, string>? Statistics { get; set; } = default!;
public System.Collections.Generic.IList<System.Collections.Generic.IDictionary<string, string>>? Statistics { get; set; } = default!;

}

Expand Down Expand Up @@ -10093,21 +9973,6 @@ public partial class ModelDownloadUrl

}

[System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
public partial class TranslationBuildSummary
{
[Newtonsoft.Json.JsonProperty("linesTrainedOn", Required = Newtonsoft.Json.Required.Always)]
public int LinesTrainedOn { get; set; } = default!;

[Newtonsoft.Json.JsonProperty("linesPretranslated", Required = Newtonsoft.Json.Required.Always)]
public int LinesPretranslated { get; set; } = default!;

[Newtonsoft.Json.JsonProperty("totalBuildTime", Required = Newtonsoft.Json.Required.Always)]
[System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)]
public System.TimeSpan TotalBuildTime { get; set; } = default!;

}

[System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
public partial class Queue
{
Expand Down
2 changes: 1 addition & 1 deletion src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,5 +28,5 @@ public record TranslationBuildDto
/// </example>
public object? Options { get; init; }
public string? DeploymentVersion { get; init; }
public Dictionary<string, string>? Statistics { get; set; }
public Dictionary<string, string>[]? Statistics { get; init; }
}
74 changes: 1 addition & 73 deletions src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1177,79 +1177,6 @@ CancellationToken cancellationToken
return Ok(Map(modelInfo));
}

/// <summary>
/// Get a build summary for a given build ID.
/// </summary>
/// <param name="id">The translation engine id</param>
/// <param name="buildId">The build job id</param>
/// <param name="cancellationToken"></param>
/// <response code="200">The build summary</response>
/// <response code="401">The client is not authenticated.</response>
/// <response code="403">The authenticated client does not own the translation engine.</response>
/// <response code="404">The engine or build does not exist.</response>
/// <response code="503">A necessary service is currently unavailable. Check `/health` for more details.</response>
[Authorize(Scopes.ReadTranslationEngines)]
[HttpGet("{id}/builds/{buildId}/summary")]
[ProducesResponseType(StatusCodes.Status200OK)]
[ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)]
[ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)]
[ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)]
[ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)]
public async Task<ActionResult<TranslationBuildSummary>> GetBuildSummaryAsync(
[NotNull] string id,
[NotNull] string buildId,
CancellationToken cancellationToken
)
{
await AuthorizeAsync(id, cancellationToken);

Build build = await _buildService.GetAsync(buildId, cancellationToken);

// Calculate total time for build
TimeSpan totalBuildTime = build.DateFinished.HasValue
? build.DateFinished.Value - build.DateStarted
: TimeSpan.Zero;

var summary = new TranslationBuildSummary
{
LinesTrainedOn = 1,
LinesPretranslated = 2,
TotalBuildTime = totalBuildTime
};

return Ok(summary);
}

// private async Task<(int TrainCount, int PretranslateCount)> GetBuildCountsFromLogAsync(
// string buildId,
// CancellationToken cancellationToken
// )
// {
// // 1. Access the logs for the given buildId.
// // - This depends on how you store and access your logs (e.g., file system, database, logging service).
// // - Replace this with your actual log access logic.
// string logContent = await GetLogContentAsync(buildId, cancellationToken);

// // 2. Parse the log content to find the relevant log entry.
// // - This assumes the log entry format you provided in the PreprocessBuildJob.
// // - Adjust the parsing logic if your log format is different.
// int trainCount = 0;
// int pretranslateCount = 0;

// // Example parsing using regular expressions (adjust as needed):
// Match match = Regex.Match(
// logContent,
// @"{""Event"",""BuildPreprocess""},{""EngineId"",""\w+""},{""BuildId"",""\w+""},{""NumTrainRows"",(\d+)},{""NumPretranslateRows"",(\d+)},"
// );
// if (match.Success)
// {
// trainCount = int.Parse(match.Groups[1].Value);
// pretranslateCount = int.Parse(match.Groups[2].Value);
// }

// return (trainCount, pretranslateCount);
// }

private async Task AuthorizeAsync(string id, CancellationToken cancellationToken)
{
Engine engine = await _engineService.GetAsync(id, cancellationToken);
Expand Down Expand Up @@ -1585,6 +1512,7 @@ private TranslationBuildDto Map(Build source)
DateFinished = source.DateFinished,
Options = source.Options,
DeploymentVersion = source.DeploymentVersion,
Statistics = source.Statistics
};
}

Expand Down
5 changes: 2 additions & 3 deletions src/Serval/src/Serval.Translation/Models/Build.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,9 @@ public record Build : IEntity
public string? Message { get; init; }
public int? QueueDepth { get; init; }
public JobState State { get; init; } = JobState.Pending;
public DateTime DateStarted { get; init; } = DateTime.UtcNow;
public DateTime DateCreated { get; init; } = DateTime.UtcNow;
public DateTime? DateFinished { get; init; }
public IReadOnlyDictionary<string, object>? Options { get; init; }
public string? DeploymentVersion { get; init; }
public Dictionary<string, string> Statistics { get; init; } =
new Dictionary<string, string>() { { "trainCount", "0" } };
public Dictionary<string, string>[] Statistics { get; init; } = [];
}
8 changes: 0 additions & 8 deletions src/Serval/src/Serval.Translation/Models/TranslationBuildSummary.cs
View file

This file was deleted.

9 changes: 6 additions & 3 deletions src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,18 +276,21 @@ ServerCallContext context
throw new RpcException(new Status(StatusCode.NotFound, "Build not found."));
}

var newStatistics = new Dictionary<string, string>();
foreach (var entry in request.Statistics)
{
build.Statistics[entry.Key] = entry.Value;
newStatistics[entry.Key] = entry.Value;
}

var updatedStatistics = build.Statistics.Concat(new[] { newStatistics }).ToArray();

await _builds.UpdateAsync(
b => b.Id == request.BuildId,
u => u.Set(b => b.Statistics, build.Statistics),
u => u.Set(b => b.Statistics, updatedStatistics),
cancellationToken: context.CancellationToken
);

return Empty;
return new Empty();
}

public override async Task<Empty> IncrementTranslationEngineCorpusSize(
Expand Down

0 comments on commit 0812665

Please sign in to comment.