From 19016e8fd20b2f254fae4f2f1757eaee9ac519f2 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 30 Jan 2024 09:54:58 -0500 Subject: [PATCH] Add Language Info endpoint (#283) Move queue endpoint Resolve reviewer comments Updated from reviewer comments IsSupportedNatively InternalCode UpdatedNames Optional name and InternalCode Update endpoint Update controllers to kebab case - PascalCase internally Update documentation Add pascal case tests updates from reviewer comments respond to reviewer comments. Strip out name from GRPC call --- .../TranslationEngineServiceV1.cs | 10 + src/Serval.Client/Client.g.cs | 577 ++++++++++++++---- .../Protos/serval/translation/v1/engine.proto | 12 + .../Contracts/TranslationInfoDto.cs | 8 + .../TranslationEngineTypesController.cs | 104 ++++ .../TranslationEnginesController.cs | 90 +-- src/Serval.Translation/Models/Engine.cs | 2 +- src/Serval.Translation/Models/LanguageInfo.cs | 8 + .../Serval.Translation.csproj | 1 + .../Services/EngineService.cs | 60 +- .../Services/IEngineService.cs | 6 + src/Serval.Translation/Usings.cs | 1 + .../TranslationEngineTests.cs | 68 ++- tests/Serval.E2ETests/ServalApiTests.cs | 10 +- tests/Serval.E2ETests/ServalClientHelper.cs | 4 +- .../Services/EngineServiceTests.cs | 6 +- tests/Serval.Translation.Tests/Usings.cs | 1 + 17 files changed, 727 insertions(+), 241 deletions(-) create mode 100644 src/Serval.Translation/Contracts/TranslationInfoDto.cs create mode 100644 src/Serval.Translation/Controllers/TranslationEngineTypesController.cs create mode 100644 src/Serval.Translation/Models/LanguageInfo.cs diff --git a/samples/EchoTranslationEngine/TranslationEngineServiceV1.cs b/samples/EchoTranslationEngine/TranslationEngineServiceV1.cs index 605a9039..80290623 100644 --- a/samples/EchoTranslationEngine/TranslationEngineServiceV1.cs +++ b/samples/EchoTranslationEngine/TranslationEngineServiceV1.cs @@ -292,6 +292,16 @@ public override Task GetQueueSize(GetQueueSizeRequest requ return Task.FromResult(new GetQueueSizeResponse { Size = 0 }); } + public override Task GetLanguageInfo( + GetLanguageInfoRequest request, + ServerCallContext context + ) + { + return Task.FromResult( + new GetLanguageInfoResponse { InternalCode = request.Language + "_echo", IsNative = false, } + ); + } + public override async Task HealthCheck(Empty request, ServerCallContext context) { HealthReport healthReport = await _healthCheckService.CheckHealthAsync(); diff --git a/src/Serval.Client/Client.g.cs b/src/Serval.Client/Client.g.cs index f0061616..2d1f671d 100644 --- a/src/Serval.Client/Client.g.cs +++ b/src/Serval.Client/Client.g.cs @@ -1361,24 +1361,24 @@ public partial interface ITranslationEnginesClient ///
* The name does not have to be unique, as the engine is uniquely identified by the auto-generated id ///
* **sourceLanguage**: The source language code (a valid [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag) is recommended) ///
* **targetLanguage**: The target language code (a valid IETF language tag is recommended) - ///
* **type**: **SmtTransfer** or **Nmt** or **Echo** - ///
### SmtTransfer + ///
* **type**: **smt-transfer** or **nmt** or **echo** + ///
### smt-transfer ///
The Statistical Machine Translation Transfer Learning engine is primarily used for translation suggestions. Typical endpoints: translate, get-word-graph, train-segment - ///
### Nmt + ///
### nmt ///
The Neural Machine Translation engine is primarily used for pretranslations. It is fine-tuned from Meta's NLLB-200. Valid IETF language tags provided to Serval will be converted to [NLLB-200 codes](https://github.com/facebookresearch/flores/tree/main/flores200#languages-in-flores-200). See more about language tag resolution [here](https://github.com/sillsdev/serval/wiki/Language-Tag-Resolution-for-NLLB%E2%80%90200). ///
///
If you use a language among NLLB's supported languages, Serval will utilize everything the NLLB-200 model already knows about that language when translating. If the language you are working with is not among NLLB's supported languages, the language code will have no effect. ///
///
Typical endpoints: pretranslate - ///
### Echo - ///
The Echo engine has full coverage of all Nmt and SmtTransfer endpoints. Endpoints like create and build return empty responses. Endpoints like translate and get-word-graph echo the sent content back to the user in a format that mocks Nmt or Smt. For example, translating a segment "test" with the Echo engine would yield a translation response with translation "test". This engine is useful for debugging and testing purposes. + ///
### echo + ///
The echo engine has full coverage of all nmt and smt-transfer endpoints. Endpoints like create and build return empty responses. Endpoints like translate and get-word-graph echo the sent content back to the user in a format that mocks nmt or Smt. For example, translating a segment "test" with the echo engine would yield a translation response with translation "test". This engine is useful for debugging and testing purposes. ///
## Sample request: ///
///
{ ///
"name": "myTeam:myProject:myEngine", ///
"sourceLanguage": "el", ///
"targetLanguage": "en", - ///
"type": "Nmt" + ///
"type": "nmt" ///
} /// /// The translation engine configuration (see above) @@ -1404,15 +1404,6 @@ public partial interface ITranslationEnginesClient /// A server side error occurred. System.Threading.Tasks.Task DeleteAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); - /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - /// - /// Get queue information for a given engine type - /// - /// A valid engine type: SmtTransfer, Nmt, or Echo - /// Queue information for the specified engine type - /// A server side error occurred. - System.Threading.Tasks.Task GetQueueAsync(string engineType, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); - /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// /// Translate a segment of text @@ -1778,24 +1769,24 @@ public string BaseUrl ///
* The name does not have to be unique, as the engine is uniquely identified by the auto-generated id ///
* **sourceLanguage**: The source language code (a valid [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag) is recommended) ///
* **targetLanguage**: The target language code (a valid IETF language tag is recommended) - ///
* **type**: **SmtTransfer** or **Nmt** or **Echo** - ///
### SmtTransfer + ///
* **type**: **smt-transfer** or **nmt** or **echo** + ///
### smt-transfer ///
The Statistical Machine Translation Transfer Learning engine is primarily used for translation suggestions. Typical endpoints: translate, get-word-graph, train-segment - ///
### Nmt + ///
### nmt ///
The Neural Machine Translation engine is primarily used for pretranslations. It is fine-tuned from Meta's NLLB-200. Valid IETF language tags provided to Serval will be converted to [NLLB-200 codes](https://github.com/facebookresearch/flores/tree/main/flores200#languages-in-flores-200). See more about language tag resolution [here](https://github.com/sillsdev/serval/wiki/Language-Tag-Resolution-for-NLLB%E2%80%90200). ///
///
If you use a language among NLLB's supported languages, Serval will utilize everything the NLLB-200 model already knows about that language when translating. If the language you are working with is not among NLLB's supported languages, the language code will have no effect. ///
///
Typical endpoints: pretranslate - ///
### Echo - ///
The Echo engine has full coverage of all Nmt and SmtTransfer endpoints. Endpoints like create and build return empty responses. Endpoints like translate and get-word-graph echo the sent content back to the user in a format that mocks Nmt or Smt. For example, translating a segment "test" with the Echo engine would yield a translation response with translation "test". This engine is useful for debugging and testing purposes. + ///
### echo + ///
The echo engine has full coverage of all nmt and smt-transfer endpoints. Endpoints like create and build return empty responses. Endpoints like translate and get-word-graph echo the sent content back to the user in a format that mocks nmt or Smt. For example, translating a segment "test" with the echo engine would yield a translation response with translation "test". This engine is useful for debugging and testing purposes. ///
## Sample request: ///
///
{ ///
"name": "myTeam:myProject:myEngine", ///
"sourceLanguage": "el", ///
"targetLanguage": "en", - ///
"type": "Nmt" + ///
"type": "nmt" ///
} /// /// The translation engine configuration (see above) @@ -2106,106 +2097,6 @@ public string BaseUrl } } - /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - /// - /// Get queue information for a given engine type - /// - /// A valid engine type: SmtTransfer, Nmt, or Echo - /// Queue information for the specified engine type - /// A server side error occurred. - public virtual async System.Threading.Tasks.Task GetQueueAsync(string engineType, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) - { - if (engineType == null) - throw new System.ArgumentNullException("engineType"); - - var client_ = _httpClient; - var disposeClient_ = false; - try - { - using (var request_ = new System.Net.Http.HttpRequestMessage()) - { - var json_ = Newtonsoft.Json.JsonConvert.SerializeObject(engineType, _settings.Value); - var content_ = new System.Net.Http.StringContent(json_); - content_.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json"); - request_.Content = content_; - request_.Method = new System.Net.Http.HttpMethod("POST"); - request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); - - var urlBuilder_ = new System.Text.StringBuilder(); - if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); - // Operation Path: "translation/engines/queues" - urlBuilder_.Append("translation/engines/queues"); - - PrepareRequest(client_, request_, urlBuilder_); - - var url_ = urlBuilder_.ToString(); - request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); - - PrepareRequest(client_, request_, url_); - - var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - var disposeResponse_ = true; - try - { - var headers_ = new System.Collections.Generic.Dictionary>(); - foreach (var item_ in response_.Headers) - headers_[item_.Key] = item_.Value; - if (response_.Content != null && response_.Content.Headers != null) - { - foreach (var item_ in response_.Content.Headers) - headers_[item_.Key] = item_.Value; - } - - ProcessResponse(client_, response_); - - var status_ = (int)response_.StatusCode; - if (status_ == 200) - { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); - if (objectResponse_.Object == null) - { - throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); - } - return objectResponse_.Object; - } - else - if (status_ == 401) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); - } - else - if (status_ == 403) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The authenticated client cannot perform the operation", status_, responseText_, headers_, null); - } - else - if (status_ == 503) - { - string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); - } - else - { - var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); - } - } - finally - { - if (disposeResponse_) - response_.Dispose(); - } - } - } - finally - { - if (disposeClient_) - client_.Dispose(); - } - } - /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// /// Translate a segment of text @@ -4181,6 +4072,411 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c } } + [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial interface ITranslationClient + { + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get queue information for a given engine type + /// + /// A valid engine type: smt-transfer, nmt, or echo + /// Queue information for the specified engine type + /// A server side error occurred. + System.Threading.Tasks.Task GetQueueAsync(string engineType, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get infromation regarding a language for a given engine type + /// + /// + /// This endpoint is to support Nmt models. It specifies the ISO 639-3 code that the language maps to + ///
and whether it is supported in the NLLB 200 model without training. This is useful for determining if a + ///
language is an appropriate candidate for a source language or if two languages can be translated between + ///
**Base Models available** + ///
* **NLLB-200**: This is the only current base transaltion model available. + ///
* The languages included in the base model are [here](https://github.com/facebookresearch/flores/blob/main/nllb_seed/README.md) + ///
without training. + ///
Response format: + ///
* **EngineType**: See above + ///
* **IsNative**: Whether the base translation model supports this language without fine-tuning. + ///
* **InternalCode**: The translation models language code that the language maps to according to [these rules](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). + /// + /// A valid engine type: nmt or echo + /// The language to retrieve information on. + /// Language information for the specified engine type + /// A server side error occurred. + System.Threading.Tasks.Task GetLanguageInfoAsync(string engineType, string language, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + + } + + [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class TranslationClient : ITranslationClient + { + #pragma warning disable 8618 // Set by constructor via BaseUrl property + private string _baseUrl; + #pragma warning restore 8618 // Set by constructor via BaseUrl property + private System.Net.Http.HttpClient _httpClient; + private static System.Lazy _settings = new System.Lazy(CreateSerializerSettings, true); + + public TranslationClient(System.Net.Http.HttpClient httpClient) + { + BaseUrl = "/api/v1"; + _httpClient = httpClient; + } + + private static Newtonsoft.Json.JsonSerializerSettings CreateSerializerSettings() + { + var settings = new Newtonsoft.Json.JsonSerializerSettings(); + UpdateJsonSerializerSettings(settings); + return settings; + } + + public string BaseUrl + { + get { return _baseUrl; } + set + { + _baseUrl = value; + if (!string.IsNullOrEmpty(_baseUrl) && !_baseUrl.EndsWith("/")) + _baseUrl += '/'; + } + } + + protected Newtonsoft.Json.JsonSerializerSettings JsonSerializerSettings { get { return _settings.Value; } } + + static partial void UpdateJsonSerializerSettings(Newtonsoft.Json.JsonSerializerSettings settings); + + partial void PrepareRequest(System.Net.Http.HttpClient client, System.Net.Http.HttpRequestMessage request, string url); + partial void PrepareRequest(System.Net.Http.HttpClient client, System.Net.Http.HttpRequestMessage request, System.Text.StringBuilder urlBuilder); + partial void ProcessResponse(System.Net.Http.HttpClient client, System.Net.Http.HttpResponseMessage response); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get queue information for a given engine type + /// + /// A valid engine type: smt-transfer, nmt, or echo + /// Queue information for the specified engine type + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetQueueAsync(string engineType, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (engineType == null) + throw new System.ArgumentNullException("engineType"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "translation/engine-types/{engineType}/queues" + urlBuilder_.Append("translation/engine-types/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(engineType, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/queues"); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation", status_, responseText_, headers_, null); + } + else + if (status_ == 503) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("A necessary service is currently unavailable. Check `/health` for more details. ", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Get infromation regarding a language for a given engine type + /// + /// + /// This endpoint is to support Nmt models. It specifies the ISO 639-3 code that the language maps to + ///
and whether it is supported in the NLLB 200 model without training. This is useful for determining if a + ///
language is an appropriate candidate for a source language or if two languages can be translated between + ///
**Base Models available** + ///
* **NLLB-200**: This is the only current base transaltion model available. + ///
* The languages included in the base model are [here](https://github.com/facebookresearch/flores/blob/main/nllb_seed/README.md) + ///
without training. + ///
Response format: + ///
* **EngineType**: See above + ///
* **IsNative**: Whether the base translation model supports this language without fine-tuning. + ///
* **InternalCode**: The translation models language code that the language maps to according to [these rules](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). + /// + /// A valid engine type: nmt or echo + /// The language to retrieve information on. + /// Language information for the specified engine type + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetLanguageInfoAsync(string engineType, string language, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + { + if (engineType == null) + throw new System.ArgumentNullException("engineType"); + + if (language == null) + throw new System.ArgumentNullException("language"); + + var client_ = _httpClient; + var disposeClient_ = false; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var urlBuilder_ = new System.Text.StringBuilder(); + if (!string.IsNullOrEmpty(BaseUrl)) urlBuilder_.Append(BaseUrl); + // Operation Path: "translation/engine-types/{engineType}/languages/{language}" + urlBuilder_.Append("translation/engine-types/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(engineType, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/languages/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(language, System.Globalization.CultureInfo.InvariantCulture))); + + PrepareRequest(client_, request_, urlBuilder_); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + PrepareRequest(client_, request_, url_); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = new System.Collections.Generic.Dictionary>(); + foreach (var item_ in response_.Headers) + headers_[item_.Key] = item_.Value; + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + ProcessResponse(client_, response_); + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 401) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null); + } + else + if (status_ == 403) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The authenticated client cannot perform the operation", status_, responseText_, headers_, null); + } + else + if (status_ == 405) + { + string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The method is not supported", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ServalApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + protected struct ObjectResponseResult + { + public ObjectResponseResult(T responseObject, string responseText) + { + this.Object = responseObject; + this.Text = responseText; + } + + public T Object { get; } + + public string Text { get; } + } + + public bool ReadResponseAsString { get; set; } + + protected virtual async System.Threading.Tasks.Task> ReadObjectResponseAsync(System.Net.Http.HttpResponseMessage response, System.Collections.Generic.IReadOnlyDictionary> headers, System.Threading.CancellationToken cancellationToken) + { + if (response == null || response.Content == null) + { + return new ObjectResponseResult(default(T)!, string.Empty); + } + + if (ReadResponseAsString) + { + var responseText = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + try + { + var typedBody = Newtonsoft.Json.JsonConvert.DeserializeObject(responseText, JsonSerializerSettings); + return new ObjectResponseResult(typedBody!, responseText); + } + catch (Newtonsoft.Json.JsonException exception) + { + var message = "Could not deserialize the response body string as " + typeof(T).FullName + "."; + throw new ServalApiException(message, (int)response.StatusCode, responseText, headers, exception); + } + } + else + { + try + { + using (var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) + using (var streamReader = new System.IO.StreamReader(responseStream)) + using (var jsonTextReader = new Newtonsoft.Json.JsonTextReader(streamReader)) + { + var serializer = Newtonsoft.Json.JsonSerializer.Create(JsonSerializerSettings); + var typedBody = serializer.Deserialize(jsonTextReader); + return new ObjectResponseResult(typedBody!, string.Empty); + } + } + catch (Newtonsoft.Json.JsonException exception) + { + var message = "Could not deserialize the response body stream as " + typeof(T).FullName + "."; + throw new ServalApiException(message, (int)response.StatusCode, string.Empty, headers, exception); + } + } + } + + private string ConvertToString(object? value, System.Globalization.CultureInfo cultureInfo) + { + if (value == null) + { + return ""; + } + + if (value is System.Enum) + { + var name = System.Enum.GetName(value.GetType(), value); + if (name != null) + { + var field = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); + if (field != null) + { + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + as System.Runtime.Serialization.EnumMemberAttribute; + if (attribute != null) + { + return attribute.Value != null ? attribute.Value : name; + } + } + + var converted = System.Convert.ToString(System.Convert.ChangeType(value, System.Enum.GetUnderlyingType(value.GetType()), cultureInfo)); + return converted == null ? string.Empty : converted; + } + } + else if (value is bool) + { + return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); + } + else if (value is byte[]) + { + return System.Convert.ToBase64String((byte[]) value); + } + else if (value is string[]) + { + return string.Join(",", (string[])value); + } + else if (value.GetType().IsArray) + { + var valueArray = (System.Array)value; + var valueTextArray = new string[valueArray.Length]; + for (var i = 0; i < valueArray.Length; i++) + { + valueTextArray[i] = ConvertToString(valueArray.GetValue(i), cultureInfo); + } + return string.Join(",", valueTextArray); + } + + var result = System.Convert.ToString(value, cultureInfo); + return result == null ? "" : result; + } + } + [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] public partial interface IWebhooksClient { @@ -4891,18 +5187,6 @@ public partial class TranslationEngineConfig } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class Queue - { - [Newtonsoft.Json.JsonProperty("size", Required = Newtonsoft.Json.Required.Always)] - public int Size { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("engineType", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string EngineType { get; set; } = default!; - - } - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] public partial class TranslationResult { @@ -5317,6 +5601,33 @@ public partial class PretranslateCorpusConfig } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class Queue + { + [Newtonsoft.Json.JsonProperty("size", Required = Newtonsoft.Json.Required.Always)] + public int Size { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("engineType", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string EngineType { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class LanguageInfo + { + [Newtonsoft.Json.JsonProperty("engineType", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string EngineType { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("isNative", Required = Newtonsoft.Json.Required.Always)] + public bool IsNative { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("internalCode", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? InternalCode { get; set; } = default!; + + } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.0.2.0 (NJsonSchema v11.0.0.0 (Newtonsoft.Json v13.0.0.0))")] public partial class Webhook { diff --git a/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto b/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto index f3a02456..3553d28a 100644 --- a/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto +++ b/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto @@ -13,6 +13,7 @@ service TranslationEngineApi { rpc StartBuild(StartBuildRequest) returns (google.protobuf.Empty); rpc CancelBuild(CancelBuildRequest) returns (google.protobuf.Empty); rpc GetQueueSize(GetQueueSizeRequest) returns (GetQueueSizeResponse); + rpc GetLanguageInfo(GetLanguageInfoRequest) returns (GetLanguageInfoResponse); rpc HealthCheck(google.protobuf.Empty) returns (HealthCheckResponse); } @@ -79,6 +80,17 @@ message GetQueueSizeResponse { int32 size = 1; } +message GetLanguageInfoRequest { + string engine_type = 1; + string language = 2; +} + +message GetLanguageInfoResponse { + bool is_native = 3; + optional string internal_code = 1; +} + + message AlignedWordPair { int32 source_index = 1; int32 target_index = 2; diff --git a/src/Serval.Translation/Contracts/TranslationInfoDto.cs b/src/Serval.Translation/Contracts/TranslationInfoDto.cs new file mode 100644 index 00000000..ff16e3ff --- /dev/null +++ b/src/Serval.Translation/Contracts/TranslationInfoDto.cs @@ -0,0 +1,8 @@ +namespace Serval.Translation.Contracts; + +public class LanguageInfoDto +{ + public string EngineType { get; set; } = default!; + public bool IsNative { get; set; } = default!; + public string? InternalCode { get; set; } = default!; +} diff --git a/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs b/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs new file mode 100644 index 00000000..22880c70 --- /dev/null +++ b/src/Serval.Translation/Controllers/TranslationEngineTypesController.cs @@ -0,0 +1,104 @@ +namespace Serval.Translation.Controllers; + +[ApiVersion(1.0)] +[Route("api/v{version:apiVersion}/translation/engine-types")] +[OpenApiTag("Translation Engines")] +public class TranslationController(IAuthorizationService authService, IEngineService engineService) + : ServalControllerBase(authService) +{ + private readonly IEngineService _engineService = engineService; + + /// + /// Get queue information for a given engine type + /// + /// A valid engine type: smt-transfer, nmt, or echo + /// + /// Queue information for the specified engine type + /// The client is not authenticated + /// The authenticated client cannot perform the operation + /// A necessary service is currently unavailable. Check `/health` for more details. + [Authorize(Scopes.ReadTranslationEngines)] + [HttpGet("{engineType}/queues")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] + public async Task> GetQueueAsync( + [NotNull] string engineType, + CancellationToken cancellationToken + ) + { + try + { + return Map( + await _engineService.GetQueueAsync(engineType.ToPascalCase(), cancellationToken: cancellationToken) + ); + } + catch (InvalidOperationException ioe) + { + return BadRequest(ioe.Message); + } + } + + /// + /// Get infromation regarding a language for a given engine type + /// + /// + /// This endpoint is to support Nmt models. It specifies the ISO 639-3 code that the language maps to + /// and whether it is supported in the NLLB 200 model without training. This is useful for determining if a + /// language is an appropriate candidate for a source language or if two languages can be translated between + /// **Base Models available** + /// * **NLLB-200**: This is the only current base transaltion model available. + /// * The languages included in the base model are [here](https://github.com/facebookresearch/flores/blob/main/nllb_seed/README.md) + /// without training. + /// Response format: + /// * **EngineType**: See above + /// * **IsNative**: Whether the base translation model supports this language without fine-tuning. + /// * **InternalCode**: The translation models language code that the language maps to according to [these rules](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). + /// + /// A valid engine type: nmt or echo + /// The language to retrieve information on. + /// + /// Language information for the specified engine type + /// The client is not authenticated + /// The authenticated client cannot perform the operation + /// The method is not supported + [Authorize(Scopes.ReadTranslationEngines)] + [HttpGet("{engineType}/languages/{language}")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] + [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] + [ProducesResponseType(typeof(void), StatusCodes.Status405MethodNotAllowed)] + public async Task> GetLanguageInfoAsync( + [NotNull] string engineType, + [NotNull] string language, + CancellationToken cancellationToken + ) + { + try + { + return Map( + await _engineService.GetLanguageInfoAsync( + engineType: engineType.ToPascalCase(), + language: language, + cancellationToken: cancellationToken + ) + ); + } + catch (InvalidOperationException ioe) + { + return BadRequest(ioe.Message); + } + } + + private static QueueDto Map(Queue source) => + new() { Size = source.Size, EngineType = source.EngineType.ToKebabCase() }; + + private static LanguageInfoDto Map(LanguageInfo source) => + new() + { + EngineType = source.EngineType.ToKebabCase(), + IsNative = source.IsNative, + InternalCode = source.InternalCode + }; +} diff --git a/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval.Translation/Controllers/TranslationEnginesController.cs index f46b7f5b..75819f60 100644 --- a/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -1,34 +1,22 @@ -using System.Net.Sockets; - -namespace Serval.Translation.Controllers; +namespace Serval.Translation.Controllers; [ApiVersion(1.0)] [Route("api/v{version:apiVersion}/translation/engines")] [OpenApiTag("Translation Engines")] -public class TranslationEnginesController : ServalControllerBase +public class TranslationEnginesController( + IAuthorizationService authService, + IEngineService engineService, + IBuildService buildService, + IPretranslationService pretranslationService, + IOptionsMonitor apiOptions, + IUrlService urlService +) : ServalControllerBase(authService) { - private readonly IEngineService _engineService; - private readonly IBuildService _buildService; - private readonly IPretranslationService _pretranslationService; - private readonly IOptionsMonitor _apiOptions; - private readonly IUrlService _urlService; - - public TranslationEnginesController( - IAuthorizationService authService, - IEngineService engineService, - IBuildService buildService, - IPretranslationService pretranslationService, - IOptionsMonitor apiOptions, - IUrlService urlService - ) - : base(authService) - { - _engineService = engineService; - _buildService = buildService; - _pretranslationService = pretranslationService; - _apiOptions = apiOptions; - _urlService = urlService; - } + private readonly IEngineService _engineService = engineService; + private readonly IBuildService _buildService = buildService; + private readonly IPretranslationService _pretranslationService = pretranslationService; + private readonly IOptionsMonitor _apiOptions = apiOptions; + private readonly IUrlService _urlService = urlService; /// /// Get all translation engines @@ -91,24 +79,24 @@ CancellationToken cancellationToken /// * The name does not have to be unique, as the engine is uniquely identified by the auto-generated id /// * **sourceLanguage**: The source language code (a valid [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag) is recommended) /// * **targetLanguage**: The target language code (a valid IETF language tag is recommended) - /// * **type**: **SmtTransfer** or **Nmt** or **Echo** - /// ### SmtTransfer + /// * **type**: **smt-transfer** or **nmt** or **echo** + /// ### smt-transfer /// The Statistical Machine Translation Transfer Learning engine is primarily used for translation suggestions. Typical endpoints: translate, get-word-graph, train-segment - /// ### Nmt + /// ### nmt /// The Neural Machine Translation engine is primarily used for pretranslations. It is fine-tuned from Meta's NLLB-200. Valid IETF language tags provided to Serval will be converted to [NLLB-200 codes](https://github.com/facebookresearch/flores/tree/main/flores200#languages-in-flores-200). See more about language tag resolution [here](https://github.com/sillsdev/serval/wiki/Language-Tag-Resolution-for-NLLB%E2%80%90200). /// /// If you use a language among NLLB's supported languages, Serval will utilize everything the NLLB-200 model already knows about that language when translating. If the language you are working with is not among NLLB's supported languages, the language code will have no effect. /// /// Typical endpoints: pretranslate - /// ### Echo - /// The Echo engine has full coverage of all Nmt and SmtTransfer endpoints. Endpoints like create and build return empty responses. Endpoints like translate and get-word-graph echo the sent content back to the user in a format that mocks Nmt or Smt. For example, translating a segment "test" with the Echo engine would yield a translation response with translation "test". This engine is useful for debugging and testing purposes. + /// ### echo + /// The echo engine has full coverage of all nmt and smt-transfer endpoints. Endpoints like create and build return empty responses. Endpoints like translate and get-word-graph echo the sent content back to the user in a format that mocks nmt or Smt. For example, translating a segment "test" with the echo engine would yield a translation response with translation "test". This engine is useful for debugging and testing purposes. /// ## Sample request: /// /// { /// "name": "myTeam:myProject:myEngine", /// "sourceLanguage": "el", /// "targetLanguage": "en", - /// "type": "Nmt" + /// "type": "nmt" /// } /// /// @@ -182,36 +170,6 @@ public async Task DeleteAsync([NotNull] string id, CancellationTok return Ok(); } - /// - /// Get queue information for a given engine type - /// - /// A valid engine type: SmtTransfer, Nmt, or Echo - /// - /// Queue information for the specified engine type - /// The client is not authenticated - /// The authenticated client cannot perform the operation - /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.ReadTranslationEngines)] - [HttpPost("queues")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] - [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] - [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> GetQueueAsync( - [FromBody] string engineType, - CancellationToken cancellationToken - ) - { - try - { - return Map(await _engineService.GetQueueAsync(engineType, cancellationToken)); - } - catch (InvalidOperationException ioe) - { - return BadRequest(ioe.Message); - } - } - /// /// Translate a segment of text /// @@ -997,9 +955,9 @@ private Engine Map(TranslationEngineConfigDto source) Name = source.Name, SourceLanguage = source.SourceLanguage, TargetLanguage = source.TargetLanguage, - Type = source.Type, + Type = source.Type.ToPascalCase(), Owner = Owner, - Corpora = new List() + Corpora = [] }; } @@ -1048,8 +1006,6 @@ private static Build Map(Engine engine, TranslationBuildConfigDto source) return build; } - private QueueDto Map(Queue source) => new() { Size = source.Size, EngineType = source.EngineType }; - private TranslationEngineDto Map(Engine source) { return new TranslationEngineDto @@ -1059,7 +1015,7 @@ private TranslationEngineDto Map(Engine source) Name = source.Name, SourceLanguage = source.SourceLanguage, TargetLanguage = source.TargetLanguage, - Type = source.Type, + Type = source.Type.ToKebabCase(), IsBuilding = source.IsBuilding, ModelRevision = source.ModelRevision, Confidence = Math.Round(source.Confidence, 8), diff --git a/src/Serval.Translation/Models/Engine.cs b/src/Serval.Translation/Models/Engine.cs index f9d0dbbd..d1178126 100644 --- a/src/Serval.Translation/Models/Engine.cs +++ b/src/Serval.Translation/Models/Engine.cs @@ -1,6 +1,6 @@ namespace Serval.Translation.Models; -public class Engine : IOwnedEntity +public partial class Engine : IOwnedEntity { public string Id { get; set; } = default!; public int Revision { get; set; } = 1; diff --git a/src/Serval.Translation/Models/LanguageInfo.cs b/src/Serval.Translation/Models/LanguageInfo.cs new file mode 100644 index 00000000..d0afb916 --- /dev/null +++ b/src/Serval.Translation/Models/LanguageInfo.cs @@ -0,0 +1,8 @@ +namespace Serval.Translation.Models; + +public class LanguageInfo +{ + public string EngineType { get; set; } = default!; + public bool IsNative { get; set; } = default!; + public string? InternalCode { get; set; } = default!; +} diff --git a/src/Serval.Translation/Serval.Translation.csproj b/src/Serval.Translation/Serval.Translation.csproj index 8818f1cc..aa9f3a87 100644 --- a/src/Serval.Translation/Serval.Translation.csproj +++ b/src/Serval.Translation/Serval.Translation.csproj @@ -10,6 +10,7 @@ + diff --git a/src/Serval.Translation/Services/EngineService.cs b/src/Serval.Translation/Services/EngineService.cs index 5f493951..8113b2f9 100644 --- a/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval.Translation/Services/EngineService.cs @@ -2,33 +2,22 @@ namespace Serval.Translation.Services; -public class EngineService : EntityServiceBase, IEngineService +public class EngineService( + IRepository engines, + IRepository builds, + IRepository pretranslations, + GrpcClientFactory grpcClientFactory, + IOptionsMonitor dataFileOptions, + IDataAccessContext dataAccessContext, + ILoggerFactory loggerFactory +) : EntityServiceBase(engines), IEngineService { - private readonly IRepository _builds; - private readonly IRepository _pretranslations; - private readonly GrpcClientFactory _grpcClientFactory; - private readonly IOptionsMonitor _dataFileOptions; - private readonly IDataAccessContext _dataAccessContext; - private readonly ILogger _logger; - - public EngineService( - IRepository engines, - IRepository builds, - IRepository pretranslations, - GrpcClientFactory grpcClientFactory, - IOptionsMonitor dataFileOptions, - IDataAccessContext dataAccessContext, - ILoggerFactory loggerFactory - ) - : base(engines) - { - _builds = builds; - _pretranslations = pretranslations; - _grpcClientFactory = grpcClientFactory; - _dataFileOptions = dataFileOptions; - _dataAccessContext = dataAccessContext; - _logger = loggerFactory.CreateLogger(); - } + private readonly IRepository _builds = builds; + private readonly IRepository _pretranslations = pretranslations; + private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; + private readonly IOptionsMonitor _dataFileOptions = dataFileOptions; + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly ILogger _logger = loggerFactory.CreateLogger(); public async Task TranslateAsync( string engineId, @@ -350,6 +339,25 @@ public async Task GetQueueAsync(string engineType, CancellationToken canc return new Queue { Size = response.Size, EngineType = engineType }; } + public async Task GetLanguageInfoAsync( + string engineType, + string language, + CancellationToken cancellationToken = default + ) + { + var client = _grpcClientFactory.CreateClient(engineType); + GetLanguageInfoResponse response = await client.GetLanguageInfoAsync( + new GetLanguageInfoRequest { EngineType = engineType, Language = language }, + cancellationToken: cancellationToken + ); + return new LanguageInfo + { + InternalCode = response.InternalCode, + IsNative = response.IsNative, + EngineType = engineType + }; + } + private Models.TranslationResult Map(V1.TranslationResult source) { return new Models.TranslationResult diff --git a/src/Serval.Translation/Services/IEngineService.cs b/src/Serval.Translation/Services/IEngineService.cs index a3397eef..fa88dfa7 100644 --- a/src/Serval.Translation/Services/IEngineService.cs +++ b/src/Serval.Translation/Services/IEngineService.cs @@ -48,4 +48,10 @@ Task TrainSegmentPairAsync( Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cancellationToken = default); Task GetQueueAsync(string engineType, CancellationToken cancellationToken = default); + + Task GetLanguageInfoAsync( + string engineType, + string language, + CancellationToken cancellationToken = default + ); } diff --git a/src/Serval.Translation/Usings.cs b/src/Serval.Translation/Usings.cs index 79e2783d..77bb4439 100644 --- a/src/Serval.Translation/Usings.cs +++ b/src/Serval.Translation/Usings.cs @@ -3,6 +3,7 @@ global using System.Text.Json; global using System.Text.Json.Nodes; global using Asp.Versioning; +global using CaseExtensions; global using Grpc.Core; global using Grpc.Net.ClientFactory; global using MassTransit; diff --git a/tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs b/tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs index bc0f4cee..85257e29 100644 --- a/tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs +++ b/tests/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs @@ -1144,7 +1144,7 @@ public void AddCorpusWithSameSourceAndTargetLangs() [TestCase("Echo")] public async Task GetQueueAsync(string engineType) { - ITranslationEnginesClient client = _env!.CreateClient(); + TranslationClient client = _env!.CreateTranslationClient(); Client.Queue queue = await client.GetQueueAsync(engineType); Assert.That(queue.Size, Is.EqualTo(0)); } @@ -1152,7 +1152,7 @@ public async Task GetQueueAsync(string engineType) [Test] public void GetQueueAsync_NotAuthorized() { - ITranslationEnginesClient client = _env!.CreateClient(new string[] { Scopes.ReadFiles }); + TranslationClient client = _env!.CreateTranslationClient([Scopes.ReadFiles]); ServalApiException? ex = Assert.ThrowsAsync(async () => { Client.Queue queue = await client.GetQueueAsync("Echo"); @@ -1161,6 +1161,30 @@ public void GetQueueAsync_NotAuthorized() Assert.That(ex.StatusCode, Is.EqualTo(403)); } + [Test] + public async Task GetLanguageInfoAsync() + { + TranslationClient client = _env!.CreateTranslationClient(); + Client.LanguageInfo languageInfo = await client.GetLanguageInfoAsync("Nmt", "Alphabet"); + Assert.Multiple(() => + { + Assert.That(languageInfo.InternalCode, Is.EqualTo("abc_123")); + Assert.That(languageInfo.IsNative, Is.EqualTo(true)); + }); + } + + [Test] + public void GetLanguageInfo_Error() + { + TranslationClient client = _env!.CreateTranslationClient([Scopes.ReadFiles]); + ServalApiException? ex = Assert.ThrowsAsync(async () => + { + Client.LanguageInfo languageInfo = await client.GetLanguageInfoAsync("Nmt", "abc"); + }); + Assert.That(ex, Is.Not.Null); + Assert.That(ex.StatusCode, Is.EqualTo(403)); + } + [TearDown] public void TearDown() { @@ -1357,9 +1381,6 @@ public TestEnvironment() NmtClient .TranslateAsync(Arg.Any(), null, null, Arg.Any()) .Returns(CreateAsyncUnaryCall(StatusCode.Unimplemented)); - NmtClient - .GetQueueSizeAsync(Arg.Any(), null, null, Arg.Any()) - .Returns(CreateAsyncUnaryCall(new GetQueueSizeResponse() { Size = 0 })); } ServalWebApplicationFactory Factory { get; } @@ -1399,6 +1420,43 @@ public TranslationEnginesClient CreateClient(IEnumerable? scope = null) return new TranslationEnginesClient(httpClient); } + public TranslationClient CreateTranslationClient(IEnumerable? scope = null) + { + scope ??= new[] + { + Scopes.CreateTranslationEngines, + Scopes.ReadTranslationEngines, + Scopes.UpdateTranslationEngines, + Scopes.DeleteTranslationEngines + }; + HttpClient httpClient = Factory + .WithWebHostBuilder(builder => + { + builder.ConfigureTestServices(services => + { + var grpcClientFactory = Substitute.For(); + grpcClientFactory + .CreateClient("Echo") + .Returns(EchoClient); + grpcClientFactory + .CreateClient("Nmt") + .Returns(NmtClient); + services.AddSingleton(grpcClientFactory); + }); + }) + .CreateClient(); + NmtClient + .GetQueueSizeAsync(Arg.Any(), null, null, Arg.Any()) + .Returns(CreateAsyncUnaryCall(new GetQueueSizeResponse() { Size = 0 })); + NmtClient + .GetLanguageInfoAsync(Arg.Any(), null, null, Arg.Any()) + .Returns( + CreateAsyncUnaryCall(new GetLanguageInfoResponse() { InternalCode = "abc_123", IsNative = true }) + ); + httpClient.DefaultRequestHeaders.Add("Scope", string.Join(" ", scope)); + return new TranslationClient(httpClient); + } + public void ResetDatabases() { _mongoClient.DropDatabase("serval_test"); diff --git a/tests/Serval.E2ETests/ServalApiTests.cs b/tests/Serval.E2ETests/ServalApiTests.cs index 10e88f7d..06eb74ef 100644 --- a/tests/Serval.E2ETests/ServalApiTests.cs +++ b/tests/Serval.E2ETests/ServalApiTests.cs @@ -58,7 +58,7 @@ public async Task GetSmtTranslation() public async Task GetSmtAddSegment() { await _helperClient!.ClearEngines(); - string engineId = await _helperClient.CreateNewEngine("SmtTransfer", "es", "en", "SMT3"); + string engineId = await _helperClient.CreateNewEngine("smt-transfer", "es", "en", "SMT3"); var books = new string[] { "1JN.txt", "2JN.txt", "3JN.txt" }; await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); await _helperClient.BuildEngine(engineId); @@ -157,7 +157,7 @@ public async Task NmtQueueMultiple() builds += $"{JsonSerializer.Serialize(build)}\n"; } - builds += "Depth = " + (await _helperClient.translationEnginesClient.GetQueueAsync("Nmt")).Size.ToString(); + builds += "Depth = " + (await _helperClient.translationClient.GetQueueAsync("Nmt")).Size.ToString(); //Status message of last started build says that there is at least one job ahead of it in the queue // (this variable due to how many jobs may already exist in the production queue from other Serval instances) @@ -165,7 +165,7 @@ public async Task NmtQueueMultiple() engineIds[NUM_ENGINES - 1] ); int? queueDepth = newestEngineCurrentBuild.QueueDepth; - Queue queue = await _helperClient.translationEnginesClient.GetQueueAsync("Nmt"); + Queue queue = await _helperClient.translationClient.GetQueueAsync("Nmt"); for (int i = 0; i < NUM_ENGINES; i++) { try @@ -270,7 +270,7 @@ public async Task CircuitousRouteTranslateTopNAsync() const int N = 3; //Create engine - string engineId = await _helperClient!.CreateNewEngine("SmtTransfer", "en", "fa", "SMT6"); + string engineId = await _helperClient!.CreateNewEngine("smt-transfer", "en", "fa", "SMT6"); //Retrieve engine TranslationEngine? engine = await _helperClient.translationEnginesClient.GetAsync(engineId); @@ -313,7 +313,7 @@ public async Task CircuitousRouteTranslateTopNAsync() public async Task GetSmtCancelAndRestartBuild() { await _helperClient!.ClearEngines(); - string engineId = await _helperClient.CreateNewEngine("SmtTransfer", "es", "en", "SMT7"); + string engineId = await _helperClient.CreateNewEngine("smt-transfer", "es", "en", "SMT7"); var books = new string[] { "1JN.txt", "2JN.txt", "3JN.txt" }; await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false); diff --git a/tests/Serval.E2ETests/ServalClientHelper.cs b/tests/Serval.E2ETests/ServalClientHelper.cs index df1b209f..fd1c3c7e 100644 --- a/tests/Serval.E2ETests/ServalClientHelper.cs +++ b/tests/Serval.E2ETests/ServalClientHelper.cs @@ -5,8 +5,9 @@ public class ServalClientHelper { public readonly DataFilesClient dataFilesClient; public readonly TranslationEnginesClient translationEnginesClient; + public readonly TranslationClient translationClient; private readonly HttpClient _httpClient; - readonly Dictionary EnginePerUser = new Dictionary(); + readonly Dictionary EnginePerUser = []; private string _prefix; public ServalClientHelper(string audience, string prefix = "SCE_", bool ignoreSSLErrors = false) @@ -24,6 +25,7 @@ public ServalClientHelper(string audience, string prefix = "SCE_", bool ignoreSS _httpClient.Timeout = TimeSpan.FromSeconds(60); dataFilesClient = new DataFilesClient(_httpClient); translationEnginesClient = new TranslationEnginesClient(_httpClient); + translationClient = new TranslationClient(_httpClient); _httpClient.DefaultRequestHeaders.Add( "authorization", $"Bearer {GetAuth0Authentication(env["authUrl"], audience, env["clientId"], env["clientSecret"]).Result}" diff --git a/tests/Serval.Translation.Tests/Services/EngineServiceTests.cs b/tests/Serval.Translation.Tests/Services/EngineServiceTests.cs index 3426dd83..311cfa2b 100644 --- a/tests/Serval.Translation.Tests/Services/EngineServiceTests.cs +++ b/tests/Serval.Translation.Tests/Services/EngineServiceTests.cs @@ -76,7 +76,7 @@ public async Task CreateAsync() Id = "engine1", SourceLanguage = "es", TargetLanguage = "en", - Type = "smt" + Type = "Smt" }; await env.Service.CreateAsync(engine); @@ -273,7 +273,7 @@ public TestEnvironment() .Returns(CreateAsyncUnaryCall(new Empty())); var grpcClientFactory = Substitute.For(); grpcClientFactory - .CreateClient("smt") + .CreateClient("Smt") .Returns(translationServiceClient); var dataFileOptions = Substitute.For>(); dataFileOptions.CurrentValue.Returns(new DataFileOptions()); @@ -299,7 +299,7 @@ public async Task CreateEngineAsync() Id = "engine1", SourceLanguage = "es", TargetLanguage = "en", - Type = "smt", + Type = "Smt", Corpora = new List { new Models.Corpus diff --git a/tests/Serval.Translation.Tests/Usings.cs b/tests/Serval.Translation.Tests/Usings.cs index 22d7d58e..54113655 100644 --- a/tests/Serval.Translation.Tests/Usings.cs +++ b/tests/Serval.Translation.Tests/Usings.cs @@ -1,3 +1,4 @@ +global using CaseExtensions; global using Grpc.Core; global using Grpc.Net.ClientFactory; global using Microsoft.Extensions.Options;