diff --git a/Cognite.Extensions/TimeSeries/DataPointExtensions.cs b/Cognite.Extensions/TimeSeries/DataPointExtensions.cs index 282e113f..64ee1b4b 100644 --- a/Cognite.Extensions/TimeSeries/DataPointExtensions.cs +++ b/Cognite.Extensions/TimeSeries/DataPointExtensions.cs @@ -49,14 +49,14 @@ public static DataPointInsertionRequest ToInsertRequest(this IDictionary dp.IsString && dp.Status.IsGood) + .Where(dp => dp.IsString) .Select(dp => new StringDatapoint { Timestamp = dp.Timestamp, @@ -68,7 +68,7 @@ public static DataPointInsertionRequest ToInsertRequest(this IDictionary !dp.IsString && dp.Status.IsGood) + .Where(dp => !dp.IsString) .Select(dp => new NumericDatapoint { Timestamp = dp.Timestamp, @@ -284,7 +284,7 @@ private static async Task> InsertDataPointsH try { bool useGzip = false; - int count = points.Sum(kvp => kvp.Value.Count()); + int count = request.Items.Sum(r => r.NumericDatapoints?.Datapoints?.Count ?? 0 + r.StringDatapoints?.Datapoints?.Count ?? 0); if (gzipCountLimit >= 0 && count >= gzipCountLimit) { useGzip = true; diff --git a/Cognite.Extensions/TimeSeries/DataPointSanitation.cs b/Cognite.Extensions/TimeSeries/DataPointSanitation.cs index 4c003e87..2a24dc45 100644 --- a/Cognite.Extensions/TimeSeries/DataPointSanitation.cs +++ b/Cognite.Extensions/TimeSeries/DataPointSanitation.cs @@ -109,6 +109,10 @@ public static (IDictionary>, IEnumerable>, IEnumerable>, IEnumerable 0) { badDpGroups.AddRange(badDps .GroupBy(pair => pair.type) @@ -152,7 +163,7 @@ public static (IDictionary>, IEnumerable> errors; - if (badDpGroups.Any()) + if (badDpGroups.Count > 0) { errors = badDpGroups .GroupBy(group => group.type) diff --git a/ExtractorUtils.Test/unit/DatapointsTest.cs b/ExtractorUtils.Test/unit/DatapointsTest.cs index 9cd987da..c4eb0120 100644 --- a/ExtractorUtils.Test/unit/DatapointsTest.cs +++ b/ExtractorUtils.Test/unit/DatapointsTest.cs @@ -25,6 +25,7 @@ using Xunit; using Xunit.Abstractions; using TimeRange = Cognite.Extractor.Common.TimeRange; +using StatusCode = Cognite.Extensions.StatusCode; namespace ExtractorUtils.Test.Unit { @@ -67,12 +68,13 @@ public async Task TestInsertDataPoints() services.AddConfig(path, 2); services.AddTestLogging(_output); services.AddCogniteClient("testApp"); - using (var provider = services.BuildServiceProvider()) { + using (var provider = services.BuildServiceProvider()) + { var cogniteDestination = provider.GetRequiredService(); double[] doublePoints = { 0.0, 1.1, 2.2, double.NaN, 3.3, 4.4, double.NaN, 5.5, double.NegativeInfinity }; - string[] stringPoints = { "0", null, "1", new string('!', CogniteUtils.StringLengthMax), new string('2', CogniteUtils.StringLengthMax + 1), "3"}; - + string[] stringPoints = { "0", null, "1", new string('!', CogniteUtils.StringLengthMax), new string('2', CogniteUtils.StringLengthMax + 1), "3" }; + var datapoints = new Dictionary>() { { new Identity("A"), new Datapoint[] { new Datapoint(DateTime.UtcNow, "1"), new Datapoint(DateTime.UtcNow, "2") }}, { new Identity(1), doublePoints.Select(d => new Datapoint(DateTime.UtcNow, d))}, @@ -157,9 +159,10 @@ public async Task TestDeleteDataPoints() services.AddConfig(path, 2); services.AddTestLogging(_output); services.AddCogniteClient("testApp"); - using (var provider = services.BuildServiceProvider()) { + using (var provider = services.BuildServiceProvider()) + { var cogniteDestination = provider.GetRequiredService(); - + var ranges = new Dictionary>() { { new Identity("A"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow), new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(4), DateTime.UtcNow - TimeSpan.FromDays(2))}}, @@ -169,10 +172,10 @@ public async Task TestDeleteDataPoints() var errors = await cogniteDestination.DeleteDataPointsIgnoreErrorsAsync( ranges, CancellationToken.None); - + mockHttpMessageHandler.Protected() .Verify>( - "SendAsync", + "SendAsync", Times.Exactly(1), // 1 delete ItExpr.IsAny(), ItExpr.IsAny()); @@ -180,17 +183,17 @@ public async Task TestDeleteDataPoints() Assert.Empty(errors.IdsDeleteNotConfirmed); Assert.Empty(errors.IdsNotFound); - ranges.Add(new Identity("missing-C"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow)}); - ranges.Add(new Identity("D"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow)}); - ranges.Add(new Identity("nc-E"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow)}); - ranges.Add(new Identity("missing-F"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow)}); - ranges.Add(new Identity("G"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow)}); - ranges.Add(new Identity("nc-H"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow)}); - + ranges.Add(new Identity("missing-C"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow) }); + ranges.Add(new Identity("D"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow) }); + ranges.Add(new Identity("nc-E"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow) }); + ranges.Add(new Identity("missing-F"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow) }); + ranges.Add(new Identity("G"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow) }); + ranges.Add(new Identity("nc-H"), new TimeRange[] { new TimeRange(DateTime.UtcNow - TimeSpan.FromDays(2), DateTime.UtcNow) }); + errors = await cogniteDestination.DeleteDataPointsIgnoreErrorsAsync( ranges, CancellationToken.None); - + Assert.Contains(new Identity("missing-C"), errors.IdsNotFound); Assert.Contains(new Identity("missing-F"), errors.IdsNotFound); } @@ -262,7 +265,8 @@ public async Task TestUploadQueue() })) { queue.AddStateStorage(stateMap, stateStore, "test-states"); - var enqueueTask = Task.Run(async () => { + var enqueueTask = Task.Run(async () => + { while (index < 13) { var dps = uploadGenerator(index); @@ -292,7 +296,8 @@ public async Task TestUploadQueue() })) { queue.AddStateStorage(stateMap, stateStore, "test-states"); - var enqueueTask = Task.Run(async () => { + var enqueueTask = Task.Run(async () => + { while (index < 23) { var dps = uploadGenerator(index); @@ -398,11 +403,11 @@ public async Task TestUploadQueueBuffer() System.IO.File.Delete(path); } - #region mock private Dictionary> _createdDataPoints = new Dictionary>(); - private async Task mockInsertDataPointsAsync(HttpRequestMessage message , CancellationToken token) { + private async Task mockInsertDataPointsAsync(HttpRequestMessage message, CancellationToken token) + { var uri = message.RequestUri.ToString(); var responseBody = "{ }"; @@ -439,7 +444,7 @@ private async Task mockInsertDataPointsAsync(HttpRequestMes .Select(i => i.DatapointTypeCase == DataPointInsertionItem.DatapointTypeOneofCase.NumericDatapoints ? i.NumericDatapoints.Datapoints.Count : i.StringDatapoints.Datapoints.Count) .Sum() <= 4); // data-points chunk size - + @@ -469,7 +474,8 @@ private async Task mockInsertDataPointsAsync(HttpRequestMes { mismatchedResponse.error.message = "Expected string value for datapoint"; } - else { + else + { mismatchedResponse.error.message = "Expected numeric value for datapoint"; } break; @@ -480,11 +486,12 @@ private async Task mockInsertDataPointsAsync(HttpRequestMes statusCode = HttpStatusCode.BadRequest; responseBody = JsonConvert.SerializeObject(mismatchedResponse); } - else if (missingResponse.error.missing.Count > 0) { + else if (missingResponse.error.missing.Count > 0) + { statusCode = HttpStatusCode.BadRequest; responseBody = JsonConvert.SerializeObject(missingResponse); } - else + else { foreach (var item in data.Items) { @@ -514,14 +521,15 @@ private async Task mockInsertDataPointsAsync(HttpRequestMes var response = new HttpResponseMessage { StatusCode = statusCode, - Content = new StringContent(responseBody) + Content = new StringContent(responseBody) }; response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); response.Headers.Add("x-request-id", "1"); return response; } - private static async Task mockDeleteDataPointsAsync(HttpRequestMessage message , CancellationToken token) { + private static async Task mockDeleteDataPointsAsync(HttpRequestMessage message, CancellationToken token) + { var uri = message.RequestUri.ToString(); HttpContent responseBody = new StringContent("{ }"); var statusCode = HttpStatusCode.OK; @@ -530,7 +538,8 @@ private static async Task mockDeleteDataPointsAsync(HttpReq var ids = JsonConvert.DeserializeObject(content); IEnumerable items = ids.items; - if (uri.Contains($"{_project}/timeseries/data/list")) { + if (uri.Contains($"{_project}/timeseries/data/list")) + { Assert.True(items.Count() <= 2); DataPointListResponse dpList = new DataPointListResponse(); foreach (var item in items) @@ -541,7 +550,7 @@ private static async Task mockDeleteDataPointsAsync(HttpReq dp.NumericDatapoints = new NumericDatapoints(); dpList.Items.Add(dp); } - using(MemoryStream stream = new MemoryStream()) + using (MemoryStream stream = new MemoryStream()) { dpList.WriteTo(stream); responseBody = new ByteArrayContent(stream.ToArray()); @@ -579,7 +588,7 @@ private static async Task mockDeleteDataPointsAsync(HttpReq var response = new HttpResponseMessage { StatusCode = statusCode, - Content = responseBody + Content = responseBody }; response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); response.Headers.Add("x-request-id", "1"); diff --git a/version b/version index 49e0a31d..14bee92c 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.23.1 +1.23.2