In-progress wait time reporting + time series stat fields (ydb-platfo…

…rm#6725)
GrigoriyPA · Jul 16, 2024 · 3de9c00 · 3de9c00
1 parent ef5f653
commit 3de9c00
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 1 deletion.
diff --git a/ydb/library/yql/dq/actors/protos/dq_stats.proto b/ydb/library/yql/dq/actors/protos/dq_stats.proto
@@ -241,11 +241,17 @@ message TDqComputeActorStats {
     google.protobuf.Any Extra = 200;
 }
 
+message TDqHistoryItem {
+    uint64 TimeMs = 1;
+    uint64 Value = 2;
+}
+
 message TDqStatsAggr {
     uint64 Min = 1;
     uint64 Max = 2;
     uint64 Sum = 3;
     uint64 Cnt = 4;
+    repeated TDqHistoryItem History = 5;
 }
 
 message TExtraStats {
@@ -301,6 +307,7 @@ message TDqStageStats {
     uint32 StageId = 1;
     string StageGuid = 2;
     string Program = 3;
+    uint64 BaseTimeMs = 38;
 
     uint32 TotalTasksCount = 5;
     uint32 FailedTasksCount = 6;

diff --git a/ydb/library/yql/dq/runtime/dq_async_stats.h b/ydb/library/yql/dq/runtime/dq_async_stats.h
@@ -100,8 +100,9 @@ struct TDqAsyncStats {
 
     inline void TryPause() {
         if (CollectFull()) {
+            auto now = TInstant::Now();
+
             if (!CurrentPauseTs) {
-                auto now = TInstant::Now();
                 if (ResumeMessageTs) {
                     auto delta = now - ResumeMessageTs;
                     if (delta >= MinWaitDuration) {
@@ -114,6 +115,17 @@ struct TDqAsyncStats {
                     CurrentPauseTs = now;
                 }
             }
+
+            // we want to report wait time asap (not on next message only)
+            // so we decrease wait period to minimal possible value of MinWaitDuration
+            // (used to filter out short periods) and report the rest
+
+            auto delta = now - *CurrentPauseTs;
+            if (delta >= MinWaitDuration * 2) {
+                WaitTime += delta;
+                WaitTime -= MinWaitDuration;
+                *CurrentPauseTs = now - MinWaitDuration;
+            }
         }
     }