feat: error msg根据body进行截取,url传递和查询

masastack · Apr 16, 2024 · 0ac209e · 0ac209e
1 parent af5368a
commit 0ac209e
Show file tree

Hide file tree

Showing 16 changed files with 261 additions and 229 deletions.
diff --git a/src/Infrastructure/Masa.Tsc.Storage.Clickhouse.Apm/ApmClickhouseInit.cs b/src/Infrastructure/Masa.Tsc.Storage.Clickhouse.Apm/ApmClickhouseInit.cs
@@ -24,20 +24,24 @@ private static void InitErrorTable(MasaStackClickhouseConnection connection)
     `ServiceName` String CODEC(ZSTD(1)),
     `Resource.service.namespace` String CODEC(ZSTD(1)),
     `Attributes.http.target` String CODEC(ZSTD(1)),
-    INDEX idx_log_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
-    INDEX idx_log_spanid SpanId TYPE bloom_filter(0.001) GRANULARITY 1,
-    INDEX idx_log_environment `Resource.service.namespace` TYPE bloom_filter(0.001) GRANULARITY 1,
-    INDEX idx_log_servicename ServiceName TYPE bloom_filter(0.001) GRANULARITY 1,
-    INDEX idx_log_type `Attributes.exception.type` TYPE bloom_filter(0.001) GRANULARITY 1,
-    INDEX idx_log_endpoint `Attributes.http.target` TYPE bloom_filter(0.001) GRANULARITY 1,
-    INDEX idx_string_message `Attributes.exception.message` TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1
+    `MsgGroupKey` String CODEC(ZSTD(1)),
+
+    INDEX idx_error_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_error_spanid SpanId TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_error_environment `Resource.service.namespace` TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_error_servicename ServiceName TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_error_type `Attributes.exception.type` TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_error_endpoint `Attributes.http.target` TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_error_message `Attributes.exception.message` TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1,
+    INDEX idx_error_msggroupkey MsgGroupKey TYPE bloom_filter(0.001) GRANULARITY 1
 )
 ENGINE = MergeTree
 PARTITION BY toDate(Timestamp)
 ORDER BY (Timestamp,
  ServiceName,
  `Resource.service.namespace`,
  `Attributes.exception.type`,
+ `MsgGroupKey`,
 `Attributes.http.target`)
 TTL toDateTime(Timestamp) + toIntervalDay(30)
 SETTINGS index_granularity = 8192,
@@ -46,8 +50,11 @@ TTL toDateTime(Timestamp) + toIntervalDay(30)
 $@"CREATE MATERIALIZED VIEW {Constants.ErrorTable.Replace(".",".v_")} TO {Constants.ErrorTable}
 AS
 SELECT
-Timestamp,TraceId,SpanId, Body AS `Attributes.exception.message`,LogAttributes['exception.type'] AS `Attributes.exception.type`,
-    ServiceName,ResourceAttributes['service.namespace'] AS `Resource.service.namespace`, LogAttributes['RequestPath'] AS `Attributes.http.target`
+    Timestamp,TraceId,SpanId, if(position(Body, '\n') > 0,extract(Body, '[^\n\r]+'),Body) `Attributes.exception.message`,LogAttributes['exception.type'] AS `Attributes.exception.type`,
+    ServiceName,ResourceAttributes['service.namespace'] AS `Resource.service.namespace`, LogAttributes['RequestPath'] AS `Attributes.http.target`,
+multiIf(
+  length(`Attributes.exception.message`)-150<=0,`Attributes.exception.message`,  
+  extract(`Attributes.exception.message`, '[^,:\\.£º£¬\{{\[]+')) AS MsgGroupKey
 FROM {MasaStackClickhouseConnection.LogSourceTable}
 WHERE mapContains(LogAttributes, 'exception.type')
 "};

diff --git a/src/Infrastructure/Masa.Tsc.Storage.Clickhouse.Apm/Cliclhouse/ClickhouseApmService.cs b/src/Infrastructure/Masa.Tsc.Storage.Clickhouse.Apm/Cliclhouse/ClickhouseApmService.cs
@@ -320,10 +320,10 @@ public Task<PaginatedListBase<ErrorMessageDto>> ErrorMessagePageAsync(ApmEndpoin
         query.IsServer = default;
         var (where, parameters) = AppendWhere(query);
         var groupby = $"group by Type,Message{(string.IsNullOrEmpty(query.Endpoint) ? "" : ",Endpoint")}";
-        var countSql = $"select count(1) from (select Attributes.exception.type as Type,Attributes.exception.message as Message,max(Timestamp) time,count(1) from {Constants.ErrorTable} where {where} {groupby})";
+        var countSql = $"select count(1) from (select `Attributes.exception.type` as Type,MsgGroupKey as Message,max(Timestamp) time,count(1) from {Constants.ErrorTable} where {where} {groupby})";
         PaginatedListBase<ErrorMessageDto> result = new() { Total = Convert.ToInt64(Scalar(countSql, parameters)) };
         var orderBy = GetOrderBy(query, errorOrders);
-        var sql = $@"select * from( select Attributes.exception.type as Type,Attributes.exception.message as Message,max(Timestamp) time,count(1) total from {Constants.ErrorTable} where {where} {groupby} {orderBy} @limit)";
+        var sql = $@"select * from( select `Attributes.exception.type` as Type,MsgGroupKey as Message,max(Timestamp) time,count(1) total from {Constants.ErrorTable} where {where} {groupby} {orderBy} @limit)";
         SetData(sql, parameters, result, query, reader => new ErrorMessageDto()
         {
             Type = reader[0]?.ToString()!,

diff --git a/src/Services/Masa.Tsc.Service.Admin/Extensions/AddTraceLogExtenstion.cs b/src/Services/Masa.Tsc.Service.Admin/Extensions/AddTraceLogExtenstion.cs
@@ -23,11 +23,6 @@ private static void AddClickHouse()
         if (ConfigConst.StorageConst.HasInit || !ConfigConst.IsClickhouse) return;
         if (string.IsNullOrEmpty(ConfigConst.ClikhouseConnection)) return;
         _services.AddMASAStackApmClickhouse(ConfigConst.ClikhouseConnection, ConfigConst.ClickhouseTableSuffix, ConfigConst.ClickHouseLogSourceTable, ConfigConst.ClickHouseTaceSourceTable);
-        //_services.AddMASAStackApmClickhouse(
-        //    "Compress=True;CheckCompressedHash=False;Compressor=lz4;SocketTimeout=10000;Host=10.130.0.106;Port=4229;User=ss;Password=Ckapp@2023;Database=lonsid_fusion", 
-        //    "masav1", 
-        //    "otel_logs_masa",
-        //    "otel_traces_masa");
         ConfigConst.StorageConst.SetClickhouse();
     }
 

diff --git a/src/Services/Masa.Tsc.Service.Admin/Infrastructure/Const/ConfigConst.cs b/src/Services/Masa.Tsc.Service.Admin/Infrastructure/Const/ConfigConst.cs
@@ -1,8 +1,6 @@
 // Copyright (c) MASA Stack All rights reserved.
 // Licensed under the MIT License. See LICENSE.txt in the project root for license information.
 
-using Nest;
-
 namespace Masa.Tsc.Service.Admin.Infrastructure.Const;
 
 internal class ConfigConst

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Data/Apm/SearchData.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Data/Apm/SearchData.cs
@@ -20,6 +20,10 @@ public class SearchData
     public ApmComparisonTypes ComparisonType { get; set; }
 
     public long Timestamp { get; set; }
+
+    public string ExceptionType { get; set; }
+
+    public string ExceptionMsg { get; set; }
 }
 
 public enum ApmComparisonTypes

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/ErrorDetail.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/ErrorDetail.razor
@@ -47,7 +47,7 @@
             </div>
             <div class="pt-4">
                 <div class="text-black Body-1 pt-4">@I18n.Apm("Error.Detail.Message")</div>
-                <div class="ex_context pl-8 py-4 Body-2 text-black">@currentLog?.Attributes["exception.message"]</div>
+                <div class="ex_context pl-8 py-4 Body-2 text-black">@currentLog?.Body</div>
             </div>
         </MCardSubtitle>
         <MCardText>

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/ErrorDetail.razor.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/ErrorDetail.razor.cs
@@ -47,14 +47,13 @@ protected override void OnInitialized()
 
         if (!string.IsNullOrEmpty(exceptionType))
         {
-            //append.AppendFormat(" and exception.type='{0}'", exceptionType);
-            append.AppendFormat(" and LogAttributesValues[indexOf(LogAttributesKeys,'exception.type')]='{0}'", exceptionType);
+            append.AppendFormat(" and `Attributes.exception.type`='{0}'", exceptionType);
         }
-        //if (!string.IsNullOrEmpty(exceptionMessage))
-        //{
-        //    append.AppendFormat(" and LogAttributesValues[indexOf(LogAttributesKeys,'exception.message')]='{0}'", exceptionMessage);
-        //}
-        if (string.IsNullOrEmpty(Search.Text))
+        if (!string.IsNullOrEmpty(exceptionMessage))
+        {
+            append.AppendFormat(" and `Body` like '%{0}%'", exceptionMessage.Replace("x2E", ".").Replace("'", "''"));
+        }
+        if (string.IsNullOrEmpty(Search.Text) && append.Length > 0)
             append.Remove(0, 5);
         Search.Text = append.ToString();
     }

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Errors.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Errors.razor
@@ -32,7 +32,12 @@
         else if (context.Header.Value == nameof(ErrorMessageDto.Message))
         {
             <div style="width:400px" class="text-truncate">
-                @context.Item.Message
+                <a target="_blank"
+                   style="text-decoration:none"
+                   title="@context.Item.Message"
+                   href="/apm/errors/@(HttpUtility.UrlEncode(context.Item.Message).Replace(".","x2E")+GetUrlParam(service:Search.Service,env: GetSearchEnv(Search.Environment), comparisonType: Search.ComparisonType, start: Search.Start, end: Search.End,exType:context.Item.Type,exMsg:context.Item.Message,search:Search.Text))">
+                    @context.Item.Message
+                </a>
             </div>
         }
         else if (context.Header.Value == nameof(ErrorMessageDto.LastTime))

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceEndpoints.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceEndpoints.razor
@@ -103,7 +103,7 @@
             </ItemColContent>
         </SDataTable>
         <MCard Rounded=true Class="px-6" Style="border-top-left-radius: 0 !important; border-top-right-radius: 0 !important;">
-            <SPagination Class="mt-2" @bind-Page="page" @bind-PageSize=defaultSize Total=total PageSizeSelect="@(new List<int>{defaultSize})" />
+            <SPagination Class="mt-2" Page="page" PageSize=defaultSize Total=total PageSizeSelect="@(new List<int>{defaultSize})" OnChange="OnPageChange" />
         </MCard>
     </div>
 </div>
diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceEndpoints.razor.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceEndpoints.razor.cs
@@ -71,8 +71,7 @@ private async Task LoadASync(SearchData data = null)
             return;
         }
         StateHasChanged();
-        await LoadPageDataAsync();
-        isTableLoading = false;
+        await LoadPageDataAsync();        
         StateHasChanged();
         await LoadChartDataAsync();
     }
@@ -107,6 +106,14 @@ private async Task LoadPageDataAsync()
             }));
         }
         total = (int)result.Total;
+        isTableLoading = false;
+    }
+
+    private async Task OnPageChange((int page, int pageSize) pageData)
+    {
+        page = pageData.page;
+        defaultSize = pageData.pageSize;
+        await LoadPageDataAsync();
     }
 
     private async Task LoadChartDataAsync()

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceErrors.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceErrors.razor
@@ -39,7 +39,14 @@
                 else if (context.Header.Value == nameof(ErrorMessageDto.Message))
                 {
                     <div style="width:400px" class="text-truncate">
-                        @context.Item.Message
+                        <div style="width:400px" class="text-truncate">
+                            <a target="_blank"
+                               style="text-decoration:none"
+                               title="@context.Item.Message"
+                               href="/apm/errors/@(HttpUtility.UrlEncode(context.Item.Message).Replace(".","x2E")+GetUrlParam(service:Search.Service,env: GetSearchEnv(Search.Environment), comparisonType: Search.ComparisonType, start: Search.Start, end: Search.End,exType:context.Item.Type,exMsg:context.Item.Message,search:Search.Text))">
+                                @context.Item.Message
+                            </a>
+                        </div>
                     </div>
                 }
                 @if (context.Header.Value == nameof(ErrorMessageDto.LastTime))

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceLogs.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceLogs.razor
@@ -51,7 +51,7 @@
             </ItemColContent>
         </SDataTable>
         <MCard Rounded=true Class="px-6" Style="border-top-left-radius: 0 !important; border-top-right-radius: 0 !important;">
-            <SPagination Class="mt-2" @bind-Page="page" @bind-PageSize=defaultSize Total=total PageSizeSelect="@(new List<int>{defaultSize})" />
+            <SPagination Class="mt-2" @bind-Page="page" @bind-PageSize=defaultSize Total=total PageSizeSelect="@(new List<int>{defaultSize})" OnChange="OnPageChange" />
         </MCard>
     </div>
 </div>

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceLogs.razor.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceLogs.razor.cs
@@ -70,8 +70,7 @@ private async Task LoadAsync(SearchData data = null)
         {
             return;
         }
-        await LoadPageDataAsync();
-        isTableLoading = false;
+        await LoadPageDataAsync();       
     }
 
     private async Task LoadChartDataAsync()
@@ -114,6 +113,14 @@ private async Task LoadPageDataAsync()
             data.AddRange(result.Result);
         }
         total = (int)result.Total;
+        isTableLoading = false;
+    }
+
+    private async Task OnPageChange((int page, int pageSize) pageData)
+    {
+        page = pageData.page;
+        defaultSize = pageData.pageSize;
+        await LoadPageDataAsync();
     }
 
     private EChartType ConvertLatencyChartData(List<ChartLineCountDto> data, string lineColor = null, string areaLineColor = null, string? unit = null, string? lineName = null)

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ShortEndpoint.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ShortEndpoint.razor
@@ -1,4 +1,4 @@
-@namespace Masa.Tsc.Web.Admin.Rcl.Pages.Apm.Services
+@* @namespace Masa.Tsc.Web.Admin.Rcl.Pages.Apm.Services
 @inherits ApmComponentBase
 
 <SDataTable Headers="headers"
@@ -78,4 +78,4 @@
 </SDataTable>
 <MCard Rounded=true Class="px-6" Style="border-top-left-radius: 0 !important; border-top-right-radius: 0 !important;">
     <SPagination Class="mt-2" @bind-Page="page" @bind-PageSize=defaultSize Total=total PageSizeSelect="@(new List<int>{defaultSize})" />
-</MCard>
+</MCard> *@