Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(tsc-clickhouse): fix query bugs #679

Merged
merged 14 commits into from
Nov 16, 2023
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public static PaginatedListBase<TraceResponseDto> QueryTrace(this IDbConnection
var result = new PaginatedListBase<TraceResponseDto>() { Total = total, Result = new() };
if (total > 0 && start - total < 0)
{
var querySql = CombineOrs($"select ServiceName,Timestamp,TraceId,SpanId,ParentSpanId,TraceState,SpanKind,Duration,SpanName,Spans,Resources from {MasaStackClickhouseConnection.TraceTable} where {where}", ors, orderBy);
var querySql = CombineOrs($"select ServiceName,Timestamp,TraceId,SpanId,ParentSpanId,TraceState,SpanKind,Duration,SpanName,Spans,Resources from {MasaStackClickhouseConnection.TraceTable} where {where}", ors,orderBy);
result.Result = Query(connection, $"select * from {querySql} as t limit {start},{query.PageSize}", parameters?.ToArray(), ConvertTraceDto);
}
return result;
Expand Down Expand Up @@ -76,7 +76,7 @@ public static List<TraceResponseDto> GetTraceByTraceId(this IDbConnection connec

public static string AppendOrderBy(BaseRequestDto query, bool isLog)
{
var str = query.Sort?.IsDesc ?? false ? " desc" : "";
var str = query.Sort?.IsDesc ?? true ? " desc" : "";
return $" order by Timestamp{str}";
}

Expand Down Expand Up @@ -242,7 +242,10 @@ private static void AppendField(FieldConditionDto item, List<IDataParameter> @pa
private static void ParseWhere(StringBuilder sql, object value, List<IDataParameter> @paramerters, string fieldName, string paramName, string compare)
{
DbType dbType = value is DateTime ? DbType.DateTime2 : DbType.AnsiString;
sql.Append($" and {fieldName} {compare} @{paramName}");
if (value is IEnumerable)
sql.Append($" and {fieldName} {compare} (@{paramName})");
else
sql.Append($" and {fieldName} {compare} @{paramName}");
@paramerters.Add(new ClickHouseParameter { ParameterName = $"{paramName}", Value = value, DbType = dbType });
}

Expand Down Expand Up @@ -350,7 +353,7 @@ public static LogResponseDto ConvertLogDto(IDataReader reader)
SeverityText = reader["SeverityText"].ToString()!,
TraceFlags = Convert.ToInt32(reader["TraceFlags"]),
SpanId = reader["SpanId"].ToString()!,
Timestamp = Convert.ToDateTime(reader["Timestamp"]),
Timestamp = Convert.ToDateTime(reader["Timestamp"]).ToLocalTime(),
};
if (!string.IsNullOrEmpty(resource))
result.Resource = JsonSerializer.Deserialize<Dictionary<string, object>>(resource)!;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ private static void InitTable(MasaStackClickhouseConnection connection)
var database = connection.ConnectionSettings?.Database;
database ??= new ClickHouseConnectionSettings(connection.ConnectionString).Database;

if (Convert.ToInt32(connection.ExecuteScalar($"select * from system.tables where database ='{database}' and name in ['{MasaStackClickhouseConnection.TraceTable}','{MasaStackClickhouseConnection.LogTable}']")) > 0)
if (Convert.ToInt32(connection.ExecuteScalar($"select count() from system.tables where database ='{database}' and name in ['{MasaStackClickhouseConnection.TraceTable.Split('.')[1]}','{MasaStackClickhouseConnection.LogTable.Split('.')[1]}']")) > 0)
return;

var createTableSqls = new string[]{
Expand All @@ -59,7 +59,7 @@ private static void InitTable(MasaStackClickhouseConnection connection)
`Resource.service.version` String CODEC(ZSTD(1)),
`Resource.service.instance.id` String CODEC(ZSTD(1)),

`Attributes.taskId` String CODEC(ZSTD(1)),
`Attributes.TaskId` String CODEC(ZSTD(1)),
`Attributes.exception.message` String CODEC(ZSTD(1)),

ResourceAttributesKeys Array(String) CODEC(ZSTD(1)),
Expand All @@ -71,7 +71,7 @@ INDEX idx_log_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_log_servicename ServiceName TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_log_serviceinstanceid `Resource.service.instance.id` TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_log_severitytext SeverityText TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_log_taskid `Attributes.taskId` TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_log_taskid `Attributes.TaskId` TYPE bloom_filter(0.001) GRANULARITY 1,

INDEX idx_string_body Body TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1,
INDEX idx_string_exceptionmessage Attributes.exception.message TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1
Expand Down Expand Up @@ -155,7 +155,7 @@ TTL toDateTime(Timestamp) + toIntervalDay(30)
ScopeSchemaUrl,ScopeName,ScopeVersion,toJSONString(ScopeAttributes) as Scopes,toJSONString(LogAttributes) as Logs,
ResourceAttributes['service.namespace'] as `Resource.service.namespace`,ResourceAttributes['service.version'] as `Resource.service.version`,
ResourceAttributes['service.instance.id'] as `Resource.service.instance.id`,
LogAttributes['TaskId'] as `Attributes.taskId`,LogAttributes['exception.message'] as `Attributes.exception.message`,
LogAttributes['TaskId'] as `Attributes.TaskId`,LogAttributes['exception.message'] as `Attributes.exception.message`,
mapKeys(ResourceAttributes) as ResourceAttributesKeys,mapValues(ResourceAttributes) as ResourceAttributesValues,
mapKeys(LogAttributes) as LogAttributesKeys,mapValues(LogAttributes) as LogAttributesValues
FROM {MasaStackClickhouseConnection.LogSourceTable};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
global using Masa.Utils.Models;
global using Microsoft.Extensions.DependencyInjection;
global using Microsoft.Extensions.Logging;
global using System.Collections;
global using System.Data;
global using System.Data.Common;
global using System.Text;
Expand Down
Loading